From f12d846e9f91bd8009777225f7789f3b87a39b56 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 04:55:27 -0400 Subject: [PATCH 01/54] native: build pipeline for the in-process toolchain MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit First step of replacing the embedded Alpine system with the native toolchain libraries. This adds the driver the app needs in order to build without a guest; the guest code is still present and still what XForgeEnvironment hands to the Build screen, so nothing behaves differently yet — but everything the native path needs now exists. - `NativeBuildPlan` / `NativeBuildPlanFactory`: read a package directory into a fully resolved plan (module, sources, resources, frameworks, search paths). It refuses, by name, the two things an in-process driver cannot do — SwiftPM dependency resolution and asset catalogs — instead of failing halfway. - `NativeToolchainInvocation`: pure argument construction for `swift-frontend` and `ld64.lld`. Pure on purpose: the compiler only exists in a device build, so a wrong argument list is otherwise undiscoverable until a build runs on a phone. The frontend convention is documented where it is implemented — the driver hands `performFrontend` everything after `-frontend`, so neither the program name nor `-frontend` may appear. - `NativeBuildExecutor`: the `BuildExecutor` the pipeline will use. Compiles Swift and C/Obj-C through the bridge, links with LLD, assembles `.app`, packages an `.ipa`, and keeps the UI responsive by running the blocking compiler calls off the main actor. - `NativeToolchainCapabilities`: reports clang / LLD / swift-frontend / SDK honestly. Clang and LLD alone can build, so the Swift frontend is not required to start — only to compile Swift, which it says plainly. - `NativeSDK.install(fromRemote:)`: install a Darwin SDK from an explicit URL, so a caller that already resolved the asset does not resolve it twice. - `NativeToolchainBridge.mm`: accept a CLI-shaped `-frontend …` argument list by stripping the flag, rather than handing `parseArgs` an argument it rejects. - Projects move to the app's own container: `Project.rootURL` derives the project directory from its name, and `XForgeEnvironment.projectsDirectory` is the single place that decides where that is. `rootPath` still holds the guest-shaped string this model was built around; removing it is the next step, and two stored locations for one project is how a build ends up reading a directory nobody created. Verified by 15 new unit tests (`NativeBuildTests`) covering the plan factory's acceptance and refusals, the frontend/linker argument lists, the main.swift and Swift-runtime cases, and search-path filtering. They run on a simulator, where the compiler is absent, which is exactly why the tested half is argument construction. --- App/Build/NativeBuildExecutor.swift | 455 +++++++++++++++++++ App/Build/NativeBuildPlan.swift | 279 ++++++++++++ App/Build/NativeToolchainCapabilities.swift | 55 +++ App/Build/NativeToolchainInvocation.swift | 117 +++++ App/Build/XForgeEnvironment.swift | 10 + App/Models/Project.swift | 9 + App/NativeToolchain/NativeSDK.swift | 36 ++ App/NativeToolchain/NativeToolchain.swift | 51 ++- App/NativeToolchain/NativeToolchainBridge.mm | 12 +- AppTests/NativeBuildTests.swift | 366 +++++++++++++++ 10 files changed, 1382 insertions(+), 8 deletions(-) create mode 100644 App/Build/NativeBuildExecutor.swift create mode 100644 App/Build/NativeBuildPlan.swift create mode 100644 App/Build/NativeToolchainCapabilities.swift create mode 100644 App/Build/NativeToolchainInvocation.swift create mode 100644 AppTests/NativeBuildTests.swift diff --git a/App/Build/NativeBuildExecutor.swift b/App/Build/NativeBuildExecutor.swift new file mode 100644 index 0000000..92d5045 --- /dev/null +++ b/App/Build/NativeBuildExecutor.swift @@ -0,0 +1,455 @@ +import Foundation + +/// Builds a project with the toolchain linked into this process — no guest, no +/// subprocess. +/// +/// The stages mirror the Alpine executor it replaces, because the pipeline's +/// shape (provision → SDK → configure → resolve → compile → package) is what the +/// Build screen presents; what changed is where the work happens. Everything here +/// is a library call in XForge's own process: +/// +/// - `swift-frontend` through `swift::performFrontend`, +/// - `clang` through `CompilerInvocation` + `EmitObjAction`, +/// - `ld64.lld` through `lld::lldMain`, +/// - the Darwin SDK read straight out of the app sandbox. +/// +/// What it deliberately does *not* do is invent capability. Resolving SwiftPM +/// dependencies and compiling asset catalogs both need a package manager and +/// `actool`, neither of which can run on iOS, so the plan factory refuses those +/// projects by name instead of failing halfway through a build. +@MainActor +final class NativeBuildExecutor: BuildExecutor { + private let stagingDirectory: URL + private let fileManager: FileManager + + init( + stagingDirectory: URL = XForgeEnvironment.stagingDirectory, + fileManager: FileManager = .default + ) { + self.stagingDirectory = stagingDirectory + self.fileManager = fileManager + } + + // MARK: - Bootstrap + + func bootstrap() async throws -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + Task { @MainActor in + let capabilities = NativeToolchainCapabilities.current + continuation.yield(.plan("Checking the native toolchain…")) + for line in capabilities.report { + continuation.yield(.output(line)) + } + + guard capabilities.canCompile else { + continuation.yield(.failed(capabilities.unavailableReason ?? "The native compiler is unavailable.")) + continuation.finish() + return + } + + continuation.yield(.plan("Checking the Darwin SDK…")) + do { + let layout = try NativeSDK.layout() + continuation.yield(.output("✓ SDK \(layout.sdkRoot.lastPathComponent)")) + if !capabilities.hasSwiftFrontend { + continuation.yield(.output( + "! The Swift frontend is not linked into this build: " + + "C and Objective-C sources compile, Swift does not." + )) + } + } catch { + continuation.yield(.failed( + "\(error.localizedDescription) Install it from the Toolchain screen." + )) + continuation.finish() + return + } + + continuation.yield(.finished) + continuation.finish() + } + } + } + + // MARK: - SDK + + func installSDK(from source: SDKSource) async throws { + switch source { + case .hostedRemote(let url): + try await NativeSDK.install(fromRemote: url) + case .bundled: + // A guest path is meaningless without the guest; say so instead of + // looking for a file that can never exist here. + throw NativeBuildError.unusableSDKSource + } + } + + // MARK: - Project creation + + /// Write a new package into the app's own container. + /// + /// The template is the smallest thing that both SwiftPM and this driver + /// understand: a `Package.swift` with one executable product, one SwiftUI + /// entry point, and the app metadata XForge keeps in `xtool.yml`. + func createProject(named name: String, organizationIdentifier: String) async throws -> Project { + let validated = try Project.validatedName(name) + let project = Project( + name: validated, + organizationIdentifier: organizationIdentifier, + rootPath: Project.path(forValidatedName: validated) + ) + let root = project.rootURL + let sources = root.appendingPathComponent("Sources/\(validated)", isDirectory: true) + + guard !fileManager.fileExists(atPath: root.path) else { + throw NativeBuildError.projectAlreadyExists(root) + } + try fileManager.createDirectory(at: sources, withIntermediateDirectories: true) + + let manifest = """ + // swift-tools-version: 6.0 + import PackageDescription + + let package = Package( + name: "\(validated)", + platforms: [.iOS(.v17)], + products: [ + .executable(name: "\(validated)", targets: ["\(validated)"]) + ], + targets: [ + .executableTarget( + name: "\(validated)", + path: "Sources/\(validated)" + ) + ] + ) + """ + try manifest.write( + to: root.appendingPathComponent("Package.swift"), + atomically: true, + encoding: .utf8 + ) + + let entry = """ + import SwiftUI + + @main + struct \(validated)App: App { + var body: some Scene { + WindowGroup { + ContentView() + } + } + } + + struct ContentView: View { + var body: some View { + VStack(spacing: 12) { + Image(systemName: "hammer.fill").font(.largeTitle) + Text("\(validated)").font(.title2.bold()) + Text("Built by XForge on device.").foregroundStyle(.secondary) + } + .padding() + } + } + """ + try entry.write( + to: sources.appendingPathComponent("\(validated)App.swift"), + atomically: true, + encoding: .utf8 + ) + + let metadata = """ + name: \(validated) + bundleIdentifier: \(organizationIdentifier).\(validated) + deploymentTarget: "17.0" + """ + try metadata.write( + to: root.appendingPathComponent("xtool.yml"), + atomically: true, + encoding: .utf8 + ) + + return project + } + + // MARK: - Resolve + + func resolve(_ project: Project) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + Task { @MainActor in + do { + let manifest = project.rootURL.appendingPathComponent("Package.swift") + let dependencies = try NativeBuildPlanFactory.swiftPackageDependencies( + manifestURL: manifest + ) + guard dependencies.isEmpty else { + continuation.yield(.plan("Resolving dependencies…")) + throw NativeBuildPlanError.unsupportedSwiftPackageDependencies(dependencies) + } + continuation.yield(.plan("Resolving dependencies…")) + continuation.yield(.output("No package dependencies: nothing to resolve.")) + continuation.yield(.finished) + continuation.finish() + } catch { + continuation.yield(.failed(error.localizedDescription)) + continuation.finish() + } + } + } + } + + // MARK: - Build + + func build( + _ project: Project, + configuration: BuildConfiguration + ) -> AsyncThrowingStream { + AsyncThrowingStream { continuation in + Task { @MainActor in + do { + try await self.build(project, configuration: configuration) { event in + continuation.yield(event) + } + continuation.finish() + } catch { + continuation.yield(.failed(error.localizedDescription)) + continuation.finish() + } + } + } + } + + /// The pipeline proper. Kept off the stream so failures can use `throw` and + /// carry their diagnostics, rather than being flattened into a string. + private func build( + _ project: Project, + configuration: BuildConfiguration, + emit: @MainActor (BuildEvent) -> Void + ) async throws { + let capabilities = NativeToolchainCapabilities.current + guard capabilities.canCompile else { + throw NativeBuildError.toolchainUnavailable(capabilities.unavailableReason) + } + + let sdk = try NativeSDK.layout() + let appInfo = project.appInfo ?? .default(for: project) + let plan = try NativeBuildPlanFactory.makePlan( + root: project.rootURL, + appInfo: appInfo, + configuration: configuration, + sdk: sdk + ) + + emit(.plan("Compiling \(plan.moduleName) for arm64-apple-ios\(plan.minimumIOSVersion)…")) + emit(.output("\(plan.swiftSources.count) Swift, \(plan.clangSources.count) C, \(plan.resources.count) resource files")) + + if !plan.swiftSources.isEmpty, !capabilities.hasSwiftFrontend { + throw NativeBuildError.swiftFrontendMissing(plan.swiftSources.count) + } + + let buildDirectory = project.rootURL.appendingPathComponent(".xforge-build", isDirectory: true) + try? fileManager.removeItem(at: buildDirectory) + try fileManager.createDirectory(at: buildDirectory, withIntermediateDirectories: true) + + var objects: [URL] = [] + + for source in plan.swiftSources { + let object = buildDirectory + .appendingPathComponent(objectName(for: source)) + let arguments = try NativeToolchainInvocation.swiftFrontendArguments( + source: source, + object: object, + plan: plan, + sdk: sdk + ) + emit(.output("swift-frontend \(source.lastPathComponent)")) + let result = try await runDetached { + NativeToolchain.runSwiftFrontend(arguments: arguments) + } + try check(result, tool: "swift-frontend", source: source) + objects.append(object) + } + + for source in plan.clangSources { + let object = buildDirectory + .appendingPathComponent(objectName(for: source.url)) + emit(.output("clang \(source.url.lastPathComponent)")) + let result = try await runDetached { + try NativeToolchain.compileC( + source: source.url, + object: object, + sdk: sdk.sdkRoot, + target: NativeToolchainInvocation.targetTriple( + minimumIOSVersion: plan.minimumIOSVersion + ), + language: source.language + ) + } + try check(result, tool: "clang", source: source.url) + objects.append(object) + } + + let executable = buildDirectory.appendingPathComponent(plan.executableName) + emit(.plan("Linking \(plan.executableName)…")) + let linkerArguments = NativeToolchainInvocation.linkerArguments( + objects: objects, + plan: plan, + sdk: sdk, + executable: executable + ) + let linkResult = try await runDetached { + try NativeToolchain.linkMachO(arguments: linkerArguments) + } + guard linkResult.succeeded else { + throw NativeBuildError.linkFailed(linkResult.diagnostics) + } + emit(.output("ld64.lld → \(plan.executableName)")) + + let bundle = try assembleBundle(plan: plan, executable: executable, emit: emit) + let ipa = try IPABuilder.buildIPA( + appBundle: bundle, + appInfo: appInfo, + outputDir: buildDirectory + ) + + try fileManager.createDirectory(at: stagingDirectory, withIntermediateDirectories: true) + let staged = stagingDirectory.appendingPathComponent("\(project.name)-\(appInfo.version).ipa") + try? fileManager.removeItem(at: staged) + try fileManager.copyItem(at: ipa, to: staged) + + emit(.output("Packaged \(staged.lastPathComponent) (unsigned — sign it on the Signing screen).")) + emit(.artifact(staged)) + emit(.finished) + } + + // MARK: - Bundle assembly + + /// `.app` with the executable, an Info.plist and the resources. + private func assembleBundle( + plan: NativeBuildPlan, + executable: URL, + emit: @MainActor (BuildEvent) -> Void + ) throws -> URL { + let bundle = plan.root + .appendingPathComponent(".xforge-build", isDirectory: true) + .appendingPathComponent("\(plan.appName).app", isDirectory: true) + if fileManager.fileExists(atPath: bundle.path) { + try fileManager.removeItem(at: bundle) + } + try fileManager.createDirectory(at: bundle, withIntermediateDirectories: true) + + let destination = bundle.appendingPathComponent(plan.executableName) + try fileManager.copyItem(at: executable, to: destination) + try fileManager.setAttributes([.posixPermissions: 0o755], ofItemAtPath: destination.path) + + var info: [String: Any] = [ + "CFBundleIdentifier": plan.bundleIdentifier, + "CFBundleName": plan.moduleName, + "CFBundleDisplayName": plan.appName, + "CFBundleExecutable": plan.executableName, + "CFBundlePackageType": "APPL", + "CFBundleShortVersionString": plan.version, + "CFBundleVersion": plan.buildNumber, + "MinimumOSVersion": plan.minimumIOSVersion, + "LSRequiresIPhoneOS": true, + "UIDeviceFamily": [1], + "UILaunchScreen": [String: Any](), + "UIRequiredDeviceCapabilities": ["arm64"], + ] + // A project's own Info.plist wins over the generated keys, which is what + // makes hand-written scene manifests and orientations survive a build. + if let template = plan.infoPlistTemplate, + let data = try? Data(contentsOf: template), + let decoded = try? PropertyListSerialization.propertyList(from: data, format: nil) + as? [String: Any] { + info.merge(decoded) { _, project in project } + } + + let plist = try PropertyListSerialization.data( + fromPropertyList: info, + format: .xml, + options: 0 + ) + try plist.write(to: bundle.appendingPathComponent("Info.plist"), options: .atomic) + + for resource in plan.resources { + if resource.pathExtension.lowercased() == "xcassets" { + // There is no actool on iOS: asset catalogs are compiled by a + // macOS-only tool. Copy the catalog so the project round-trips + // and say plainly what is missing rather than shipping an app + // whose icon silently vanishes. + emit(.output( + "! \(resource.lastPathComponent) was copied uncompiled: " + + "asset catalogs need actool, which cannot run on iOS." + )) + } + let target = bundle.appendingPathComponent(resource.lastPathComponent) + try? fileManager.removeItem(at: target) + try fileManager.copyItem(at: resource, to: target) + } + + return bundle + } + + // MARK: - Helpers + + private func objectName(for source: URL) -> String { + // Path separators cannot appear in a file name, and two sources with the + // same last path component would otherwise collide. + source.path + .replacingOccurrences(of: "/", with: "_") + .replacingOccurrences(of: "..", with: "_") + + ".o" + } + + private func check(_ result: NativeToolchainResult, tool: String, source: URL) throws { + guard result.succeeded else { + throw NativeBuildError.compileFailed( + tool: tool, + source: source, + diagnostics: result.diagnostics + ) + } + } + + /// Run a blocking compiler call off the main actor. The frontend and linker + /// are synchronous C++ entry points: a large module can take seconds, and the + /// UI must keep drawing while it does. + private func runDetached(_ work: @escaping @Sendable () throws -> T) async throws -> T { + try await Task.detached(priority: .userInitiated, operation: work).value + } +} + +enum NativeBuildError: LocalizedError { + case toolchainUnavailable(String?) + case swiftFrontendMissing(Int) + case unusableSDKSource + case projectAlreadyExists(URL) + case compileFailed(tool: String, source: URL, diagnostics: String) + case linkFailed(String) + + var errorDescription: String? { + switch self { + case .toolchainUnavailable(let reason): + return reason ?? "The native compiler toolchain is not linked into this build." + case .swiftFrontendMissing(let count): + return """ + \(count) Swift source file(s) must be compiled, but the Swift frontend \ + is not linked into this build of XForge. Only the C/C++/Objective-C \ + half of the toolchain is present. + """ + case .unusableSDKSource: + return "That SDK source belongs to the embedded Linux guest, which no longer exists. Install the Darwin SDK bundle instead." + case .projectAlreadyExists(let url): + return "A project already exists at \(url.path)." + case .compileFailed(let tool, let source, let diagnostics): + let text = diagnostics.trimmingCharacters(in: .whitespacesAndNewlines) + let detail = text.isEmpty ? "\(tool) reported a failure without diagnostics." : text + return "\(source.lastPathComponent): \(detail)" + case .linkFailed(let diagnostics): + let text = diagnostics.trimmingCharacters(in: .whitespacesAndNewlines) + return text.isEmpty ? "The linker failed without diagnostics." : text + } + } +} diff --git a/App/Build/NativeBuildPlan.swift b/App/Build/NativeBuildPlan.swift new file mode 100644 index 0000000..f434628 --- /dev/null +++ b/App/Build/NativeBuildPlan.swift @@ -0,0 +1,279 @@ +import Foundation + +/// A resolved, self-contained description of what to compile and link. +/// +/// The plan is the boundary between "reading a project" and "running a compiler". +/// It exists so the on-device driver is *not* a second build system: every source +/// file, framework and search path is decided here, and the executor only turns +/// this value into `swift-frontend` / `clang` / `ld64.lld` argument lists. That +/// split is what makes the argument construction testable on a simulator, where +/// no compiler libraries exist. +struct NativeBuildPlan: Sendable, Equatable { + /// Directory holding `Package.swift` / the sources. + var root: URL + /// The product being built, e.g. `Sources/MyApp` → `MyApp`. + var moduleName: String + /// Display name of the produced app bundle (`.app`). + var appName: String + /// Mach-O executable name inside the bundle. + var executableName: String + var bundleIdentifier: String + var minimumIOSVersion: String + var version: String + var buildNumber: String + var configuration: BuildConfiguration + + /// Swift translation units, in a stable order. + var swiftSources: [URL] + /// C / Objective-C / Objective-C++ translation units. + var clangSources: [NativeClangSource] + /// Files copied into the bundle unchanged. + var resources: [URL] + /// A hand-written Info.plist in the project, merged over the generated one. + var infoPlistTemplate: URL? + /// Frameworks every app needs; the plan adds the project's own on top. + var frameworks: [String] + /// Extra `-L` paths, e.g. the Swift runtime inside the Darwin SDK. + var librarySearchPaths: [URL] + /// Extra `-F` paths. + var frameworkSearchPaths: [URL] + + var isEmpty: Bool { + swiftSources.isEmpty && clangSources.isEmpty + } + + var hasSwift: Bool { !swiftSources.isEmpty } +} + +/// One non-Swift translation unit plus the language clang must be told to use. +struct NativeClangSource: Sendable, Equatable, Hashable { + var url: URL + /// `c`, `objective-c`, `c++`, `objective-c++`. + var language: String + + static func language(for url: URL) -> String? { + switch url.pathExtension.lowercased() { + case "c": return "c" + case "m": return "objective-c" + case "mm": return "objective-c++" + case "cc", "cpp", "cxx": return "c++" + default: return nil + } + } +} + +enum NativeBuildPlanError: LocalizedError { + case noSources(URL) + case missingTargetsDirectory(URL) + case unsupportedSwiftPackageDependencies([String]) + case missingPackageManifest(URL) + + var errorDescription: String? { + switch self { + case .noSources(let root): + return "No compilable sources were found under \(root.path)." + case .missingTargetsDirectory(let root): + return "This project has no Sources/ directory, so XForge cannot tell what to build (\(root.path))." + case .unsupportedSwiftPackageDependencies(let names): + let list = names.prefix(5).joined(separator: ", ") + let more = names.count > 5 ? " and \(names.count - 5) more" : "" + return """ + This project declares SwiftPM dependencies (\(list)\(more)). Resolving \ + them needs a package manager, which cannot run inside an app that may \ + not spawn processes. Remove the dependencies, or build a project whose \ + sources are self-contained. + """ + case .missingPackageManifest(let root): + return "No Package.swift was found at \(root.path)." + } + } +} + +/// Turns a project directory into a `NativeBuildPlan`. +/// +/// Deliberately conservative: it reads the layout SwiftPM and `xtool new` produce +/// (`Package.swift`, `Sources//**`) and refuses anything it cannot resolve +/// exactly, rather than guessing and producing a binary that links the wrong code. +enum NativeBuildPlanFactory { + static func makePlan( + root: URL, + appInfo: AppInfo, + configuration: BuildConfiguration, + sdk: NativeSDKLayout, + fileManager: FileManager = .default + ) throws -> NativeBuildPlan { + let manifest = root.appendingPathComponent("Package.swift") + guard fileManager.fileExists(atPath: manifest.path) else { + throw NativeBuildPlanError.missingPackageManifest(root) + } + + // Dependencies are the one thing an in-process driver cannot resolve: + // SwiftPM resolves them by fetching and *running* package manifests. + // Refuse them by name instead of failing somewhere further down. + let dependencies = try swiftPackageDependencies(manifestURL: manifest) + if !dependencies.isEmpty { + throw NativeBuildPlanError.unsupportedSwiftPackageDependencies(dependencies) + } + + let sourcesRoot = root.appendingPathComponent("Sources", isDirectory: true) + var isDirectory: ObjCBool = false + guard fileManager.fileExists(atPath: sourcesRoot.path, isDirectory: &isDirectory), + isDirectory.boolValue else { + throw NativeBuildPlanError.missingTargetsDirectory(root) + } + + let (moduleName, sourcesDirectory) = try target(in: sourcesRoot, fileManager: fileManager) + let swiftSources = try files(in: sourcesDirectory, extensions: ["swift"], fileManager: fileManager) + let clangSources = try files( + in: sourcesDirectory, + extensions: ["c", "m", "mm", "cc", "cpp", "cxx"], + fileManager: fileManager + ).compactMap { url -> NativeClangSource? in + guard let language = NativeClangSource.language(for: url) else { return nil } + return NativeClangSource(url: url, language: language) + } + + guard !swiftSources.isEmpty || !clangSources.isEmpty else { + throw NativeBuildPlanError.noSources(sourcesDirectory) + } + + // Resources live next to the sources in the layouts XForge creates and in + // the ones `xtool new` produces. + let resources = try files( + in: sourcesDirectory, + extensions: ["xcassets", "json", "plist", "strings", "storyboard", "xib", "png", "jpg"], + fileManager: fileManager + ).filter { !$0.lastPathComponent.hasPrefix("Info.plist") } + + var searchPaths: [URL] = [] + var frameworkSearchPaths: [URL] = [] + var frameworks = ["Foundation"] + if !swiftSources.isEmpty { + // A Swift executable links the Swift runtime statically: the Darwin + // SDK bundle carries it, and there is no dyld-shared runtime on a + // sideloaded device build. + searchPaths.append(contentsOf: sdk.swiftRuntimeLibraryPaths) + frameworks.append("UIKit") + } + + let template = root.appendingPathComponent("Support/Info.plist") + let hasTemplate = fileManager.fileExists(atPath: template.path) + + return NativeBuildPlan( + root: root, + moduleName: moduleName, + appName: appInfo.displayName, + executableName: moduleName, + bundleIdentifier: appInfo.bundleIdentifier, + minimumIOSVersion: appInfo.minimumOSVersion, + version: appInfo.version, + buildNumber: appInfo.buildNumber, + configuration: configuration, + swiftSources: swiftSources, + clangSources: clangSources, + resources: resources, + infoPlistTemplate: hasTemplate ? template : nil, + frameworks: frameworks, + librarySearchPaths: searchPaths, + frameworkSearchPaths: frameworkSearchPaths + ) + } + + /// The package's declared `.package(url: "…")` dependencies, by name. + /// + /// A text scan on purpose: a full manifest evaluation means compiling and + /// *running* `Package.swift`, which is exactly what this path cannot do. The + /// scan only needs to answer "are there any", which it does reliably; the + /// names are for the error message. + static func swiftPackageDependencies(manifestURL: URL) throws -> [String] { + guard let text = try? String(contentsOf: manifestURL, encoding: .utf8) else { + return [] + } + var names: [String] = [] + for line in text.split(separator: "\n") { + let trimmed = line.trimmingCharacters(in: .whitespaces) + guard trimmed.hasPrefix(".package(") else { continue } + if let name = quotedName(in: trimmed) { + names.append(name) + } else { + names.append("an unnamed package") + } + } + return names + } + + /// The last quoted path-ish component of a `.package(...)` line, e.g. + /// `…/swift-argument-parser.git` → `swift-argument-parser`. + private static func quotedName(in line: String) -> String? { + guard let firstQuote = line.firstIndex(of: "\"") else { return nil } + let rest = line[line.index(after: firstQuote)...] + guard let lastQuote = rest.firstIndex(of: "\"") else { return nil } + let value = String(rest[..` when there is exactly one, the + /// project's own name when several exist, and the only directory otherwise. + private static func target( + in sourcesRoot: URL, + fileManager: FileManager + ) throws -> (name: String, directory: URL) { + let entries = (try? fileManager.contentsOfDirectory( + at: sourcesRoot, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + )) ?? [] + let directories = entries.filter { + (try? $0.resourceValues(forKeys: [.isDirectoryKey]))?.isDirectory == true + } + guard !directories.isEmpty else { + throw NativeBuildPlanError.noSources(sourcesRoot) + } + guard directories.count > 1 else { + return (directories[0].lastPathComponent, directories[0]) + } + // Several targets: the app target is the one named after the package, + // which is the layout `xtool new` and XForge both produce. + let packageName = sourcesRoot.deletingLastPathComponent().lastPathComponent + if let match = directories.first(where: { $0.lastPathComponent == packageName }) { + return (match.lastPathComponent, match) + } + throw NativeBuildPlanError.noSources(sourcesRoot) + } + + /// Regular files with one of `extensions`, recursively, sorted by path. + private static func files( + in directory: URL, + extensions: Set, + fileManager: FileManager + ) throws -> [URL] { + guard let enumerator = fileManager.enumerator( + at: directory, + includingPropertiesForKeys: [.isRegularFileKey], + options: [.skipsHiddenFiles, .skipsPackageDescendants] + ) else { + return [] + } + var found: [URL] = [] + for case let url as URL in enumerator { + guard extensions.contains(url.pathExtension.lowercased()) else { continue } + // Asset catalogs are directories, and they are copied whole. + if extensions.contains("xcassets") { + if url.pathExtension.lowercased() == "xcassets" { + found.append(url) + enumerator.skipDescendants() + } else if (try? url.resourceValues(forKeys: [.isRegularFileKey]))?.isRegularFile == true { + found.append(url) + } + continue + } + if (try? url.resourceValues(forKeys: [.isRegularFileKey]))?.isRegularFile == true { + found.append(url) + } + } + return found.sorted { $0.path < $1.path } + } +} diff --git a/App/Build/NativeToolchainCapabilities.swift b/App/Build/NativeToolchainCapabilities.swift new file mode 100644 index 0000000..2a1f4bc --- /dev/null +++ b/App/Build/NativeToolchainCapabilities.swift @@ -0,0 +1,55 @@ +import Foundation + +/// What this build of XForge can actually compile with. +/// +/// The toolchain is linked in at *link* time through +/// `Support/NativeToolchain.generated.xcconfig`, so a plain clone — and every CI +/// build — has no compiler at all, while a build that installed the toolchain +/// bundle has some or all of it (Clang and LLD for a long time, the Swift +/// frontend only once its libraries exist for iOS). Reporting that honestly is +/// the difference between "this project needs a toolchain" and a build that fails +/// somewhere in the middle with a linker error. +struct NativeToolchainCapabilities: Sendable, Equatable { + let hasClang: Bool + let hasLLDMachO: Bool + let hasSwiftFrontend: Bool + let backendDescription: String + let sdkInstalled: Bool + + static var current: NativeToolchainCapabilities { + NativeToolchainCapabilities( + hasClang: NativeToolchain.isAvailable, + hasLLDMachO: NativeToolchain.isAvailable, + hasSwiftFrontend: NativeToolchain.isSwiftAvailable, + backendDescription: NativeToolchain.isAvailable + ? NativeToolchain.version + : "not linked", + sdkInstalled: NativeSDK.isInstalled + ) + } + + /// Whether anything can be compiled at all. The Swift frontend is deliberately + /// not required: a C/Objective-C target links fine without it, and refusing to + /// start would be wrong about why the build fails. + var canCompile: Bool { hasClang && hasLLDMachO } + + var unavailableReason: String? { + guard !canCompile else { return nil } + return """ + The native compiler is not linked into this build of XForge, so it cannot \ + compile anything. Install the toolchain bundle (NativeToolchain/install-bundle.sh) \ + and rebuild, or use a release that ships it. + """ + } + + /// Lines for the build console: one per component, present or missing. + var report: [String] { + [ + "clang: \(hasClang ? "linked" : "missing")", + "ld64.lld: \(hasLLDMachO ? "linked" : "missing")", + "swift-frontend: \(hasSwiftFrontend ? "linked" : "missing")", + "darwin SDK: \(sdkInstalled ? "installed" : "missing")", + "backend: \(backendDescription)", + ] + } +} diff --git a/App/Build/NativeToolchainInvocation.swift b/App/Build/NativeToolchainInvocation.swift new file mode 100644 index 0000000..14d7e82 --- /dev/null +++ b/App/Build/NativeToolchainInvocation.swift @@ -0,0 +1,117 @@ +import Foundation + +/// Argument construction for the in-process toolchain. +/// +/// Pure functions, on purpose: the compiler itself only exists inside a device +/// build (the LLVM/Swift libraries are linked in by `NativeToolchain.generated.xcconfig`, +/// which is disabled on a plain clone), so a mis-built argument list could only be +/// discovered on a phone with a real SDK. Splitting it out means the unit tests +/// exercise the exact strings the frontend and linker receive, on a simulator, +/// with no compiler present. +enum NativeToolchainInvocation { + /// `arm64-apple-ios17.0` — the triple every iOS arm64 tool in the stack uses. + static func targetTriple(minimumIOSVersion: String) -> String { + "arm64-apple-ios\(minimumIOSVersion)" + } + + /// `swift-frontend` arguments, exactly as `swift::performFrontend` expects + /// them. + /// + /// The convention is not obvious and is worth stating: the driver's own main + /// calls `performFrontend(argv[2…], argv[0], …)` for a `swift-frontend -frontend …` + /// command line, so the array must contain neither the program name nor + /// `-frontend`. Passing `-frontend` makes `CompilerInvocation::parseArgs` + /// reject the whole invocation. + static func swiftFrontendArguments( + source: URL, + object: URL, + plan: NativeBuildPlan, + sdk: NativeSDKLayout + ) throws -> [String] { + guard let swiftResources = sdk.swiftResources else { + throw NativeToolchainError.io( + "The Darwin SDK does not declare swiftResourcesPath, so the Swift standard library cannot be found." + ) + } + + var arguments = [ + // `-parse-as-library` unless the file *is* main.swift: an app's entry + // point is `@main`, and without this the frontend treats top-level + // code as the entry point and rejects the attribute. + "-parse-as-library", + "-c", source.path, + "-target", targetTriple(minimumIOSVersion: plan.minimumIOSVersion), + "-sdk", sdk.sdkRoot.path, + "-resource-dir", swiftResources.path, + "-module-name", plan.moduleName, + "-o", object.path, + ] + if source.lastPathComponent == "main.swift" { + arguments.removeFirst() + } + arguments.append(plan.configuration == .release ? "-O" : "-Onone") + // DebugInfo makes a crash in a built app diagnosable; it costs object size + // only, and the linker strips what the app does not use. + arguments.append(contentsOf: ["-g", "-debug-info-format=dwarf"]) + return arguments + } + + /// `ld64.lld` arguments for an arm64 iOS executable. + /// + /// `fileExists` is injected so the Swift runtime libraries are only named when + /// the SDK bundle actually carries them: naming `-lswift_Concurrency` for a + /// bundle that predates the split would fail the link with "library not + /// found", and a Swift 5.7+ runtime has it inside `libswiftCore` anyway. + static func linkerArguments( + objects: [URL], + plan: NativeBuildPlan, + sdk: NativeSDKLayout, + executable: URL, + fileExists: (URL) -> Bool = { FileManager.default.fileExists(atPath: $0.path) } + ) -> [String] { + let deploymentTarget = plan.minimumIOSVersion + var arguments = [ + "-arch", "arm64", + // lld needs the platform *and* the version pair explicitly; without + // it the Mach-O records no LC_BUILD_VERSION and iOS refuses to load + // the binary. + "-platform_version", "ios", deploymentTarget, deploymentTarget, + "-syslibroot", sdk.sdkRoot.path, + "-o", executable.path, + ] + + for path in plan.librarySearchPaths where fileExists(path) { + arguments.append(contentsOf: ["-L", path.path]) + } + for path in plan.frameworkSearchPaths where fileExists(path) { + arguments.append(contentsOf: ["-F", path.path]) + } + + if plan.hasSwift { + arguments.append(contentsOf: ["-lswiftCore", "-lswift_Concurrency", "-lswift_StringProcessing"] + .filter { library in + // -lswiftCore is unconditional: a Swift binary cannot run + // without it. The other two merged into it over time. + library == "-lswiftCore" || swiftLibraryExists(library, sdk: sdk, fileExists: fileExists) + }) + } + + arguments.append(contentsOf: ["-lSystem"]) + for framework in plan.frameworks { + arguments.append(contentsOf: ["-framework", framework]) + } + arguments.append(contentsOf: objects.map(\.path)) + return arguments + } + + private static func swiftLibraryExists( + _ flag: String, + sdk: NativeSDKLayout, + fileExists: (URL) -> Bool + ) -> Bool { + let name = String(flag.dropFirst(2)) // "-lswift_Concurrency" → "swift_Concurrency" + return sdk.swiftRuntimeLibraryPaths.contains { directory in + fileExists(directory.appendingPathComponent("lib\(name).a")) + } + } +} diff --git a/App/Build/XForgeEnvironment.swift b/App/Build/XForgeEnvironment.swift index 39a0ee2..f2f9c87 100644 --- a/App/Build/XForgeEnvironment.swift +++ b/App/Build/XForgeEnvironment.swift @@ -47,6 +47,16 @@ enum XForgeEnvironment { documentDirectory.appendingPathComponent("native-sdk", isDirectory: true) } + /// Projects live in the app's own container. + /// + /// They used to live inside the embedded Linux filesystem (`/root/projects`), + /// which is why `Project.rootPath` still carries a guest-shaped string. Now + /// that the toolchain runs in-process, a project is a directory of files this + /// process can open, and this is the one place that decides where. + nonisolated static var projectsDirectory: URL { + documentDirectory.appendingPathComponent("projects", isDirectory: true) + } + /// Diagnostics (`XForgeLog` writes the engine log here). static var logsDirectory: URL { XForgeLog.directory } diff --git a/App/Models/Project.swift b/App/Models/Project.swift index 0cc9641..00aca36 100644 --- a/App/Models/Project.swift +++ b/App/Models/Project.swift @@ -15,6 +15,15 @@ struct Project: Identifiable, Hashable, Codable { var packageManifestPath: String { "\(rootPath)/Package.swift" } var ipaOutputPath: String { "\(rootPath)/.build/xforge-\(name).ipa" } + /// The project directory in XForge's own container. + /// + /// Derived from `name` rather than stored. `rootPath` is the guest path this + /// model was designed around, and two stored locations for one project is how + /// a build ends up reading a directory nobody created. + var rootURL: URL { + XForgeEnvironment.projectsDirectory.appendingPathComponent(name, isDirectory: true) + } + static func validatedName(_ value: String) throws -> String { let name = value.trimmingCharacters(in: .whitespacesAndNewlines) let allowed = CharacterSet.alphanumerics.union(CharacterSet(charactersIn: "-_")) diff --git a/App/NativeToolchain/NativeSDK.swift b/App/NativeToolchain/NativeSDK.swift index b54b33a..56c389b 100644 --- a/App/NativeToolchain/NativeSDK.swift +++ b/App/NativeToolchain/NativeSDK.swift @@ -9,6 +9,30 @@ struct NativeSDKLayout: Sendable { let platformLibrarySearchPaths: [URL] } +extension NativeSDKLayout { + /// `-L` paths that make `-lswiftCore` and friends resolvable when linking + /// statically. + /// + /// A sideloaded app links the Swift runtime into the binary — there is no + /// system-wide `/usr/lib/swift` on iOS to load it from at runtime — so the + /// linker has to be pointed at the static runtime the Darwin SDK carries. + /// `swift-sdk.json` names it as `swiftStaticResourcesPath`, and the compiled + /// libraries for the device live one directory below it (`…/iphoneos`). + /// `librarySearchPaths` from the same file is the fallback for bundles that + /// lay it out differently, and both are merged rather than chosen between: + /// a duplicate `-L` costs nothing, a missing one fails the link. + var swiftRuntimeLibraryPaths: [URL] { + var paths: [URL] = [] + if let swiftStaticResources { + paths.append(swiftStaticResources) + paths.append(swiftStaticResources.appendingPathComponent("iphoneos", isDirectory: true)) + } + paths.append(contentsOf: platformLibrarySearchPaths) + var seen = Set() + return paths.filter { seen.insert($0.path).inserted } + } +} + enum NativeSDKError: LocalizedError { case missingBundle case invalidMetadata @@ -107,11 +131,23 @@ enum NativeSDK { static func installLatestPrebuilt() async throws { let remote = try await XForgeReleases.darwinSDKURL() + try await install(fromRemote: remote) + } + + /// Download and install a Darwin SDK archive from an explicit URL. + /// + /// The URL-parameterised form is the one the build pipeline uses: the release + /// lookup happens in the caller, so a caller that already resolved an asset + /// (or pinned one) does not pay for a second lookup. + static func install(fromRemote remote: URL) async throws { let (downloaded, response) = try await URLSession.shared.download(from: remote) if let http = response as? HTTPURLResponse, !(200..<300).contains(http.statusCode) { throw DownloadError.http(status: http.statusCode, url: remote) } + try installDownloadedArchive(at: downloaded) + } + private static func installDownloadedArchive(at downloaded: URL) throws { let fm = FileManager.default let extractionRoot = XForgeEnvironment.nativeSDKDirectory .appendingPathComponent("download-\(UUID().uuidString)", isDirectory: true) diff --git a/App/NativeToolchain/NativeToolchain.swift b/App/NativeToolchain/NativeToolchain.swift index 46f1e50..e2b963a 100644 --- a/App/NativeToolchain/NativeToolchain.swift +++ b/App/NativeToolchain/NativeToolchain.swift @@ -139,22 +139,59 @@ enum NativeToolchain { throw NativeToolchainError.link("No LLD arguments were supplied.") } + return try marshal(arguments, tool: "ld64.lld", failure: { + NativeToolchainError.link($0) + }) { argc, argv, diagnostics, capacity in + xf_native_lld_link(argc, argv, diagnostics, capacity) + } + } + + /// Compile through `swift::performFrontend` with a caller-supplied argument + /// list. + /// + /// The argument list is built by `NativeToolchainInvocation`, not here: it is + /// pure, so it can be unit-tested on a simulator, where this call cannot even + /// be linked. The convention `performFrontend` expects is documented there. + static func runSwiftFrontend(arguments: [String]) throws -> NativeToolchainResult { + guard isAvailable, isSwiftAvailable else { + throw NativeToolchainError.unavailable + } + guard !arguments.isEmpty else { + throw NativeToolchainError.compile("No swift-frontend arguments were supplied.") + } + return try marshal(arguments, tool: "swift-frontend", failure: { + NativeToolchainError.compile($0) + }) { argc, argv, diagnostics, capacity in + xf_native_swift_frontend(argc, argv, diagnostics, capacity) + } + } + + /// Marshal a Swift `[String]` into the C ABI's `char *const *` and collect the + /// diagnostics buffer both entry points share. + private static func marshal( + _ arguments: [String], + tool: String, + failure: (String) -> Error, + body: (Int32, UnsafePointer?>?, UnsafeMutablePointer, Int) -> Int32 + ) throws -> NativeToolchainResult { let duplicated: [UnsafeMutablePointer] = arguments.compactMap { strdup($0) } defer { duplicated.forEach { free($0) } } guard duplicated.count == arguments.count else { - throw NativeToolchainError.io("Could not allocate linker arguments.") + throw failure("Could not allocate \(tool) arguments.") } let argv: [UnsafePointer?] = duplicated.map { UnsafePointer($0) } var diagnostics = [CChar](repeating: 0, count: 64 * 1024) let diagnosticsCapacity = diagnostics.count let code = argv.withUnsafeBufferPointer { buffer -> Int32 in - Int32(xf_native_lld_link( - Int32(arguments.count), - buffer.baseAddress, - &diagnostics, - diagnosticsCapacity - )) + diagnostics.withUnsafeMutableBufferPointer { storage in + body( + Int32(arguments.count), + buffer.baseAddress, + storage.baseAddress!, + diagnosticsCapacity + ) + } } return NativeToolchainResult( exitCode: code, diff --git a/App/NativeToolchain/NativeToolchainBridge.mm b/App/NativeToolchain/NativeToolchainBridge.mm index 79fc1cb..4d3441e 100644 --- a/App/NativeToolchain/NativeToolchainBridge.mm +++ b/App/NativeToolchain/NativeToolchainBridge.mm @@ -65,7 +65,17 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity xf_copy_diag("native swift: empty frontend argument list", diagnostics, diagnostics_capacity); return 64; } - llvm::ArrayRef args(argv, static_cast(argc)); + // `swift-frontend -frontend …` is how the driver invokes the frontend, and it + // hands performFrontend everything *after* `-frontend`. Accept both shapes, so + // a caller that mirrors the command line does not hand parseArgs a flag it + // rejects ("error: unknown argument: '-frontend'"). + const char *const *argsBegin = argv; + size_t argsCount = static_cast(argc); + if (argsCount > 0 && std::strcmp(argsBegin[0], "-frontend") == 0) { + argsBegin += 1; + argsCount -= 1; + } + llvm::ArrayRef args(argsBegin, argsCount); const int rc = swift::performFrontend( args, "swift-frontend", diff --git a/AppTests/NativeBuildTests.swift b/AppTests/NativeBuildTests.swift new file mode 100644 index 0000000..10a12da --- /dev/null +++ b/AppTests/NativeBuildTests.swift @@ -0,0 +1,366 @@ +import XCTest +@testable import XForge + +/// Tests for the native build path's pure half: reading a project into a plan and +/// turning that plan into compiler/linker argument lists. +/// +/// This is the only part of the native pipeline a simulator can exercise — the +/// compiler itself is linked into device builds only — and it is deliberately the +/// largest part, because a wrong argument is invisible until a build runs on a +/// phone with a real SDK. +final class NativeBuildTests: XCTestCase { + private var root: URL! + + override func setUpWithError() throws { + root = FileManager.default.temporaryDirectory + .appendingPathComponent("xforge-native-tests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: root, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + try? FileManager.default.removeItem(at: root) + } + + // MARK: - Fixtures + + private func sdkFixture() -> NativeSDKLayout { + let bundle = root.appendingPathComponent("darwin.artifactbundle", isDirectory: true) + return NativeSDKLayout( + bundle: bundle, + sdkRoot: bundle.appendingPathComponent("iPhoneOS.sdk", isDirectory: true), + swiftResources: bundle.appendingPathComponent("usr/lib/swift", isDirectory: true), + swiftStaticResources: bundle.appendingPathComponent("usr/lib/swift_static", isDirectory: true), + platformLibrarySearchPaths: [bundle.appendingPathComponent("usr/lib", isDirectory: true)] + ) + } + + private func appInfo() -> AppInfo { + AppInfo( + bundleIdentifier: "com.example.Demo", + displayName: "Demo", + version: "2.1", + buildNumber: "7", + minimumOSVersion: "17.0" + ) + } + + /// A package laid out the way `xtool new` and `XForge` both produce one. + @discardableResult + private func writePackage( + name: String = "Demo", + packageSwift: String? = nil, + files: [String] = ["Demo.swift"] + ) throws -> URL { + let project = root.appendingPathComponent(name, isDirectory: true) + let sources = project.appendingPathComponent("Sources/\(name)", isDirectory: true) + try FileManager.default.createDirectory(at: sources, withIntermediateDirectories: true) + + let manifest = packageSwift ?? """ + // swift-tools-version: 6.0 + import PackageDescription + + let package = Package( + name: "\(name)", + products: [.executable(name: "\(name)", targets: ["\(name)"])], + targets: [.executableTarget(name: "\(name)", path: "Sources/\(name)")] + ) + """ + try manifest.write( + to: project.appendingPathComponent("Package.swift"), + atomically: true, + encoding: .utf8 + ) + for file in files { + try "// \(file)\n".write( + to: sources.appendingPathComponent(file), + atomically: true, + encoding: .utf8 + ) + } + return project + } + + // MARK: - Plan factory + + func testPlanFindsSourcesAndModule() throws { + let project = try writePackage(files: ["Demo.swift", "README.md", "helper.c"]) + + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdkFixture() + ) + + XCTAssertEqual(plan.moduleName, "Demo") + XCTAssertEqual(plan.executableName, "Demo") + XCTAssertEqual(plan.appName, "Demo") + XCTAssertEqual(plan.swiftSources.count, 1) + XCTAssertEqual(plan.clangSources.map(\.url.lastPathComponent), ["helper.c"]) + XCTAssertEqual(plan.clangSources.first?.language, "c") + // A Swift app links UIKit; a C-only one does not, which is why the plan + // decides this rather than the executor. + XCTAssertTrue(plan.frameworks.contains("UIKit")) + XCTAssertTrue(plan.frameworks.contains("Foundation")) + XCTAssertTrue(plan.hasSwift) + } + + func testPlanRejectsPackageWithoutSourcesDirectory() throws { + let project = root.appendingPathComponent("Empty", isDirectory: true) + try FileManager.default.createDirectory(at: project, withIntermediateDirectories: true) + try "// swift-tools-version: 6.0\n" + .write(to: project.appendingPathComponent("Package.swift"), + atomically: true, encoding: .utf8) + + XCTAssertThrowsError(try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdkFixture() + )) { error in + guard case NativeBuildPlanError.missingTargetsDirectory = error else { + return XCTFail("unexpected error: \(error)") + } + } + } + + func testPlanRejectsMissingManifest() throws { + XCTAssertThrowsError(try NativeBuildPlanFactory.makePlan( + root: root, + appInfo: appInfo(), + configuration: .debug, + sdk: sdkFixture() + )) { error in + guard case NativeBuildPlanError.missingPackageManifest = error else { + return XCTFail("unexpected error: \(error)") + } + } + } + + /// The load-bearing refusal: a package manager cannot run inside an app that + /// may not spawn processes, so a project with dependencies fails by name. + func testPlanRefusesSwiftPackageDependencies() throws { + let manifest = """ + // swift-tools-version: 6.0 + import PackageDescription + + let package = Package( + name: "Demo", + dependencies: [ + .package(url: "https://github.com/apple/swift-argument-parser.git", from: "1.0.0") + ], + targets: [.executableTarget(name: "Demo", path: "Sources/Demo")] + ) + """ + let project = try writePackage(packageSwift: manifest) + + XCTAssertThrowsError(try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdkFixture() + )) { error in + guard case NativeBuildPlanError.unsupportedSwiftPackageDependencies(let names) = error else { + return XCTFail("unexpected error: \(error)") + } + XCTAssertEqual(names, ["swift-argument-parser"]) + XCTAssertTrue(error.localizedDescription.contains("swift-argument-parser")) + } + } + + func testDependencyScanIgnoresLocalTargets() throws { + let manifest = """ + // swift-tools-version: 6.0 + import PackageDescription + + let package = Package( + name: "Demo", + targets: [.executableTarget(name: "Demo", path: "Sources/Demo")] + ) + """ + let url = root.appendingPathComponent("Package.swift") + try manifest.write(to: url, atomically: true, encoding: .utf8) + XCTAssertEqual(try NativeBuildPlanFactory.swiftPackageDependencies(manifestURL: url), []) + } + + func testClangLanguageMapping() { + XCTAssertEqual(NativeClangSource.language(for: URL(fileURLWithPath: "/a/b.c")), "c") + XCTAssertEqual(NativeClangSource.language(for: URL(fileURLWithPath: "/a/b.m")), "objective-c") + XCTAssertEqual(NativeClangSource.language(for: URL(fileURLWithPath: "/a/b.mm")), "objective-c++") + XCTAssertEqual(NativeClangSource.language(for: URL(fileURLWithPath: "/a/b.cpp")), "c++") + XCTAssertNil(NativeClangSource.language(for: URL(fileURLWithPath: "/a/b.swift"))) + } + + // MARK: - swift-frontend arguments + + func testSwiftFrontendArgumentsMatchTheEmbeddedFrontendContract() throws { + let project = try writePackage() + let sdk = sdkFixture() + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdk + ) + let source = try XCTUnwrap(plan.swiftSources.first) + let object = root.appendingPathComponent("Demo.o") + + let arguments = try NativeToolchainInvocation.swiftFrontendArguments( + source: source, + object: object, + plan: plan, + sdk: sdk + ) + + let target = "arm64-apple-ios\(plan.minimumIOSVersion)" + XCTAssertEqual(NativeToolchainInvocation.targetTriple(minimumIOSVersion: "17.0"), + "arm64-apple-ios17.0") + XCTAssertTrue(arguments.contains(target)) + XCTAssertTrue(arguments.contains("-sdk")) + XCTAssertTrue(arguments.contains(sdk.sdkRoot.path)) + XCTAssertTrue(arguments.contains("-resource-dir")) + XCTAssertTrue(arguments.contains("-module-name")) + XCTAssertTrue(arguments.contains(plan.moduleName)) + XCTAssertTrue(arguments.contains("-o")) + XCTAssertTrue(arguments.contains(object.path)) + // -parse-as-library for a file that is not main.swift, and never the + // driver's own `-frontend`: performFrontend receives argv[2…] and would + // reject it. + XCTAssertEqual(arguments.first, "-parse-as-library") + XCTAssertFalse(arguments.contains("-frontend")) + XCTAssertTrue(arguments.contains("-Onone")) + } + + func testMainSwiftDoesNotGetParseAsLibrary() throws { + let project = try writePackage(files: ["main.swift"]) + let sdk = sdkFixture() + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .release, + sdk: sdk + ) + let source = try XCTUnwrap(plan.swiftSources.first) + + let arguments = try NativeToolchainInvocation.swiftFrontendArguments( + source: source, + object: root.appendingPathComponent("main.o"), + plan: plan, + sdk: sdk + ) + + XCTAssertFalse(arguments.contains("-parse-as-library")) + XCTAssertTrue(arguments.contains("-O")) + XCTAssertFalse(arguments.contains("-Onone")) + } + + func testSwiftFrontendArgumentsRequireSwiftResourceDirectory() throws { + let project = try writePackage() + var sdk = sdkFixture() + sdk = NativeSDKLayout( + bundle: sdk.bundle, + sdkRoot: sdk.sdkRoot, + swiftResources: nil, + swiftStaticResources: sdk.swiftStaticResources, + platformLibrarySearchPaths: sdk.platformLibrarySearchPaths + ) + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdk + ) + + XCTAssertThrowsError(try NativeToolchainInvocation.swiftFrontendArguments( + source: try XCTUnwrap(plan.swiftSources.first), + object: root.appendingPathComponent("Demo.o"), + plan: plan, + sdk: sdk + )) + } + + // MARK: - Linker arguments + + func testLinkerArgumentsDescribeAnIOSExecutable() throws { + let project = try writePackage() + let sdk = sdkFixture() + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdk + ) + let object = root.appendingPathComponent("Demo.o") + let executable = root.appendingPathComponent("Demo") + + let arguments = NativeToolchainInvocation.linkerArguments( + objects: [object], + plan: plan, + sdk: sdk, + executable: executable, + fileExists: { _ in true } + ) + + XCTAssertEqual(arguments[0], "-arch") + XCTAssertEqual(arguments[1], "arm64") + XCTAssertTrue(arguments.contains("-platform_version")) + let platformIndex = try XCTUnwrap(arguments.firstIndex(of: "-platform_version")) + XCTAssertEqual(arguments[platformIndex + 1], "ios") + XCTAssertEqual(arguments[platformIndex + 2], plan.minimumIOSVersion) + XCTAssertTrue(arguments.contains("-syslibroot")) + XCTAssertTrue(arguments.contains(sdk.sdkRoot.path)) + XCTAssertTrue(arguments.contains("-lswiftCore")) + XCTAssertTrue(arguments.contains("-lSystem")) + XCTAssertTrue(arguments.contains("-framework")) + XCTAssertTrue(arguments.contains(object.path)) + XCTAssertTrue(arguments.contains(executable.path)) + XCTAssertEqual(arguments[arguments.count - 1], object.path) + } + + func testLinkerSkipsSearchPathsThatDoNotExist() throws { + let project = try writePackage() + let sdk = sdkFixture() + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdk + ) + + let arguments = NativeToolchainInvocation.linkerArguments( + objects: [root.appendingPathComponent("Demo.o")], + plan: plan, + sdk: sdk, + executable: root.appendingPathComponent("Demo"), + fileExists: { _ in false } + ) + + // -lswiftCore stays (a Swift binary cannot run without it) but nothing + // that names a file that is not there. + XCTAssertTrue(arguments.contains("-lswiftCore")) + XCTAssertFalse(arguments.contains("-L")) + XCTAssertFalse(arguments.contains("-lswift_Concurrency")) + } + + func testCLanguageLibraryPathsComeFromThePlan() throws { + let project = try writePackage(files: ["main.c"]) + var sdk = sdkFixture() + sdk = NativeSDKLayout( + bundle: sdk.bundle, + sdkRoot: sdk.sdkRoot, + swiftResources: sdk.swiftResources, + swiftStaticResources: sdk.swiftStaticResources, + platformLibrarySearchPaths: sdk.platformLibrarySearchPaths + ) + let plan = try NativeBuildPlanFactory.makePlan( + root: project, + appInfo: appInfo(), + configuration: .debug, + sdk: sdk + ) + + XCTAssertFalse(plan.hasSwift) + XCTAssertFalse(plan.frameworks.contains("UIKit")) + XCTAssertTrue(plan.librarySearchPaths.isEmpty) + } +} From d9dd41b8a1737c62e3346a87c9af640dc50299b9 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 05:03:33 -0400 Subject: [PATCH 02/54] native: remove the embedded Alpine system MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit XForge no longer boots a Linux guest. The toolchain that builds an app is the LLVM/Clang/LLD and Swift libraries linked into the process, and the app's own code does everything the guest used to be asked for. Removed outright (with the guest they belonged to): - `App/EmbeddedVM/**` — the ish-arm64 engine bridge, the fakefs rootfs installer, the emulator, the console/shell sessions. The app links no Linux kernel now. - `App/Build/EmbeddedLinuxExecutor.swift` and `App/Build/SDKInstaller.swift` — replaced by `NativeBuildExecutor` and `NativeSDK`. - `App/Services/{SystemComponents,GuestNetwork,ToolchainManager,IPAConfigureSignService}.swift`, `HostDNS.{c,h}`, `DownloadManager.swift` — guest provisioning, guest DNS, the in-guest component installer and the zsign job. - `App/Views/Terminal/**`, `EngineLogView`, `DownloadsView` — the terminal had nothing behind it once the guest was gone, and the download screen existed to drive `curl` inside it. - `AppTests/{TerminalSessionTests,TerminalKeyConfigurationTests,TerminalBufferTests}.swift` and everything in `ReleaseResolutionTests` that asserted guest behaviour. Replaced rather than deleted: - **Build path.** `NativeBuildExecutor` compiles each translation unit through the bridge, links with LLD, assembles the `.app`, and packages the `.ipa`, streaming every step into the Build screen's console. `BuildManager`'s stages lost their guest commands: the SDK stage installs the bundle into the app container, and "Configure" now validates the project directory instead of `mkdir`-ing inside a guest. - **Signing.** `IPASigningJob` + `AppBundleSigner` sign an IPA with XKit, in process. The private key is imported through the Security framework and used in memory: the `.p12` password file that `zsign` needed on disk is gone with it. - **Project files.** `ProjectFiles` replaces `GuestProjectFiles`, with the same relative-path discipline (a `..` is rejected, not normalized). The file browser, the source browser/editor and the Package.swift editor all use it. - **Export.** `ProjectExporter` zips the project directory with ZIPFoundation (skipping `.xforge-build`/`.build`) instead of `tar`ring it inside the guest. - **Log.** `XForgeLog` is an ordinary file the user can share; the log file used to be the engine's kernel messages through the ish bridge. - **Import.** A folder picked in Files is copied in, since an iOS process cannot run `git`. The dependencies editor is gone: a project that declares SwiftPM dependencies cannot be resolved without a package manager, which cannot run here, so `NativeBuildPlanFactory` refuses such a project with a message that names the packages — an editor that could only produce unbuildable projects was worse than no editor. `project.yml` no longer builds an engine, links `-lish`/`-lz`/`-liconv`, sets `GUEST_ARM64`, or needs `Vendor/ish-arm64*` to exist. A plain clone builds. **What this costs, stated plainly:** the Swift frontend libraries are not part of the toolchain artifact yet (`.github/workflows/native-toolchain.yml` builds Clang and LLD only), so a project whose sources are Swift still cannot be compiled — the build now fails at the Swift compile step with `swiftFrontendMissing`, naming the count of files, instead of silently needing a guest. C/Objective-C targets compile and link today. The guest is not coming back as a fallback: it was 1.6 GB of carefully provisioned Linux, and the honest replacement is a toolchain artifact that carries the frontend. The Build screen's stages, the Toolchain screen (capabilities + SDK + smoke test) and the Settings screen (log, project files, diagnostics) all report from the in-process toolchain now. --- App/Build/BuildManager.swift | 71 +- App/Build/EmbeddedLinuxExecutor.swift | 211 ---- App/Build/NativeBuildExecutor.swift | 12 +- App/Build/SDKInstaller.swift | 134 --- App/Build/XForgeEnvironment.swift | 81 +- App/EmbeddedVM/ConsoleShellSession.swift | 50 - App/EmbeddedVM/EmbeddedLinuxVM.swift | 527 --------- App/EmbeddedVM/ISHBridge.c | 1039 ----------------- App/EmbeddedVM/ISHBridge.h | 178 --- App/EmbeddedVM/ISHEmulator.swift | 427 ------- App/EmbeddedVM/InteractiveShellSession.swift | 53 - App/EmbeddedVM/LinuxEmulator.swift | 125 -- App/EmbeddedVM/LinuxVM.swift | 61 - App/EmbeddedVM/RootfsInstaller.swift | 316 ----- App/Models/Project.swift | 12 +- App/Models/XcodeProject.swift | 2 +- App/Services/AppBundleSigner.swift | 158 +++ App/Services/DirectorySize.swift | 41 - App/Services/DownloadManager.swift | 237 ---- App/Services/GuestNetwork.swift | 67 -- App/Services/HostDNS.c | 158 --- App/Services/HostDNS.h | 20 - App/Services/IPAConfigureSignService.swift | 186 --- App/Services/IPASigningJob.swift | 179 +++ App/Services/ProjectExporter.swift | 102 +- App/Services/ProjectFiles.swift | 151 +++ App/Services/SystemComponents.swift | 247 ---- App/Services/ToolchainManager.swift | 261 ----- App/Services/XForgeLog.swift | 61 +- App/Views/Build/SigningView.swift | 132 +-- App/Views/Projects/DependenciesView.swift | 154 --- App/Views/Projects/FileBrowserView.swift | 8 +- App/Views/Projects/ImportProjectView.swift | 108 +- App/Views/Projects/ManifestEditorView.swift | 8 +- App/Views/Projects/ProjectDetailView.swift | 15 +- App/Views/Projects/SourceBrowserView.swift | 140 +-- App/Views/RootTabView.swift | 27 +- App/Views/Settings/DownloadsView.swift | 138 --- App/Views/Settings/EngineLogView.swift | 81 -- App/Views/Settings/SettingsView.swift | 268 +---- App/Views/Terminal/GuestFileBrowserView.swift | 216 ---- App/Views/Terminal/TerminalKeys.swift | 152 --- App/Views/Terminal/TerminalSession.swift | 296 ----- App/Views/Terminal/TerminalText.swift | 446 ------- App/Views/Terminal/TerminalView.swift | 533 --------- App/Views/Toolchain/ToolchainView.swift | 351 +++--- AppTests/ProjectTests.swift | 11 +- AppTests/ReleaseResolutionTests.swift | 240 +--- AppTests/TerminalBufferTests.swift | 139 --- AppTests/TerminalKeyConfigurationTests.swift | 91 -- AppTests/TerminalSessionTests.swift | 165 --- Support/XForge-Bridging-Header.h | 7 +- project.yml | 85 +- 53 files changed, 1030 insertions(+), 7948 deletions(-) delete mode 100644 App/Build/EmbeddedLinuxExecutor.swift delete mode 100644 App/Build/SDKInstaller.swift delete mode 100644 App/EmbeddedVM/ConsoleShellSession.swift delete mode 100644 App/EmbeddedVM/EmbeddedLinuxVM.swift delete mode 100644 App/EmbeddedVM/ISHBridge.c delete mode 100644 App/EmbeddedVM/ISHBridge.h delete mode 100644 App/EmbeddedVM/ISHEmulator.swift delete mode 100644 App/EmbeddedVM/InteractiveShellSession.swift delete mode 100644 App/EmbeddedVM/LinuxEmulator.swift delete mode 100644 App/EmbeddedVM/LinuxVM.swift delete mode 100644 App/EmbeddedVM/RootfsInstaller.swift create mode 100644 App/Services/AppBundleSigner.swift delete mode 100644 App/Services/DirectorySize.swift delete mode 100644 App/Services/DownloadManager.swift delete mode 100644 App/Services/GuestNetwork.swift delete mode 100644 App/Services/HostDNS.c delete mode 100644 App/Services/HostDNS.h delete mode 100644 App/Services/IPAConfigureSignService.swift create mode 100644 App/Services/IPASigningJob.swift create mode 100644 App/Services/ProjectFiles.swift delete mode 100644 App/Services/SystemComponents.swift delete mode 100644 App/Services/ToolchainManager.swift delete mode 100644 App/Views/Projects/DependenciesView.swift delete mode 100644 App/Views/Settings/DownloadsView.swift delete mode 100644 App/Views/Settings/EngineLogView.swift delete mode 100644 App/Views/Terminal/GuestFileBrowserView.swift delete mode 100644 App/Views/Terminal/TerminalKeys.swift delete mode 100644 App/Views/Terminal/TerminalSession.swift delete mode 100644 App/Views/Terminal/TerminalText.swift delete mode 100644 App/Views/Terminal/TerminalView.swift delete mode 100644 AppTests/TerminalBufferTests.swift delete mode 100644 AppTests/TerminalKeyConfigurationTests.swift delete mode 100644 AppTests/TerminalSessionTests.swift diff --git a/App/Build/BuildManager.swift b/App/Build/BuildManager.swift index 5d0a5b7..d4b06ab 100644 --- a/App/Build/BuildManager.swift +++ b/App/Build/BuildManager.swift @@ -2,8 +2,8 @@ import Foundation import Combine /// Orchestrates the on-device IPA build pipeline. Exposes a `PipelineSnapshot` that -/// drives the GUI's stage UI. Toolchain checks, dependency resolution, compilation, -/// packaging, and SDK tooling all run inside Alpine via `BuildExecutor`. +/// drives the GUI's stage UI. Every stage runs in this process, through the +/// toolchain linked into the app (`NativeBuildExecutor`). @MainActor final class BuildManager: ObservableObject { let project: Project @@ -76,33 +76,27 @@ final class BuildManager: ObservableObject { } catch { markFailed(.provision, error) } } + /// The Darwin SDK is installed into the app's own container. + /// + /// It used to be installed by `swift sdk install` inside the guest; the SDK + /// bundle is the same artifact either way, so this only had to stop going + /// through a guest to get it. private func ensureSDK() async { let executor = makeExecutor() markRunning(.sdk) do { - // Ask the guest before the network. Every published rootfs ships a - // darwin SDK, and the release lookup below is a network call — so a - // device that is offline (or rate-limited by the GitHub API) failed a - // build at this stage for an SDK it already had. The probe mirrors - // ToolchainManager's: `swift sdk list` is a program, so its output goes - // to the pipe rather than /dev/null, which this engine kills. - let vm = XForgeEnvironment.makeVM() - if !vm.isBooted { try await vm.boot() } - let installed = try await vm.run( - "command -v swift >/dev/null 2>&1 && swift sdk list 2>&1 | grep -qi darwin", - environment: nil - ) { _ in } - if installed == 0 { - appendConsole("▸ darwin SDK: already installed in the guest") + if NativeSDK.isInstalled { + appendConsole("▸ darwin SDK: already installed") markSucceeded(.sdk) return } - // Resolve the published asset first: the darwin SDK lives under its own - // `darwin-sdk-*` release series, so `releases/latest/download/…` 404s. let url = try await XForgeReleases.darwinSDKURL() appendConsole("▸ darwin SDK: \(url.lastPathComponent)") try await executor.installSDK(from: .hostedRemote(url)) + guard NativeSDK.isInstalled else { + throw NativeSDKError.missingBundle + } markSucceeded(.sdk) } catch { markFailed(.sdk, error) } } @@ -110,21 +104,22 @@ final class BuildManager: ObservableObject { /// Make sure the project exists in the guest and record the app identity the /// build should produce. Previously this stage only printed a line and /// reported success unconditionally, even after provision had failed. + /// Record the app identity the build should produce. + /// + /// A project is a directory in the app container now, so there is nothing to + /// create inside a guest and nothing to copy into one; what remains is the + /// check that the directory is really there and the metadata dump that makes a + /// failed build reproducible. private func configure() async { markRunning(.configure) do { - let vm = XForgeEnvironment.makeVM() - if !vm.isBooted { try await vm.boot() } - guard project.hasSafeRootPath else { throw ProjectValidationError.unsafePath } - let dir = project.rootPath - let mkdir = try await vm.run( - "mkdir -p \(GuestShell.quote(dir))", - environment: nil - ) { _ in } - guard mkdir == 0 else { throw BuildError.stepFailed("mkdir \(dir)", mkdir) } + let root = project.rootURL + guard FileManager.default.fileExists(atPath: root.path) else { + throw NativeBuildError.missingProjectDirectory(root) + } let metadata = BuildMetadata( bundleIdentifier: appInfo.bundleIdentifier, @@ -134,14 +129,14 @@ final class BuildManager: ObservableObject { minimumOSVersion: appInfo.minimumOSVersion, configuration: configuration.rawValue ) - let temp = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-app-\(UUID().uuidString).json") - try JSONEncoder().encode(metadata).write(to: temp, options: .atomic) - defer { try? FileManager.default.removeItem(at: temp) } - try await vm.copyIn(hostURL: temp, to: "\(dir)/xforge-app.json") + try FileManager.default.createDirectory( + at: root.appendingPathComponent(".xforge-build", isDirectory: true), + withIntermediateDirectories: true + ) + let record = root.appendingPathComponent(".xforge-build/build.json") + try JSONEncoder().encode(metadata).write(to: record, options: .atomic) appendConsole("▸ bundle \(appInfo.bundleIdentifier) · \(configuration.rawValue)") - appendConsole("▸ wrote app identity to \(dir)/xforge-app.json") markSucceeded(.configure) } catch { markFailed(.configure, error) @@ -187,8 +182,8 @@ final class BuildManager: ObservableObject { } catch { markFailed(.compile, error) } } - /// Packaging belongs to xtool inside Alpine. The host receives only the final - /// IPA for export; it never runs build or SDK tooling. + /// The executor packages the IPA as the last step of compiling, so this stage + /// only has to accept the result. private func package() async { guard let compiled = compiledURL, compiled.pathExtension.lowercased() == "ipa" else { @@ -197,7 +192,7 @@ final class BuildManager: ObservableObject { } markRunning(.package) snapshot.lastIpa = compiled - appendConsole("✓ .ipa packaged by xtool inside Alpine") + appendConsole("✓ .ipa packaged by the native toolchain") markSucceeded(.package) } @@ -297,9 +292,9 @@ enum BuildError: LocalizedError { case .noArtifact: return "The build did not produce an artifact." case .notProvisioned: - return "The Alpine build toolchain is not installed. Run `sh /root/install-toolchain.sh all` in Terminal." + return "The native compiler toolchain is not linked into this build of XForge." case .stepFailed(let step, let status): - return "\(step) failed inside the embedded Linux (exit \(status))." + return "\(step) failed (exit \(status))." } } } diff --git a/App/Build/EmbeddedLinuxExecutor.swift b/App/Build/EmbeddedLinuxExecutor.swift deleted file mode 100644 index ee695da..0000000 --- a/App/Build/EmbeddedLinuxExecutor.swift +++ /dev/null @@ -1,211 +0,0 @@ -import Foundation - -/// Thread-safe accumulator for output delivered from the VM's `@Sendable` callback. -private final class ExecOutputBox: @unchecked Sendable { - private let lock = NSLock() - private var text = "" - func append(_ chunk: String) { lock.lock(); text += chunk; lock.unlock() } - var value: String { lock.lock(); defer { lock.unlock() }; return text } -} - -/// BuildExecutor backed by the embedded Linux VM. -/// -/// Commands run in the shared Alpine guest through its interactive login terminal. -/// Every guest command's exit status is checked — a build that did not run must -/// never report success. -@MainActor -final class EmbeddedLinuxExecutor: BuildExecutor { - let vm: LinuxVM - private let stagingDir: URL - private(set) var stagedOutputs: [URL] = [] - - init(vm: LinuxVM, stagingDir: URL) { - self.vm = vm - self.stagingDir = stagingDir - } - - func bootstrap() -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - Task { - do { - continuation.yield(.plan("Booting embedded Alpine Linux…")) - await vm.prepareRootfs() - try await vm.boot() - - // Whether the root carries the toolchain is the root's own - // business: the published one has xtool and Swift baked in - // (EmbeddedLinux/build-rootfs.sh, XFORGE_PROVISION=all), and a - // plain one installs them on demand with - // `install-toolchain.sh`. This check reports what the guest - // can actually do rather than what it was supposed to arrive - // with, and the message names the installer rather than - // calling a plain root broken. - continuation.yield(.plan("Checking the Alpine toolchain…")) - guard try await buildEnvironmentIsReady() else { - continuation.yield(.failed( - "The Alpine build toolchain is not ready. " - + "Run `sh /root/install-toolchain.sh all` in the Terminal tab." - )) - continuation.finish() - return - } - - continuation.yield(.finished) - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - } - } - - func installSDK(from source: SDKSource) async throws { - switch source { - case .bundled(let path): - // Already inside the guest filesystem. - let status = try await vm.run( - "swift sdk install \(GuestShell.quote(path))", - environment: nil - ) { _ in } - guard status == 0 else { throw BuildError.stepFailed("swift sdk install", status) } - case .hostedRemote(let url): - // Alpine downloads, extracts, and installs the published SDK directly - // into its rootfs. The iOS host only provides the resolved asset URL. - try await SDKInstaller.install(vm: vm, remoteURL: url) { _, _ in } - } - } - - /// Verify only; this must never invoke the installer. The executable and apk - /// checks make a stale or partially restored installation fail clearly. - /// - /// Whether the guest has the toolchain XForge builds with: the apk build - /// dependencies, Swift, and xtool. - /// - /// These are what the bundled rootfs *is* expected to carry — the published - /// root is built with `XFORGE_PROVISION=all` and arrives with them installed - /// (`EmbeddedLinux/build-rootfs.sh`) — but they are also exactly what the - /// guest's own installer puts there (`install-toolchain.sh`), so a plain root - /// that has been provisioned by hand answers yes too, and a false result is - /// answered with the installer rather than treated as a broken release. - /// - /// Every probe here is a separate guest process, and their output goes to a - /// *file*, never `/dev/null`: this engine kills a forked guest program whose - /// stdout/stderr is `/dev/null` (found with the engine-smoke harness — - /// `swift --version >/dev/null 2>&1` died where the unredirected form ran - /// fine). A probe killed that way reports "not installed" for a toolchain - /// that is, which is worse than a noisy log line. - private func buildEnvironmentIsReady() async throws -> Bool { - let silence = ">/tmp/xforge-probe.log 2>&1" - let status = try await vm.run( - "apk info -e clang lld cmake ninja git \(silence) && " - + "swift --version \(silence) && xtool --version \(silence)", - environment: nil - ) { _ in } - return status == 0 - } - - func createProject(named name: String, organizationIdentifier: String) async throws -> Project { - if !vm.isBooted { try await vm.boot() } - let validatedName = try Project.validatedName(name) - let path = Project.path(forValidatedName: validatedName) - let status = try await vm.run( - "mkdir -p \(GuestShell.quote(path)) && cd \(GuestShell.quote(path)) " - + "&& XTOOL_ORG=\(GuestShell.quote(organizationIdentifier)) " - + "xtool new --name \(GuestShell.quote(validatedName))", - environment: nil - ) { _ in } - guard status == 0 else { throw BuildError.stepFailed("xtool new", status) } - return Project(name: validatedName, organizationIdentifier: organizationIdentifier, rootPath: path) - } - - func resolve(_ project: Project) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - Task { - do { - if !vm.isBooted { try await vm.boot() } - continuation.yield(.plan("Resolving dependencies for \(project.name)…")) - guard project.hasSafeRootPath else { - throw ProjectValidationError.unsafePath - } - let status = try await vm.run( - "cd \(GuestShell.quote(project.rootPath)) && swift package resolve", - environment: nil - ) { line in - continuation.yield(.output(line)) - } - if status != 0 { - continuation.yield(.failed("Dependency resolution failed (exit \(status)).")) - continuation.finish() - return - } - continuation.yield(.finished) - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - } - } - - func build(_ project: Project, configuration: BuildConfiguration) -> AsyncThrowingStream { - AsyncThrowingStream { continuation in - Task { - do { - if !vm.isBooted { try await vm.boot() } - - var flags = "-s -i" // sign + output .ipa - if configuration == .release { flags = "-c release -s -i" } - - continuation.yield(.plan("Building \(project.name) (\(configuration.rawValue))…")) - guard project.hasSafeRootPath else { - throw ProjectValidationError.unsafePath - } - let code = try await vm.run( - "cd \(GuestShell.quote(project.rootPath)) && xtool dev build \(flags)", - environment: nil - ) { line in - continuation.yield(.output(line)) - } - - // Report the build's own failure before trying to collect an - // artifact, otherwise the user sees a misleading copy error. - guard code == 0 else { - continuation.yield(.failed("Build failed (exit \(code)).")) - continuation.finish() - return - } - - guard let guestIPA = try await newestIPA(in: project) else { - continuation.yield(.failed("The build finished but produced no .ipa in .build.")) - continuation.finish() - return - } - - let hostURL = stagingDir.appendingPathComponent((guestIPA as NSString).lastPathComponent) - try await vm.copyOut(guestPath: guestIPA, to: hostURL) - - stagedOutputs = [hostURL] - continuation.yield(.artifact(hostURL)) - continuation.yield(.finished) - continuation.finish() - } catch { - continuation.finish(throwing: error) - } - } - } - } - - /// Newest `*.ipa` under the project's `.build`, rather than assuming a name - /// (xtool's output filename is not guaranteed). - private func newestIPA(in project: Project) async throws -> String? { - let box = ExecOutputBox() - let status = try await vm.run( - "find \(GuestShell.quote("\(project.rootPath)/.build")) -maxdepth 1 -type f " - + "-name '*.ipa' -print0 2>/dev/null | xargs -0 ls -t 2>/dev/null | head -1", - environment: nil - ) { box.append($0) } - guard status == 0 else { return nil } - let path = box.value.trimmingCharacters(in: .whitespacesAndNewlines) - return path.isEmpty ? nil : path - } -} diff --git a/App/Build/NativeBuildExecutor.swift b/App/Build/NativeBuildExecutor.swift index 92d5045..08682de 100644 --- a/App/Build/NativeBuildExecutor.swift +++ b/App/Build/NativeBuildExecutor.swift @@ -77,10 +77,6 @@ final class NativeBuildExecutor: BuildExecutor { switch source { case .hostedRemote(let url): try await NativeSDK.install(fromRemote: url) - case .bundled: - // A guest path is meaningless without the guest; say so instead of - // looking for a file that can never exist here. - throw NativeBuildError.unusableSDKSource } } @@ -265,7 +261,7 @@ final class NativeBuildExecutor: BuildExecutor { ) emit(.output("swift-frontend \(source.lastPathComponent)")) let result = try await runDetached { - NativeToolchain.runSwiftFrontend(arguments: arguments) + try NativeToolchain.runSwiftFrontend(arguments: arguments) } try check(result, tool: "swift-frontend", source: source) objects.append(object) @@ -424,8 +420,8 @@ final class NativeBuildExecutor: BuildExecutor { enum NativeBuildError: LocalizedError { case toolchainUnavailable(String?) case swiftFrontendMissing(Int) - case unusableSDKSource case projectAlreadyExists(URL) + case missingProjectDirectory(URL) case compileFailed(tool: String, source: URL, diagnostics: String) case linkFailed(String) @@ -439,10 +435,10 @@ enum NativeBuildError: LocalizedError { is not linked into this build of XForge. Only the C/C++/Objective-C \ half of the toolchain is present. """ - case .unusableSDKSource: - return "That SDK source belongs to the embedded Linux guest, which no longer exists. Install the Darwin SDK bundle instead." case .projectAlreadyExists(let url): return "A project already exists at \(url.path)." + case .missingProjectDirectory(let url): + return "The project directory is missing: \(url.path)." case .compileFailed(let tool, let source, let diagnostics): let text = diagnostics.trimmingCharacters(in: .whitespacesAndNewlines) let detail = text.isEmpty ? "\(tool) reported a failure without diagnostics." : text diff --git a/App/Build/SDKInstaller.swift b/App/Build/SDKInstaller.swift deleted file mode 100644 index a05d45e..0000000 --- a/App/Build/SDKInstaller.swift +++ /dev/null @@ -1,134 +0,0 @@ -import Foundation - -/// Installs the Darwin Swift SDK entirely inside the embedded Alpine rootfs. -/// -/// iOS resolves release metadata and displays progress, but the archive download, -/// extraction, validation, and Swift SDK installation are Linux commands. Temporary -/// files live under /root/.cache and the installed SDK lives in SwiftPM's guest-side -/// SDK directory; /host is never used as an installation location. -@MainActor -enum SDKInstaller { - static let guestCacheDirectory = "/root/.cache/xforge-sdk" - static let guestBundlePath = guestCacheDirectory + "/darwin.artifactbundle" - - /// Conservative free-space requirement for the archive, expanded bundle, and - /// installed copy. The fakefs root lives in the app container, so host capacity - /// is also the capacity available to Alpine. - static let requiredFreeBytes: Int64 = 2_000_000_000 - - static func install( - vm: LinuxVM, - remoteURL: URL? = nil, - advance: (Double, String) -> Void - ) async throws { - advance(0.02, "Checking the Alpine Swift SDK installation…") - try await vm.boot() - - let alreadyInstalled = try await vm.run( - "swift sdk list 2>&1 | grep -qi darwin", - environment: nil - ) { _ in } - if alreadyInstalled == 0 { - advance(1.0, "Darwin SDK is already installed in Alpine") - XForgeLog.note("sdk: darwin is already installed in the guest rootfs") - return - } - - let free = XForgeEnvironment.availableBytes - if free > 0, free < requiredFreeBytes { - throw ToolchainError.notEnoughSpace(needed: requiredFreeBytes, free: free) - } - - advance(0.05, "Resolving the latest Darwin SDK release…") - let url: URL - if let remoteURL { - url = remoteURL - } else { - url = try await XForgeReleases.darwinSDKURL() - } - XForgeLog.note("sdk: Alpine will download \(url.absoluteString)") - - let archiveName = url.lastPathComponent.isEmpty ? "darwin-sdk.zip" : url.lastPathComponent - let archivePath = guestCacheDirectory + "/" + archiveName - let script = """ - set -eu - cache=\(GuestShell.quote(guestCacheDirectory)) - archive=\(GuestShell.quote(archivePath)) - bundle=\(GuestShell.quote(guestBundlePath)) - - command -v curl >/dev/null 2>&1 - command -v unzip >/dev/null 2>&1 - command -v swift >/dev/null 2>&1 - - mkdir -p "$cache" - rm -f "$archive" - rm -rf "$bundle" - - echo "Downloading Darwin SDK inside Alpine..." - curl -fL --retry 3 --progress-bar \ - \(GuestShell.quote(url.absoluteString)) -o "$archive" - - echo "Extracting Darwin SDK inside the rootfs..." - unzip -q "$archive" -d "$cache" - test -f "$bundle/info.json" - - echo "Installing Darwin SDK into the Alpine SwiftPM store..." - swift sdk install "$bundle" - - rm -f "$archive" - rm -rf "$bundle" - swift sdk list - """ - - advance(0.1, "Downloading and installing the SDK inside Alpine…") - let output = OutputCollector() - let status = try await vm.run(script, environment: nil) { chunk in - output.append(chunk) - let text = chunk.trimmingCharacters(in: .whitespacesAndNewlines) - if !text.isEmpty { - XForgeLog.note("guest: " + text) - } - } - guard status == 0 else { - let tail = output.tail - if tail.contains("info.json") { - throw ToolchainError.sdkLayoutUnexpected - } - throw ToolchainError.sdkInstallFailed(status, output.tail) - } - - let verified = try await vm.run( - "swift sdk list 2>&1 | grep -qi darwin", - environment: nil - ) { _ in } - guard verified == 0 else { - throw ToolchainError.sdkInstallFailed(verified, "") - } - - advance(1.0, "Darwin SDK installed in Alpine") - XForgeLog.note("sdk: installed and verified in the guest rootfs") - } -} - -/// Thread-safe string accumulator for output delivered from a `@Sendable` callback. -final class OutputCollector: @unchecked Sendable { - private let lock = NSLock() - private var text = "" - - func append(_ chunk: String) { - lock.lock() - text += chunk - lock.unlock() - } - - var value: String { - lock.lock() - defer { lock.unlock() } - return text - } - - /// Last few lines — what an error message should show. - var tail: String { - value.split(separator: "\n").suffix(6).joined(separator: "\n") - } -} diff --git a/App/Build/XForgeEnvironment.swift b/App/Build/XForgeEnvironment.swift index f2f9c87..c04788b 100644 --- a/App/Build/XForgeEnvironment.swift +++ b/App/Build/XForgeEnvironment.swift @@ -1,7 +1,7 @@ import Foundation -/// Application-level wiring: where the embedded Linux lives, how a build executor is -/// constructed for a project, and where staged artifacts land. +/// Application-level wiring: where the app's directories are, how a build executor +/// is constructed for a project, and where staged artifacts land. @MainActor enum XForgeEnvironment { /// App sandbox root. @@ -13,30 +13,6 @@ enum XForgeEnvironment { FileManager.default.urls(for: .documentDirectory, in: .userDomainMask)[0] } - /// App sandbox subdirectory holding the embedded Linux userspace. - static var embeddedRoot: URL { - documentDirectory.appendingPathComponent("embedded-linux", isDirectory: true) - } - - /// Installed ish-arm64 `fakefs` root filesystems. The bundled Alpine rootfs is - /// imported here at launch and reused afterwards (the first boot imports it - /// as a fallback). - static var rootsDirectory: URL { - embeddedRoot.appendingPathComponent("roots", isDirectory: true) - } - - /// Whether the bundled Alpine rootfs has already been imported. - static var isRootfsInstalled: Bool { - RootfsInstaller.isInstalled(in: rootsDirectory) - } - - /// Directory shared into the guest at `/host` (read-write, realfs). Large - /// artifacts are staged here by the host instead of being pushed through the - /// guest command pipe. - static var hostShareDirectory: URL { - embeddedRoot.appendingPathComponent("host", isDirectory: true) - } - /// Host-side downloads (SDK archives, toolchain bundles). static var downloadsDirectory: URL { documentDirectory.appendingPathComponent("downloads", isDirectory: true) @@ -80,26 +56,21 @@ enum XForgeEnvironment { /// iOS data storage guidelines forbid — it bloats every backup and can get an /// app rejected — so: /// - /// - the downloads, staged artifacts and engine log are excluded: each is - /// produced again by downloading, rebuilding or running; - /// - **the guest filesystem is excluded too.** It is the bundled rootfs - /// imported into fakefs, and with `XFORGE_PROVISION=all` that is several - /// gigabytes of Alpine, Swift, xtool and the SDK. The user's projects live - /// *inside* it — nothing here can separate one from the other — so this is - /// a real trade-off: projects are no longer in device backups. The way out - /// is `ProjectExporter` ("Export project" on a project's screen, and the - /// archive appears in `Documents/exports`, which *is* backed up), plus the - /// `/host` share and the Terminal for anything else. + /// - the downloads, the staged artifacts and the log are excluded: each is + /// produced again by downloading, rebuilding or running. /// - /// See Docs/DESIGN.md, "What is backed up", for why the two cannot be split. + /// Projects are *not* excluded. They used to be inside the guest filesystem, + /// which had to be kept out of backup as a whole and took the user's work with + /// it; now that they are plain directories in the container, they are backed up + /// like any other document. static func prepareStorage() { let fm = FileManager.default - let excluded = [embeddedRoot, downloadsDirectory, nativeSDKDirectory, stagingDirectory, logsDirectory] + let excluded = [downloadsDirectory, nativeSDKDirectory, stagingDirectory, logsDirectory] for directory in excluded { try? fm.createDirectory(at: directory, withIntermediateDirectories: true) excludeFromBackup(directory) } - // Exports are the user's work leaving the guest, so they are left alone: + // Exports are the user's work leaving the app, so they are left alone: // backed up, and visible in the Files app. try? fm.createDirectory(at: exportsDirectory, withIntermediateDirectories: true) } @@ -147,34 +118,12 @@ enum XForgeEnvironment { .sorted { $0.date > $1.date } } - /// The single embedded Linux VM for the whole app. + /// Construct the build executor: the toolchain linked into this process. /// - /// ish-arm64 can only boot one guest per process, so every screen (Terminal, - /// Toolchain, Build) must share this instance rather than creating its own. - private static var sharedVM: LinuxVM? - - static func makeVM() -> LinuxVM { - if let sharedVM { return sharedVM } - for dir in [embeddedRoot, rootsDirectory, hostShareDirectory, downloadsDirectory] { - try? FileManager.default.createDirectory(at: dir, withIntermediateDirectories: true) - } - let vm = EmbeddedLinuxVM(root: embeddedRoot, - hostShare: hostShareDirectory, - emulator: makeEmulator()) - sharedVM = vm - return vm - } - - /// Construct the build executor. `Local` uses the embedded Linux VM. + /// There is exactly one implementation now. It used to choose between the + /// embedded Linux and a "remote" backend that never existed; both the choice + /// and the guest it offered are gone. static func makeExecutor(for project: Project? = nil) -> BuildExecutor { - EmbeddedLinuxExecutor(vm: makeVM(), stagingDir: stagingDirectory) - } - - /// The in-process Linux emulator that runs the embedded Alpine guest. - /// ish-arm64 runs a real aarch64 Linux guest in-process; its threaded-code - /// interpreter dispatches guest instructions to pre-compiled "gadgets", so it - /// needs no JIT entitlement and works in a sideloaded app. - static func makeEmulator() -> LinuxEmulator { - ISHEmulator(rootsDirectory: rootsDirectory, hostDirectory: hostShareDirectory) + NativeBuildExecutor(stagingDirectory: stagingDirectory) } } diff --git a/App/EmbeddedVM/ConsoleShellSession.swift b/App/EmbeddedVM/ConsoleShellSession.swift deleted file mode 100644 index 905f0d8..0000000 --- a/App/EmbeddedVM/ConsoleShellSession.swift +++ /dev/null @@ -1,50 +0,0 @@ -import Foundation - -/// The terminal's shell, as it actually is: the guest's console. -/// -/// There is nothing to attach and nothing to wait for — the console is up as soon -/// as the guest's init has opened it, and it stays up. Typing goes into the tty; -/// the guest's own line discipline does the rest. See `InteractiveShellSession`. -@MainActor -final class ConsoleShellSession: InteractiveShellSession { - private let console: any GuestConsole - - /// init owns the console, and it respawns the session the user types into, so - /// init is the process this session speaks for. - let pid: Int32 = 1 - private(set) var isRunning = true - - /// Ctrl-C, the byte the guest's tty turns into SIGINT for the foreground - /// group. - private static let interruptByte: UInt8 = 0x03 - - init(console: any GuestConsole) { - self.console = console - } - - @discardableResult - func send(_ text: String) -> Bool { - guard isRunning else { return false } - return console.write(text) - } - - @discardableResult - func sendControl(_ scalar: UInt8) -> Bool { - guard isRunning else { return false } - return console.write(String(UnicodeScalar(scalar))) - } - - func interruptForeground() async { - _ = sendControl(Self.interruptByte) - } - - func resize(cols: Int, rows: Int) { - guard isRunning else { return } - console.resize(cols: cols, rows: rows) - } - - func stop() { - isRunning = false - console.stop() - } -} diff --git a/App/EmbeddedVM/EmbeddedLinuxVM.swift b/App/EmbeddedVM/EmbeddedLinuxVM.swift deleted file mode 100644 index c09df0f..0000000 --- a/App/EmbeddedVM/EmbeddedLinuxVM.swift +++ /dev/null @@ -1,527 +0,0 @@ -import Foundation - -/// Thread-safe string accumulator for output captured from a `@Sendable` callback. -private final class OutputBox: @unchecked Sendable { - private let lock = NSLock() - private var text = "" - - func append(_ chunk: String) { - lock.lock() - text += chunk - lock.unlock() - } - - var value: String { - lock.lock() - defer { lock.unlock() } - return text - } -} - -/// Concrete `LinuxVM` that drives an in-process `LinuxEmulator` (ish-arm64). -/// -/// iOS cannot spawn subprocesses, so the guest runs inside the app and this type -/// provides the command/file bridge to it. -/// -/// File transfer goes through the directory shared with the guest at `/host` -/// (`hostShare`), *not* through the command pipe: an `.ipa` is easily tens of -/// megabytes, and piping that as base64 through a shell would both truncate and -/// exhaust memory. The pipe is only a fallback for when no share is configured. -@MainActor -final class EmbeddedLinuxVM: LinuxVM { - let root: URL - private let hostShare: URL? - private let emulator: LinuxEmulator - private(set) var isBooted = false - private var bootTask: Task? - - /// No timeout: builds run in the foreground and may legitimately take minutes. - private static let noTimeout: TimeInterval = 0 - /// Capture cap for the base64 fallback path (the share path is uncapped). - private static let outputCap = 64 * 1024 * 1024 - /// Guest path of the shared host directory. - private static let guestShare = "/host" - /// Staging subdirectory inside the share used for transfers. - private static let transferDir = ".xforge-transfer" - /// Launch shell for probes and one-shot commands. Not the guest's boot: pid 1 - /// is `/sbin/init` (see `initCommand`), which is what starts a shell on the - /// console. - static let launchCommand = "/bin/sh" - /// The program XForge boots pid 1 with. It reads `/etc/inittab`, which runs - /// `/sbin/xforge-login root` on the console — that script execs root's login - /// shell from `/etc/passwd`, which is the shell the Terminal tab shows. - static let initCommand = "/sbin/init" - - /// Why the guest's init did not start, if it did not. Kept rather than thrown - /// from `boot()` because the rest of the app — builds, file transfers — runs - /// as children of pid 1 whatever it is, and works either way; it is the - /// terminal that cannot exist without a console, so the terminal reports it. - private var initProblem: String? - - init(root: URL, hostShare: URL? = nil, emulator: LinuxEmulator) { - self.root = root - self.hostShare = hostShare - self.emulator = emulator - } - - func boot() async throws { - guard !isBooted else { return } - if let bootTask { - try await bootTask.value - return - } - - let task = Task { - if !emulator.isRunning { - try await emulator.boot() - } - guard emulator.isRunning else { - throw LinuxVMError.guestDidNotStart - } - // `xf_ish_boot` has mounted the fakefs, but that alone does not mean - // the guest can execute a shell or reach its /host share. The latter - // is the transport used for every large SDK import and for streamed - // terminal output, so do not report Alpine as ready until both paths - // have completed a bounded round trip. - try await verifyRootfs() - try await verifyCommandBridge() - await configureGuestResolver() - await startInit() - isBooted = true - XForgeLog.note("boot: verified Alpine guest is ready") - } - bootTask = task - do { - try await task.value - bootTask = nil - } catch { - isBooted = false - bootTask = nil - throw error - } - } - - /// Import the bundled rootfs before anything needs the guest, so the first - /// build or terminal session is not the one that pays for it. Best effort: - /// `boot()` imports it anyway when this has not run or has failed. - func prepareRootfs() async { - do { - try await emulator.prepareRootfs() - } catch { - XForgeLog.note("rootfs: pre-install failed: \(error.localizedDescription)") - } - } - - /// Bring up the guest's own init, which is what turns the mounted root into a - /// booted system: `/sbin/init` reads `/etc/inittab` and respawns - /// `/sbin/xforge-login root` on the console. - /// - /// A failure is recorded, not thrown. Everything except the terminal works - /// without it — the command runner makes its own children of pid 1, who that - /// is makes no difference — so failing the whole boot would take the build - /// pipeline down with the terminal. - private func startInit() async { - do { - try await emulator.startInit(Self.initCommand) - initProblem = nil - XForgeLog.note("boot: \(Self.initCommand) is pid 1") - } catch { - initProblem = error.localizedDescription - XForgeLog.note("boot: \(Self.initCommand) did not start: \(error.localizedDescription)") - } - } - - /// Confirm that the mounted filesystem is a usable Alpine guest before any - /// app feature is allowed to issue commands. This deliberately calls the - /// emulator directly because the normal command path waits for boot. - private func verifyRootfs() async throws { - let probe = """ - set -eu - test -x /bin/sh - test -r /etc/alpine-release - test -r /proc/version - test -e /dev/null - test -d /host - __xf_probe=/root/.xforge-health-$$ - printf x > "$__xf_probe" - test "$(cat "$__xf_probe")" = x - rm -f "$__xf_probe" - printf 'Alpine %s; kernel %s; shell %s\\n' \ - "$(cat /etc/alpine-release)" "$(uname -r)" "/bin/sh" - """ - let result = try await emulator.runCommand( - probe, - shell: Self.launchCommand, - timeout: 15, - maxOutput: 64 * 1024 - ) - let detail = result.output.trimmingCharacters(in: .whitespacesAndNewlines) - guard result.status == 0 else { - throw LinuxVMError.guestHealthCheckFailed( - detail.isEmpty ? "health check exited \(result.status)" : detail) - } - XForgeLog.note("rootfs: \(detail); /proc, /dev, /host, and writable root verified") - } - - /// Exercise the same guest-to-host streaming path used by Terminal and every - /// app command. A mounted rootfs is not enough if command output cannot cross - /// the /host realfs bridge back into Swift. - private func verifyCommandBridge() async throws { - let token = "__XFORGE_BRIDGE_OK__" - let output = OutputBox() - let status = try await runLoginStreamingAfterBoot( - "printf '%s\\n' \(GuestShell.quote(token))", - environment: nil, - onOutput: output.append - ) - let received = output.value - guard status == 0, received.contains(token) else { - let detail = received.trimmingCharacters(in: .whitespacesAndNewlines) - throw LinuxVMError.commandBridgeFailed( - detail.isEmpty ? "no output returned from the guest" : detail) - } - XForgeLog.note("bridge: interactive /host command round trip verified") - } - - /// Write the guest's `/etc/resolv.conf` from the device's own DNS. - /// - /// Resolution happens in the guest, and the bundled minirootfs ships no - /// nameservers, so without this `apk add` — the first thing provisioning - /// runs — cannot reach its repositories. Failure is not fatal: the guest - /// still boots, and a user can see what happened in the engine log. - private func configureGuestResolver() async { - guard let share = hostShare else { return } - let (text, source) = GuestNetwork.resolvConfForDevice() - - let staging = share.appendingPathComponent(Self.transferDir, isDirectory: true) - let file = staging.appendingPathComponent("resolv.conf") - do { - try FileManager.default.createDirectory(at: staging, withIntermediateDirectories: true) - try text.write(to: file, atomically: true, encoding: .utf8) - } catch { - XForgeLog.note("dns: could not stage resolv.conf: \(error.localizedDescription)") - return - } - - let status = (try? await runLoginStreamingAfterBoot( - // The file can be a dangling symlink into /run in newer roots, so - // replace it rather than writing through it (same reason the engine's - // app unlinks first). - "rm -f /etc/resolv.conf && cp -f /host/\(Self.transferDir)/resolv.conf /etc/resolv.conf && cat /etc/resolv.conf", - environment: nil - ) { _ in }) ?? -1 - - XForgeLog.note("dns: resolv.conf from \(source) servers (exit \(status)): " - + text.split(separator: "\n").joined(separator: " ")) - - // Do not make Alpine startup wait on a network query. The resolver is - // now installed, and the download command that needs it reports any - // real DNS/network error in its own output. - } - - /// Run every app command through the same interactive login-shell transport - /// used by Terminal, forwarding output while the command is running. - func run( - _ command: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - try await runLoginStreaming( - command, - environment: environment, - onOutput: onOutput - ) - } - - /// Low-level one-shot primitive used only to launch the login-shell transport. - /// App features must call `run` so their commands use the interactive terminal. - private func runCaptured( - _ command: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - try await boot() - return try await runCapturedAfterBoot( - command, - environment: environment, - onOutput: onOutput - ) - } - - private func runCapturedAfterBoot( - _ command: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - let env = GuestShell.environment(environment) - - // Always execute through Alpine's non-login shell. A login wrapper is - // inappropriate for the headless bridge (no controlling TTY/PAM - // session) and causes otherwise valid commands such as cp, nslookup and - // terminal probes to exit 1 before the script body runs. - let result = try await emulator.runCommand( - env + command, - shell: "/bin/sh", - timeout: Self.noTimeout, - maxOutput: Self.outputCap - ) - if !result.output.isEmpty { - onOutput(result.output) - } - if result.truncated { - onOutput("\n[output truncated at \(Self.outputCap / (1024 * 1024)) MB]\n") - } - return result.status - } - - /// Run a script through XForge's default Alpine launch shell, streaming its - /// merged stdout+stderr to `onOutput` **as the guest writes it**. - /// - /// The engine's command primitive returns a command's output only once it - /// has finished, so live output cannot come from there. Instead the guest - /// redirects the session's output into the shared folder (`/host`), which is - /// realfs, and the host tails that file while the (blocking) command is still - /// running. The command's exit status still comes back from the engine. - func runLoginStreaming( - _ script: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - try await boot() - return try await runLoginStreamingAfterBoot( - script, - environment: environment, - onOutput: onOutput - ) - } - - /// Run the shared-folder terminal transport after the caller has completed - /// boot. Keeping this separate prevents the boot health check from awaiting - /// its own in-flight boot task. - private func runLoginStreamingAfterBoot( - _ script: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - guard let share = hostShare else { - // No shared folder to tail; fall back to the captured output. - return try await runCapturedAfterBoot(script, environment: environment, onOutput: onOutput) - } - - let transfer = share.appendingPathComponent(Self.transferDir, isDirectory: true) - try FileManager.default.createDirectory(at: transfer, withIntermediateDirectories: true) - let name = "term-\(UUID().uuidString).log" - let logURL = transfer.appendingPathComponent(name) - FileManager.default.createFile(atPath: logURL.path, contents: nil) - let guestLog = "\(Self.guestShare)/\(Self.transferDir)/\(name)" - - // The C bridge already runs this text as `/bin/sh -c`. A login - // program requires a TTY and exits 1 in this headless command runner. - let wrapped = """ - { - \(script) - } > \(GuestShell.quote(guestLog)) 2>&1 - """ - - let tailer = FileTailer(url: logURL, onChunk: onOutput) - tailer.start() - do { - let status = try await runCapturedAfterBoot(wrapped, environment: environment) { _ in } - tailer.stop() - try? FileManager.default.removeItem(at: logURL) - return status - } catch { - tailer.stop() - try? FileManager.default.removeItem(at: logURL) - throw error - } - } - - // MARK: - Interactive shell - - /// Start the terminal's shell. - /// - /// The shell is not a process this code launches: it is the guest's own - /// console session, started by `/sbin/init` from `/etc/inittab`, which - /// respawns `/sbin/xforge-login root` on the console. So all this does is - /// attach the screen to the console the guest is already driving. - func startInteractiveShell( - onOutput: @escaping @Sendable (String) -> Void - ) async throws -> any InteractiveShellSession { - try await boot() - if let initProblem { - throw LinuxVMError.initFailed(initProblem) - } - let console = try await emulator.openConsole(onOutput: onOutput) - XForgeLog.note("terminal: attached to the guest console") - return ConsoleShellSession(console: console) - } - - // MARK: - File transfer - - /// Copy a file out of the guest into a host URL. - func copyOut(guestPath: String, to hostURL: URL) async throws { - if let share = hostShare { - let name = "out-\(UUID().uuidString)" - let status = try await run( - "mkdir -p \(Self.guestShare)/\(Self.transferDir) && " - + "cp -f \(GuestShell.quote(guestPath)) " - + "\(GuestShell.quote("\(Self.guestShare)/\(Self.transferDir)/\(name)"))", - environment: nil - ) { _ in } - guard status == 0 else { throw LinuxVMError.fileCopyFailed } - - let staged = share - .appendingPathComponent(Self.transferDir, isDirectory: true) - .appendingPathComponent(name) - // Whatever happens next, the staged copy is ours to clean up: a - // *failed* copy used to leave it in the share folder for good, and the - // things copied through here are large (an Xcode.xip is gigabytes, and - // the container has no room to keep two of them). - defer { try? FileManager.default.removeItem(at: staged) } - guard FileManager.default.fileExists(atPath: staged.path) else { - throw LinuxVMError.fileCopyFailed - } - try? FileManager.default.removeItem(at: hostURL) - try FileManager.default.createDirectory( - at: hostURL.deletingLastPathComponent(), withIntermediateDirectories: true) - try FileManager.default.moveItem(at: staged, to: hostURL) - return - } - - // Fallback: base64 over the shell. - let box = OutputBox() - let code = try await run( - "base64 -w0 \(GuestShell.quote(guestPath)) 2>/dev/null || echo __XF_COPY_ERR__", - environment: nil - ) { box.append($0) } - - let trimmed = box.value.trimmingCharacters(in: .whitespacesAndNewlines) - guard code == 0, !trimmed.contains("__XF_COPY_ERR__"), - let data = Data(base64Encoded: trimmed) else { - throw LinuxVMError.fileCopyFailed - } - try FileManager.default.createDirectory( - at: hostURL.deletingLastPathComponent(), withIntermediateDirectories: true) - try data.write(to: hostURL) - } - - /// Copy a host file into the guest. - func copyIn(hostURL: URL, to guestPath: String) async throws { - if let share = hostShare { - let name = "in-\(UUID().uuidString)" - let staging = share - .appendingPathComponent(Self.transferDir, isDirectory: true) - .appendingPathComponent(name) - try FileManager.default.createDirectory( - at: staging.deletingLastPathComponent(), withIntermediateDirectories: true) - try? FileManager.default.removeItem(at: staging) - try FileManager.default.copyItem(at: hostURL, to: staging) - // The guest deletes the shared copy when it has taken it, and this - // covers the case where it does not get that far. - defer { try? FileManager.default.removeItem(at: staging) } - - let dir = (guestPath as NSString).deletingLastPathComponent - let status = try await run( - "mkdir -p \(GuestShell.quote(dir)) && " - + "cp -f \(GuestShell.quote("\(Self.guestShare)/\(Self.transferDir)/\(name)")) " - + "\(GuestShell.quote(guestPath)) && " - + "rm -f \(GuestShell.quote("\(Self.guestShare)/\(Self.transferDir)/\(name)"))", - environment: nil - ) { _ in } - guard status == 0 else { throw LinuxVMError.fileCopyFailed } - return - } - - let data = try Data(contentsOf: hostURL) - let b64 = data.base64EncodedString() - let dir = (guestPath as NSString).deletingLastPathComponent - let status = try await run( - "mkdir -p \(GuestShell.quote(dir)) && base64 -d > \(GuestShell.quote(guestPath)) " - + "<<'__XF_B64__'\n\(b64)\n__XF_B64__\n", - environment: nil - ) { _ in } - guard status == 0 else { throw LinuxVMError.fileCopyFailed } - } -} - -enum LinuxVMError: LocalizedError { - case notImplemented(String) - case fileCopyFailed - case guestDidNotStart - case guestHealthCheckFailed(String) - case commandBridgeFailed(String) - case initFailed(String) - - var errorDescription: String? { - switch self { - case .notImplemented(let m): return m - case .fileCopyFailed: - return "Could not copy the file to/from the embedded Linux." - case .guestDidNotStart: - return "The embedded Linux engine did not start." - case .guestHealthCheckFailed(let detail): - return "The Alpine system started but is not usable: \(detail)" - case .commandBridgeFailed(let detail): - return "Linux started, but XForge could not communicate with it: \(detail)" - case .initFailed(let detail): - return "The guest's init did not start, so there is no console to type " - + "into: \(detail)\nThe bundled rootfs has to provide /sbin/init and " - + "/etc/inittab — an older rootfs is replaced on next launch." - } - } -} - -/// Polls a file as the guest appends to it and hands each new chunk to `onChunk`. -/// -/// This is what makes the terminal live despite the engine's one-shot command -/// primitive: the guest writes its output into the shared folder, and this reads -/// it back while the command is still running. `onChunk` runs on a private queue -/// and must not block (hop to the main actor asynchronously instead). -final class FileTailer: @unchecked Sendable { - private let url: URL - private let onChunk: @Sendable (String) -> Void - private let queue = DispatchQueue(label: "org.xforge.terminal.tail") - private var offset: UInt64 = 0 - private var running = false - - init(url: URL, onChunk: @escaping @Sendable (String) -> Void) { - self.url = url - self.onChunk = onChunk - } - - func start() { - queue.async { [self] in - guard !running else { return } - running = true - poll() - } - } - - func stop() { - queue.sync { [self] in - guard running else { return } - running = false - drain() - } - } - - private func poll() { - guard running else { return } - drain() - queue.asyncAfter(deadline: .now() + .milliseconds(80)) { [self] in poll() } - } - - private func drain() { - guard let handle = try? FileHandle(forReadingFrom: url) else { return } - defer { try? handle.close() } - do { - try handle.seek(toOffset: offset) - guard let data = try handle.readToEnd(), !data.isEmpty else { return } - offset += UInt64(data.count) - onChunk(String(decoding: data, as: UTF8.self)) - } catch { - // The file may not exist (or have new bytes) yet; the next tick retries. - } - } -} diff --git a/App/EmbeddedVM/ISHBridge.c b/App/EmbeddedVM/ISHBridge.c deleted file mode 100644 index 75d7b6d..0000000 --- a/App/EmbeddedVM/ISHBridge.c +++ /dev/null @@ -1,1039 +0,0 @@ -// -// ISHBridge.c -// XForge -// -// C shim over the embedded Linux engine, ish-arm64. See ISHBridge.h. -// -// The boot sequence mirrors what the engine's own iOS app does in -// -[AppDelegate boot] (mount the root, create init, mount the virtual -// filesystems and create device nodes) and how its ISHShellExecutor spawns a -// process (become_new_init_child, wire fds, do_execve, task_start), minus the -// UIKit/terminal machinery XForge does not need. -// -// The engine is built for the iOS *device* only (see -// EmbeddedLinux/build-ish-core.sh), so simulator builds — which the unit tests -// use — compile the stub below instead of linking the engine. -// - -#include "ISHBridge.h" -#include - -#if TARGET_OS_SIMULATOR - -// --- simulator stub --------------------------------------------------------- -// No engine is linked in simulator builds. Every entry point reports clearly. - -#include - -static char s_sim_error[256]; - -const char *xf_ish_last_error(void) { return s_sim_error; } - -static int xf_sim_unsupported(void) { - strncpy(s_sim_error, - "The ish-arm64 Linux engine runs on device only (simulator builds do not link it).", - sizeof(s_sim_error) - 1); - return -1; -} - -int xf_ish_set_log_file(const char *path) { - (void) path; - return 0; -} - -int xf_ish_log(const char *text) { - (void) text; - return 0; -} - -int xf_ish_boot(const char *root_dir, const char *host_dir) { - (void) root_dir; (void) host_dir; - return xf_sim_unsupported(); -} -int xf_ish_is_booted(void) { return 0; } - -int xf_ish_run(const char *command, const char *shell, int timeout_ms, - size_t max_output, xf_guest_result *result) { - (void) command; (void) shell; (void) timeout_ms; (void) max_output; - if (result != NULL) memset(result, 0, sizeof(*result)); - return xf_sim_unsupported(); -} - -int xf_ish_run_detached(const char *command, const char *shell, const char *stdin_path) { - (void) command; (void) shell; (void) stdin_path; - return xf_sim_unsupported(); -} - -int xf_ish_process_alive(int pid) { - // No guest in the simulator, so nothing is running — which is also the right - // answer for a caller polling a pid that does not exist. - (void) pid; - return 0; -} - -int xf_ish_kill_process(int pid, int signal) { - (void) pid; (void) signal; - return xf_sim_unsupported(); -} - -void xf_guest_result_free(xf_guest_result *result) { - if (result != NULL) memset(result, 0, sizeof(*result)); -} - -int xf_ish_start_init(const char *program) { - (void) program; - return xf_sim_unsupported(); -} - -int xf_ish_console_ready(void) { return 0; } - -ssize_t xf_ish_console_read(char *buf, size_t len, int timeout_ms) { - (void) buf; (void) len; (void) timeout_ms; - return xf_sim_unsupported(); -} - -ssize_t xf_ish_console_write(const char *buf, size_t len) { - (void) buf; (void) len; - return xf_sim_unsupported(); -} - -int xf_ish_console_resize(int cols, int rows) { - (void) cols; (void) rows; - return xf_sim_unsupported(); -} - -void xf_ish_shutdown(void) {} - -#else - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -// --- ish-arm64 core headers ------------------------------------------------- -#include "kernel/init.h" // mount_root, become_first_process, become_new_init_child -#include "kernel/fs.h" // struct fs_ops, do_mount, fs_register, fakefs/procfs/... -#include "kernel/task.h" // struct task, current, task_start, exit_hook -#include "kernel/calls.h" // do_execve -#include "kernel/signal.h" // send_signal, SIGINFO_NIL -#include "fs/real.h" // realfs -#include "fs/fd.h" // adhoc_fd_create, realfs_fdops -#include "fs/devices.h" // MEM_MAJOR, DEV_NULL_MINOR, TTY_ALTERNATE_MAJOR, ... -#include "fs/tty.h" // struct tty, tty_drivers, tty_input, tty_set_winsize -#include "fs/path.h" // AT_PWD -#include "util/sync.h" // lock, unlock - -// --- state ------------------------------------------------------------------ - -// The engine's `current` is thread-local, so the thread that boots must be the -// one that runs commands. Swift funnels every call through one permanent OS -// thread. -static __thread char s_error[512]; -static bool s_global_inited = false; -static bool s_booted = false; - -static void xf_fail(const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - vsnprintf(s_error, sizeof(s_error), fmt, ap); - va_end(ap); -} - -const char *xf_ish_last_error(void) { - return s_error[0] ? s_error : ""; -} - -// --- engine log -------------------------------------------------------------- -// -// The engine's own printk goes through the log handler it was built with -// (-Dlog_handler=nslog here, so it reaches the device console). This file is for -// XForge's own breadcrumbs, which is what a bug report actually needs: the boot -// stage it reached, and the output of the command that failed. - -// Hard ceiling on what the log may grow to. A provisioning run that compiles or -// downloads for an hour would otherwise fill the app container. -#define XF_LOG_MAX_BYTES (4 * 1024 * 1024) - -static int s_log_fd = -1; -static long long s_log_bytes = 0; -static bool s_log_capped = false; - -int xf_ish_log(const char *text) { - if (text == NULL) - return -EINVAL; - if (s_log_fd < 0) - return -ENODEV; - if (s_log_capped) - return -EFBIG; - - struct timeval tv; - gettimeofday(&tv, NULL); - struct tm tm; - time_t secs = tv.tv_sec; - localtime_r(&secs, &tm); - - char line[2048]; - int head = snprintf(line, sizeof(line), "[%02d:%02d:%02d.%03d] ", - tm.tm_hour, tm.tm_min, tm.tm_sec, (int) (tv.tv_usec / 1000)); - if (head < 0) - head = 0; - // Truncate rather than split: a partial line is worse than a clipped one. - size_t room = sizeof(line) - (size_t) head - 2; - size_t len = strlen(text); - if (len > room) - len = room; - memcpy(line + head, text, len); - line[head + (int) len] = '\n'; - ssize_t written = write(s_log_fd, line, (size_t) head + len + 1); - if (written > 0) { - s_log_bytes += written; - if (s_log_bytes > XF_LOG_MAX_BYTES) { - s_log_capped = true; - static const char full[] = - "[xforge] log reached its size limit; further lines are dropped\n"; - write(s_log_fd, full, sizeof(full) - 1); - } - } - return written < 0 ? -errno : 0; -} - -// printf-style breadcrumb into the log file. -static void xf_logf(const char *fmt, ...) { - char buf[1024]; - va_list ap; - va_start(ap, fmt); - vsnprintf(buf, sizeof(buf), fmt, ap); - va_end(ap); - xf_ish_log(buf); -} - -int xf_ish_set_log_file(const char *path) { - if (s_log_fd >= 0) { - close(s_log_fd); - s_log_fd = -1; - } - s_log_bytes = 0; - s_log_capped = false; - if (path == NULL || path[0] == '\0') - return 0; - - int fd = open(path, O_WRONLY | O_CREAT | O_APPEND, 0644); - if (fd < 0) - return -errno; - // Continue counting from whatever is already in the file, so the ceiling - // holds across the session rather than per call. - struct stat st; - if (fstat(fd, &st) == 0 && st.st_size > 0) - s_log_bytes = st.st_size; - s_log_fd = fd; - return 0; -} - -// --- command exit status ----------------------------------------------------- -// -// The engine reports a process's exit through its global `exit_hook`, which is -// the only place the status of a guest process is observable from the host -// (there is no host-side waitpid for guest pids). XForge runs one command at a -// time on one thread, so the hook records the status of the pid it is waiting -// for and ignores everything else — the guest starts several tasks of its own -// during boot, and those exits must not be mistaken for the command's. -static pthread_mutex_t s_exit_lock = PTHREAD_MUTEX_INITIALIZER; -static pthread_cond_t s_exit_cond = PTHREAD_COND_INITIALIZER; -static pid_t_ s_wait_pid = 0; -static int s_wait_status = 0; -static bool s_wait_done = false; -/// Pids observed to exit, so a detached process can be distinguished from one -/// still running. A small fixed ring: the app starts a handful of detached -/// processes at most (one terminal shell, occasionally an install), and an -/// over-full ring only means an old entry is forgotten — which degrades to -/// asking the engine, not to a wrong answer. -#define XF_EXITED_MAX 16 -static pid_t_ s_exited[XF_EXITED_MAX]; -static int s_exited_count = 0; - -static bool xf_exit_was_seen(pid_t_ pid) { - for (int i = 0; i < s_exited_count; i++) { - if (s_exited[i] == pid) return true; - } - return false; -} - -static void xf_exit_hook(struct task *task, int code) { - pthread_mutex_lock(&s_exit_lock); - if (task != NULL) { - if (!xf_exit_was_seen(task->pid)) { - if (s_exited_count < XF_EXITED_MAX) { - s_exited[s_exited_count++] = task->pid; - } else { - // Drop the oldest to make room; see the comment above. - for (int i = 1; i < XF_EXITED_MAX; i++) - s_exited[i - 1] = s_exited[i]; - s_exited[XF_EXITED_MAX - 1] = task->pid; - } - } - if (s_wait_pid != 0 && task->pid == s_wait_pid) { - s_wait_status = code; - s_wait_done = true; - pthread_cond_broadcast(&s_exit_cond); - } - } - pthread_mutex_unlock(&s_exit_lock); -} - -// --- guest console ----------------------------------------------------------- -// -// --- pid 1's stdio ---------------------------------------------------------- -// -// The engine's `create_stdio` opens the file it is given with plain O_RDWR, and -// for the console that has a consequence the guest can see: -// * pid 1 is a session leader when this runs (the engine's `construct_task` -// calls `task_setsid`), so the open makes the console *pid 1's* controlling -// terminal, pinning the tty's session to pid 1 (`tty_open` in fs/tty.c -// hands a tty over only while `tty->session == 0`); -// * the login session init later starts does `setsid()` and opens /dev/tty1 — -// and by then the tty has a session, so it never gets it. The shell runs with -// no controlling terminal, which bash reports on the console as -// "cannot set terminal process group (-1): Not a tty" followed by -// "no job control in this shell": no Ctrl-C to the foreground program, no -// Ctrl-Z, no SIGWINCH to the shell on resize. -// -// Linux's init has no controlling terminal either — the console belongs to the -// login session that opens it — and the only difference is O_NOCTTY. So this is -// `create_stdio` with that flag, kept here rather than patching the vendored -// engine: the engine's version is right for every other caller. -static int xf_create_console_stdio(void) { - struct fd *fd = generic_open("/dev/console", O_RDWR_ | O_NOCTTY_, 0); - if (!IS_ERR(fd) && !S_ISCHR(fd->stat.mode)) { - // Opened a regular file instead of a character device (realfs cannot - // create device nodes), so close it and take the same fallback the engine - // does. - fd_close(fd); - fd = ERR_PTR(_ENOENT); - } - if (IS_ERR(fd)) { - fd = adhoc_fd_create(NULL); - if (fd == NULL) - return _ENOMEM; - fd->stat.rdev = dev_make(TTY_CONSOLE_MAJOR, 1); - fd->stat.mode = S_IFCHR | S_IRUSR; - // O_NOCTTY again: the fallback opens the tty directly, and it must not - // claim the session either. - fd->flags = O_RDWR_ | O_NOCTTY_; - int dev_err = dev_open(TTY_CONSOLE_MAJOR, 1, DEV_CHAR, fd); - if (dev_err < 0) - return dev_err; - } - - fd->refcount = 0; - current->files->files[0] = fd_retain(fd); - current->files->files[1] = fd_retain(fd); - current->files->files[2] = fd_retain(fd); - xf_logf("boot: pid 1's stdio is the console (O_NOCTTY — the login session owns it)"); - return 0; -} - -// The guest's console is a tty the *host* implements. Everything above the -// driver — the line discipline, echo, signal generation, job control — is the -// guest kernel's, so the app gets a real terminal rather than a pipe: Ctrl-C -// becomes SIGINT for the foreground program, the shell does its own line -// editing and history, and a full-screen program can put the tty in raw mode. -// -// The engine calls the driver's write op from whichever guest thread is doing -// the writing, so this side is only a byte buffer plus a condition variable. The -// host reader blocks in xf_ish_console_read on a thread of its own, which is -// safe: that call touches no per-thread engine state. - -#define XF_CONSOLE_BUFFER_BYTES (256 * 1024) - -static pthread_mutex_t s_console_lock = PTHREAD_MUTEX_INITIALIZER; -static pthread_cond_t s_console_cond = PTHREAD_COND_INITIALIZER; -static char s_console_buffer[XF_CONSOLE_BUFFER_BYTES]; -static size_t s_console_head = 0; // index of the oldest unread byte -static size_t s_console_length = 0; // unread bytes -static long long s_console_dropped = 0; -static bool s_console_stopping = false; -static struct tty *s_console_tty = NULL; - -/// Append what the guest wrote, waking the host reader. -static void xf_console_push(const char *data, size_t len) { - pthread_mutex_lock(&s_console_lock); - bool first_drop = false; - for (size_t i = 0; i < len; i++) { - if (s_console_length == sizeof(s_console_buffer)) { - // The reader has fallen a quarter of a megabyte behind. Drop the - // oldest byte rather than block: a writer blocked inside the kernel - // stalls the whole guest, which is much worse than a gap in the - // scrollback. Only report the first loss, to keep this cheap. - s_console_head = (s_console_head + 1) % sizeof(s_console_buffer); - s_console_length--; - s_console_dropped++; - first_drop = first_drop || s_console_dropped == 1; - } - s_console_buffer[(s_console_head + s_console_length) % sizeof(s_console_buffer)] = data[i]; - s_console_length++; - } - pthread_cond_broadcast(&s_console_cond); - pthread_mutex_unlock(&s_console_lock); - if (first_drop) - xf_logf("console: output is arriving faster than it is read; dropping bytes"); -} - -static int xf_console_tty_init(struct tty *tty) { - // Called from tty_get with ttys_lock held, so record the tty and nothing - // else — no behaviour that could take another lock or block. - if (tty->num == 1) { - pthread_mutex_lock(&s_console_lock); - s_console_tty = tty; - pthread_mutex_unlock(&s_console_lock); - xf_logf("console: tty1 attached"); - } - return 0; -} - -static int xf_console_tty_write(struct tty *tty, const void *buf, size_t len, bool blocking) { - (void) blocking; - if (tty->num == 1 && len > 0) - xf_console_push((const char *) buf, len); - // Claim every byte. Under-reporting would leave the kernel waiting for room - // that only this side could free. - return (int) len; -} - -static void xf_console_tty_cleanup(struct tty *tty) { - if (tty->num != 1) - return; - pthread_mutex_lock(&s_console_lock); - if (s_console_tty == tty) - s_console_tty = NULL; - pthread_cond_broadcast(&s_console_cond); - pthread_mutex_unlock(&s_console_lock); -} - -static struct tty_driver_ops xf_console_ops = { - .init = xf_console_tty_init, - .write = xf_console_tty_write, - .cleanup = xf_console_tty_cleanup, -}; - -// Major 4 (TTY_CONSOLE_MAJOR), minors 1..7. Only minor 1 has a screen; the rest -// accept and discard, so a program writing to /dev/tty2 cannot wedge. -DEFINE_TTY_DRIVER(xf_console_driver, &xf_console_ops, TTY_CONSOLE_MAJOR, 8); - -static struct tty *xf_console_tty(void) { - pthread_mutex_lock(&s_console_lock); - struct tty *tty = s_console_tty; - pthread_mutex_unlock(&s_console_lock); - return tty; -} - -int xf_ish_console_ready(void) { - return xf_console_tty() != NULL ? 1 : 0; -} - -ssize_t xf_ish_console_read(char *buf, size_t len, int timeout_ms) { - if (buf == NULL || len == 0) - return -EINVAL; - - pthread_mutex_lock(&s_console_lock); - if (timeout_ms != 0) { - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - if (timeout_ms > 0) { - deadline.tv_sec += timeout_ms / 1000; - deadline.tv_nsec += (long) (timeout_ms % 1000) * 1000000L; - if (deadline.tv_nsec >= 1000000000L) { - deadline.tv_sec++; - deadline.tv_nsec -= 1000000000L; - } - } - while (s_console_length == 0 && !s_console_stopping) { - if (timeout_ms < 0) { - pthread_cond_wait(&s_console_cond, &s_console_lock); - } else if (pthread_cond_timedwait(&s_console_cond, &s_console_lock, &deadline) == ETIMEDOUT) { - break; - } - } - } - - size_t take = s_console_length < len ? s_console_length : len; - for (size_t i = 0; i < take; i++) - buf[i] = s_console_buffer[(s_console_head + i) % sizeof(s_console_buffer)]; - s_console_head = (s_console_head + take) % sizeof(s_console_buffer); - s_console_length -= take; - pthread_mutex_unlock(&s_console_lock); - return (ssize_t) take; -} - -ssize_t xf_ish_console_write(const char *buf, size_t len) { - if (buf == NULL) - return -EINVAL; - if (len == 0) - return 0; - - struct tty *tty = xf_console_tty(); - if (tty == NULL) - return -ENODEV; - // tty_input takes the tty's own lock and reads no per-thread engine state, - // so input can be pushed from the UI thread while the guest is running — - // which is the whole point of a terminal you can type into. - return tty_input(tty, buf, len, false); -} - -int xf_ish_console_resize(int cols, int rows) { - if (cols <= 0 || rows <= 0) - return -EINVAL; - - struct tty *tty = xf_console_tty(); - if (tty == NULL) - return -ENODEV; - // tty_set_winsize does not lock; it also raises SIGWINCH for the foreground - // group, which is what makes a full-screen program redraw itself. - lock(&tty->lock); - tty_set_winsize(tty, (struct winsize_) { - .row = (word_t) rows, - .col = (word_t) cols, - }); - unlock(&tty->lock); - return 0; -} - -// --- command output ---------------------------------------------------------- - -/// Read chunk for the blocking command runner's pipe. -#define XF_READ_CHUNK 8192 - -// --- API -------------------------------------------------------------------- - -int xf_ish_is_booted(void) { - return s_booted ? 1 : 0; -} - -// One-time process-global emulator init. Mirrors what the engine's app does -// before its first guest process: arm the memory governor, then seed the -// emulator. Kept to what a headless host needs. -static void xf_global_init(void) { - if (s_global_inited) return; - s_global_inited = true; - xf_logf("engine: global init"); -} - -// The bundled minirootfs ships an empty /dev, and the guest userland needs the -// standard nodes before anything else runs. Mirrors the device nodes the -// engine's app creates in -[AppDelegate boot]. -static void xf_ensure_devices(void) { - generic_mkdirat(AT_PWD, "/dev", 0755); - generic_mkdirat(AT_PWD, "/dev/pts", 0755); - generic_mkdirat(AT_PWD, "/dev/shm", 01777); - generic_mkdirat(AT_PWD, "/run", 0755); - - static const struct { const char *path; mode_t_ mode; int major; int minor; } nodes[] = { - {"/dev/null", S_IFCHR | 0666, MEM_MAJOR, DEV_NULL_MINOR}, - {"/dev/zero", S_IFCHR | 0666, MEM_MAJOR, DEV_ZERO_MINOR}, - {"/dev/full", S_IFCHR | 0666, MEM_MAJOR, DEV_FULL_MINOR}, - {"/dev/random", S_IFCHR | 0666, MEM_MAJOR, DEV_RANDOM_MINOR}, - {"/dev/urandom", S_IFCHR | 0666, MEM_MAJOR, DEV_URANDOM_MINOR}, - {"/dev/tty", S_IFCHR | 0666, TTY_ALTERNATE_MAJOR, DEV_TTY_MINOR}, - {"/dev/console", S_IFCHR | 0666, TTY_ALTERNATE_MAJOR, DEV_CONSOLE_MINOR}, - {"/dev/ptmx", S_IFCHR | 0666, TTY_ALTERNATE_MAJOR, DEV_PTMX_MINOR}, - }; - for (size_t i = 0; i < sizeof(nodes) / sizeof(nodes[0]); i++) { - // EEXIST is expected on every boot after the first. - generic_mknodat(AT_PWD, nodes[i].path, nodes[i].mode, - dev_make(nodes[i].major, nodes[i].minor)); - } - - // Linux guarantees these modes; a tarball can leave them stricter. - generic_setattrat(AT_PWD, "/tmp", (struct attr) { .type = attr_mode, .mode = S_IFDIR | 01777 }, false); - generic_setattrat(AT_PWD, "/", (struct attr) { .type = attr_mode, .mode = S_IFDIR | 0755 }, false); -} - -int xf_ish_boot(const char *root_dir, const char *host_dir) { - if (root_dir == NULL) return -EINVAL; - if (s_booted) return 0; - - xf_global_init(); - xf_logf("boot: mounting %s/data as /", root_dir); - - // The fakefs filesystem is the root's `data` directory; `meta.db` sits - // beside it. This is the same source the engine's app passes to mount_root. - char data_dir[4096]; - int n = snprintf(data_dir, sizeof(data_dir), "%s/data", root_dir); - if (n <= 0 || (size_t) n >= sizeof(data_dir)) { - xf_fail("root path too long"); - return -ENAMETOOLONG; - } - - intptr_t err = mount_root(&fakefs, data_dir); - if (err < 0) { - xf_logf("boot: mount_root failed: %s (%ld)", strerror((int) -err), (long) err); - xf_fail("mount_root failed: %s (%ld)", strerror((int) -err), (long) err); - return (int) err; - } - xf_logf("boot: root mounted"); - - // NOTE: nothing is registered here on purpose. The engine's fs/mount.c - // already carries a static table of the filesystems it uses (realfs, - // procfs, devptsfs, tmpfs) and MAX_FILESYSTEMS is 10. A registration path - // that overflows it hits `assert(!"reached filesystem limit")`, which is an - // abort() — and in an iOS app stderr goes nowhere, so that presents as the - // app simply dying during boot with no message. Registration is only for - // filesystems that are *not* in that table, so there is nothing to add. - - // The engine reports guest exits through this global hook; it is how - // xf_ish_run learns a command's exit status. - exit_hook = xf_exit_hook; - - // Creates pid 1. After this the guest has an init task, which is all the - // headless command runner needs — XForge does not run /sbin/init, it runs - // build commands as fresh children of init. - xf_logf("boot: become_first_process"); - err = become_first_process(); - if (err < 0) { - xf_logf("boot: become_first_process failed: %s (%ld)", strerror((int) -err), (long) err); - xf_fail("become_first_process failed: %s (%ld)", strerror((int) -err), (long) err); - return (int) err; - } - xf_logf("boot: pid 1 created"); - - // Virtual filesystems and device nodes the guest userland expects. Failures - // are not fatal: the rootfs may already provide some of these. - xf_logf("boot: device nodes + /proc /dev/pts"); - xf_ensure_devices(); - do_mount(&procfs, "proc", "/proc", "", 0); - do_mount(&devptsfs, "devpts", "/dev/pts", "", 0); - - // The console. Registering the host's tty driver *before* pid 1's stdio is - // opened is what makes /dev/console and /dev/tty1 the same terminal, and that - // terminal is the app's screen — so init, the console session it respawns, and - // the shell the user types into all appear there. - xf_logf("boot: console tty driver"); - pthread_mutex_lock(&s_console_lock); - s_console_stopping = false; - pthread_mutex_unlock(&s_console_lock); - tty_drivers[TTY_CONSOLE_MAJOR] = &xf_console_driver; - generic_mknodat(AT_PWD, "/dev/tty1", S_IFCHR | 0666, dev_make(TTY_CONSOLE_MAJOR, 1)); - set_console_device(TTY_CONSOLE_MAJOR, 1); - - // Share the app's own container into the guest at /host (realfs) so large - // artifacts — the darwin Swift SDK is hundreds of megabytes — can be staged - // by the host instead of being pushed through the command pipe. Not fatal. - if (host_dir != NULL && host_dir[0] != '\0') { - generic_mkdirat(AT_PWD, "/host", 0777); - int merr = do_mount(&realfs, host_dir, "/host", "", 0); - if (merr < 0) { - xf_logf("boot: could not share %s at /host: %s", host_dir, strerror(-merr)); - } else { - xf_logf("boot: %s shared at /host", host_dir); - } - } - - // pid 1's stdin/stdout/stderr are the console. Nothing inherits them by - // accident — the headless command runner gives every child its own fds — but - // init and everything init starts do, which is how the guest's own boot - // appears on the screen. - // - // Opened with O_NOCTTY, which is the difference between a console that works - // and one whose shell has no job control — see xf_create_console_stdio. - int stdio_err = xf_create_console_stdio(); - if (stdio_err < 0) { - // Not fatal for the command runner, which does not use pid 1's stdio. - // `xf_ish_start_init` will report the real problem if the console is - // missing, and the terminal shows that to the user. - xf_logf("boot: could not wire pid 1's stdio to /dev/console: %s", - strerror((int) -stdio_err)); - } - - s_booted = true; - xf_logf("boot: guest is up (pid 1 created; %s)", - xf_ish_console_ready() ? "console attached" : "no console"); - return 0; -} - -int xf_ish_start_init(const char *program) { - if (!s_booted) { - xf_fail("guest is not booted"); - return -ENODEV; - } - - const char *path = (program != NULL && program[0] != '\0') ? program : "/sbin/init"; - xf_logf("init: exec %s as pid 1", path); - - // argv, as do_execve wants it: NUL-separated entries and a final NUL. - size_t path_len = strlen(path); - char *argv = malloc(path_len + 2); - if (argv == NULL) { - xf_fail("out of memory building init's argv"); - return -ENOMEM; - } - memcpy(argv, path, path_len + 1); - argv[path_len + 1] = '\0'; - - // init gets the environment a console session expects; /sbin/xforge-login - // replaces TERM/HOME/PATH with its own anyway, and /etc/profile re-exports - // them for the shell it execs. - static const char *const envp = - "TERM=xterm-256color\0" - "HOME=/root\0" - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\0"; - - int err = do_execve(path, 1, argv, envp); - free(argv); - if (err < 0) { - xf_logf("init: do_execve(%s) failed: %s (%d)", path, strerror(-err), err); - xf_fail("do_execve(%s) failed: %s (%d)", path, strerror(-err), err); - return err; - } - // init runs on a thread of its own, like every other guest task, so the - // engine thread is free again as soon as this returns. - int start_err = task_start(current); - if (start_err != 0) { - // task_start reports a guest errno (negative), like the rest of the engine. - xf_logf("init: could not start pid 1: %s (%d)", strerror(-start_err), start_err); - xf_fail("could not start pid 1: %s (%d)", strerror(-start_err), start_err); - return start_err; - } - return 0; -} -// --- spawning a guest child -------------------------------------------------- -// -// Shared by the blocking and the detached entry points below: everything up to -// and including `task_start`, which is the point where the two diverge (one waits -// for the child, the other returns its pid and lets it run). -// -// `stdout_fd` is where the child's stdout and stderr go; pass -1 for none (a -// detached command redirects its own output in its command line). -// `stdin_path` is opened for the child's stdin; NULL means /dev/null. -// On success the child's pid is written to `*pid_out` and 0 is returned; on -// failure a negative errno is returned and nothing has been started. -static int xf_spawn_child(const char *command, const char *shell, - int stdout_fd, const char *stdin_path, - pid_t_ *pid_out) { - struct task *saved_current = current; - - int err = become_new_init_child(); - if (err < 0) { - current = saved_current; - xf_fail("become_new_init_child failed: %s (%d)", strerror(-err), err); - return err; - } - struct task *task = current; - - // stdin. A detached interactive shell is given a real file here: that file is - // its standard input for as long as it lives, which is what lets the host - // feed it a byte stream. - struct fd *stdin_fd = adhoc_fd_create(&realfs_fdops); - if (stdin_fd != NULL) { - const char *path = (stdin_path != NULL && stdin_path[0] != '\0') - ? stdin_path : "/dev/null"; - int fd = open(path, O_RDONLY); - if (fd < 0 && stdin_path != NULL) { - // Falling back to /dev/null would leave a shell reading EOF and - // exiting immediately, so report it instead of starting something - // that cannot work. - int open_err = errno; - current = saved_current; - xf_fail("could not open stdin %s: %s", path, strerror(open_err)); - return -open_err; - } - stdin_fd->real_fd = fd; - task->files->files[0] = stdin_fd; - } - - for (int idx = 1; idx <= 2; idx++) { - if (stdout_fd < 0) - break; - struct fd *out_fd = adhoc_fd_create(&realfs_fdops); - if (out_fd != NULL) { - out_fd->real_fd = dup(stdout_fd); - task->files->files[idx] = out_fd; - } - } - - // argv, as the NUL-separated, double-NUL-terminated block do_execve wants. - // `-c` is what turns the command into something /bin/sh runs and reports the - // exit status of. - static const char *const argv0 = "sh"; - static const char *const flag = "-c"; - size_t argv_len = strlen(argv0) + 1 + strlen(flag) + 1 + strlen(command) + 1 + 1; - char *argv_buf = calloc(1, argv_len); - if (argv_buf == NULL) { - current = saved_current; - xf_fail("out of memory building argv"); - return -ENOMEM; - } - size_t pos = 0; - memcpy(argv_buf + pos, argv0, strlen(argv0) + 1); pos += strlen(argv0) + 1; - memcpy(argv_buf + pos, flag, strlen(flag) + 1); pos += strlen(flag) + 1; - memcpy(argv_buf + pos, command, strlen(command) + 1); pos += strlen(command) + 1; - argv_buf[pos] = '\0'; - - // A build environment needs a real PATH, and HOME for the toolchains. TERM is - // set so an interactive shell does not refuse to prompt. - static const char *const envp = - "TERM=xterm-256color\0" - "HOME=/root\0" - "PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin\0"; - - err = do_execve(shell, 3, argv_buf, envp); - if (err < 0) { - free(argv_buf); - current = saved_current; - xf_fail("do_execve(%s) failed: %s (%d)", shell, strerror(-err), err); - return err; - } - free(argv_buf); - - *pid_out = task->pid; - task_start(task); - current = saved_current; - return 0; -} - -int xf_ish_run(const char *command, const char *shell, int timeout_ms, - size_t max_output, xf_guest_result *result) { - if (command == NULL || result == NULL) return -EINVAL; - memset(result, 0, sizeof(*result)); - - if (!s_booted) { - xf_fail("guest is not booted"); - return -ENODEV; - } - - const char *exec_path = (shell != NULL && shell[0] != '\0') ? shell : "/bin/sh"; - xf_logf("run: %.200s", command); - - int pipe_fds[2]; - if (pipe(pipe_fds) != 0) { - xf_fail("pipe failed: %s", strerror(errno)); - return -errno; - } - - // stdout+stderr to the pipe, merged — XForge presents one combined - // transcript, and separating them would need a second reader thread for no - // gain here. - pid_t_ pid = 0; - int err = xf_spawn_child(command, exec_path, pipe_fds[1], NULL, &pid); - close(pipe_fds[1]); - if (err < 0) { - close(pipe_fds[0]); - return err; - } - - result->launched = 1; - - // Register interest in this pid BEFORE reading, so a command that finishes - // immediately cannot report its exit before anyone is listening. - pthread_mutex_lock(&s_exit_lock); - s_wait_pid = pid; - s_wait_status = 0; - s_wait_done = false; - pthread_mutex_unlock(&s_exit_lock); - - // Read until the child closes the pipe (it does so when it exits), which - // both collects the output and waits for the command. The timeout is - // enforced with poll() so a hung guest command cannot wedge XForge. - char *buffer = malloc(XF_READ_CHUNK * 4); - if (buffer == NULL) { - close(pipe_fds[0]); - pthread_mutex_lock(&s_exit_lock); - s_wait_pid = 0; - pthread_mutex_unlock(&s_exit_lock); - xf_fail("out of memory reading command output"); - return -ENOMEM; - } - size_t length = 0, capacity = XF_READ_CHUNK * 4; - int truncated = 0; - struct timeval start; - gettimeofday(&start, NULL); - bool pipe_eof = false; - - while (!pipe_eof) { - if (timeout_ms > 0) { - struct timeval now; - gettimeofday(&now, NULL); - long long elapsed = (long long) (now.tv_sec - start.tv_sec) * 1000 - + (now.tv_usec - start.tv_usec) / 1000; - if (elapsed >= timeout_ms) { - result->timed_out = 1; - xf_logf("run: timeout after %d ms (pid %d)", timeout_ms, (int) pid); - break; - } - struct pollfd pfd = { .fd = pipe_fds[0], .events = POLLIN }; - int pr = poll(&pfd, 1, (int) (timeout_ms - elapsed)); - if (pr < 0 && errno != EINTR) { - pipe_eof = true; - break; - } - if (pr == 0) { - result->timed_out = 1; - xf_logf("run: timeout waiting for pid %d", (int) pid); - break; - } - } - // poll() said there is data (or the write end closed, which shows up as - // POLLHUP). Drain what is available; EOF ends the loop. - char chunk[XF_READ_CHUNK]; - ssize_t n = read(pipe_fds[0], chunk, sizeof(chunk)); - if (n < 0) { - if (errno == EINTR) - continue; - break; - } - if (n == 0) { - pipe_eof = true; - break; - } - if (max_output == 0 || length < max_output) { - size_t room = (max_output == 0 ? (size_t) n : max_output - length); - size_t take = (size_t) n < room ? (size_t) n : room; - if (length + take + 1 > capacity) { - size_t want = capacity * 2; - while (want < length + take + 1) - want *= 2; - char *grown = realloc(buffer, want); - if (grown == NULL) - break; - buffer = grown; - capacity = want; - } - memcpy(buffer + length, chunk, take); - length += take; - if (take < (size_t) n) - truncated = 1; - } else { - truncated = 1; - } - } - - close(pipe_fds[0]); - buffer[length] = '\0'; - - // Wait for the exit status, briefly: the pipe closes when the process exits, - // but the hook that records the status may run a moment later. - pthread_mutex_lock(&s_exit_lock); - if (!s_wait_done) { - struct timeval then; - gettimeofday(&then, NULL); - struct timespec deadline; - clock_gettime(CLOCK_REALTIME, &deadline); - deadline.tv_sec += 5; - while (!s_wait_done) { - if (pthread_cond_timedwait(&s_exit_cond, &s_exit_lock, &deadline) != 0) - break; - } - (void) then; - } - bool have_status = s_wait_done; - int status = s_wait_status; - s_wait_pid = 0; - s_wait_done = false; - pthread_mutex_unlock(&s_exit_lock); - - result->truncated = truncated; - result->output = buffer; - result->output_len = length; - if (have_status) { - result->exited = 1; - // The guest reports a wait(2)-style status; expose the plain exit code. - result->exit_code = (status & 0x7f) == 0 ? ((status >> 8) & 0xff) : 0; - result->term_signal = (status & 0x7f) == 0 ? 0 : (status & 0x7f); - } else if (!result->timed_out) { - // Output stopped but no exit was observed: the command was killed or the - // engine never reported it. Say so rather than reporting success. - result->exited = 1; - result->exit_code = -1; - } - - xf_logf("run: collected %zu bytes (truncated=%d timed_out=%d exited=%d " - "code=%d signal=%d)", - length, truncated, result->timed_out, result->exited, - result->exit_code, result->term_signal); - return 0; -} - -// --- detached processes ------------------------------------------------------ - -int xf_ish_run_detached(const char *command, const char *shell, const char *stdin_path) { - if (command == NULL) return -EINVAL; - if (!s_booted) { - xf_fail("guest is not booted"); - return -ENODEV; - } - - const char *exec_path = (shell != NULL && shell[0] != '\0') ? shell : "/bin/sh"; - xf_logf("run-detached: %.200s", command); - - // No stdout fd: a detached command redirects its own output, because there is - // no reader here to drain a pipe. That matters — a child writing into a pipe - // nobody reads fills the pipe buffer and blocks forever. - pid_t_ pid = 0; - int err = xf_spawn_child(command, exec_path, -1, stdin_path, &pid); - if (err < 0) - return err; - - xf_logf("run-detached: started pid %d", (int) pid); - return (int) pid; -} - -int xf_ish_process_alive(int pid) { - if (pid <= 0) return -EINVAL; - - pthread_mutex_lock(&s_exit_lock); - bool known_exited = xf_exit_was_seen((pid_t_) pid); - pthread_mutex_unlock(&s_exit_lock); - if (known_exited) return 0; - - // The engine gives no waitpid for guest pids, so ask the task table directly. - // A pid that has exited but not been reaped still appears here as a zombie, - // which is why the flag above is checked first: the exit hook is the only - // place that observes an exit reliably. - struct task *task = pid_get_task_zombie((dword_t) pid); - if (task == NULL) return 0; - return task->zombie ? 0 : 1; -} - -int xf_ish_kill_process(int pid, int signal) { - if (pid <= 0) return -EINVAL; - - struct task *task = pid_get_task_zombie((dword_t) pid); - if (task == NULL) { - xf_fail("no such process: %d", pid); - return -ESRCH; - } - // A real guest signal, not a byte on stdin. Sending the Ctrl-C *character* - // would need a tty to interpret it — the terminal's transport is a plain - // pipe, so the byte would simply be read as input. This reaches the process - // regardless of what its stdin is. - xf_logf("kill: pid %d signal %d", pid, signal); - send_signal(task, signal, SIGINFO_NIL); - return 0; -} - -void xf_guest_result_free(xf_guest_result *result) { - if (result == NULL) return; - free(result->output); - memset(result, 0, sizeof(*result)); -} - -void xf_ish_shutdown(void) { - // The engine cannot boot a second machine inside the same process, so there - // is nothing to tear down beyond forgetting that we booted — but a reader - // blocked in xf_ish_console_read has to be let go, or it would outlive the - // session it was reading for. - pthread_mutex_lock(&s_console_lock); - s_console_stopping = true; - pthread_cond_broadcast(&s_console_cond); - pthread_mutex_unlock(&s_console_lock); - s_booted = false; -} - -#endif /* TARGET_OS_SIMULATOR */ diff --git a/App/EmbeddedVM/ISHBridge.h b/App/EmbeddedVM/ISHBridge.h deleted file mode 100644 index 4aee34b..0000000 --- a/App/EmbeddedVM/ISHBridge.h +++ /dev/null @@ -1,178 +0,0 @@ -// -// ISHBridge.h -// XForge -// -// Thin C shim over the embedded Linux engine, **ish-arm64** -// (github.com/OpenMinis/ish-arm64). -// -// ish-arm64 is a fork of ish-app/ish that adds a native AArch64 guest backend -// to the threaded-code interpreter. That matters for XForge: the bundled root -// filesystem is an Alpine *aarch64* minirootfs, so with this engine it runs as -// a same-architecture guest (a handful of host instructions per guest -// instruction) instead of being cross-translated from x86. It emulates -// instructions by dispatching to pre-compiled "gadget" functions — it does not -// emit machine code at runtime and needs no executable memory, so it works in -// a sideloaded app with no JIT entitlement. -// -// XForge links the engine's core static libraries (`libish`, `libish_emu`, -// `libfakefs`) plus `tools/fakefs.c` (for `fakefs_import`) and this shim, then -// boots the root filesystem bundled in the app. -// -// Everything here is a plain C ABI so Swift can call it through the bridging -// header. All calls must be made from one dedicated permanent OS thread: -// the engine's `current` task pointer is thread-local, and a serial dispatch -// queue does not guarantee pthread affinity. -// -#ifndef XFORGE_ISH_BRIDGE_H -#define XFORGE_ISH_BRIDGE_H - -#include -#include - -/// Outcome of one headless guest command. -typedef struct xf_guest_result { - int launched; ///< the command started in the guest - int exited; ///< exit_code is valid - int exit_code; ///< exit status when exited != 0 - int term_signal; ///< terminating signal when exited == 0 - int timed_out; ///< killed because timeout_ms elapsed - int truncated; ///< output hit max_output - char *output; ///< merged stdout+stderr, malloc'd (free with xf_guest_result_free) - size_t output_len; -} xf_guest_result; - -/// Point the engine's own log at `path` (append mode) and remember it for -/// `xf_ish_log`. Pass NULL to stop logging. -/// -/// The engine's `printk` goes through its `log_handler`, chosen at build time. -/// XForge builds with `-Dlog_handler=nslog` (see EmbeddedLinux/build-ish-core.sh) -/// so kernel messages reach the device console. This file is for the messages -/// XForge itself produces around them — boot stages, import progress, and the -/// output of a command that failed — which is what makes a bug report readable. -/// -/// Returns 0 on success or a negative errno. -int xf_ish_set_log_file(const char *path); - -/// Append one timestamped line to the log set by `xf_ish_set_log_file`. -/// No-op (returning -ENODEV) when no log file is set. -int xf_ish_log(const char *text); - -/// Console — the guest's own terminal. -/// -/// XForge implements the guest console as a tty driver, which is what makes the -/// terminal a *terminal* rather than a pipe: the line discipline lives in the -/// guest's kernel, so echo, line editing, Ctrl-C (ISIG), Ctrl-D and job control -/// all behave exactly as they do on a console. The engine calls the driver when -/// the guest writes to `/dev/tty1` or `/dev/console`; the host pushes input back -/// with `xf_ish_console_write`. -/// -/// `/dev/tty1` and `/dev/console` are the same terminal (major `TTY_CONSOLE_MAJOR`, -/// minor 1), and it is the one pid 1's stdio is wired to — so `/sbin/init` and -/// everything it starts (the console session and the shell it execs) talk to the -/// screen in the app. -/// -/// Read and write are safe from any thread: the write callback only appends to a -/// host buffer, `xf_ish_console_write` takes the tty's own lock inside the engine, -/// and neither touches the engine's per-thread state. - -/// Start the guest's init as pid 1, giving it the console as its stdio. -/// -/// This is what turns the mounted root into a *booted system*: `/sbin/init` -/// reads `/etc/inittab`, which respawns `/sbin/xforge-login root` on the console. -/// Must be called on the engine thread, after `xf_ish_boot`. Returns 0, or a -/// negative errno. -int xf_ish_start_init(const char *program); - -/// 1 once the guest console exists (something has opened `/dev/console`). -int xf_ish_console_ready(void); - -/// Copy output produced by the guest console into `buf`, waiting up to -/// `timeout_ms` for at least one byte (-1 waits indefinitely, 0 returns at once). -/// -/// Returns the number of bytes copied, 0 if the wait timed out, or a negative -/// errno. Callers poll this in a loop; it does not touch the engine's thread- -/// local state, so it may block on a thread of its own. -ssize_t xf_ish_console_read(char *buf, size_t len, int timeout_ms); - -/// Feed host input into the guest console (keystrokes, pastes, control bytes). -/// Returns the number of bytes accepted, or a negative errno. -ssize_t xf_ish_console_write(const char *buf, size_t len); - -/// Tell the guest console how big the screen is, so the guest's programs wrap -/// and lay out correctly. Returns 0, or a negative errno. -int xf_ish_console_resize(int cols, int rows); - -/// Boot the guest from an installed fakefs root directory. -/// -/// The root is *already* fakefs — a `data/` tree plus `meta.db`, converted on the -/// build machine by `tools/fakefsify` (see EmbeddedLinux/build-rootfs.sh). The app -/// only unzips it, so there is no import step here: `fakefs_import` is a build-time -/// tool in this design, not a runtime one. -/// -/// `host_dir` (may be NULL) is mounted read-write into the guest at `/host`, so -/// the app's own container is reachable from inside Linux. That is how large -/// artifacts get in without pushing megabytes through the shell pipe. Returns 0 -/// on success or a negative errno. -/// -/// This creates pid 1, mounts the virtual filesystems, wires pid 1's stdio to the -/// console tty and brings up the host's end of that terminal — but it does not -/// exec anything: the caller decides when the guest's own `init` takes over, with -/// `xf_ish_start_init`. Keeping the two apart is what lets XForge verify the -/// mounted root before handing the machine to `/sbin/init`. -int xf_ish_boot(const char *root_dir, const char *host_dir); - -/// 1 once the guest is booted and commands can be run. -int xf_ish_is_booted(void); - -/// Run one command headlessly as a fresh child of init and capture its merged -/// stdout+stderr. `shell` is an absolute guest path (e.g. "/bin/sh") or NULL for -/// `/bin/sh`. `timeout_ms` of 0 means no timeout. Returns 0 if the command was -/// launched (inspect `result`), or a negative errno if it could not start. -/// -/// This call blocks until the command exits. -int xf_ish_run(const char *command, const char *shell, int timeout_ms, - size_t max_output, xf_guest_result *result); - -/// Start a command and return its guest pid immediately, without waiting for it. -/// -/// This exists because `xf_ish_run` blocks until the child exits, and the guest -/// engine may only be driven from one thread at a time — so a long-lived process -/// (an interactive shell) started through `xf_ish_run` would hold that thread -/// forever and nothing else could ever run. Starting it detached leaves the -/// engine free, and the caller polls the pid instead. -/// -/// Returns the guest pid (> 0) on success, or a negative errno. -/// -/// The process reads its stdin from `stdin_path` when that is not NULL: it is -/// opened once, when the child is created. Output is *not* captured — a detached -/// command is expected to redirect its own stdout/stderr (that is how the -/// terminal's transport works), so pass a path in the command itself. -int xf_ish_run_detached(const char *command, const char *shell, const char *stdin_path); - -/// Whether a detached process is still running. Returns 1 while it is alive, -/// 0 once it has exited, or a negative errno for an unknown pid. -int xf_ish_process_alive(int pid); - -// There is deliberately no blocking "wait for a detached process": the engine -// may only be driven from one thread, and a wait would hold it for the process's -// whole lifetime — the starvation that detached runs exist to avoid. Callers poll -// `xf_ish_process_alive` instead, which answers without blocking. - -/// Ask a detached process to stop, by delivering `signal` to it. Returns 0 if the -/// signal was delivered or a negative errno. -/// -/// This is a real guest signal, so Ctrl-C on an interactive shell reaches the -/// foreground program rather than merely detaching the screen. -int xf_ish_kill_process(int pid, int signal); - -/// Free `result->output` and zero the struct. -void xf_guest_result_free(xf_guest_result *result); - -/// Shut the guest down for this launch. The engine cannot boot a second machine -/// in the same process, so this only tears down local state. -void xf_ish_shutdown(void); - -/// Human-readable description of the last failure on this thread. -const char *xf_ish_last_error(void); - -#endif /* XFORGE_ISH_BRIDGE_H */ diff --git a/App/EmbeddedVM/ISHEmulator.swift b/App/EmbeddedVM/ISHEmulator.swift deleted file mode 100644 index 7c350f4..0000000 --- a/App/EmbeddedVM/ISHEmulator.swift +++ /dev/null @@ -1,427 +0,0 @@ -import Foundation - -/// A serial executor backed by one permanent OS thread. -/// -/// A DispatchQueue is not sufficient here: it preserves ordering but may move -/// work between pthreads. ish-arm64 stores its active guest process in thread-local -/// storage, so importing, booting, and every command must run on the exact same -/// pthread for the lifetime of the guest. -private final class GuestThreadExecutor: @unchecked Sendable { - private let condition = NSCondition() - private var jobs: [@Sendable () -> Void] = [] - private var thread: Thread? - - init() { - let thread = Thread { [weak self] in - self?.run() - } - thread.name = "org.xforge.ish.guest" - thread.qualityOfService = .userInitiated - self.thread = thread - thread.start() - } - - func submit(_ job: @escaping @Sendable () -> Void) { - condition.lock() - jobs.append(job) - condition.signal() - condition.unlock() - } - - private func run() { - while true { - condition.lock() - while jobs.isEmpty { - condition.wait() - } - let job = jobs.removeFirst() - condition.unlock() - job() - } - } -} - -/// `LinuxEmulator` backed by the embedded ish-arm64 engine. -/// -/// ish-arm64 runs a real Linux guest in-process: its aarch64 backend dispatches -/// guest instructions to pre-compiled "gadget" functions rather than emitting -/// machine code, so it needs no JIT entitlement and works in a sideloaded app. -/// The root filesystem is the Alpine aarch64 minirootfs bundled in the app; on -/// first boot it is imported into the engine's `fakefs` format, then mounted as `/`. -/// -/// Threading: ish-arm64 keeps its current guest task in a thread-local, so boot and -/// every command run serialized on one dedicated queue. -@MainActor -final class ISHEmulator: LinuxEmulator { - let name = "ish-arm64 (arm64 guest)" - private(set) var isRunning = false - - private let rootsDirectory: URL - private let hostDirectory: URL - private let guestThread = GuestThreadExecutor() - - init(rootsDirectory: URL, hostDirectory: URL) { - self.rootsDirectory = rootsDirectory - self.hostDirectory = hostDirectory - } - - func boot() async throws { - guard !isRunning else { return } - let roots = rootsDirectory - let host = hostDirectory - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - guestThread.submit { - do { - // Capture XForge's own breadcrumbs before the engine can - // write anything. The engine's kernel messages go through - // its build-time log handler (nslog, so they reach the - // device console); this file is the boot/import/command - // trace that makes a bug report readable. - XForgeLog.prepare() - XForgeLog.note("emulator: boot requested (rootfs \(roots.lastPathComponent))") - - // Import the bundled Alpine rootfs into fakefs the first - // time; subsequent launches reuse it. - let root = try RootfsInstaller.installIfNeeded(into: roots) - XForgeLog.note("emulator: rootfs ready at \(root.lastPathComponent)") - let rc = root.path.withCString { rootPath in - host.path.withCString { hostPath in - xf_ish_boot(rootPath, hostPath) - } - } - guard rc == 0 else { - throw LinuxVMError.notImplemented(ishLastError(fallback: "Could not boot the guest (errno \(rc)).")) - } - XForgeLog.note("emulator: boot complete") - continuation.resume() - } catch { - XForgeLog.note("emulator: boot failed: \(error.localizedDescription)") - continuation.resume(throwing: error) - } - } - } - isRunning = true - } - - /// Import the bundled rootfs on the guest's permanent serial thread, - /// without booting the emulator. The importer itself is host-only; the - /// engine's one-time initialization happens later in `boot()`. - func prepareRootfs() async throws { - let roots = rootsDirectory - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - guestThread.submit { - do { - XForgeLog.prepare() - XForgeLog.note("emulator: rootfs pre-install requested") - let root = try RootfsInstaller.installIfNeeded(into: roots) - XForgeLog.note("emulator: rootfs pre-installed at \(root.lastPathComponent)") - continuation.resume() - } catch { - XForgeLog.note("emulator: rootfs pre-install failed: \(error.localizedDescription)") - continuation.resume(throwing: error) - } - } - } - } - - func runCommand( - _ command: String, - shell: String?, - timeout: TimeInterval, - maxOutput: Int - ) async throws -> GuestCommandResult { - if !isRunning { try await boot() } - let timeoutMs = timeout <= 0 ? 0 : Int32(min(timeout * 1000, Double(Int32.max))) - let maxOut = maxOutput > 0 ? maxOutput : 256 * 1024 - - return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - guestThread.submit { - var raw = xf_guest_result() - let rc: Int32 = command.withCString { cCommand in - if let shell { - return shell.withCString { cShell in - xf_ish_run(cCommand, cShell, timeoutMs, maxOut, &raw) - } - } - return xf_ish_run(cCommand, nil, timeoutMs, maxOut, &raw) - } - guard rc == 0 else { - continuation.resume(throwing: LinuxVMError.notImplemented( - ishLastError(fallback: "The command could not start (errno \(rc))."))) - return - } - - let output = raw.output.map { String(cString: $0) } ?? "" - let result = GuestCommandResult( - launched: raw.launched != 0, - exited: raw.exited != 0, - exitCode: raw.exit_code, - termSignal: raw.term_signal, - timedOut: raw.timed_out != 0, - truncated: raw.truncated != 0, - output: output - ) - var mutable = raw - xf_guest_result_free(&mutable) - continuation.resume(returning: result) - } - } - } - - func startInit(_ program: String) async throws { - if !isRunning { try await boot() } - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - guestThread.submit { - let rc = program.withCString { xf_ish_start_init($0) } - guard rc == 0 else { - continuation.resume(throwing: LinuxVMError.notImplemented( - ishLastError(fallback: "The guest's init could not start (errno \(rc))."))) - return - } - XForgeLog.note("emulator: \(program) is pid 1") - continuation.resume() - } - } - } - - func openConsole( - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> any GuestConsole { - if !isRunning { try await boot() } - let console = BridgeConsole(onOutput: onOutput) - console.startReading() - XForgeLog.note("emulator: console reader started") - return console - } - - func startDetached( - _ command: String, - shell: String?, - stdinPath: String? - ) async throws -> DetachedProcess { - if !isRunning { try await boot() } - return try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - guestThread.submit { - let rc: Int32 = command.withCString { cCommand in - let run: (UnsafePointer?) -> Int32 = { cShell in - if let stdinPath { - return stdinPath.withCString { cStdin in - xf_ish_run_detached(cCommand, cShell, cStdin) - } - } - return xf_ish_run_detached(cCommand, cShell, nil) - } - if let shell { return shell.withCString { run($0) } } - return run(nil) - } - guard rc > 0 else { - // Negative: the process never started. Report it rather than - // handing back a handle to something that does not exist. - continuation.resume(throwing: LinuxVMError.notImplemented( - ishLastError(fallback: "The process could not start (errno \(rc))."))) - return - } - continuation.resume(returning: GuestProcess(pid: rc, thread: self.guestThread)) - } - } - } - - func shutdown() async { - guard isRunning else { return } - await withCheckedContinuation { (continuation: CheckedContinuation) in - guestThread.submit { - xf_ish_shutdown() - continuation.resume() - } - } - isRunning = false - } -} - -/// A guest process started by `ISHEmulator.startDetached`. -/// -/// Every call hops onto the one guest thread, because the engine may only be -/// driven from there. The thread is free precisely because the process was -/// *started* detached rather than run to completion — which is the whole reason -/// this exists: a blocking run would have held that thread for the process's -/// lifetime and starved everything else in the app. -/// -/// Deliberately NOT `@MainActor`: it is a handle to something that lives on the -/// guest thread, and it is constructed from inside that thread's queue. Marking -/// it main-actor isolated made its initialiser unreachable from where it is -/// created, and it has no main-actor state to protect. -private final class GuestProcess: DetachedProcess, @unchecked Sendable { - let pid: Int32 - private let thread: GuestThreadExecutor - - init(pid: Int32, thread: GuestThreadExecutor) { - self.pid = pid - self.thread = thread - } - - var isRunning: Bool { - // Asks the guest, which means hopping onto its thread — so this is - // `async` rather than a plain property, and it does not block: the hop is - // a queued job that completes immediately, not a wait on the process. - get async { await checkAlive() } - } - - private func checkAlive() async -> Bool { - let pid = self.pid - return await withCheckedContinuation { (continuation: CheckedContinuation) in - thread.submit { - continuation.resume(returning: xf_ish_process_alive(pid) == 1) - } - } - } - - func signal(_ number: Int32) async { - let pid = self.pid - await withCheckedContinuation { (continuation: CheckedContinuation) in - thread.submit { - _ = xf_ish_kill_process(pid, number) - continuation.resume() - } - } - } - - /// Poll for exit rather than blocking the guest thread. - /// - /// A blocking wait would occupy the one thread the engine allows, for as long - /// as the process runs — the very starvation that detached processes exist to - /// avoid. Polling asks a cheap question repeatedly instead, leaving the thread - /// free between asks. - @discardableResult - func waitForExit(timeout: TimeInterval) async -> Bool { - let deadline = timeout > 0 ? Date().addingTimeInterval(timeout) : nil - while true { - if !(await checkAlive()) { return true } - if let deadline, Date() >= deadline { return false } - try? await Task.sleep(for: .milliseconds(200)) - } - } -} - -/// The host end of the guest's console tty. -/// -/// Two directions, two threads, because that is what a terminal is: -/// - **out**: a thread of its own blocks in `xf_ish_console_read` and hands each -/// chunk to `onOutput` as the guest writes it. It is not the engine's thread: -/// reading the console touches none of the engine's per-thread guest state, and -/// the engine thread has to stay free to run the guest (and any command -/// XForge has started) while output streams out. -/// - **in**: writes are handed to a serial queue. A tty's input buffer is small, -/// so a paste larger than it has to be pushed in pieces, waiting for the guest -/// to drain each one — and that waiting must not happen on the thread that is -/// handling a keystroke. -final class BridgeConsole: GuestConsole, @unchecked Sendable { - /// Big enough that a screenful arrives in a single read, small enough that a - /// burst of output does not allocate. - private static let readChunk = 32 * 1024 - /// How long a single read waits before the loop checks whether it should stop. - private static let readTimeoutMs: Int32 = 500 - /// Give up on an input burst the guest has not drained — a tty nobody is - /// reading from must not wedge the keyboard. - private static let inputDeadline: TimeInterval = 5 - - private let onOutput: @Sendable (String) -> Void - private let inputQueue = DispatchQueue(label: "org.xforge.console.input") - private let state = NSLock() - private var stopped = false - - init(onOutput: @escaping @Sendable (String) -> Void) { - self.onOutput = onOutput - } - - deinit { - stop() - } - - var isReady: Bool { xf_ish_console_ready() == 1 } - - /// Begin delivering the console's output. Called once, by the emulator. - func startReading() { - let thread = Thread { [weak self] in self?.readLoop() } - thread.name = "org.xforge.console.reader" - thread.qualityOfService = .userInitiated - thread.stackSize = 256 * 1024 - thread.start() - } - - @discardableResult - func write(_ text: String) -> Bool { - guard !text.isEmpty, let data = text.data(using: .utf8), !data.isEmpty else { - return false - } - // `GuestConsole` promises `false` for "there is no console to write to", - // and the terminal reports that to the user. Returning `true` here - // unconditionally meant input was accepted, queued against a console that - // was stopped or not up yet, and silently dropped — the one failure mode a - // terminal must not have. - guard !isStopped, isReady else { return false } - inputQueue.async { [weak self] in self?.push(data) } - return true - } - - func resize(cols: Int, rows: Int) { - guard cols > 0, rows > 0 else { return } - let result = xf_ish_console_resize(Int32(cols), Int32(rows)) - if result != 0 { - XForgeLog.note("console: could not set \(cols)x\(rows) (error \(result))") - } - } - - func stop() { - state.lock() - stopped = true - state.unlock() - } - - private var isStopped: Bool { - state.lock() - defer { state.unlock() } - return stopped - } - - /// Read until stopped. `xf_ish_console_read` returns 0 when its short wait - /// expires, which is what gives this loop the chance to notice `stop()`. - private func readLoop() { - var buffer = [CChar](repeating: 0, count: Self.readChunk) - while !isStopped { - let count = buffer.withUnsafeMutableBufferPointer { pointer -> Int in - guard let base = pointer.baseAddress else { return 0 } - return Int(xf_ish_console_read(base, pointer.count, Self.readTimeoutMs)) - } - guard count > 0 else { continue } - let bytes = buffer.prefix(count).map { UInt8(bitPattern: $0) } - onOutput(String(decoding: bytes, as: UTF8.self)) - } - } - - /// Push input in pieces, waiting for the guest to drain the tty between them. - private func push(_ data: Data) { - var offset = 0 - let deadline = Date().addingTimeInterval(Self.inputDeadline) - while offset < data.count, !isStopped, Date() < deadline { - let accepted = data.withUnsafeBytes { raw -> Int in - guard let base = raw.bindMemory(to: CChar.self).baseAddress else { return -1 } - return Int(xf_ish_console_write(base + offset, data.count - offset)) - } - if accepted > 0 { - offset += accepted - continue - } - // No console yet, or its buffer is full while the guest reads. - Thread.sleep(forTimeInterval: 0.01) - } - if offset < data.count { - XForgeLog.note("console: \(data.count - offset) byte(s) of input not delivered") - } - } -} - -/// Reads the bridge's thread-local last error. Must be called on the guest thread. -private func ishLastError(fallback: String) -> String { - let message = String(cString: xf_ish_last_error()) - return message.isEmpty ? fallback : message -} diff --git a/App/EmbeddedVM/InteractiveShellSession.swift b/App/EmbeddedVM/InteractiveShellSession.swift deleted file mode 100644 index 0585b06..0000000 --- a/App/EmbeddedVM/InteractiveShellSession.swift +++ /dev/null @@ -1,53 +0,0 @@ -import Foundation - -/// A live shell in the embedded Linux: what the terminal types into. -/// -/// The shell runs on the guest's **console**, which the host implements as a tty -/// (`GuestConsole`). That is the whole design, and it is what makes the terminal a -/// terminal rather than a pipe: -/// - the host writes keystrokes into the guest's tty, so the guest kernel's line -/// discipline supplies echo, backspace, line editing, `Ctrl-D` and the rest; -/// - `Ctrl-C` is a *byte* again, turned into `SIGINT` for the foreground program -/// by the tty — so an interrupt reaches whatever is running, including -/// `Ctrl-Z`'s job control; -/// - output appears as the guest writes it, including prompts that do not end in -/// a newline, which is what makes a password prompt or a progress bar usable. -/// -/// The guest's `/sbin/init` owns that console and respawns `/sbin/xforge-login -/// root` on it, so the shell is that script's child — it execs root's login shell -/// from `/etc/passwd`: logging out gives a fresh session rather than a dead -/// screen. -@MainActor -protocol InteractiveShellSession: AnyObject { - /// Guest pid the session speaks for. For a console login this is init, which - /// owns the console; the shell itself is init's child and may be replaced. - var pid: Int32 { get } - /// False once the session has stopped; `send` reports failure rather than - /// appearing to accept input nobody will read. - var isRunning: Bool { get } - - /// Send text to the console. Returns false when the session is gone or there - /// is no console to write to, so a caller can say so instead of silently - /// dropping it. - @discardableResult - func send(_ text: String) -> Bool - - /// Send a single control byte — `0x03` for Ctrl-C, `0x04` for Ctrl-D — or any - /// character the key bar cannot produce. - @discardableResult - func sendControl(_ scalar: UInt8) -> Bool - - /// Interrupt whatever is running in the foreground, with a real `SIGINT`. - /// - /// This is the Ctrl-C *character* written to the console: the guest's tty - /// turns it into the signal for the foreground process group, which is exactly - /// what a terminal is supposed to do. - func interruptForeground() async - - /// Tell the guest how big the screen is, so its programs wrap and lay out - /// correctly. - func resize(cols: Int, rows: Int) - - /// Stop the session and its console reader. - func stop() -} diff --git a/App/EmbeddedVM/LinuxEmulator.swift b/App/EmbeddedVM/LinuxEmulator.swift deleted file mode 100644 index 2f4fd12..0000000 --- a/App/EmbeddedVM/LinuxEmulator.swift +++ /dev/null @@ -1,125 +0,0 @@ -import Foundation - -/// Result of one headless command run in the guest. -struct GuestCommandResult: Sendable { - var launched: Bool - var exited: Bool - var exitCode: Int32 - var termSignal: Int32 - var timedOut: Bool - var truncated: Bool - var output: String - - /// Exit status the caller should branch on (non-zero also for timeouts/signals). - var status: Int32 { - if timedOut { return 124 } // like coreutils `timeout` - if !exited && termSignal != 0 { return 128 + termSignal } - return exitCode - } -} - -/// The host end of the guest's console — a real tty, so what the host writes goes -/// through the guest kernel's line discipline: it echoes, it edits, it turns -/// Ctrl-C into a signal for the foreground program, and it lets a full-screen -/// program put the terminal in raw mode. -/// -/// Deliberately not main-actor isolated: reading blocks until the guest writes, -/// so it happens on a thread of its own, and the calls it makes touch no -/// per-thread engine state. -protocol GuestConsole: AnyObject, Sendable { - /// Whether the guest has opened its console yet. - var isReady: Bool { get } - /// Push host input into the guest's console. Returns false only when there is - /// no console to write to. - @discardableResult func write(_ text: String) -> Bool - /// Tell the guest how big the screen is, so its programs wrap and lay out - /// correctly (and redraw when the size changes). - func resize(cols: Int, rows: Int) - /// Stop reading. Idempotent. - func stop() -} - -/// An in-process Linux execution engine. -/// -/// iOS cannot spawn subprocesses, so the embedded Linux runs as a **library** -/// inside the app. XForge uses ish-arm64 (github.com/OpenMinis/ish-arm64): a real Linux -/// kernel + userspace emulator whose threaded-code interpreter dispatches aarch64 -/// guest instructions to pre-compiled functions ("gadgets"). It emits no machine -/// code and needs no executable memory, so no JIT entitlement is required and it -/// works in a sideloaded app. -/// -/// The engine's primitive is "run this command line, give me its merged output -/// and exit status" — the engine's guest command runner — not a byte -/// pipe. Conformers boot a guest and then execute commands through that. The one -/// exception is the guest console, which *is* a byte pipe in both directions (see -/// `GuestConsole`), because a terminal has to be a tty to behave like one. -@MainActor -protocol LinuxEmulator: AnyObject { - var name: String { get } - var isRunning: Bool { get } - /// Boot the guest; returns once commands can be run. - func boot() async throws - /// Import the bundled root filesystem into the engine's on-disk format - /// ahead of first use, without booting the guest. Idempotent: a root that - /// is already installed is left untouched. - func prepareRootfs() async throws - /// Run one command headlessly in the guest. - func runCommand( - _ command: String, - shell: String?, - timeout: TimeInterval, - maxOutput: Int - ) async throws -> GuestCommandResult - - /// Start the guest's own init as pid 1, giving it the console as its stdio. - /// This is what boots the guest as a *system*: init reads /etc/inittab, which - /// puts a root shell on the console. Safe to call once per boot. - func startInit(_ program: String) async throws - - /// Attach to the guest console, delivering its output as the guest produces - /// it. Returns the host end, which is what the terminal types into. - func openConsole(onOutput: @Sendable @escaping (String) -> Void) async throws -> any GuestConsole - - /// Start a command and return immediately, without waiting for it to exit. - /// - /// `stdinPath` is a *guest* path the child reads its stdin from (NULL for - /// `/dev/null`). A detached command redirects its own stdout/stderr, so the - /// caller supplies a complete command line. - /// - /// Returns a handle for waiting on and signalling the process. - func startDetached( - _ command: String, - shell: String?, - stdinPath: String? - ) async throws -> DetachedProcess - - func shutdown() async -} - -/// A guest process started with `startDetached`. -/// -/// It is deliberately thin: the engine reports no exit *status* for a process -/// nobody waits on, so a detached command communicates its result through -/// whatever its command line redirects to. This handle answers the two questions -/// the host actually needs — is it still running, and how do I stop it. -/// Not `@MainActor`: the handle itself is thread-agnostic and does its own -/// hopping onto the guest thread, so callers may hold and use it from anywhere. -/// `Sendable` is what carries that guarantee. -protocol DetachedProcess: AnyObject, Sendable { - /// Guest pid, for logging and for `kill`. - var pid: Int32 { get } - /// Whether it is still running. - /// - /// `async`, because answering it means asking the guest — every engine call - /// has to hop onto the one thread the engine allows, so this cannot be a - /// synchronous property even though it reads like one. - var isRunning: Bool { get async } - /// Send a real guest signal. `SIGINT` (2) interrupts the foreground program, - /// `SIGTERM` (15) asks it to stop, `SIGKILL` (9) cannot be caught. - func signal(_ number: Int32) async - /// Wait for exit, up to `timeout` seconds (0 waits indefinitely). - /// Returns true if it exited. - @discardableResult - func waitForExit(timeout: TimeInterval) async -> Bool -} - diff --git a/App/EmbeddedVM/LinuxVM.swift b/App/EmbeddedVM/LinuxVM.swift deleted file mode 100644 index 81f4c5d..0000000 --- a/App/EmbeddedVM/LinuxVM.swift +++ /dev/null @@ -1,61 +0,0 @@ -import Foundation - -enum GuestShell { - static func quote(_ value: String) -> String { - "'" + value.replacingOccurrences(of: "'", with: "'\\''") + "'" - } - - static func environment(_ values: [String: String]?) -> String { - guard let values else { return "" } - return values.sorted { $0.key < $1.key } - .map { "\($0.key)=\(quote($0.value))" } - .joined(separator: " ") + " " - } -} - -/// Interface to the embedded Linux VM. The concrete implementation boots a -/// bundled iSH-style userspace and gives us a shell to run commands in plus -/// access to the guest filesystem (read/write files, stage artifacts). -@MainActor -protocol LinuxVM: AnyObject { - /// Boot the embedded Linux (blocking until a shell is ready). - func boot() async throws - /// Install the bundled rootfs ahead of first use, so the first build does - /// not pay for the import. Best effort; a no-op once it is installed. - func prepareRootfs() async - /// Run a command through the guest's interactive login terminal, streaming output. - func run( - _ command: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 - /// Run a shell script through the terminal's login launch command - /// (`/bin/sh`), delivering output as the guest writes it. - func runLoginStreaming( - _ script: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 - /// Read a file out of the guest filesystem into a host URL. - func copyOut(guestPath: String, to hostURL: URL) async throws - /// Stage a host file into the guest filesystem. - func copyIn(hostURL: URL, to guestPath: String) async throws - /// Attach the screen to the guest's console shell: the session `/sbin/init` - /// respawns there. Output is delivered as the guest writes it. - func startInteractiveShell( - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> any InteractiveShellSession - var isBooted: Bool { get } -} - -extension LinuxVM { - /// Default for VMs with no shared folder to tail: fall back to the one-shot - /// capture, which delivers the output when the command finishes. - func runLoginStreaming( - _ script: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - try await run(script, environment: environment, onOutput: onOutput) - } -} diff --git a/App/EmbeddedVM/RootfsInstaller.swift b/App/EmbeddedVM/RootfsInstaller.swift deleted file mode 100644 index 048ec9e..0000000 --- a/App/EmbeddedVM/RootfsInstaller.swift +++ /dev/null @@ -1,316 +0,0 @@ -import Foundation -import ZIPFoundation - -/// Installs the bundled Alpine aarch64 root filesystem into the app container -/// ahead of first use — at launch, with the first boot as the fallback. -/// -/// The archive ships *inside the app*, so nothing is downloaded after install. -/// -/// **The archive is already a fakefs**, not a plain tarball: it contains a -/// `data/` tree and a `meta.db` SQLite database, converted on the build machine -/// by the engine's own `tools/fakefsify` (see `EmbeddedLinux/build-rootfs.sh`). -/// That is deliberate, and it is what the engine's reference implementation does: -/// the conversion touches thousands of files, and doing it here means first -/// launch is a plain unzip instead of a multi-minute import on a phone. -/// -/// The root is a full Alpine userspace. The published root also carries the build -/// toolchain — xtool, swiftly, the Swift toolchain and the Darwin SDK — installed -/// at build time by `EmbeddedLinux/install-toolchain.sh`, run in a chroot of the -/// root before the fakefs conversion (see `EmbeddedLinux/build-rootfs.sh`). A root -/// built without it (`XFORGE_PROVISION=none`) is provisioned *by the guest* -/// instead, on demand, with the same script — see the Toolkit screen. -enum RootfsInstaller { - /// The ZIP of the pre-converted fakefs, built by `EmbeddedLinux/build-rootfs.sh`. - static let archiveName = "alpine-rootfs" - static let archiveExtension = "zip" - - /// Directory name of the installed root inside `/embedded-linux/roots`. - /// The Alpine version is in the name so a distribution bump starts clean. - static let rootName = "alpine-3.21" - - /// Roots from earlier layouts that must not be reused. XForge used to import a - /// tarball at runtime (`alpine-3.24.2`) and, before that, a different root - /// entirely (`alpine`). They are removed only once the replacement is valid. - static let legacyRootNames = ["alpine", "alpine-3.24.2", "alpine-3.23.3"] - - /// The ZIP holds `alpine-rootfs/` at its top level (the builder packs the - /// directory, so the paths are relative and unzip into Documents directly). - private static let archiveEntryPrefix = "alpine-rootfs" - - /// The manifest the builder writes into every root, relative to `data/`. - /// Its `stamp:` line names the revision — see `EmbeddedLinux/build-rootfs.sh`. - private static let manifestPath = "usr/local/share/xforge/rootfs-manifest.txt" - - static func bundledArchiveURL() -> URL? { - Bundle.main.url(forResource: archiveName, withExtension: archiveExtension) - } - - /// Whether the installed root is a different revision from the one bundled in - /// this build of the app, and therefore has to be replaced. - /// - /// Only a root that carries a manifest — which is every root XForge has ever - /// installed itself — can be compared, so anything unidentified is left in - /// place rather than replaced by something that might be worse. - private static func installedRootIsStale(_ root: URL) -> Bool { - let manifest = root.appendingPathComponent("data/\(manifestPath)") - guard let installed = stamp(inManifestAt: manifest) else { return false } - guard let bundled = bundledStamp() else { return false } - return installed != bundled - } - - /// The revision named by the manifest inside the bundled ZIP, read straight - /// out of the archive so asking the question costs no unpacking. - private static func bundledStamp() -> String? { - guard let archiveURL = bundledArchiveURL(), - let zip = try? Archive(url: archiveURL, accessMode: .read), - let entry = zip["\(archiveEntryPrefix)/\(manifestPath)"] else { - return nil - } - var contents = Data() - guard (try? zip.extract(entry, consumer: { contents.append($0) })) != nil else { - return nil - } - guard let text = String(data: contents, encoding: .utf8) else { return nil } - return stamp(inManifestText: text) - } - - private static func stamp(inManifestAt url: URL) -> String? { - guard let data = try? Data(contentsOf: url), - let text = String(data: data, encoding: .utf8) else { return nil } - return stamp(inManifestText: text) - } - - private static func stamp(inManifestText text: String) -> String? { - for line in text.split(separator: "\n") where line.hasPrefix("stamp:") { - let value = line.dropFirst("stamp:".count) - .trimmingCharacters(in: .whitespaces) - if !value.isEmpty { return value } - } - return nil - } - - static func installedRoot(in rootsDirectory: URL) -> URL { - rootsDirectory.appendingPathComponent(rootName, isDirectory: true) - } - - /// A root is usable when both halves of the fakefs are present and non-empty: - /// the `data/` tree and the `meta.db` database that indexes it. - static func isInstalled(in rootsDirectory: URL) -> Bool { - let fm = FileManager.default - let root = installedRoot(in: rootsDirectory) - let metadata = root.appendingPathComponent("meta.db") - let data = root.appendingPathComponent("data", isDirectory: true) - - var isDirectory: ObjCBool = false - guard fm.fileExists(atPath: data.path, isDirectory: &isDirectory), - isDirectory.boolValue, - fm.fileExists(atPath: metadata.path), - let attributes = try? fm.attributesOfItem(atPath: metadata.path), - let size = attributes[.size] as? NSNumber, - size.intValue > 0 else { - return false - } - return true - } - - /// Returns the ready-to-boot fakefs root, unpacking the bundled archive the - /// first time — or when the bundled archive is a newer revision than the one - /// already installed. - @discardableResult - static func installIfNeeded(into rootsDirectory: URL, - progress: RootfsImportProgress? = nil) throws -> URL { - let root = installedRoot(in: rootsDirectory) - if isInstalled(in: rootsDirectory) { - if installedRootIsStale(root) { - // The guest's own setup lives in the root — its /etc/inittab, its - // default shell, its profile. Without this check, a device that - // had booted once would keep the first root it ever installed and - // never see any of that change. - XForgeLog.note("rootfs: the installed root is not the bundled revision; reinstalling") - } else { - removeLegacyRoots(in: rootsDirectory, fileManager: .default) - return root - } - } - - guard let archive = bundledArchiveURL() else { - throw RootfsError.archiveMissing( - "No Alpine rootfs is bundled in the app (looked for " - + "\(archiveName).\(archiveExtension)). Run EmbeddedLinux/build-rootfs.sh " - + "before building, or rebuild via CI.") - } - - XForgeLog.note("rootfs: unpacking the bundled \(archive.lastPathComponent)") - return try install(archive: archive, into: rootsDirectory, progress: progress) - } - - /// Unpack a rootfs ZIP. The existing root is retained until the new archive - /// has completely unpacked and passed fakefs validation. - /// - /// `progress` is called from the unpacking thread with a 0..1 fraction. - @discardableResult - static func install(archive: URL, into rootsDirectory: URL, - progress: RootfsImportProgress? = nil) throws -> URL { - let fm = FileManager.default - let root = installedRoot(in: rootsDirectory) - guard archive.pathExtension.lowercased() == archiveExtension else { - throw RootfsError.importFailed( - "Choose an \(archiveName).\(archiveExtension) rootfs archive.") - } - - try fm.createDirectory(at: rootsDirectory, withIntermediateDirectories: true) - - // Unpack into a staging directory and move it into place afterwards, so a - // failure part-way cannot leave a half-populated root that later looks - // installed. (The old import path needed the same care; an interrupted - // staging directory is as large as the whole root filesystem.) - let staging = rootsDirectory.appendingPathComponent( - ".import-\(UUID().uuidString)", isDirectory: true) - try? fm.removeItem(at: staging) - try fm.createDirectory(at: staging, withIntermediateDirectories: true) - - progress?.report(fraction: 0, message: "Opening \(archive.lastPathComponent)") - - guard let zip = try? Archive(url: archive, accessMode: .read) else { - try? fm.removeItem(at: staging) - throw RootfsError.importFailed( - "\(archive.lastPathComponent) could not be opened — it is not a readable ZIP.") - } - - // ZIPFoundation walks entries in order; progress is entry count, which is - // the honest denominator here (entries are small and uniform enough). - let entries = Array(zip) - let total = max(entries.count, 1) - var index = 0 - - for entry in entries { - index += 1 - // Strip the leading `alpine-rootfs/` so the ZIP unpacks to - // `staging/data`, `staging/meta.db` — the layout `mount_root` wants. - let relative = stripArchivePrefix(entry.path) - if relative.isEmpty { continue } - let destination = staging.appendingPathComponent(relative) - - do { - if entry.type == .directory { - try fm.createDirectory(at: destination, withIntermediateDirectories: true) - continue - } - try fm.createDirectory(at: destination.deletingLastPathComponent(), - withIntermediateDirectories: true) - // `.overwrite` keeps a re-unpack into a dirty staging dir honest. - _ = try zip.extract(entry, to: destination, skipCRC32: false) - } catch { - try? fm.removeItem(at: staging) - throw RootfsError.importFailed( - "unpacking \(relative) failed: \(error.localizedDescription)") - } - - if index % 64 == 0 || index == total { - progress?.report(fraction: Double(index) / Double(total), message: relative) - } - } - - guard isValidImportedRoot(staging, fileManager: fm) else { - try? fm.removeItem(at: staging) - throw RootfsError.importFailed( - "the unpacked rootfs is missing its data directory or metadata database") - } - - // Safe to remove the old root only now: `staging` has already passed - // validation, so a failure cannot leave the app with no root at all. - if fm.fileExists(atPath: root.path) { - try fm.removeItem(at: root) - } - try fm.moveItem(at: staging, to: root) - removeLegacyRoots(in: rootsDirectory, fileManager: fm) - progress?.report(fraction: 1, message: nil) - XForgeLog.note("rootfs: ready at \(root.lastPathComponent)") - return root - } - - /// Drops the builder's top-level directory from an archive entry path. - private static func stripArchivePrefix(_ path: String) -> String { - var relative = path - let prefix = archiveEntryPrefix + "/" - if relative == archiveEntryPrefix { return "" } - if relative.hasPrefix(prefix) { - relative.removeFirst(prefix.count) - } - return relative.trimmingCharacters(in: CharacterSet(charactersIn: "/")) - } - - private static func removeLegacyRoots(in rootsDirectory: URL, fileManager fm: FileManager) { - for name in legacyRootNames where name != rootName { - let stale = rootsDirectory.appendingPathComponent(name, isDirectory: true) - if fm.fileExists(atPath: stale.path) { - XForgeLog.note("rootfs: removing the superseded \(name) root") - try? fm.removeItem(at: stale) - } - } - } - - /// Validates an unpacked fakefs: the `data/` tree plus a non-empty `meta.db`. - private static func isValidImportedRoot(_ directory: URL, fileManager fm: FileManager) -> Bool { - var isDirectory: ObjCBool = false - guard fm.fileExists(atPath: directory.appendingPathComponent("data").path, - isDirectory: &isDirectory), - isDirectory.boolValue else { - return false - } - let metadata = directory.appendingPathComponent("meta.db") - guard fm.fileExists(atPath: metadata.path), - let attributes = try? fm.attributesOfItem(atPath: metadata.path), - let size = attributes[.size] as? NSNumber, - size.intValue > 0 else { - return false - } - return true - } - - enum RootfsError: LocalizedError { - case archiveMissing(String) - case importFailed(String) - - var errorDescription: String? { - switch self { - case .archiveMissing(let detail): return detail - case .importFailed(let detail): return detail - } - } - } -} - -/// Reports the progress of a rootfs unpack. -/// -/// Deliberately a plain `@unchecked Sendable` type and NOT a `@MainActor` one: -/// it is called from the unpacking thread, and a main-actor type cannot be -/// referenced from a non-isolated closure under Swift 6's strict concurrency. -/// `report` does its own hop if the caller needs one. -final class RootfsImportProgress: @unchecked Sendable { - /// One update per this much progress. - private static let step = 0.005 - - private let lock = NSLock() - private var lastPublished: Double = -1 - private let onUpdate: @Sendable (Double, String?) -> Void - - init(onUpdate: @escaping @Sendable (Double, String?) -> Void) { - self.onUpdate = onUpdate - } - - /// Called from the unpacking thread. `fraction` is clamped to 0...1. - func report(fraction: Double, message: String?) { - let clamped = fraction.isFinite ? min(max(fraction, 0), 1) : 0 - lock.lock() - // Always let the final update through, so the row can reach 100%. - let isLast = clamped >= 1 - if !isLast && clamped - lastPublished < Self.step { - lock.unlock() - return - } - lastPublished = clamped - lock.unlock() - onUpdate(clamped, message) - } -} diff --git a/App/Models/Project.swift b/App/Models/Project.swift index 00aca36..8131fb6 100644 --- a/App/Models/Project.swift +++ b/App/Models/Project.swift @@ -2,11 +2,14 @@ import Foundation /// A SwiftPM package that can be built into an iOS app. struct Project: Identifiable, Hashable, Codable { - static let projectsRoot = "/root/projects" + /// Project directories live under `/projects`; this is the stored + /// prefix a project's `rootPath` is built from, kept so a project record and + /// its directory can be checked against each other. + static let projectsRoot = "projects" var id: UUID = UUID() var name: String var organizationIdentifier: String = "com.example" - /// Path of the package root inside the embedded Linux filesystem. + /// The project's stored path, relative to the app container (`projects/`). var rootPath: String var createdAt: Date = Date() /// Configured Info.plist settings for the produced app (editable in the GUI). @@ -89,7 +92,8 @@ protocol BuildExecutor { enum SDKSource { /// A prebuilt `darwin.artifactbundle` we host (built in CI from Xcode). + /// + /// It used to have a `.bundled` case for an SDK already inside the guest + /// filesystem. There is no guest, so there is one way to get an SDK: fetch it. case hostedRemote(URL) - /// Already inside the embedded Linux filesystem. - case bundled(String) } diff --git a/App/Models/XcodeProject.swift b/App/Models/XcodeProject.swift index 3d1a021..8c4867b 100644 --- a/App/Models/XcodeProject.swift +++ b/App/Models/XcodeProject.swift @@ -20,7 +20,7 @@ import Foundation /// a parser instead of fighting the graph. See Docs/XCODE-ALTERNATIVE.md. /// /// Deliberately *not* here yet: -/// - anything that compiles (see `NativeToolchain`/`EmbeddedLinuxExecutor`); +/// - anything that compiles (see `NativeToolchain`/`NativeBuildExecutor`); /// - `$(...)` evaluation, `[config=…]` conditions and Xcode's huge default /// settings table: values are reported as written; /// - schemes, asset catalogs and storyboards. `actool`/`ibtool` are Xcode's own diff --git a/App/Services/AppBundleSigner.swift b/App/Services/AppBundleSigner.swift new file mode 100644 index 0000000..85c74b8 --- /dev/null +++ b/App/Services/AppBundleSigner.swift @@ -0,0 +1,158 @@ +import Foundation +import XKit + +/// Signing an app bundle with XKit, in this process. +/// +/// This is the replacement for running `zsign` inside the embedded Linux guest. +/// XKit is xtool's own library and the app already depends on it; its signer is +/// the bundled C implementation, so this stays in-process, needs no subprocess, +/// and never has to copy a private key anywhere the app cannot see it. +/// +/// What it does *not* replace: the Apple ID session. Obtaining a certificate +/// without one means anisette + GrandSlam + 2FA (XKit's `SigningContext` / +/// `DeveloperServices` path), which is a separate, device-only piece of work. This +/// signs with an identity the user already has (a `.p12` and a profile). +enum AppBundleSigner { + struct Identity { + /// DER bytes of the signing certificate. + var certificateDER: Data + /// The private key, in the format `PrivateKey(data:)` accepts (PEM). + var privateKey: Data + /// `embedded.mobileprovision` contents, when the identity has one. + var provisioningProfile: Data? + /// Entitlements to seal into the signature. Empty means none beyond the + /// profile's own. + var entitlements: [String: Any]? + } + + enum SigningError: LocalizedError { + case noSignerAvailable + case entitlementsUnreadable(String) + + var errorDescription: String? { + switch self { + case .noSignerAvailable: + return "XKit reported no usable codesigning backend in this build." + case .entitlementsUnreadable(let detail): + return "The entitlements could not be read: \(detail)" + } + } + } + + /// Sign `appBundleURL` in place. + /// + /// The profile has to be inside the bundle *before* signing: the signer reads + /// it to decide which entitlements the signature may carry, and a bundle whose + /// signature does not match its profile is rejected at install time rather than + /// at signing time. + static func sign(appBundleURL: URL, identity: Identity) async throws { + let certificate = try Certificate(data: identity.certificateDER) + let privateKey = try PrivateKey(data: identity.privateKey) + + if let profile = identity.provisioningProfile { + try profile.write( + to: appBundleURL.appendingPathComponent("embedded.mobileprovision"), + options: .atomic + ) + } + + let entitlements = try entitlements(from: identity.entitlements) + + let signer = try Signer.first() + try await signer.sign( + app: appBundleURL, + identity: .real(certificate, privateKey), + entitlementMapping: [appBundleURL: entitlements], + progress: { _ in } + ) + } + + /// `Entitlements` decodes from a property list, so a dictionary has to be + /// serialized back into one. Building the type directly is not public API. + private static func entitlements(from dictionary: [String: Any]?) throws -> Entitlements { + guard let dictionary, !dictionary.isEmpty else { + return try Entitlements(entitlements: []) + } + do { + let data = try PropertyListSerialization.data( + fromPropertyList: dictionary, + format: .xml, + options: 0 + ) + return try PropertyListDecoder().decode(Entitlements.self, from: data) + } catch { + throw SigningError.entitlementsUnreadable(error.localizedDescription) + } + } +} + +/// Loading a `.p12` into an `AppBundleSigner.Identity` using the Security +/// framework, which is the only PKCS#12 reader available on iOS. +enum PKCS12Identity { + enum LoadError: LocalizedError { + case badPassword + case noIdentity + case missingCertificate(Int32) + case missingKey(Int32) + + var errorDescription: String? { + switch self { + case .badPassword: + return "The PKCS#12 password is wrong, or the file is not a .p12." + case .noIdentity: + return "That file holds no signing identity (certificate + private key)." + case .missingCertificate(let status): + return "Could not export the certificate (OSStatus \(status))." + case .missingKey(let status): + return "Could not export the private key (OSStatus \(status))." + } + } + } + + /// Import a `.p12` and return the certificate and key bytes. + /// + /// The key is exported as PKCS#1 DER and then wrapped in a PEM envelope, + /// which is the form the signer's parser reads. A key that cannot be exported + /// this way (a hardware-backed one, say) fails here with the Security status + /// rather than somewhere inside the signer. + static func load(data: Data, password: String) throws -> (certificate: Data, key: Data) { + var items: CFArray? + let options = [kSecImportExportPassphrase as String: password] as CFDictionary + let status = SecPKCS12Import(data as CFData, options, &items) + guard status == errSecSuccess else { throw LoadError.badPassword } + + guard let entries = items as? [[String: Any]] else { throw LoadError.noIdentity } + for entry in entries { + guard let identityRef = entry[kSecImportItemIdentity as String] else { continue } + let identity = identityRef as! SecIdentity + + var certificateRef: SecCertificate? + let certStatus = SecIdentityCopyCertificate(identity, &certificateRef) + guard certStatus == errSecSuccess, let certificate = certificateRef else { + throw LoadError.missingCertificate(certStatus) + } + + var keyRef: SecKey? + let keyStatus = SecIdentityCopyPrivateKey(identity, &keyRef) + guard keyStatus == errSecSuccess, let key = keyRef else { + throw LoadError.missingKey(keyStatus) + } + + let certificateData = SecCertificateCopyData(certificate) as Data + var error: Unmanaged? + guard let keyData = SecKeyCopyExternalRepresentation(key, &error) as Data? else { + let code = (error?.takeRetainedValue()).map { CFErrorGetCode($0) } ?? -1 + throw LoadError.missingKey(Int32(code)) + } + return (certificateData, pem(keyData)) + } + throw LoadError.noIdentity + } + + /// Wrap raw private-key bytes in a PEM envelope. + private static func pem(_ der: Data) -> Data { + let base64 = der.base64EncodedString(options: [.lineLength64Characters, .endLineWithLineFeed]) + let text = "-----BEGIN PRIVATE KEY-----\n\(base64)\n-----END PRIVATE KEY-----\n" + return Data(text.utf8) + } +} diff --git a/App/Services/DirectorySize.swift b/App/Services/DirectorySize.swift deleted file mode 100644 index 8030ba2..0000000 --- a/App/Services/DirectorySize.swift +++ /dev/null @@ -1,41 +0,0 @@ -import Foundation - -/// Recursive on-disk size of a directory. -/// -/// Deliberately a plain `nonisolated` function so callers can run it off the main -/// actor: walking the imported rootfs is tens of thousands of file stats, and doing -/// that during a SwiftUI `body` evaluation can freeze — or watchdog-kill — the app. -enum DirectorySize { - static func bytes(at url: URL, limit: Int = 400_000) -> Int64 { - let fm = FileManager.default - var isDirectory: ObjCBool = false - guard fm.fileExists(atPath: url.path, isDirectory: &isDirectory) else { return 0 } - guard isDirectory.boolValue else { - let size = (try? url.resourceValues(forKeys: [.fileSizeKey]))?.fileSize ?? 0 - return Int64(size) - } - // No `.skipsHiddenFiles`: in the guest rootfs the interesting things *are* - // hidden — the Swift toolchain is /root/.local/share/swiftly, the SDK is - // /root/.swiftpm/swift-sdks, the caches are /root/.cache — so skipping - // hidden entries reported a few MB for a download that is several GB, and - // reported almost none of the rootfs's own size. (Symlinks are not - // followed by this enumerator, so counting them cannot loop.) - guard let enumerator = fm.enumerator( - at: url, - includingPropertiesForKeys: [.fileSizeKey, .isRegularFileKey], - options: [] - ) else { return 0 } - - var total: Int64 = 0 - var visited = 0 - for case let fileURL as URL in enumerator { - visited += 1 - if visited > limit { break } - guard let values = try? fileURL.resourceValues(forKeys: [.fileSizeKey, .isRegularFileKey]), - values.isRegularFile == true, - let size = values.fileSize else { continue } - total += Int64(size) - } - return total - } -} diff --git a/App/Services/DownloadManager.swift b/App/Services/DownloadManager.swift deleted file mode 100644 index 13dd0d1..0000000 --- a/App/Services/DownloadManager.swift +++ /dev/null @@ -1,237 +0,0 @@ -import Foundation -import Combine - -/// One downloadable item with live state. -struct DownloadItem: Identifiable, Equatable { - let id = UUID() - var name: String - var url: URL - var state: State = .idle - var progress: Double = 0 - var destination: URL? - var error: String? - - enum State: String, Equatable { - case idle, downloading, done, failed - } -} - -/// Real host-side downloads with state tracking. Files land in `folder` (default the -/// app's Documents/downloads), which is shared into the guest Linux at `/host/downloads`. -@MainActor -final class DownloadManager: ObservableObject { - @Published private(set) var items: [DownloadItem] = [] - let folder: URL - - init(folder: URL? = nil) { - let base = folder ?? XForgeEnvironment.downloadsDirectory - self.folder = base - try? FileManager.default.createDirectory(at: self.folder, withIntermediateDirectories: true) - } - - @discardableResult - func enqueue(name: String, url: URL) -> UUID { - if let existing = items.firstIndex(where: { $0.name == name }) { - items[existing].url = url - items[existing].state = .idle - items[existing].progress = 0 - items[existing].error = nil - return items[existing].id - } - let item = DownloadItem(name: name, url: url) - items.append(item) - return item.id - } - - func start(_ id: UUID) async { - guard let idx = items.firstIndex(where: { $0.id == id }) else { return } - items[idx].state = .downloading - items[idx].progress = 0 - items[idx].error = nil - let name = items[idx].name - let url = items[idx].url - do { - let dest = folder.appendingPathComponent(name) - let saved = try await DownloadManager.download(url, to: dest) { fraction in - // Called from the session's delegate queue. - Task { @MainActor [weak self] in - guard let self, let i = self.items.firstIndex(where: { $0.id == id }) else { return } - self.items[i].progress = fraction - } - } - if let i = items.firstIndex(where: { $0.id == id }) { - items[i].state = .done - items[i].progress = 1 - items[i].destination = saved - } - } catch { - if let i = items.firstIndex(where: { $0.id == id }) { - items[i].state = .failed - items[i].error = error.localizedDescription - } - } - } - - func retry(_ id: UUID) async { - await start(id) - } - - func remove(_ id: UUID) { - guard let idx = items.firstIndex(where: { $0.id == id }) else { return } - if let dest = items[idx].destination { - try? FileManager.default.removeItem(at: dest) - } - items.remove(at: idx) - } - - /// Download `url` to `destination`, replacing anything already there and - /// reporting byte progress. - /// - /// Retries on failure: a 456 MB download dropped its connection 22 seconds - /// in on a real device, and one dropped connection is not a reason to make - /// the user start over. A server-side answer (404/403) is not retried -- - /// it will not change. - @discardableResult - nonisolated static func download( - _ url: URL, - to destination: URL, - attempts: Int = 3, - progress: @escaping @Sendable (Double) -> Void - ) async throws -> URL { - try FileManager.default.createDirectory( - at: destination.deletingLastPathComponent(), withIntermediateDirectories: true) - - var lastError: Error = DownloadError.invalidResponse(url) - for attempt in 1...max(1, attempts) { - do { - let temp = try await ProgressDownload(url: url, progress: progress).start() - - try? FileManager.default.removeItem(at: destination) - try FileManager.default.moveItem(at: temp, to: destination) - progress(1.0) - return destination - } catch let error as DownloadError { - throw error - } catch { - lastError = error - XForgeLog.note("download: attempt \(attempt)/\(attempts) of " - + "\(url.lastPathComponent) failed: \(error.localizedDescription)") - if attempt < attempts { - try? await Task.sleep(nanoseconds: UInt64(attempt) * 2_000_000_000) - } - } - } - throw lastError - } -} - -/// A single download that reports progress. -/// -/// `URLSession.download(from:)` cannot report anything until it is finished, and -/// the files here are hundreds of megabytes — so a delegate does the work and -/// this object keeps the session, the delegate and the continuation together -/// for as long as the transfer lasts. -private final class ProgressDownload: NSObject, URLSessionDownloadDelegate, @unchecked Sendable { - private let url: URL - private let progress: @Sendable (Double) -> Void - - private let lock = NSLock() - private var continuation: CheckedContinuation? - private var session: URLSession? - private var finished = false - - init(url: URL, progress: @escaping @Sendable (Double) -> Void) { - self.url = url - self.progress = progress - } - - /// Returns the finished download's temporary file, which the caller owns. - func start() async throws -> URL { - try await withCheckedThrowingContinuation { (continuation: CheckedContinuation) in - lock.lock() - self.continuation = continuation - let session = URLSession(configuration: .default, delegate: self, delegateQueue: nil) - self.session = session - lock.unlock() - session.downloadTask(with: url).resume() - } - } - - private func finish(_ result: Result) { - lock.lock() - defer { lock.unlock() } - guard !finished, let continuation else { return } - finished = true - self.continuation = nil - session?.finishTasksAndInvalidate() - session = nil - continuation.resume(with: result) - } - - func urlSession( - _ session: URLSession, - downloadTask: URLSessionDownloadTask, - didWriteData bytesWritten: Int64, - totalBytesWritten: Int64, - totalBytesExpectedToWrite: Int64 - ) { - guard totalBytesExpectedToWrite > 0 else { return } - progress(min(1.0, Double(totalBytesWritten) / Double(totalBytesExpectedToWrite))) - } - - func urlSession( - _ session: URLSession, - downloadTask: URLSessionDownloadTask, - didFinishDownloadingTo location: URL - ) { - // The file at `location` is deleted when this returns, so take it now. - if let http = downloadTask.response as? HTTPURLResponse, - !(200..<300).contains(http.statusCode) { - finish(.failure(DownloadError.http(status: http.statusCode, url: url))) - return - } - let kept = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-\(UUID().uuidString).download") - do { - try? FileManager.default.removeItem(at: kept) - try FileManager.default.moveItem(at: location, to: kept) - finish(.success(kept)) - } catch { - finish(.failure(error)) - } - } - - func urlSession( - _ session: URLSession, - task: URLSessionTask, - didCompleteWithError error: Error? - ) { - // Success already went through didFinishDownloadingTo; a non-nil error - // here is the transfer giving up (connection lost, timeout, ...). - if let error { - finish(.failure(error)) - } else { - finish(.failure(DownloadError.invalidResponse(url))) - } - } -} - -enum DownloadError: LocalizedError { - case http(status: Int, url: URL) - case invalidResponse(URL) - case notDownloaded - case assetNotFound(String) - - var errorDescription: String? { - switch self { - case .http(let status, let url): - return "The server returned HTTP \(status) for \(url.lastPathComponent)." - case .invalidResponse(let url): - return "Unexpected response from \(url.host ?? "the server")." - case .notDownloaded: - return "The file has not finished downloading." - case .assetNotFound(let what): - return "No published release asset found: \(what)." - } - } -} diff --git a/App/Services/GuestNetwork.swift b/App/Services/GuestNetwork.swift deleted file mode 100644 index 5a6cc57..0000000 --- a/App/Services/GuestNetwork.swift +++ /dev/null @@ -1,67 +0,0 @@ -import Foundation - -/// The guest's `nameserver` configuration. -/// -/// The engine hands the guest the host's sockets, but **name resolution happens -/// inside the guest**: musl (and busybox's `wget`, and `apk`) read -/// `/etc/resolv.conf`, and the Alpine minirootfs XForge bundles ships no -/// nameservers at all. ish-arm64's own app writes that file from the device's DNS -/// on every boot for exactly this reason; without the same step the guest -/// resolves nothing — `apk add`, and therefore all of -/// `install-toolchain.sh`, fails with `DNS: transient error`. -/// -/// **Order matters, and it is not the obvious one.** A home network hands out -/// its router (`192.168.x.1`), which is a *local* address: iOS refuses -/// connections to those unless the user has granted Local Network access, and -/// the guest's DNS queries are made by this very process, so the refusal lands -/// as an in-guest "DNS: transient error" with a correct-looking resolv.conf. -/// Observed exactly that on a device whose resolver was `192.168.4.1`: the file -/// was right, `apk` still failed. -/// -/// So the public resolvers go first — they are reachable without any -/// permission — and the device's own servers are kept as the fallback for -/// networks where public DNS is blocked (captive portals, some VPNs). -enum GuestNetwork { - /// The device's DNS servers, in preference order. Empty when the system - /// will not say (no network, or the dnsinfo SPI is unavailable) — see - /// HostDNS.c for why this is not SystemConfiguration. - static func systemDNSServers() -> [String] { - var buffer = [CChar](repeating: 0, count: 512) - let count = buffer.withUnsafeMutableBufferPointer { pointer -> Int32 in - guard let base = pointer.baseAddress else { return 0 } - return xf_host_dns_servers(base, pointer.count) - } - guard count > 0 else { return [] } - return String(cString: buffer) - .split(separator: "\n") - .map(String.init) - .filter { !$0.isEmpty } - } - - /// Reachable without Local Network permission, so they are tried first. - static let publicServers = ["1.1.1.1", "8.8.8.8"] - - /// musl reads every `nameserver` line and tries them in order, so the list - /// is short on purpose: a dead entry costs a timeout before the next one. - static let maxServers = 3 - - static func resolvConf(servers: [String]) -> String { - var ordered: [String] = [] - for server in publicServers + servers where !ordered.contains(server) { - ordered.append(server) - if ordered.count == maxServers { break } - } - return ordered.map { "nameserver \($0)" }.joined(separator: "\n") + "\n" - } - - /// The file the guest should have. `source` says which servers were - /// actually used, for the engine log. - static func resolvConfForDevice() -> (text: String, source: String) { - let deviceServers = systemDNSServers() - let text = resolvConf(servers: deviceServers) - let source = deviceServers.isEmpty - ? "public (the device published none)" - : "public + device (\(deviceServers.joined(separator: ", ")))" - return (text, source) - } -} diff --git a/App/Services/HostDNS.c b/App/Services/HostDNS.c deleted file mode 100644 index dfeb313..0000000 --- a/App/Services/HostDNS.c +++ /dev/null @@ -1,158 +0,0 @@ -// -// HostDNS.c -// XForge -// -// The device's DNS servers, for the guest's /etc/resolv.conf. -// -// Resolution happens *inside* the guest, and the Alpine minirootfs XForge -// bundles ships no nameservers, so the guest cannot resolve anything until -// something writes that file — `apk add`, and therefore all of -// install-toolchain.sh, fails with DNS errors without it. -// -// Where the servers come from, on iOS: -// -// - SystemConfiguration's dynamic-store API (SCDynamicStoreCopyValue on -// State:/Network/Global/DNS) is the obvious answer and is what the first -// version of this used. It is macOS-only: on iOS those symbols are marked -// unavailable at compile time. -// - libresolv's res_ninit() would be next, but it is not usable from an app. -// - What *is* available is libSystem's dnsinfo SPI: -// dns_configuration_copy(), which is exactly what ish-arm64's own app uses -// for this same job. It is resolved with dlsym rather than linked, so the -// app carries no reference to a private symbol; if it ever disappears, -// this returns 0 and the caller falls back to public resolvers. -// -// The struct layouts below must match the SPI's. They are the ones ish-arm64 -// ships (app/AppDelegate.m), including the #pragma pack(4). -// -#include "HostDNS.h" - -#include -#include -#include -#include -#include -#include -#include - -#pragma pack(push, 4) -typedef struct { - struct in_addr address; - struct in_addr mask; -} xf_dns_sortaddr_t; - -typedef struct { - char *domain; - int32_t n_nameserver; - struct sockaddr **nameserver; - uint16_t port; - int32_t n_search; - char **search; - int32_t n_sortaddr; - xf_dns_sortaddr_t **sortaddr; - char *options; - uint32_t timeout; - uint32_t search_order; - uint32_t if_index; - uint32_t flags; - uint32_t reach_flags; - uint32_t reserved[5]; -} xf_dns_resolver_t; - -typedef struct { - int32_t n_resolver; - xf_dns_resolver_t **resolver; - int32_t n_scoped_resolver; - xf_dns_resolver_t **scoped_resolver; - uint32_t reserved[5]; -} xf_dns_config_t; -#pragma pack(pop) - -typedef xf_dns_config_t *(*xf_dns_copy_fn)(void); -typedef void (*xf_dns_free_fn)(xf_dns_config_t *); - -// Copies rather than casts: the resolver and nameserver arrays are arrays of -// pointers that the SPI does not promise are aligned for the destination type. -static xf_dns_resolver_t *xf_resolver_at(xf_dns_resolver_t **resolvers, int index) { - xf_dns_resolver_t *resolver = NULL; - if (resolvers == NULL || index < 0) - return NULL; - memcpy(&resolver, resolvers + index, sizeof(resolver)); - return resolver; -} - -static struct sockaddr *xf_nameserver_at(struct sockaddr **nameservers, int index) { - struct sockaddr *address = NULL; - if (nameservers == NULL || index < 0) - return NULL; - memcpy(&address, nameservers + index, sizeof(address)); - return address; -} - -int xf_host_dns_servers(char *out, size_t cap) { - static xf_dns_copy_fn copy_fn; - static xf_dns_free_fn free_fn; - static int looked_up = 0; - if (!looked_up) { - looked_up = 1; - copy_fn = (xf_dns_copy_fn) dlsym(RTLD_DEFAULT, "dns_configuration_copy"); - free_fn = (xf_dns_free_fn) dlsym(RTLD_DEFAULT, "dns_configuration_free"); - } - if (copy_fn == NULL || free_fn == NULL || out == NULL || cap == 0) - return 0; - - xf_dns_config_t *config = copy_fn(); - if (config == NULL) - return 0; - - size_t used = 0; - int written = 0; - out[0] = '\0'; - - for (int r = 0; r < config->n_resolver; r++) { - xf_dns_resolver_t *resolver = xf_resolver_at(config->resolver, r); - if (resolver == NULL || resolver->n_nameserver <= 0) - continue; - // mDNS resolvers (.local) list 224.0.0.251, which is not a nameserver - // to hand to musl. - if (resolver->options != NULL && strcmp(resolver->options, "mdns") == 0) - continue; - - for (int i = 0; i < resolver->n_nameserver; i++) { - struct sockaddr *address = xf_nameserver_at(resolver->nameserver, i); - if (address == NULL) - continue; - char text[INET6_ADDRSTRLEN] = {0}; - void *source = NULL; - if (address->sa_family == AF_INET) - source = &((struct sockaddr_in *) address)->sin_addr; - else if (address->sa_family == AF_INET6) - source = &((struct sockaddr_in6 *) address)->sin6_addr; - else - continue; - if (inet_ntop(address->sa_family, source, text, sizeof(text)) == NULL) - continue; - if (text[0] == '\0') - continue; - - // Skip duplicates: the SPI lists one resolver per interface. - if (strstr(out, text) != NULL) - continue; - - size_t len = strlen(text); - if (used + len + 2 > cap) - break; - memcpy(out + used, text, len); - used += len; - out[used++] = '\n'; - out[used] = '\0'; - written++; - if (written >= 4) - goto done; - } - } - -done: - free_fn(config); - return written; -} diff --git a/App/Services/HostDNS.h b/App/Services/HostDNS.h deleted file mode 100644 index 8c11723..0000000 --- a/App/Services/HostDNS.h +++ /dev/null @@ -1,20 +0,0 @@ -// -// HostDNS.h -// XForge -// -// The device's DNS servers, for the guest's /etc/resolv.conf. -// -// See HostDNS.c for why this is not SystemConfiguration. -// -#ifndef XFORGE_HOST_DNS_H -#define XFORGE_HOST_DNS_H - -#include - -/// Write up to `cap` bytes of the device's DNS servers into `out`, one IPv4 or -/// IPv6 address per line, and return how many were written. Returns 0 when the -/// system does not publish any (no network, or the SPI is unavailable), in -/// which case the caller should fall back to public resolvers. -int xf_host_dns_servers(char *out, size_t cap); - -#endif /* XFORGE_HOST_DNS_H */ diff --git a/App/Services/IPAConfigureSignService.swift b/App/Services/IPAConfigureSignService.swift deleted file mode 100644 index f95105f..0000000 --- a/App/Services/IPAConfigureSignService.swift +++ /dev/null @@ -1,186 +0,0 @@ -import Foundation -import SwiftUI - -/// Host-side state for the zsign workflow. zsign is a Linux C++ command-line -/// signer, so it runs inside the embedded Alpine guest; the host UI only stages -/// files, invokes it, and copies the signed IPA back. -@MainActor -final class IPAConfigureSignService: ObservableObject { - @Published var inputIPA: URL? - @Published var provisioningProfile: URL? - @Published var p12: URL? - @Published var entitlements: URL? - /// Never persisted. Cleared immediately after the signing attempt. - @Published var password = "" - @Published var bundleIdentifier = "" - @Published var displayName = "" - @Published var version = "" - @Published var status = "Ready to sign." - @Published var signedIPA: URL? - @Published var isWorking = false - @Published var error: String? - - private let makeVM: @MainActor () -> any LinuxVM - - init(makeVM: @escaping @MainActor () -> any LinuxVM = { XForgeEnvironment.makeVM() }) { - self.makeVM = makeVM - } - - var canSign: Bool { - inputIPA != nil && provisioningProfile != nil && p12 != nil && !password.isEmpty && !isWorking - } - - func sign() async { - guard let inputIPA, let provisioningProfile, let p12 else { return } - let oneShotPassword = password - guard !oneShotPassword.contains("\n"), !oneShotPassword.contains("\r") else { - password = "" - error = "The PKCS#12 password cannot contain a newline." - return - } - let ipaScoped = inputIPA.startAccessingSecurityScopedResource() - let profileScoped = provisioningProfile.startAccessingSecurityScopedResource() - let p12Scoped = p12.startAccessingSecurityScopedResource() - let entitlementsScoped = entitlements?.startAccessingSecurityScopedResource() ?? false - defer { - if ipaScoped { inputIPA.stopAccessingSecurityScopedResource() } - if profileScoped { provisioningProfile.stopAccessingSecurityScopedResource() } - if p12Scoped { p12.stopAccessingSecurityScopedResource() } - if entitlementsScoped { entitlements?.stopAccessingSecurityScopedResource() } - } - - isWorking = true - error = nil - signedIPA = nil - defer { - // Do not leave the signing secret in the view model after the attempt. - password = "" - isWorking = false - } - - let id = UUID().uuidString - let dir = "/root/xforge/zsign-jobs/\(id)" - let ipaPath = "\(dir)/input.ipa" - let p12Path = "\(dir)/signing.p12" - let profilePath = "\(dir)/profile.mobileprovision" - let passwordPath = "\(dir)/.p12-password" - let scriptPath = "\(dir)/run-sign.sh" - let outputPath = "\(dir)/signed.ipa" - var vm: (any LinuxVM)? - - do { - status = "Starting Alpine…" - let guest = makeVM() - vm = guest - await guest.prepareRootfs() - try await guest.boot() - try await run(guest, "mkdir -p \(GuestShell.quote(dir)) && chmod 700 \(GuestShell.quote(dir))") - - status = "Copying signing inputs into the guest…" - try await guest.copyIn(hostURL: inputIPA, to: ipaPath) - try await guest.copyIn(hostURL: p12, to: p12Path) - try await guest.copyIn(hostURL: provisioningProfile, to: profilePath) - try await SystemComponents.ensureZsignInstaller(in: guest) - - // zsign's upstream -p accepts a password as a process argument, which - // a same-user guest process could read through /proc//cmdline. - // A short-lived file is narrower exposure: random name inside a 700 - // job directory, chmod 600, consumed by the pinned -Q password-file - // patch and removed with the rest of this per-signing directory. - let passwordTemp = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-p12-\(UUID().uuidString)") - try Data(oneShotPassword.utf8).write(to: passwordTemp, options: .atomic) - try FileManager.default.setAttributes([.posixPermissions: 0o600], - ofItemAtPath: passwordTemp.path) - defer { try? FileManager.default.removeItem(at: passwordTemp) } - try await guest.copyIn(hostURL: passwordTemp, to: passwordPath) - try await run(guest, "chmod 600 \(GuestShell.quote(passwordPath))") - - var args = "-f -k \(GuestShell.quote(p12Path)) -m \(GuestShell.quote(profilePath))" - if let entitlements { - let path = "\(dir)/entitlements.plist" - try await guest.copyIn(hostURL: entitlements, to: path) - args += " -e \(GuestShell.quote(path))" - } - if !bundleIdentifier.isEmpty { args += " -b \(GuestShell.quote(bundleIdentifier))" } - if !displayName.isEmpty { args += " -n \(GuestShell.quote(displayName))" } - if !version.isEmpty { args += " -r \(GuestShell.quote(version))" } - - // The patched zsign reads its password from this mode-600 file rather - // than argv, so the signing secret is not exposed in the guest process - // table. The per-job directory is deleted after the sign on success - // or failure. - let script = """ - #!/bin/sh - set -eu - umask 077 - exec zsign \(args) -Q \(GuestShell.quote(passwordPath)) -o \(GuestShell.quote(outputPath)) \(GuestShell.quote(ipaPath)) - """ - let hostScript = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-zsign-\(UUID().uuidString).sh") - try Data(script.utf8).write(to: hostScript, options: .atomic) - defer { try? FileManager.default.removeItem(at: hostScript) } - try await guest.copyIn(hostURL: hostScript, to: scriptPath) - try await run(guest, "chmod 700 \(GuestShell.quote(scriptPath))") - - status = "Installing the pinned zsign build in Alpine (first use only)…" - try await run(guest, "sh /root/xforge/install-zsign.sh") - status = "Signing in the guest…" - try await run(guest, "sh \(GuestShell.quote(scriptPath))") - - status = "Copying the signed IPA back…" - let destination = XForgeEnvironment.documentDirectory - .appendingPathComponent("Signed-\(inputIPA.deletingPathExtension().lastPathComponent).ipa") - try await guest.copyOut(guestPath: outputPath, to: destination) - signedIPA = destination - status = "Signed IPA ready: \(destination.lastPathComponent)" - } catch { - self.error = error.localizedDescription - status = "Signing failed." - } - - // Remove the private key, provisioning profile and password-bearing script - // from the guest even when signing failed. Keep the output IPA (if any). - if let vm { - _ = try? await vm.run("rm -rf \(GuestShell.quote(dir))", environment: nil) { _ in } - } - } - - private func run(_ vm: any LinuxVM, _ command: String) async throws { - let box = OutputBox() - let status = try await vm.run(command, environment: nil) { [weak self] line in - box.append(line) - Task { @MainActor in - // The status panel is a live short line, not a transcript of - // secrets or command text. Only show guest output that does not - // contain arguments from the signing script. - self?.status = line - } - } - guard status == 0 else { - throw SignError.commandFailed(status, detail: box.value) - } - } - - enum SignError: LocalizedError { - case commandFailed(Int32, detail: String) - var errorDescription: String? { - switch self { - case .commandFailed(let code, let detail): - let safe = detail.trimmingCharacters(in: .whitespacesAndNewlines) - return safe.isEmpty - ? "Guest signing command failed (exit \(code))." - : "Guest signing command failed (exit \(code)): \(safe)" - } - } - } -} - -/// Thread-safe output accumulator used while a guest command streams from its -/// background tailer. -private final class OutputBox: @unchecked Sendable { - private let lock = NSLock() - private var stored = "" - func append(_ value: String) { lock.lock(); stored += value; lock.unlock() } - var value: String { lock.lock(); defer { lock.unlock() }; return stored } -} diff --git a/App/Services/IPASigningJob.swift b/App/Services/IPASigningJob.swift new file mode 100644 index 0000000..c98f9ae --- /dev/null +++ b/App/Services/IPASigningJob.swift @@ -0,0 +1,179 @@ +import Foundation +import Combine + +/// Sign an `.ipa` in this process with XKit. +/// +/// This replaces the `zsign` job that used to run inside the embedded Linux. The +/// shape is deliberately the same — IPA + `.p12` + profile + entitlements in, a +/// signed IPA out — because that is the workflow a user has: they have a built +/// (unsigned) IPA from the Build screen and an Apple Developer certificate. +/// +/// Two things are narrower than before, both on purpose: +/// +/// - the private key never leaves the app. It is imported through the Security +/// framework and handed to the signer in memory; nothing is written to disk, +/// which is why the `zsign` password-file workaround is gone with `zsign`; +/// - the signature is produced by XKit's own signer, so a signed app is one the +/// same code path would produce on a Mac (zsign was a copy of it). +@MainActor +final class IPASigningJob: ObservableObject { + @Published var inputIPA: URL? + @Published var p12: URL? + @Published var provisioningProfile: URL? + @Published var entitlements: URL? + /// Never persisted. Cleared immediately after the signing attempt. + @Published var password = "" + @Published var bundleIdentifier = "" + @Published var displayName = "" + @Published var version = "" + @Published var status = "Ready to sign." + @Published var signedIPA: URL? + @Published var isWorking = false + @Published var error: String? + + var canSign: Bool { + inputIPA != nil && p12 != nil && !password.isEmpty && !isWorking + } + + func sign() async { + guard let inputIPA, let p12 else { return } + let oneShotPassword = password + + let scoped = [inputIPA, p12, provisioningProfile, entitlements].compactMap { $0 } + .map { ($0, $0.startAccessingSecurityScopedResource()) } + defer { + for (url, accessed) in scoped where accessed { + url.stopAccessingSecurityScopedResource() + } + } + + isWorking = true + error = nil + signedIPA = nil + defer { + // Do not leave the signing secret in the view model after the attempt. + password = "" + isWorking = false + } + + do { + status = "Reading the signing identity…" + let keyMaterial = try PKCS12Identity.load( + data: try Data(contentsOf: p12), + password: oneShotPassword + ) + + let profileData = try provisioningProfile.map { try Data(contentsOf: $0) } + let entitlementsDictionary = try entitlements.map { try readEntitlements($0) } + + status = "Unpacking \(inputIPA.lastPathComponent)…" + let work = FileManager.default.temporaryDirectory + .appendingPathComponent("xforge-sign-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: work) } + try FileManager.default.createDirectory(at: work, withIntermediateDirectories: true) + try FileManager.default.unzipItem(at: inputIPA, to: work) + + let payload = work.appendingPathComponent("Payload", isDirectory: true) + let apps = try apps(in: payload) + guard apps.count == 1, let app = apps.first else { + throw SigningJobError.expectedOneApp(apps.map(\.lastPathComponent)) + } + + try applyIdentityOverrides(to: app) + + status = "Signing \(app.lastPathComponent)…" + try await AppBundleSigner.sign( + appBundleURL: app, + identity: .init( + certificateDER: keyMaterial.certificate, + privateKey: keyMaterial.key, + provisioningProfile: profileData, + entitlements: entitlementsDictionary + ) + ) + + status = "Repackaging…" + let destination = XForgeEnvironment.documentDirectory.appendingPathComponent( + "Signed-" + inputIPA.deletingPathExtension().lastPathComponent + ".ipa" + ) + try? FileManager.default.removeItem(at: destination) + try FileManager.default.zipItem( + at: payload, + to: destination, + shouldKeepParent: true + ) + signedIPA = destination + status = "Signed IPA ready: \(destination.lastPathComponent)" + } catch { + self.error = error.localizedDescription + status = "Signing failed." + } + } + + // MARK: - Helpers + + private func apps(in payload: URL) throws -> [URL] { + let entries = (try? FileManager.default.contentsOfDirectory( + at: payload, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + )) ?? [] + return entries.filter { $0.pathExtension == "app" } + } + + /// Apply the identity fields the user typed over the bundle's own plist. + /// + /// Only these three: replacing the plist wholesale would drop + /// `CFBundleExecutable` and produce an app that cannot launch. + private func applyIdentityOverrides(to app: URL) throws { + var overrides: [String: Any] = [:] + if !bundleIdentifier.isEmpty { overrides["CFBundleIdentifier"] = bundleIdentifier } + if !displayName.isEmpty { + overrides["CFBundleDisplayName"] = displayName + overrides["CFBundleName"] = displayName + } + if !version.isEmpty { overrides["CFBundleShortVersionString"] = version } + guard !overrides.isEmpty else { return } + + let plistURL = app.appendingPathComponent("Info.plist") + guard let data = try? Data(contentsOf: plistURL), + var plist = try? PropertyListSerialization.propertyList(from: data, format: nil) + as? [String: Any] else { + throw SigningJobError.unreadableInfoPlist + } + for (key, value) in overrides { plist[key] = value } + let updated = try PropertyListSerialization.data( + fromPropertyList: plist, + format: .xml, + options: 0 + ) + try updated.write(to: plistURL, options: .atomic) + } + + private func readEntitlements(_ url: URL) throws -> [String: Any] { + let data = try Data(contentsOf: url) + guard let plist = try PropertyListSerialization.propertyList(from: data, format: nil) + as? [String: Any] else { + throw SigningJobError.unreadableEntitlements + } + return plist + } +} + +enum SigningJobError: LocalizedError { + case expectedOneApp([String]) + case unreadableInfoPlist + case unreadableEntitlements + + var errorDescription: String? { + switch self { + case .expectedOneApp(let found): + let list = found.isEmpty ? "none" : found.joined(separator: ", ") + return "The IPA must contain exactly one .app in Payload/; found \(list)." + case .unreadableInfoPlist: + return "The app's Info.plist could not be read." + case .unreadableEntitlements: + return "The entitlements file is not a property list." + } + } +} diff --git a/App/Services/ProjectExporter.swift b/App/Services/ProjectExporter.swift index d2ab589..382c0bd 100644 --- a/App/Services/ProjectExporter.swift +++ b/App/Services/ProjectExporter.swift @@ -1,68 +1,82 @@ import Foundation +import ZIPFoundation -/// Getting a project out of the guest. +/// Getting a project out of the app. /// -/// Projects live inside the guest filesystem's `fakefs`, and that filesystem is -/// deliberately kept *out* of iCloud backup: it is several gigabytes of toolchain -/// wrapped around a few kilobytes of source, and nothing in the app can mark one -/// without the other (see Docs/DESIGN.md, "What is backed up"). That makes an -/// export the way work leaves the device, so it is a first-class action rather -/// than something to be assembled from `tar` in the Terminal. +/// Projects are directories in the app's container, so an export is a ZIP of one +/// directory — no guest, no `tar` in a shell. The archive lands in +/// `/exports`, which the Files app shows and the share sheet can hand on. /// -/// The archive is built *inside* the guest — that is where the files are — and -/// copied out into the app's own `Documents/exports`, where the Files app shows it -/// and the share sheet can hand it on. +/// The walk skips build products: `.xforge-build` and `.build` are output, and an +/// export that carries a stale binary is worse than one that does not. enum ProjectExporter { - /// Where exported archives land: `/exports`, which is user data and - /// is backed up (unlike the guest filesystem it came from). Defined by + /// Where exported archives land: `/exports`. Defined by /// `XForgeEnvironment`, which owns the storage rules. static var exportsDirectory: URL { XForgeEnvironment.exportsDirectory } - /// Archive `project` inside the guest and copy the archive out. - /// - /// Returns the host URL of the archive, ready to share. - /// - /// `@MainActor` because the VM it is handed is (`LinuxVM` is main-actor isolated - /// and not `Sendable`, so a nonisolated function taking one is a Swift 6 error). - /// Only the unit-test build reports it: the Release build compiles the module - /// whole and does not type-check every body. - @MainActor - static func export(_ project: Project, via vm: LinuxVM) async throws -> URL { - guard project.hasSafeRootPath else { throw ProjectValidationError.unsafePath } + /// Archive `project` and return the host URL of the `.zip`. + static func export(_ project: Project) throws -> URL { let name = try Project.validatedName(project.name) - try await vm.boot() - - let guestArchive = "/tmp/\(name).tar.gz" - let parent = (project.rootPath as NSString).deletingLastPathComponent - // `-C parent name` keeps the archive to one top-level directory, so - // unpacking it cannot scatter files over wherever it is opened. - let status = try await vm.run( - "rm -f \(GuestShell.quote(guestArchive)) && " - + "tar -czf \(GuestShell.quote(guestArchive)) " - + "-C \(GuestShell.quote(parent)) \(GuestShell.quote(name))", - environment: nil - ) { _ in } - guard status == 0 else { throw ExportError.archiveFailed } + let root = project.rootURL + guard FileManager.default.fileExists(atPath: root.path) else { + throw ExportError.projectMissing(root) + } let directory = exportsDirectory try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) - let destination = directory.appendingPathComponent("\(name).tar.gz") - try await vm.copyOut(guestPath: guestArchive, to: destination) + let destination = directory.appendingPathComponent("\(name).zip") + try? FileManager.default.removeItem(at: destination) + + let staging = FileManager.default.temporaryDirectory + .appendingPathComponent("xforge-export-\(UUID().uuidString)", isDirectory: true) + defer { try? FileManager.default.removeItem(at: staging) } - // The guest copy is a second copy of the project; the host has it now. - _ = try? await vm.run("rm -f \(GuestShell.quote(guestArchive))", environment: nil) { _ in } + let copy = staging.appendingPathComponent(name, isDirectory: true) + try FileManager.default.createDirectory(at: copy, withIntermediateDirectories: true) + try copyContents(of: root, to: copy) + + // `shouldKeepParent` on the *staging* directory keeps one top-level folder, + // so unpacking the archive cannot scatter files wherever it is opened. + try FileManager.default.zipItem( + at: copy, + to: destination, + shouldKeepParent: true + ) XForgeLog.note("export: \(name) archived to \(destination.lastPathComponent)") return destination } + /// Copy a project directory, skipping build output. + private static func copyContents(of source: URL, to destination: URL) throws { + let fileManager = FileManager.default + let excluded: Set = [".xforge-build", ".build", ".swiftpm", ".git"] + + let entries = try fileManager.contentsOfDirectory( + at: source, + includingPropertiesForKeys: [.isDirectoryKey], + options: [.skipsHiddenFiles] + ) + for entry in entries { + let name = entry.lastPathComponent + guard !excluded.contains(name) else { continue } + let values = try entry.resourceValues(forKeys: [.isDirectoryKey]) + let target = destination.appendingPathComponent(name) + if values.isDirectory == true { + try fileManager.createDirectory(at: target, withIntermediateDirectories: true) + try copyContents(of: entry, to: target) + } else { + try fileManager.copyItem(at: entry, to: target) + } + } + } + enum ExportError: LocalizedError { - case archiveFailed + case projectMissing(URL) var errorDescription: String? { switch self { - case .archiveFailed: - return "The project could not be archived inside the embedded Linux. " - + "The Terminal tab has the guest's own output." + case .projectMissing(let url): + return "The project directory is missing: \(url.path)" } } } diff --git a/App/Services/ProjectFiles.swift b/App/Services/ProjectFiles.swift new file mode 100644 index 0000000..3cdb5d5 --- /dev/null +++ b/App/Services/ProjectFiles.swift @@ -0,0 +1,151 @@ +import Foundation + +/// Reading and writing a project's files. +/// +/// This replaces `GuestProjectFiles`, which listed and copied files through the +/// embedded Linux. A project is now an ordinary directory in the app's container, +/// so the whole thing is `FileManager` — but the *relative-path* discipline stays: +/// every path is resolved against the project root and rejected if it escapes, +/// because these paths come from editable text in the UI. +@MainActor +enum ProjectFiles { + struct File: Identifiable, Hashable { + let id: String + let name: String + let relativePath: String + var contents: String + } + + /// Files that are never project content: build products and editor state. + private static let excludedDirectories: Set = [".xforge-build", ".git", ".build", ".swiftpm"] + + static func load(project: Project, sourcesOnly: Bool) async throws -> [File] { + let root = sourcesOnly + ? project.rootURL.appendingPathComponent("Sources", isDirectory: true) + : project.rootURL + return try await loadFiles(under: root, project: project) + } + + static func load(relativePath: String, project: Project) async throws -> File { + let url = try resolve(relativePath: relativePath, in: project) + return try read(url, relativePath: relativePath) + } + + static func save(_ contents: String, file: File, project: Project) async throws { + let url = try resolve(relativePath: file.relativePath, in: project) + try contents.write(to: url, atomically: true, encoding: .utf8) + } + + /// Write a file that may not exist yet, creating intermediate directories. + static func write(_ contents: String, relativePath: String, project: Project) throws { + let url = try resolve(relativePath: relativePath, in: project) + try FileManager.default.createDirectory( + at: url.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + try contents.write(to: url, atomically: true, encoding: .utf8) + } + + static func createDirectory(relativePath: String, project: Project) throws { + let url = try resolve(relativePath: relativePath, in: project) + try FileManager.default.createDirectory(at: url, withIntermediateDirectories: true) + } + + static func delete(relativePath: String, project: Project) throws { + let url = try resolve(relativePath: relativePath, in: project) + try FileManager.default.removeItem(at: url) + } + + // MARK: - Internals + + private static func loadFiles(under root: URL, project: Project) async throws -> [File] { + let fileManager = FileManager.default + guard fileManager.fileExists(atPath: root.path) else { return [] } + + guard let enumerator = fileManager.enumerator( + at: root, + includingPropertiesForKeys: [.isRegularFileKey, .isDirectoryKey, .fileSizeKey], + options: [.skipsHiddenFiles, .skipsPackageDescendants] + ) else { return [] } + + var files: [File] = [] + for case let url as URL in enumerator { + let values = try? url.resourceValues(forKeys: [.isRegularFileKey, .isDirectoryKey]) + if values?.isDirectory == true { + if excludedDirectories.contains(url.lastPathComponent) { + enumerator.skipDescendants() + } + continue + } + guard values?.isRegularFile == true else { continue } + guard let relative = relativePath(of: url, in: project) else { continue } + // Text only: the editor cannot show a binary, and a project full of + // build products should not be read into memory to find that out. + guard let contents = try? String(contentsOf: url, encoding: .utf8) else { continue } + files.append(File( + id: relative, + name: url.lastPathComponent, + relativePath: relative, + contents: contents + )) + } + return files.sorted { $0.relativePath < $1.relativePath } + } + + private static func read(_ url: URL, relativePath: String) throws -> File { + let contents = try String(contentsOf: url, encoding: .utf8) + return File( + id: relativePath, + name: (relativePath as NSString).lastPathComponent, + relativePath: relativePath, + contents: contents + ) + } + + /// A project-relative path for `url`, or nil when it is outside the project. + private static func relativePath(of url: URL, in project: Project) -> String? { + let root = project.rootURL.standardizedFileURL.path + let path = url.standardizedFileURL.path + guard path.hasPrefix(root + "/") else { return nil } + return String(path.dropFirst(root.count + 1)) + } + + /// Resolve a user-supplied relative path inside the project. + /// + /// `..` is rejected rather than merely normalized: an editor that can write + /// outside its own project is a sandbox escape with a nice UI, and the + /// standardized path check catches the symlink-free cases the textual check + /// would miss. + private static func resolve(relativePath: String, in project: Project) throws -> URL { + guard isSafe(relativePath: relativePath) else { + throw ProjectFileError.unsafeRelativePath + } + let root = project.rootURL + let url = root.appendingPathComponent(relativePath) + guard relativePath(of: url, in: project) != nil else { + throw ProjectFileError.unsafeRelativePath + } + return url + } + + static func isSafe(relativePath: String) -> Bool { + !relativePath.isEmpty + && !relativePath.hasPrefix("/") + && !relativePath.split(separator: "/").contains("..") + && !relativePath.contains("\n") + } +} + +enum ProjectFileError: LocalizedError { + case unsafeRelativePath + case unreadable(String) + + var errorDescription: String? { + switch self { + case .unsafeRelativePath: + return "That path is outside the project." + case .unreadable(let path): + return "Could not read \(path) as text." + } + } +} diff --git a/App/Services/SystemComponents.swift b/App/Services/SystemComponents.swift deleted file mode 100644 index d9b0434..0000000 --- a/App/Services/SystemComponents.swift +++ /dev/null @@ -1,247 +0,0 @@ -import Foundation - -/// The commands that install the embedded system's components, as commands. -/// -/// Everything in the guest is managed *from the terminal*. That is deliberate: -/// the guest is a real Alpine system, and the tools that provision it are -/// ordinary Linux programs. XForge's job is to know the right commands, get the -/// user's files into the guest's own storage, and run them where the output can -/// be watched — not to reimplement them behind a button. -/// -/// These strings are shared by the Terminal screen (its Components menu) and the -/// Toolchain screen, so both offer exactly the same thing. -@MainActor -enum SystemComponents { - /// Where the guest's own copy of the provisioning script lives. - static let guestInstallerScript = "/root/install-toolchain.sh" - - /// Where a user-supplied Xcode.xip is stored inside the guest filesystem. - static let guestXIPDirectory = "/root/xforge/xip" - - /// xtool's Darwin-SDK cache inside the guest. - static let guestSDKCache = "/root/.cache/xforge-sdk" - - enum Component: String, CaseIterable, Identifiable { - case glibc - case xtool - case swift - case darwinSDK - - var id: String { rawValue } - - var title: String { - switch self { - case .glibc: return "glibc compatibility layer" - case .xtool: return "xtool" - case .swift: return "Swift toolchain" - case .darwinSDK: return "Darwin SDK" - } - } - - var detail: String { - switch self { - case .glibc: - return "The glibc runtime xtool and the Swift toolchain link against (Alpine is musl)." - case .xtool: - return "xtool builds iOS apps from Swift packages on Linux." - case .swift: - return "Swift 6 for Linux, installed with swiftly from swift.org." - case .darwinSDK: - return "The arm64-apple-ios Swift SDK, built from your own Xcode.xip with xtool." - } - } - } - - /// The one-line install for a component that is just a step of the bundled - /// provisioning script. - static func scriptCommand(_ component: Component) -> String { - "sh \(guestInstallerScript) \(component.rawValue)" - } - - /// The Swift toolchain, installed by the guest's own provisioning script. - /// - /// This used to spell out swift.org's documented steps inline — `curl` the - /// swiftly tarball, `tar zxf`, `./swiftly init`, `env.sh`, `hash -r` — which - /// had two problems that only showed up on a device: - /// - /// - it was *unconditional*, so on a root that already carried the toolchain - /// (every published root does now: `EmbeddedLinux/build-rootfs.sh` with - /// `XFORGE_PROVISION=all`) it downloaded 28 MB and ran `swiftly init` over a - /// working installation; - /// - `swiftly init` launches a child process, and this engine does not let it: - /// it fails with "Failed to launch the new process. Underlying error: - /// Invalid argument" before downloading anything. - /// - /// The script's steps are the same work with the guards that belong on them — - /// `swiftly` is a no-op when `/usr/local/bin/swiftly` is already there, and - /// `swift` installs a toolchain only when the toolchains directory is empty — - /// so a provisioned guest answers instantly and an empty one still gets a - /// toolchain. It is also the same code the rootfs is built with, which is the - /// point: one implementation of provisioning, not two that drift. - static var swiftInstallCommand: String { - """ - sh \(guestInstallerScript) glibc && \ - sh \(guestInstallerScript) swiftly && \ - sh \(guestInstallerScript) swift && \ - swift --version - """ - } - - /// xtool installs with its own step (it needs the glibc layer, which the same - /// script installs), then verifies itself. - static var xtoolInstallCommand: String { - """ - sh \(guestInstallerScript) glibc && \ - sh \(guestInstallerScript) xtool && \ - sh \(guestInstallerScript) verify - """ - } - - /// The command that turns an Xcode.xip the user picked into an installed - /// Darwin Swift SDK. - /// - /// The `.xip` is copied into the guest's *own* storage first (`/root/xforge`), - /// and xtool is pointed at that path: xtool does the extraction and the SDK - /// post-processing itself, so nothing has to be unpacked on the host. - /// - /// The removal first is not tidiness. SwiftPM refuses to install a bundle - /// whose artifact ID is already present (`swiftSDKArtifactAlreadyInstalled`, - /// which tells the user to remove one of them) — and the bundled rootfs ships - /// the darwin artifact, so without this the whole point of the import, using - /// *your* Xcode, was unreachable: the install failed with nothing in the UI to - /// explain it. `sdk-remove` deletes SwiftPM's store entry, which is what it - /// consults before installing, and it is the guest's own script so there is - /// one implementation of this rather than one here and one in the rootfs - /// build. - static func darwinSDKInstallCommand(guestXIPPath: String) -> String { - """ - sh \(guestInstallerScript) sdk-remove && \ - xtool sdk install \(GuestShell.quote(guestXIPPath)) && \ - swift sdk list - """ - } - - /// The prebuilt-bundle alternative, for when the user has no Xcode.xip to - /// hand (it downloads XForge's own darwin.artifactbundle inside the guest). - /// - /// The same removal as the `.xip` path, for the same reason: SwiftPM will not - /// install a bundle whose artifact ID is already there, and the bundled rootfs - /// ships one — so "install the prebuilt SDK" over a provisioned guest failed - /// exactly like the import did. - static func darwinSDKDownloadCommand(from url: URL) -> String { - let archive = guestSDKCache + "/darwin-sdk.zip" - let bundle = guestSDKCache + "/darwin.artifactbundle" - return """ - set -eu - sh \(guestInstallerScript) sdk-remove - mkdir -p \(GuestShell.quote(guestSDKCache)) - curl -fL --retry 3 \(GuestShell.quote(url.absoluteString)) -o \(GuestShell.quote(archive)) - rm -rf \(GuestShell.quote(bundle)) - unzip -q \(GuestShell.quote(archive)) -d \(GuestShell.quote(guestSDKCache)) - test -f \(GuestShell.quote(bundle))/info.json - swift sdk install \(GuestShell.quote(bundle)) - rm -f \(GuestShell.quote(archive)) - swift sdk list - """ - } - - // MARK: - Host → guest files - - /// Copy the bundled provisioning script into the guest, where the terminal's - /// commands expect it. - static func ensureInstallerScript(in vm: LinuxVM) async throws { - guard let script = Bundle.main.url(forResource: "install-toolchain", - withExtension: "sh") else { - throw SystemComponentsError.installerScriptMissing - } - try await vm.copyIn(hostURL: script, to: guestInstallerScript) - } - - /// Copy the bundled zsign installer into the guest. - static func ensureZsignInstaller(in vm: LinuxVM) async throws { - guard let script = Bundle.main.url(forResource: "install-zsign", - withExtension: "sh") else { - throw SystemComponentsError.zsignInstallerMissing - } - try await vm.run("mkdir -p /root/xforge", environment: nil) { _ in } - try await vm.copyIn(hostURL: script, to: "/root/xforge/install-zsign.sh") - guard let patch = Bundle.main.url(forResource: "patch-zsign-password-file", - withExtension: "py") else { - throw SystemComponentsError.zsignInstallerMissing - } - try await vm.copyIn(hostURL: patch, to: "/root/xforge/patch-zsign-password-file.py") - let status = try await vm.run( - "chmod 700 /root/xforge/install-zsign.sh && chmod 600 /root/xforge/patch-zsign-password-file.py", - environment: nil - ) { _ in } - guard status == 0 else { throw SystemComponentsError.zsignInstallerMissing } - } - - /// Copy the chosen `.xip` into the guest's own filesystem and return the path the guest should install from. - static func stageXIP(_ source: URL, in vm: LinuxVM, report: @escaping (String) -> Void) async throws -> String { - let scoped = source.startAccessingSecurityScopedResource() - defer { if scoped { source.stopAccessingSecurityScopedResource() } } - - let size = (try? FileManager.default.attributesOfItem(atPath: source.path))?[.size] as? Int64 ?? 0 - if size > 0 { - // The xip is copied *and* xtool expands it inside the guest, so the - // guest needs room for both plus the installed SDK. - let needed = size * 2 + 1_000_000_000 - let free = XForgeEnvironment.availableBytes - if free > 0, free < needed { - throw SystemComponentsError.notEnoughSpace(needed: needed, free: free) - } - } - - let guestPath = "\(guestXIPDirectory)/\(safeName(source.lastPathComponent))" - report("creating \(guestXIPDirectory) in the guest filesystem") - _ = try await vm.run("mkdir -p \(GuestShell.quote(guestXIPDirectory))", environment: nil) { _ in } - - report("copying \(source.lastPathComponent) into the guest (\(byteString(size)))") - XForgeLog.note("components: copying \(source.lastPathComponent) to \(guestPath)") - try await vm.copyIn(hostURL: source, to: guestPath) - - let guestSize = try await vm.run( - "wc -c < \(GuestShell.quote(guestPath))", - environment: nil - ) { _ in } - guard guestSize == 0 else { - throw SystemComponentsError.copyFailed(guestPath) - } - return guestPath - } - - private static func safeName(_ name: String) -> String { - let allowed = CharacterSet.alphanumerics.union(CharacterSet(charactersIn: ".-_")) - let cleaned = name.unicodeScalars.map { allowed.contains($0) ? Character($0) : "-" } - let result = String(cleaned) - return result.isEmpty ? "Xcode.xip" : result - } - - private static func byteString(_ size: Int64) -> String { - size > 0 ? ByteCountFormatter.string(fromByteCount: size, countStyle: .file) : "unknown size" - } -} - -enum SystemComponentsError: LocalizedError { - case installerScriptMissing - case zsignInstallerMissing - case notEnoughSpace(needed: Int64, free: Int64) - case copyFailed(String) - - var errorDescription: String? { - switch self { - case .installerScriptMissing: - return "install-toolchain.sh is not bundled in the app." - case .zsignInstallerMissing: - return "The guest zsign installer is not bundled in the app." - case .notEnoughSpace(let needed, let free): - let formatter = ByteCountFormatter() - return "Not enough free space: this needs about " - + "\(formatter.string(fromByteCount: needed)) and " - + "\(formatter.string(fromByteCount: free)) is free." - case .copyFailed(let path): - return "The file did not arrive in the guest at \(path)." - } - } -} diff --git a/App/Services/ToolchainManager.swift b/App/Services/ToolchainManager.swift deleted file mode 100644 index 9344609..0000000 --- a/App/Services/ToolchainManager.swift +++ /dev/null @@ -1,261 +0,0 @@ -import Foundation -import Combine - -/// Manages the on-device build toolchain. -/// -/// The pieces live in two places, and this type is careful to check the right one: -/// -/// - **Alpine rootfs** — bundled inside the app and imported into the engine's fakefs -/// on first boot. Nothing is downloaded; "installing" it just performs the import. -/// - **Swift / xtool / darwin SDK** — inside the guest Linux, reached over the VM -/// bridge. Installing them means running commands *in the guest*; this type -/// checks what is there and stages the files those commands need, while the -/// commands themselves run in the Terminal (`SystemComponents` holds them). -@MainActor -final class ToolchainManager: ObservableObject { - enum Component: String, CaseIterable, Identifiable { - case rootfs = "Alpine Rootfs" - case swift = "Swift Toolchain" - case xtool = "xtool" - case sdk = "Darwin SDK" - - var id: String { rawValue } - - var icon: String { - switch self { - case .rootfs: return "shippingbox" - case .swift: return "swift" - case .xtool: return "hammer" - case .sdk: return "externaldrive.connected.to.line.below" - } - } - - var blurb: String { - switch self { - case .rootfs: return "Alpine aarch64 userspace · bundled in the app" - case .swift: return "Swift toolchain, installed with swiftly" - case .xtool: return "xtool aarch64 (glibc, via the compatibility layer)" - case .sdk: return "arm64-apple-ios Swift SDK" - } - } - - /// Every component is stored in the embedded Alpine guest. The rootfs is - /// imported into ish-arm64 fakefs; Swift, xtool, and SDKs install below it. - var livesInGuest: Bool { true } - } - - @Published private(set) var installed: Set = [] - @Published private(set) var isInstalling: Component? - @Published private(set) var guestChecked = false - @Published private(set) var activity: String? - @Published var message: String? - - /// 0…1 while a host-side preparation (staging a file into the guest) runs. - @Published private(set) var progress: Double = 0 - @Published private(set) var progressLabel: String? - - private let vm: LinuxVM - - init(vm: LinuxVM? = nil) { - self.vm = vm ?? XForgeEnvironment.makeVM() - } - - func isInstalled(_ component: Component) -> Bool { installed.contains(component) } - - /// Exposed for the import UI: ish-arm64 cannot replace a mounted rootfs. - var isGuestBooted: Bool { vm.isBooted } - - // MARK: - Status - - /// Refresh component status. - /// - /// Host-side facts are always checked. Guest-side components are only probed - /// when `probeGuest` is true (or the guest is already running), because probing - /// boots the embedded Linux — which imports the rootfs the first time. - func refresh(probeGuest: Bool = false) async { - var found: Set = [] - - // RootTabView starts this at launch, but Settings can be presented before - // that task has completed. Await the same idempotent preparation here so - // the row reflects the actual bundled rootfs rather than a stale snapshot. - if !RootfsInstaller.isInstalled(in: XForgeEnvironment.rootsDirectory) { - activity = "Preparing the bundled Alpine rootfs…" - await vm.prepareRootfs() - activity = nil - } - if RootfsInstaller.isInstalled(in: XForgeEnvironment.rootsDirectory) { - found.insert(.rootfs) - } - - if probeGuest || vm.isBooted { - activity = vm.isBooted ? "Checking the embedded Linux…" : "Starting the embedded Linux…" - defer { activity = nil } - do { - XForgeLog.prepare() - XForgeLog.note("refresh: probing the guest") - try await vm.boot() - for component in [Component.swift, .xtool, .sdk] { - if await guestHas(component) { found.insert(component) } - } - guestChecked = true - } catch { - XForgeLog.note("refresh: guest probe failed: \(error.localizedDescription)") - message = "Could not check the embedded Linux: \(error.localizedDescription)" - // Keep whatever we already knew rather than reporting everything missing. - found.formUnion(guestKnown) - } - } else { - found.formUnion(guestKnown) - } - - guestKnown = found.subtracting([.rootfs]) - installed = found - } - - /// Last known guest-side state, so an un-probed refresh doesn't lose it. - private var guestKnown: Set = [] - - /// True when the guest reports the component present (exit status 0). - private func guestHas(_ component: Component) async -> Bool { - let command: String - switch component { - case .rootfs: - return RootfsInstaller.isInstalled(in: XForgeEnvironment.rootsDirectory) - case .swift: - command = "command -v swift >/dev/null 2>&1" - case .xtool: - command = "command -v xtool >/dev/null 2>&1" - case .sdk: - // `swift sdk list` is a glibc binary being asked a question, so its - // stderr is piped rather than sent to /dev/null: this engine kills a - // forked guest program whose output points at /dev/null, and the - // answer here decides whether the SDK row reads "installed". - command = "command -v swift >/dev/null 2>&1 && swift sdk list 2>&1 | grep -qi darwin" - } - do { - let status = try await vm.run(command, environment: nil) { _ in } - return status == 0 - } catch { - return false - } - } - - // MARK: - Getting files into the guest - - /// Copy the user's `Xcode.xip` into the guest's own storage and return the - /// command that installs it there. - /// - /// The `.xip` is never unpacked on the host. The guest keeps the file in its - /// own filesystem (`/root/xforge/xip`), and `xtool sdk install ` — run - /// in the Terminal, where the user can watch it — does the extraction and the - /// SDK post-processing inside Alpine. The returned string is the command to - /// hand to `TerminalSession`. - func installSDKFromXcode(xip: URL) async throws -> String { - isInstalling = .sdk - defer { isInstalling = nil } - XForgeLog.prepare() - let keepAwake = InstallAssertion.begin(reason: "stage Xcode.xip for the guest") - defer { keepAwake.end() } - beginProgress(.sdk) - defer { endProgress() } - - try await vm.boot() - activity = "Copying \(xip.lastPathComponent) into the guest…" - let guestPath = try await SystemComponents.stageXIP(xip, in: vm) { [weak self] line in - self?.advanceProgress(0.5, line) - XForgeLog.note("sdk: \(line)") - } - activity = nil - advanceProgress(1.0, "Ready to install inside the guest") - message = "\(xip.lastPathComponent) is in the guest at \(guestPath). " - + "Its install is running in the Terminal." - return SystemComponents.darwinSDKInstallCommand(guestXIPPath: guestPath) - } - - /// Replace the bundled rootfs with an Alpine `.tar.gz` chosen from Files. - /// ish-arm64 cannot switch roots after it has booted, so this is intentionally - /// limited to a fresh app launch. - func importRootfs(from archive: URL) async { - guard !vm.isBooted else { - message = "Quit and reopen XForge before replacing the Alpine rootfs. The running Linux guest cannot switch roots." - return - } - - isInstalling = .rootfs - defer { isInstalling = nil } - beginProgress(.rootfs) - let scoped = archive.startAccessingSecurityScopedResource() - defer { if scoped { archive.stopAccessingSecurityScopedResource() } } - do { - activity = "Importing \(archive.lastPathComponent)…" - advanceProgress(0.2, "Validating and importing the selected Alpine rootfs") - _ = try RootfsInstaller.install( - archive: archive, - into: XForgeEnvironment.rootsDirectory - ) - advanceProgress(1.0, "Done") - message = "Alpine rootfs imported from \(archive.lastPathComponent)." - installed.insert(.rootfs) - } catch { - message = error.localizedDescription - } - activity = nil - endProgress() - } - - // MARK: - Progress - - private func beginProgress(_ component: Component) { - progress = 0 - progressLabel = "Starting \(component.rawValue)" - } - - private func advanceProgress(_ fraction: Double, _ label: String) { - progress = max(0, min(1, fraction)) - progressLabel = label - XForgeLog.note("progress: \(Int(progress * 100))% — \(label)") - } - - private func endProgress() { - progress = 0 - progressLabel = nil - } - - // MARK: - Reset - - /// Remove the imported rootfs, the staged SDK and downloaded archives. - func reset() async { - isInstalling = nil - activity = nil - endProgress() - let fm = FileManager.default - try? fm.removeItem(at: XForgeEnvironment.rootsDirectory) - try? fm.removeItem(at: XForgeEnvironment.downloadsDirectory) - installed = [] - guestKnown = [] - guestChecked = false - message = "Removed the imported rootfs, the staged SDK and downloads. " - + "Quit and reopen XForge to boot a fresh guest." - } -} - -enum ToolchainError: LocalizedError { - case sdkLayoutUnexpected - case sdkInstallFailed(Int32, String) - case notEnoughSpace(needed: Int64, free: Int64) - - var errorDescription: String? { - switch self { - case .sdkLayoutUnexpected: - return "The downloaded Darwin SDK archive did not contain darwin.artifactbundle/info.json." - case .sdkInstallFailed(let status, let output): - return "`swift sdk install` failed inside the guest (exit \(status)). " - + (output.isEmpty ? "See Settings → Diagnostics → Engine log." : String(output.suffix(400))) - case .notEnoughSpace(let needed, let free): - let formatter = ByteCountFormatter() - return "Not enough free space: this needs about " - + "\(formatter.string(fromByteCount: needed)) and " - + "\(formatter.string(fromByteCount: free)) is free." - } - } -} diff --git a/App/Services/XForgeLog.swift b/App/Services/XForgeLog.swift index cc2299d..a468930 100644 --- a/App/Services/XForgeLog.swift +++ b/App/Services/XForgeLog.swift @@ -1,21 +1,13 @@ import Foundation -/// On-device diagnostics for the embedded Linux. +/// On-device diagnostics. /// -/// Two writers, one file: +/// Plain file writes, no `os_log`: the point is a file the user can share from the +/// app, and `os_log` messages are only readable with a console attached. /// -/// - **The engine.** Its `printk` — every kernel message, including the one -/// `die()` prints immediately before it calls `abort()` — goes to file -/// descriptor 555 (`kernel/log.c`'s dprintf handler). Nothing in an iOS app -/// opens that descriptor, so without `prepare()` a guest crash kills the app -/// and leaves no trace anywhere. -/// - **This app.** `note(_:)` writes a breadcrumb through the same bridge call, -/// so the last line of the file says how far an install got before the app -/// disappeared — which is the difference between "it crashes" and a bug -/// report. -/// -/// Plain file writes, no `os_log`: the point is a file the user can share from -/// the app, and `os_log` messages are only readable with a console attached. +/// The engine that used to share this file — the embedded Linux, whose `printk` +/// went to file descriptor 555 through the ish bridge — is gone, so this is now +/// only XForge's own breadcrumb trail. enum XForgeLog { /// `/logs` — the same container the Files app shows. static var directory: URL { @@ -23,17 +15,30 @@ enum XForgeLog { .appendingPathComponent("logs", isDirectory: true) } - static var url: URL { directory.appendingPathComponent("engine.log") } + static var url: URL { directory.appendingPathComponent("xforge.log") } /// Rotate past this size so the file stays shareable. static let maxBytes = 512 * 1024 - /// Point the bridge (and therefore the engine) at the log file. - /// Safe to call repeatedly; also safe to call after the guest has booted. + private static let timestamp: ISO8601DateFormatter = { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime] + return formatter + }() + + /// Create the log directory and rotate an oversized log. + /// + /// Kept as a separate step from `note(_:)` so the directory exists before the + /// first breadcrumb, and so a caller that wants the log path can ask for it + /// without writing anything. @discardableResult static func prepare() -> Bool { let fm = FileManager.default - try? fm.createDirectory(at: directory, withIntermediateDirectories: true) + do { + try fm.createDirectory(at: directory, withIntermediateDirectories: true) + } catch { + return false + } if let attributes = try? fm.attributesOfItem(atPath: url.path), let size = attributes[.size] as? Int, @@ -45,13 +50,25 @@ enum XForgeLog { try? String(tail).write(to: url, atomically: true, encoding: .utf8) } } - - return url.path.withCString { xf_ish_set_log_file($0) } == 0 + return true } - /// Append one breadcrumb line, timestamped by the bridge. + /// Append one timestamped breadcrumb line. + /// + /// Best effort by design: a diagnostic that can fail a build is worse than a + /// diagnostic that is missing a line. static func note(_ line: String) { - _ = line.withCString { xf_ish_log($0) } + let entry = "\(timestamp.string(from: Date())) \(line)\n" + let fm = FileManager.default + if !fm.fileExists(atPath: url.path) { + _ = prepare() + try? entry.write(to: url, atomically: true, encoding: .utf8) + return + } + guard let handle = try? FileHandle(forWritingTo: url) else { return } + defer { try? handle.close() } + _ = try? handle.seekToEnd() + try? handle.write(contentsOf: Data(entry.utf8)) #if DEBUG print("[XForge] \(line)") #endif diff --git a/App/Views/Build/SigningView.swift b/App/Views/Build/SigningView.swift index bc7b3ca..242fb91 100644 --- a/App/Views/Build/SigningView.swift +++ b/App/Views/Build/SigningView.swift @@ -1,17 +1,21 @@ import SwiftUI import UniformTypeIdentifiers -/// Full IPA configure, sign and install surface. +/// Sign a built IPA with an Apple Developer identity. /// -/// Signing is intentionally performed by zsign inside the Alpine guest. zsign is -/// a cross-platform C++ signer, not an iOS framework; running it in the guest -/// keeps the private key on the device, gives the user live command output, and -/// avoids pretending XKit's currently-unwired Apple-ID service can sign an IPA. +/// The signing itself is XKit — xtool's own signer — running in this process, so +/// the private key is read through the Security framework, used, and never written +/// anywhere. That used to be a `zsign` binary inside the embedded Linux with a +/// password file on disk; both are gone. +/// +/// The Apple ID half of xtool (obtaining a certificate, 2FA, provisioning) is a +/// separate piece of work and is not faked here: this signs with an identity the +/// user already has. struct SigningView: View { @ObservedObject var signing: XKitSigningService @ObservedObject var device: XKitDeviceService - @StateObject private var job = IPAConfigureSignService() + @StateObject private var job = IPASigningJob() @State private var importer: ImportTarget? @State private var showPassword = false @@ -35,24 +39,25 @@ struct SigningView: View { fileRow(title: "IPA to sign", url: job.inputIPA, icon: "app.badge") { importer = .ipa } - if job.inputIPA != nil { - Text("The original IPA is copied into the guest and never modified in place.") + if let ipa = job.inputIPA { + Text("The original is never modified: the signed IPA is written next to it in Documents.") .font(.caption).foregroundStyle(.secondary) + Text(ipa.lastPathComponent).font(.caption).foregroundStyle(.secondary) } } Section("Signing files") { - fileRow(title: "Provisioning profile", url: job.provisioningProfile, - icon: "checkmark.seal") { importer = .mobileprovision } fileRow(title: "Certificate / private key (.p12)", url: job.p12, icon: "key.fill") { importer = .p12 } SecureField("PKCS#12 password", text: $job.password) .textInputAutocapitalization(.never) .autocorrectionDisabled() + fileRow(title: "Provisioning profile (optional)", url: job.provisioningProfile, + icon: "checkmark.seal") { importer = .mobileprovision } fileRow(title: "Entitlements (optional)", url: job.entitlements, icon: "list.bullet.rectangle") { importer = .entitlements } - Text("Keep the .p12 password private. It is held in memory for this signing job and is never saved to the project or build history.") + Text("The password and key stay in memory for this signing job and are never written to disk.") .font(.caption).foregroundStyle(.secondary) } @@ -64,7 +69,7 @@ struct SigningView: View { TextField("Version (optional)", text: $job.version) .textInputAutocapitalization(.never) .autocorrectionDisabled() - Text("Leave a field blank to preserve the IPA's existing value. If you change the bundle identifier, the provisioning profile must authorize it.") + Text("Leave a field blank to keep the IPA's existing value. If you change the bundle identifier, the provisioning profile must authorize it.") .font(.caption).foregroundStyle(.secondary) } @@ -74,8 +79,7 @@ struct SigningView: View { } label: { HStack { if job.isWorking { ProgressView().controlSize(.small) } - Label(job.isWorking ? "Signing in Alpine…" : "Configure & Sign IPA", - systemImage: "signature") + Label(job.isWorking ? "Signing…" : "Sign IPA", systemImage: "signature") Spacer() } } @@ -90,73 +94,65 @@ struct SigningView: View { } header: { Text("Action") } footer: { - Text("zsign is installed into the embedded Linux guest on first use. It supports .p12/.pfx keys, .mobileprovision profiles, entitlements, bundle ID/name/version changes, and IPA output.") + Text(job.status) } - Section("Status") { - Text(job.status).font(.system(.footnote, design: .monospaced)) - if let error = job.error { - Text(error).foregroundStyle(.red) + if let error = job.error { + Section { + Label(error, systemImage: "exclamationmark.triangle.fill") + .font(.footnote).foregroundStyle(.red) } } - } - .navigationTitle("Configure & Sign") - .fileImporter(isPresented: Binding( - get: { importer != nil }, - set: { if !$0 { importer = nil } } - ), allowedContentTypes: importer?.types ?? [.data], allowsMultipleSelection: false) { result in - guard case .success(let urls) = result, let url = urls.first, - let target = importer else { return } - do { - let localURL = try persistImportedFile(url) - switch target { - case .ipa: job.inputIPA = localURL - case .p12: job.p12 = localURL - case .mobileprovision: job.provisioningProfile = localURL - case .entitlements: job.entitlements = localURL + + Section { + NavigationLink { + InstallView(device: device) + } label: { + Label("Install on this device", systemImage: "iphone.and.arrow.forward") } - } catch { - job.error = "Import failed: \(error.localizedDescription)" + } footer: { + Text("Installing a signed build needs a device link that an app cannot open by itself; " + + "the Install screen explains the ways that do work on device.") } - importer = nil } - } - - /// File-provider URLs returned by UIDocumentPicker are security-scoped and can - /// become unreadable as soon as the picker callback returns. Copy the selected - /// item into XForge's sandbox while access is active so signing can reliably - /// stage it into Alpine later. - private func persistImportedFile(_ source: URL) throws -> URL { - let scoped = source.startAccessingSecurityScopedResource() - defer { if scoped { source.stopAccessingSecurityScopedResource() } } - - let directory = XForgeEnvironment.documentDirectory - .appendingPathComponent("Signing Imports", isDirectory: true) - try FileManager.default.createDirectory(at: directory, - withIntermediateDirectories: true) - let destination = directory.appendingPathComponent(source.lastPathComponent) - if FileManager.default.fileExists(atPath: destination.path) { - try FileManager.default.removeItem(at: destination) + .navigationTitle("Signing") + .fileImporter( + isPresented: Binding( + get: { importer != nil }, + set: { if !$0 { importer = nil } } + ), + allowedContentTypes: importer?.types ?? [.data], + allowsMultipleSelection: false + ) { result in + guard case .success(let urls) = result, let url = urls.first else { return } + switch importer { + case .ipa: job.inputIPA = url + case .p12: job.p12 = url + case .mobileprovision: job.provisioningProfile = url + case .entitlements: job.entitlements = url + case nil: break + } + importer = nil } - try FileManager.default.copyItem(at: source, to: destination) - return destination } @ViewBuilder - private func fileRow(title: String, url: URL?, icon: String, - choose: @escaping () -> Void) -> some View { - Button(action: choose) { - HStack(spacing: 12) { - Image(systemName: icon).foregroundStyle(.tint).frame(width: 22) - VStack(alignment: .leading, spacing: 2) { - Text(title).foregroundStyle(.primary) - Text(url?.lastPathComponent ?? "Choose from Files…") - .font(.caption).foregroundStyle(url == nil ? .secondary : .primary) - .lineLimit(1) - } + private func fileRow( + title: String, + url: URL?, + icon: String, + action: @escaping () -> Void + ) -> some View { + Button(action: action) { + HStack { + Label(title, systemImage: icon) Spacer() - Image(systemName: "chevron.right").font(.caption.bold()).foregroundStyle(.tertiary) + Text(url?.lastPathComponent ?? "Choose…") + .font(.caption) + .foregroundStyle(url == nil ? .secondary : .primary) + .lineLimit(1) } } + .foregroundStyle(.primary) } } diff --git a/App/Views/Projects/DependenciesView.swift b/App/Views/Projects/DependenciesView.swift deleted file mode 100644 index 70079eb..0000000 --- a/App/Views/Projects/DependenciesView.swift +++ /dev/null @@ -1,154 +0,0 @@ -import SwiftUI - -/// Add SwiftPM dependencies directly to the project's real Package.swift. -struct DependenciesView: View { - let project: Project - - @State private var enabled: Set = [] - @State private var customPackages: [CatalogPackage] = [] - @State private var customURL = "" - @State private var customVersion = "1.0.0" - @State private var applying = false - @State private var message: String? - - struct CatalogPackage: Identifiable, Hashable { - let id: String - let name: String - let from: String - } - - private static let catalog: [CatalogPackage] = [ - CatalogPackage(id: "https://github.com/xtool-org/xtool", name: "XKit (xtool)", from: "1.17.0"), - CatalogPackage(id: "https://github.com/apple/swift-nio", name: "SwiftNIO", from: "2.77.0"), - CatalogPackage(id: "https://github.com/apple/swift-crypto", name: "swift-crypto", from: "4.5.0"), - CatalogPackage(id: "https://github.com/apple/swift-argument-parser", name: "swift-argument-parser", from: "1.5.0"), - CatalogPackage(id: "https://github.com/pointfreeco/swift-dependencies", name: "swift-dependencies", from: "1.6.2"), - CatalogPackage(id: "https://github.com/Alamofire/Alamofire", name: "Alamofire", from: "5.10.0"), - CatalogPackage(id: "https://github.com/SDWebImage/SDWebImage", name: "SDWebImage", from: "5.20.0"), - CatalogPackage(id: "https://github.com/SwiftyJSON/SwiftyJSON", name: "SwiftyJSON", from: "5.0.2"), - CatalogPackage(id: "https://github.com/krzyzanowskim/OpenSSL", name: "OpenSSL", from: "3.3.2000"), - ] - - var body: some View { - Form { - Section("Catalog") { - ForEach(Self.catalog + customPackages) { package in - Button { - toggle(package) - } label: { - HStack { - VStack(alignment: .leading) { - Text(package.name) - Text(package.id) - .font(.caption) - .foregroundStyle(.secondary) - } - Spacer() - Image(systemName: enabled.contains(package.id) - ? "checkmark.circle.fill" : "circle") - .foregroundStyle(enabled.contains(package.id) - ? .green : .secondary) - } - } - .foregroundStyle(.primary) - } - } - - Section("Custom Package") { - TextField("https://github.com/user/repo", text: $customURL) - .keyboardType(.URL) - .autocorrectionDisabled() - .textInputAutocapitalization(.never) - TextField("Version (from)", text: $customVersion) - .keyboardType(.numbersAndPunctuation) - Button("Add Custom") { addCustom() } - .disabled(customURL.trimmingCharacters(in: .whitespacesAndNewlines).isEmpty) - } - - Section { - Button { - Task { await apply() } - } label: { - Label(applying ? "Applying…" : "Apply to Package.swift", - systemImage: "shippingbox.and.arrow.backward") - } - .disabled(enabled.isEmpty || applying) - - if let message { - Text(message) - .font(.footnote) - .foregroundStyle(message.hasPrefix("Added") ? .green : .red) - } - } - } - .navigationTitle("Dependencies") - } - - private var selectedPackages: [CatalogPackage] { - (Self.catalog + customPackages).filter { enabled.contains($0.id) } - } - - private func toggle(_ package: CatalogPackage) { - if enabled.contains(package.id) { - enabled.remove(package.id) - } else { - enabled.insert(package.id) - } - } - - private func addCustom() { - let url = customURL.trimmingCharacters(in: .whitespacesAndNewlines) - let version = customVersion.trimmingCharacters(in: .whitespacesAndNewlines) - guard URL(string: url)?.scheme == "https", !version.isEmpty else { - message = "Enter an HTTPS repository URL and a version." - return - } - - let name = URL(string: url)?.deletingPathExtension().lastPathComponent ?? "Package" - let package = CatalogPackage(id: url, name: name, from: version) - if !customPackages.contains(where: { $0.id == url }) { - customPackages.append(package) - } - enabled.insert(url) - customURL = "" - } - - private func apply() async { - guard project.hasSafeRootPath else { - message = ProjectValidationError.unsafePath.localizedDescription - return - } - - applying = true - message = nil - defer { applying = false } - - do { - let vm = XForgeEnvironment.makeVM() - for package in selectedPackages { - let command = "cd \(GuestShell.quote(project.rootPath)) && " - + "swift package add-dependency \(GuestShell.quote(package.id)) " - + "--from \(GuestShell.quote(package.from))" - let status = try await vm.run(command, environment: nil) { _ in } - guard status == 0 else { - throw DependencyError.addFailed(package.name, status) - } - } - message = "Added \(selectedPackages.count) package(s) to Package.swift." - enabled.removeAll() - } catch { - message = error.localizedDescription - } - } -} - -enum DependencyError: LocalizedError { - case addFailed(String, Int32) - - var errorDescription: String? { - switch self { - case .addFailed(let name, let status): - return "Could not add \(name) (exit \(status))." - } - } -} diff --git a/App/Views/Projects/FileBrowserView.swift b/App/Views/Projects/FileBrowserView.swift index 16bf149..9b01aea 100644 --- a/App/Views/Projects/FileBrowserView.swift +++ b/App/Views/Projects/FileBrowserView.swift @@ -3,7 +3,7 @@ import SwiftUI /// Browse every readable file in the project's real guest filesystem. struct FileBrowserView: View { let project: Project - @State private var files: [GuestProjectFiles.File] = [] + @State private var files: [ProjectFiles.File] = [] @State private var error: String? @State private var loading = true @@ -34,7 +34,7 @@ struct FileBrowserView: View { } } .navigationTitle("Files") - .navigationDestination(for: GuestProjectFiles.File.self) { file in + .navigationDestination(for: ProjectFiles.File.self) { file in SourceEditorView(file: file, project: project) { updated in if let index = files.firstIndex(where: { $0.id == file.id }) { files[index].contents = updated @@ -48,13 +48,13 @@ struct FileBrowserView: View { loading = true defer { loading = false } do { - files = try await GuestProjectFiles.load(project: project, sourcesOnly: false) + files = try await ProjectFiles.load(project: project, sourcesOnly: false) } catch { self.error = error.localizedDescription } } - private func symbol(for file: GuestProjectFiles.File) -> String { + private func symbol(for file: ProjectFiles.File) -> String { file.name.hasSuffix(".swift") ? "swift" : "doc" } } diff --git a/App/Views/Projects/ImportProjectView.swift b/App/Views/Projects/ImportProjectView.swift index 63c962d..4c918ef 100644 --- a/App/Views/Projects/ImportProjectView.swift +++ b/App/Views/Projects/ImportProjectView.swift @@ -1,10 +1,17 @@ import SwiftUI +import UniformTypeIdentifiers -/// Sheet for importing an existing SwiftPM package from a git URL. +/// Import an existing package from the Files app. +/// +/// This used to `git clone` into the embedded Linux. There is no `git` in an iOS +/// process and no process to run it in, so the import is now what an app can +/// actually do: copy a folder the user picked into the projects directory. A +/// repository can still be fetched with an app that does have git — Working Copy, +/// or Files' own "Download" on a zip — and imported from there. struct ImportProjectView: View { @EnvironmentObject private var preferences: AppPreferences @Environment(\.dismiss) private var dismiss - @State private var gitURL = "" + @State private var picked: URL? @State private var name = "" @State private var orgId = "" @State private var isImporting = false @@ -13,32 +20,49 @@ struct ImportProjectView: View { let onImported: (Project) -> Void private var derivedName: String { - name.isEmpty ? (gitURL.split(separator: "/").last?.split(separator: ".").first.map(String.init) ?? "Imported") : name + if !name.isEmpty { return name } + guard let picked else { return "" } + return picked.lastPathComponent } var body: some View { NavigationStack { Form { - Section("Repository") { - TextField("https://github.com/you/repo.git", text: $gitURL) - .keyboardType(.URL).autocorrectionDisabled().textInputAutocapitalization(.never) + Section("Folder") { + Button { + showingPicker = true + } label: { + Label(picked?.lastPathComponent ?? "Choose a project folder…", + systemImage: "folder") + } + if let picked { + Text(picked.path) + .font(.caption) + .foregroundStyle(.secondary) + .lineLimit(2) + } } + Section("Project") { TextField("Name", text: $name) TextField("Organization Identifier", text: $orgId) .keyboardType(.alphabet).autocorrectionDisabled() } + Section { if isImporting { - HStack { ProgressView(); Text("Cloning into embedded Linux…") } + HStack { ProgressView(); Text("Copying into XForge…") } } if let error { Label(error, systemImage: "exclamationmark.triangle.fill") .font(.footnote).foregroundStyle(.red) } + } footer: { + Text("The folder must contain a Package.swift with a Sources/ directory. " + + "It is copied, not moved: the original stays where it is.") } } - .navigationTitle("Import from Git") + .navigationTitle("Import a project") .onAppear { if orgId.isEmpty { orgId = preferences.defaultOrgId } } @@ -48,39 +72,59 @@ struct ImportProjectView: View { } ToolbarItem(placement: .confirmationAction) { Button("Import") { startImport() } - .disabled(gitURL.isEmpty || isImporting) + .disabled(picked == nil || isImporting) } } } .interactiveDismissDisabled(isImporting) + .fileImporter( + isPresented: $showingPicker, + allowedContentTypes: [.folder], + allowsMultipleSelection: false + ) { result in + guard case .success(let urls) = result, let url = urls.first else { return } + picked = url + if name.isEmpty { name = url.lastPathComponent } + } } + @State private var showingPicker = false + private func startImport() { + guard let source = picked else { return } isImporting = true error = nil - let url = gitURL.trimmingCharacters(in: .whitespacesAndNewlines) + let projectName = derivedName let org = orgId + Task { defer { isImporting = false } do { - let projectName = try Project.validatedName(derivedName) - // Clone into the guest; without this the project path does not - // exist and every later build step fails. - let vm = XForgeEnvironment.makeVM() - if !vm.isBooted { try await vm.boot() } - let path = Project.path(forValidatedName: projectName) - let status = try await vm.run( - "mkdir -p \(GuestShell.quote(Project.projectsRoot)) && " - + "rm -rf \(GuestShell.quote(path)) && " - + "git clone --depth 1 -- \(GuestShell.quote(url)) \(GuestShell.quote(path))", - environment: nil - ) { _ in } - guard status == 0 else { - throw ImportError.cloneFailed(status, url) + let validated = try Project.validatedName(projectName) + let accessed = source.startAccessingSecurityScopedResource() + defer { if accessed { source.stopAccessingSecurityScopedResource() } } + + let manifest = source.appendingPathComponent("Package.swift") + guard FileManager.default.fileExists(atPath: manifest.path) else { + throw ImportError.notAPackage(source.lastPathComponent) } - onImported(Project(name: projectName, - organizationIdentifier: org, - rootPath: path)) + + let project = Project( + name: validated, + organizationIdentifier: org, + rootPath: Project.path(forValidatedName: validated) + ) + let destination = project.rootURL + try FileManager.default.createDirectory( + at: destination.deletingLastPathComponent(), + withIntermediateDirectories: true + ) + if FileManager.default.fileExists(atPath: destination.path) { + throw ImportError.alreadyExists(validated) + } + try FileManager.default.copyItem(at: source, to: destination) + + onImported(project) dismiss() } catch { self.error = error.localizedDescription @@ -90,13 +134,15 @@ struct ImportProjectView: View { } enum ImportError: LocalizedError { - case cloneFailed(Int32, String) + case notAPackage(String) + case alreadyExists(String) var errorDescription: String? { switch self { - case .cloneFailed(let status, let url): - return "git clone failed (exit \(status)) for \(url). Check the URL, and that the " - + "embedded Linux has network access." + case .notAPackage(let name): + return "\(name) has no Package.swift, so it is not a Swift package." + case .alreadyExists(let name): + return "A project named \(name) already exists. Rename it, or remove the existing one first." } } } diff --git a/App/Views/Projects/ManifestEditorView.swift b/App/Views/Projects/ManifestEditorView.swift index 7dffaf5..a88ad0f 100644 --- a/App/Views/Projects/ManifestEditorView.swift +++ b/App/Views/Projects/ManifestEditorView.swift @@ -1,6 +1,6 @@ import SwiftUI -/// Edit the real Package.swift stored in the embedded Alpine guest. +/// Edit the project's Package.swift. struct ManifestEditorView: View { let project: Project @Environment(\.dismiss) private var dismiss @@ -78,7 +78,7 @@ struct ManifestEditorView: View { loading = true defer { loading = false } do { - text = try await GuestProjectFiles.load( + text = try await ProjectFiles.load( relativePath: "Package.swift", project: project ).contents @@ -98,13 +98,13 @@ struct ManifestEditorView: View { error = nil defer { saving = false } do { - let file = GuestProjectFiles.File( + let file = ProjectFiles.File( id: "Package.swift", name: "Package.swift", relativePath: "Package.swift", contents: text ) - try await GuestProjectFiles.save(text, file: file, project: project) + try await ProjectFiles.save(text, file: file, project: project) onSave(text) dismiss() } catch { diff --git a/App/Views/Projects/ProjectDetailView.swift b/App/Views/Projects/ProjectDetailView.swift index bc3814a..b4edfc8 100644 --- a/App/Views/Projects/ProjectDetailView.swift +++ b/App/Views/Projects/ProjectDetailView.swift @@ -39,7 +39,6 @@ private struct OverviewSection: View { let project: Project @State private var showingFiles = false @State private var showingInfo = false - @State private var showingDeps = false @State private var exporting = false @State private var exportedArchive: URL? @State private var exportError: String? @@ -74,16 +73,12 @@ private struct OverviewSection: View { store.update(p) } } - .sheet(isPresented: $showingDeps) { - NavigationStack { DependenciesView(project: project) } - } } private var toolsRow: some View { HStack(spacing: 12) { toolButton("Files", icon: "folder", action: { showingFiles = true }) toolButton("App Info", icon: "doc.badge.gearshape", action: { showingInfo = true }) - toolButton("Dependencies", icon: "shippingbox", action: { showingDeps = true }) Spacer() } .padding(.horizontal).padding(.bottom, 10) @@ -187,7 +182,7 @@ private struct OverviewSection: View { .buttonStyle(.bordered) } else { Button { - Task { await exportProject() } + exportProject() } label: { Label(exporting ? "Exporting…" : "Export project", systemImage: "archivebox") @@ -207,14 +202,12 @@ private struct OverviewSection: View { } } - /// Archive the project inside the guest and stage it for sharing. - private func exportProject() async { + /// Archive the project's directory and stage it for sharing. + private func exportProject() { exporting = true defer { exporting = false } do { - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - exportedArchive = try await ProjectExporter.export(project, via: vm) + exportedArchive = try ProjectExporter.export(project) } catch { exportError = error.localizedDescription } diff --git a/App/Views/Projects/SourceBrowserView.swift b/App/Views/Projects/SourceBrowserView.swift index 4188a34..bdd2a08 100644 --- a/App/Views/Projects/SourceBrowserView.swift +++ b/App/Views/Projects/SourceBrowserView.swift @@ -1,132 +1,12 @@ import SwiftUI -@MainActor -enum GuestProjectFiles { - struct File: Identifiable, Hashable { - let id: String - let name: String - let relativePath: String - var contents: String - } - - static func load(project: Project, sourcesOnly: Bool) async throws -> [File] { - guard project.hasSafeRootPath else { throw ProjectValidationError.unsafePath } - - let vm = XForgeEnvironment.makeVM() - let root = sourcesOnly ? "\(project.rootPath)/Sources" : project.rootPath - let output = GuestFileOutputCollector() - let status = try await vm.run( - "find \(GuestShell.quote(root)) -type f -print 2>/dev/null | sort", - environment: nil - ) { output.append($0) } - guard status == 0 else { throw GuestProjectFileError.listFailed(status) } - - let paths = output.value - .split(separator: "\n") - .map(String.init) - .filter { $0.hasPrefix(project.rootPath + "/") } - - var files: [File] = [] - for path in paths { - let relativePath = String(path.dropFirst(project.rootPath.count + 1)) - guard !relativePath.contains("\n") else { continue } - let temporaryURL = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-source-\(UUID().uuidString)") - defer { try? FileManager.default.removeItem(at: temporaryURL) } - try await vm.copyOut(guestPath: path, to: temporaryURL) - guard let contents = try? String(contentsOf: temporaryURL, encoding: .utf8) else { - continue - } - files.append(File( - id: relativePath, - name: (relativePath as NSString).lastPathComponent, - relativePath: relativePath, - contents: contents - )) - } - return files - } - - static func load(relativePath: String, project: Project) async throws -> File { - guard project.hasSafeRootPath else { throw ProjectValidationError.unsafePath } - guard isSafe(relativePath: relativePath) else { - throw GuestProjectFileError.unsafeRelativePath - } - - let temporaryURL = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-source-\(UUID().uuidString)") - defer { try? FileManager.default.removeItem(at: temporaryURL) } - try await XForgeEnvironment.makeVM().copyOut( - guestPath: "\(project.rootPath)/\(relativePath)", - to: temporaryURL - ) - let contents = try String(contentsOf: temporaryURL, encoding: .utf8) - return File( - id: relativePath, - name: (relativePath as NSString).lastPathComponent, - relativePath: relativePath, - contents: contents - ) - } - - static func save(_ contents: String, file: File, project: Project) async throws { - guard project.hasSafeRootPath else { throw ProjectValidationError.unsafePath } - guard isSafe(relativePath: file.relativePath) else { - throw GuestProjectFileError.unsafeRelativePath - } - - let temporaryURL = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-source-\(UUID().uuidString)") - defer { try? FileManager.default.removeItem(at: temporaryURL) } - try contents.write(to: temporaryURL, atomically: true, encoding: .utf8) - try await XForgeEnvironment.makeVM().copyIn( - hostURL: temporaryURL, - to: "\(project.rootPath)/\(file.relativePath)" - ) - } - - private static func isSafe(relativePath: String) -> Bool { - !relativePath.isEmpty - && !relativePath.hasPrefix("/") - && !relativePath.split(separator: "/").contains("..") - && !relativePath.contains("\n") - } -} - -private final class GuestFileOutputCollector: @unchecked Sendable { - private let lock = NSLock() - private var storage = "" - - func append(_ text: String) { - lock.lock() - storage += text - lock.unlock() - } - - var value: String { - lock.lock() - defer { lock.unlock() } - return storage - } -} - -enum GuestProjectFileError: LocalizedError { - case listFailed(Int32) - case unsafeRelativePath - - var errorDescription: String? { - switch self { - case .listFailed(let status): - return "Could not list project files (exit \(status))." - case .unsafeRelativePath: - return "The selected file is outside the project." - } - } -} - +/// Browse and edit the project's Swift sources. +/// +/// The files are ordinary files in the app container, read through +/// `ProjectFiles`: the guest filesystem they used to live in is gone. struct SourceBrowserView: View { let project: Project - @State private var files: [GuestProjectFiles.File] = [] + @State private var files: [ProjectFiles.File] = [] @State private var error: String? @State private var loading = true @@ -157,7 +37,7 @@ struct SourceBrowserView: View { } } .navigationTitle("Sources") - .navigationDestination(for: GuestProjectFiles.File.self) { file in + .navigationDestination(for: ProjectFiles.File.self) { file in SourceEditorView(file: file, project: project) { updated in if let index = files.firstIndex(where: { $0.id == file.id }) { files[index].contents = updated @@ -171,7 +51,7 @@ struct SourceBrowserView: View { loading = true defer { loading = false } do { - files = try await GuestProjectFiles.load(project: project, sourcesOnly: true) + files = try await ProjectFiles.load(project: project, sourcesOnly: true) } catch { self.error = error.localizedDescription } @@ -184,12 +64,12 @@ struct SourceEditorView: View { @State private var saving = false @State private var error: String? - let file: GuestProjectFiles.File + let file: ProjectFiles.File let project: Project let onSave: (String) -> Void init( - file: GuestProjectFiles.File, + file: ProjectFiles.File, project: Project, onSave: @escaping (String) -> Void ) { @@ -230,7 +110,7 @@ struct SourceEditorView: View { error = nil defer { saving = false } do { - try await GuestProjectFiles.save(text, file: file, project: project) + try await ProjectFiles.save(text, file: file, project: project) onSave(text) dismiss() } catch { diff --git a/App/Views/RootTabView.swift b/App/Views/RootTabView.swift index 10a4543..bdd7a2a 100644 --- a/App/Views/RootTabView.swift +++ b/App/Views/RootTabView.swift @@ -5,16 +5,17 @@ struct RootTabView: View { @StateObject private var signing = XKitSigningService() @StateObject private var device = XKitDeviceService() @StateObject private var preferences = AppPreferences() - /// One terminal for the whole app: the Toolchain screen and the terminal - /// itself hand commands to the same session, so what a component install is - /// doing is visible where it runs. - @StateObject private var terminal = TerminalSession() @State private var selection: AppTab = .projects + /// Three tabs, not four. + /// + /// The Terminal tab existed to type into the embedded Linux. With the guest + /// gone there is nothing behind it — the toolchain runs in this process and + /// reports into the Build screen's console — so the tab is gone rather than + /// left opening an empty screen. enum AppTab: Hashable { case projects case build - case terminal case settings } @@ -31,26 +32,16 @@ struct RootTabView: View { .tabItem { Label("Build", systemImage: "hammer") } .tag(AppTab.build) - TerminalTab() - .environmentObject(terminal) - .tabItem { Label("Terminal", systemImage: "terminal.fill") } - .tag(AppTab.terminal) - SettingsTab(preferences: preferences) - .environmentObject(terminal) .tabItem { Label("Settings", systemImage: "gearshape") } .tag(AppTab.settings) } .task { // Create the app's own directories and keep the regenerable ones out - // of iCloud backup — before anything writes into them. + // of iCloud backup — before anything writes into them. Projects are + // user data and are backed up. XForgeEnvironment.prepareStorage() - // Install the bundled rootfs early, but do not make app presentation - // depend on booting the guest. The guest command bridge can take time - // to initialize on a physical device; Terminal and Toolchain perform - // the same idempotent boot and surface its result when actually used. - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() + XForgeLog.prepare() } } } diff --git a/App/Views/Settings/DownloadsView.swift b/App/Views/Settings/DownloadsView.swift deleted file mode 100644 index 7a56d72..0000000 --- a/App/Views/Settings/DownloadsView.swift +++ /dev/null @@ -1,138 +0,0 @@ -import SwiftUI - -/// Guest-backed provisioning controls. -/// -/// The app never downloads build tooling into a host-side cache: every component -/// is installed by a command inside the embedded Alpine filesystem, and those -/// commands run in the Terminal tab so their output is visible. This screen -/// shows what is present and hands the command over. -struct DownloadsView: View { - @EnvironmentObject private var terminal: TerminalSession - @StateObject private var toolchain = ToolchainManager() - - var body: some View { - List { - Section { - LinuxProvisioningRow( - component: .swift, - installed: toolchain.isInstalled(.swift), - isBusy: toolchain.isInstalling != nil - ) { - install(SystemComponents.swiftInstallCommand, "Swift toolchain") - } - - LinuxProvisioningRow( - component: .xtool, - installed: toolchain.isInstalled(.xtool), - isBusy: toolchain.isInstalling != nil - ) { - install(SystemComponents.xtoolInstallCommand, "xtool") - } - - LinuxProvisioningRow( - component: .sdk, - installed: toolchain.isInstalled(.sdk), - isBusy: toolchain.isInstalling != nil - ) { - installPrebuiltSDK() - } - } header: { - Text("Alpine Toolchain") - } footer: { - Text("Each install is an ordinary Linux command run inside the " - + "embedded Alpine filesystem, shown in the Terminal tab.") - } - - if let message = toolchain.message { - Section { - Text(message) - .font(.footnote) - .foregroundStyle(.secondary) - } - } - - Section { - Text("The Alpine terminal is the Terminal tab, where these installs " - + "run: use it to inspect them with swift --version, " - + "xtool --version and swift sdk list.") - .font(.footnote) - .foregroundStyle(.secondary) - } - } - .navigationTitle("Linux Toolchain") - .task { - await toolchain.refresh(probeGuest: true) - } - } - - /// Boot the guest, make sure the provisioning script is in it, and hand the - /// command to the shared terminal. - private func install(_ command: String, _ what: String) { - Task { - do { - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - try await vm.boot() - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(command, label: "Settings") - toolchain.message = "\(what): running in the Terminal tab." - } catch { - toolchain.message = error.localizedDescription - } - } - } - - private func installPrebuiltSDK() { - Task { - do { - let url = try await XForgeReleases.darwinSDKURL() - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - try await vm.boot() - // The command runs the guest's script first, to remove the SDK the - // rootfs already carries: same preparation as the other installs. - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(SystemComponents.darwinSDKDownloadCommand(from: url), - label: "Settings") - toolchain.message = "Darwin SDK: downloading and installing in the Terminal tab." - } catch { - toolchain.message = error.localizedDescription - } - } - } -} - -private struct LinuxProvisioningRow: View { - let component: ToolchainManager.Component - let installed: Bool - let isBusy: Bool - let install: () -> Void - - var body: some View { - HStack(alignment: .top) { - Image(systemName: component.icon) - .foregroundStyle(installed ? .green : .secondary) - .frame(width: 24) - - VStack(alignment: .leading, spacing: 2) { - Text(component.rawValue) - Text(component.blurb) - .font(.caption) - .foregroundStyle(.secondary) - if installed { - Label("Installed in Alpine", systemImage: "checkmark") - .font(.caption) - .foregroundStyle(.green) - } - } - - Spacer() - - if !installed { - Button("Install", action: install) - .buttonStyle(.bordered) - .disabled(isBusy) - } - } - } -} diff --git a/App/Views/Settings/EngineLogView.swift b/App/Views/Settings/EngineLogView.swift deleted file mode 100644 index a9a7c1c..0000000 --- a/App/Views/Settings/EngineLogView.swift +++ /dev/null @@ -1,81 +0,0 @@ -import SwiftUI -import UIKit - -/// The engine log: the engine's own kernel messages plus XForge's install -/// breadcrumbs, in one file, oldest first. -/// -/// This is the thing to share when the app dies without saying why. The engine's -/// `die()` prints its reason and then calls `abort()` — killing the whole app — -/// and that message is the last line of this file. The breadcrumbs above it say -/// how far the install got first. -@MainActor -struct EngineLogView: View { - @State private var text = "" - @State private var bytes: Int64 = 0 - @State private var copied = false - - private var url: URL { XForgeLog.url } - - var body: some View { - Group { - if text.isEmpty { - ContentUnavailableViewCompat( - title: "Nothing logged yet", - systemImage: "doc.text", - message: "Boot the embedded Linux, or install a toolchain component, " - + "and the engine's messages land here." - ) - } else { - ScrollView { - Text(text) - .font(.system(.caption, design: .monospaced)) - .textSelection(.enabled) - .frame(maxWidth: .infinity, alignment: .leading) - .padding(12) - } - } - } - .navigationTitle("Engine log") - .navigationBarTitleDisplayMode(.inline) - .toolbar { - ToolbarItem(placement: .primaryAction) { - ShareLink(item: url) { - Label("Share", systemImage: "square.and.arrow.up") - } - .disabled(text.isEmpty) - } - ToolbarItem(placement: .secondaryAction) { - Button { - UIPasteboard.general.string = text - copied = true - } label: { - Label(copied ? "Copied" : "Copy", systemImage: "doc.on.doc") - } - .disabled(text.isEmpty) - } - ToolbarItem(placement: .secondaryAction) { - Button(role: .destructive) { - XForgeLog.clear() - reload() - } label: { - Label("Clear", systemImage: "trash") - } - .disabled(text.isEmpty) - } - } - .safeAreaInset(edge: .bottom) { - if !text.isEmpty { - Text("\(ByteCountFormatter.string(fromByteCount: bytes, countStyle: .file)) · \(url.lastPathComponent)") - .font(.caption2) - .foregroundStyle(.secondary) - .padding(.vertical, 4) - } - } - .task { reload() } - } - - private func reload() { - text = XForgeLog.text() - bytes = XForgeLog.byteCount() - } -} diff --git a/App/Views/Settings/SettingsView.swift b/App/Views/Settings/SettingsView.swift index 4fda823..b9997c9 100644 --- a/App/Views/Settings/SettingsView.swift +++ b/App/Views/Settings/SettingsView.swift @@ -1,25 +1,22 @@ import SwiftUI +/// Settings: the toolchain, the app's files, build defaults and diagnostics. +/// +/// Everything here used to be a *guest* concern — install Alpine, install Swift in +/// it, watch the engine log, size the fakefs. The toolchain is now linked into the +/// app and the SDK is a folder, so what is left is what the user can actually act on. @MainActor struct SettingsView: View { @ObservedObject var preferences: AppPreferences - @EnvironmentObject private var terminal: TerminalSession - @StateObject private var toolchain = ToolchainManager() - /// On-disk sizes, filled off the main actor — never computed during `body`. - @State private var sizes: [ToolchainManager.Component: Int64] = [:] - @State private var artifactsSize: Int64 = 0 var body: some View { List { Section { NavigationLink { ToolchainView() } label: { - Label("Toolchain & Alpine", systemImage: "wrench.and.screwdriver") + Label("Toolchain & SDK", systemImage: "wrench.and.screwdriver") } NavigationLink { NativeToolchainView() } label: { - Label("Native Toolchain", systemImage: "cpu") - } - NavigationLink { DownloadsView() } label: { - Label("Downloads", systemImage: "arrow.down.circle") + Label("Native Compiler", systemImage: "cpu") } NavigationLink { SandboxBrowserView(root: XForgeEnvironment.documentDirectory) @@ -29,79 +26,20 @@ struct SettingsView: View { NavigationLink { HistoryView() } label: { Label("Build History", systemImage: "clock.arrow.circlepath") } + NavigationLink { XForgeLogView() } label: { + Label("Log", systemImage: "doc.text.magnifyingglass") + } } header: { Text("System & Files") } footer: { - Text("Alpine, the compiler toolchain, SDKs, downloads and app files are managed here.") + Text("Projects, downloads, built IPAs and the log all live in the app's " + + "container, visible in the Files app.") } preferencesSection - storageSection diagnosticsSection aboutSection } - .task { - await toolchain.refresh() - await loadSizes() - } - } - - /// Compute the storage sizes once, off the main actor. - private func loadSizes() async { - let components = ToolchainManager.Component.allCases - let paths: [(ToolchainManager.Component, URL)] = components.map { - ($0, storageURL(for: $0)) - } - let staging = XForgeEnvironment.stagingDirectory - let result = await Task.detached(priority: .utility) { () -> ([ToolchainManager.Component: Int64], Int64) in - var map: [ToolchainManager.Component: Int64] = [:] - for (component, url) in paths { - map[component] = DirectorySize.bytes(at: url) - } - return (map, DirectorySize.bytes(at: staging)) - }.value - sizes = result.0 - artifactsSize = result.1 - } - - /// Where a component's bytes live on the host. - /// - /// The guest's `/` is the installed fakefs's `data/` directory — the files are - /// there, their modes are in `meta.db` — so this maps a guest path onto the - /// app container. Naming the *component's* directory rather than the whole - /// fakefs is not a detail: every component used to report the size of the - /// entire root (the same five gigabytes four times over), and measuring it four - /// times meant four walks of a thirty-thousand-file tree on every settings - /// visit. - private func storageURL(for component: ToolchainManager.Component) -> URL { - let guest = RootfsInstaller.installedRoot(in: XForgeEnvironment.rootsDirectory) - .appendingPathComponent("data", isDirectory: true) - switch component { - case .rootfs: - return XForgeEnvironment.rootsDirectory - case .swift: - return guest.appendingPathComponent("root/.local/share/swiftly", isDirectory: true) - case .xtool: - return guest.appendingPathComponent("opt/xtool", isDirectory: true) - case .sdk: - // Whatever the install recorded, which is the only reliable answer: - // SwiftPM's store has moved between releases. - return recordedSDKPath(under: guest) ?? guest.appendingPathComponent( - "root/.swiftpm/swift-sdks", isDirectory: true) - } - } - - /// The Darwin SDK's installed path, from the record the installer writes next - /// to the manifest (`darwin-sdk-path:`), mapped into the app container. - private func recordedSDKPath(under guest: URL) -> URL? { - let record = guest.appendingPathComponent("usr/local/share/xforge/darwin-sdk.txt") - guard let text = try? String(contentsOf: record, encoding: .utf8) else { return nil } - for line in text.split(separator: "\n") where line.hasPrefix("path:") { - let value = line.dropFirst("path:".count).trimmingCharacters(in: .whitespaces) - guard value.hasPrefix("/") else { return nil } - return guest.appendingPathComponent(String(value.dropFirst()), isDirectory: true) - } - return nil } private var preferencesSection: some View { @@ -119,90 +57,6 @@ struct SettingsView: View { } } - private var storageSection: some View { - Section { - ForEach(ToolchainManager.Component.allCases) { component in - StorageRow( - title: component.rawValue, - detail: storageDetail(for: component), - installed: toolchain.isInstalled(component), - installing: toolchain.isInstalling == component, - progress: toolchain.isInstalling == component ? toolchain.progress : nil, - progressLabel: toolchain.isInstalling == component ? toolchain.progressLabel : nil - ) { - install(component) - } - } - StorageRow(title: "Build artifacts", - detail: ByteCountFormatter.string(fromByteCount: artifactsSize, countStyle: .file), - installed: true) {} - } header: { - Text("Storage & Toolchain") - } footer: { - Text("The Alpine rootfs is bundled and installs offline. Swift, xtool and " - + "the darwin SDK are installed by commands inside the embedded Linux: " - + "Install hands the command to the Terminal tab, where you can watch it.") - } - } - - /// Hand a component's install command to the Terminal, after making sure the - /// guest is running and the provisioning script is in it. - private func install(_ component: ToolchainManager.Component) { - Task { - do { - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - try await vm.boot() - switch component { - case .rootfs: - toolchain.message = "The Alpine rootfs is bundled in the app and " - + "already installed in the guest filesystem." - case .swift: - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(SystemComponents.swiftInstallCommand, label: "Settings") - toolchain.message = "Swift toolchain: installing in the Terminal tab." - case .xtool: - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(SystemComponents.xtoolInstallCommand, label: "Settings") - toolchain.message = "xtool: installing in the Terminal tab." - case .sdk: - let url = try await XForgeReleases.darwinSDKURL() - // The command removes the installed SDK first (SwiftPM will not - // install a second bundle with the same artifact ID), so the - // script it calls has to be in the guest. - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(SystemComponents.darwinSDKDownloadCommand(from: url), - label: "Settings") - toolchain.message = "Darwin SDK: downloading and installing in the Terminal tab." - } - } catch { - toolchain.message = error.localizedDescription - } - } - } - - private func storageDetail(for component: ToolchainManager.Component) -> String { - let size = ByteCountFormatter.string(fromByteCount: sizes[component] ?? 0, countStyle: .file) - switch (component, toolchain.isInstalled(component)) { - case (.rootfs, true): - return "Installed · \(size)" - case (.rootfs, false): - return "Bundled in the app · installs offline" - case (.sdk, true): - return "Installed in the guest · \(size)" - case (.swift, true), (.xtool, true): - return "Installed in the embedded Linux · \(size)" - case (.sdk, false): - return "Downloads on demand (457 MB)" - case (_, true): - return "Installed in the embedded Linux" - case (_, false): - return component.livesInGuest && !toolchain.guestChecked - ? "Not checked — tap Check on the Toolchain screen" - : "Not installed" - } - } - private var diagnosticsSection: some View { Section { LabeledContent("Device", value: "\(SystemInfo.deviceName) (\(SystemInfo.deviceModel))") @@ -213,17 +67,9 @@ struct SettingsView: View { LabeledContent("CPU", value: "\(SystemInfo.processorCount) cores") LabeledContent("Storage free", value: SystemInfo.storage.free + " of " + SystemInfo.storage.total) LabeledContent("Low Power Mode", value: SystemInfo.isLowPowerMode ? "On" : "Off") - NavigationLink { - EngineLogView() - } label: { - Label("Engine log", systemImage: "doc.text.magnifyingglass") - } + LabeledContent("Compiler", value: NativeToolchainCapabilities.current.backendDescription) } header: { Text("Diagnostics") - } footer: { - Text("The engine log holds the embedded Linux's own messages and " - + "XForge's install breadcrumbs — share it when something dies " - + "without an explanation.") } } @@ -235,61 +81,61 @@ struct SettingsView: View { } header: { Text("About") } footer: { - Text("XForge builds iOS apps on-device with xtool — a cross-platform Xcode replacement.") + Text("XForge builds iOS apps on-device with xtool's libraries — no Mac, no Linux guest.") } } } -struct StorageRow: View { - let title: String - let detail: String - var installed = true - var installing = false - /// 0…1 while this component installs, and what that fraction is measuring. - var progress: Double? - var progressLabel: String? - let onInstall: () -> Void +/// XForge's own log, with the share sheet. +/// +/// It used to be the *engine's* log — the embedded Linux's kernel messages, which +/// was where a crash left its last trace. The compiler runs in this process now, so +/// what is left is XForge's own breadcrumbs plus whatever the build console showed. +struct XForgeLogView: View { + @State private var text = "" + @State private var size: Int64 = 0 var body: some View { - VStack(alignment: .leading, spacing: 6) { - HStack { - Image(systemName: installed ? "checkmark.circle.fill" : "circle.dashed") - .foregroundStyle(installed ? .green : .secondary) - VStack(alignment: .leading, spacing: 1) { - Text(title) - Text(detail).font(.caption).foregroundStyle(.secondary) - } - Spacer() - if !installed { - Button { - onInstall() - } label: { - if installing { - ProgressView().controlSize(.small) - } else { - Label("Install", systemImage: "arrow.down.circle") - } - } - .buttonStyle(.bordered) - .disabled(installing) + Group { + if text.isEmpty { + ContentUnavailableViewCompat( + title: "No Log", + systemImage: "doc.text", + message: "Nothing has been written yet." + ) + } else { + ScrollView { + Text(text) + .font(.system(.caption, design: .monospaced)) + .textSelection(.enabled) + .frame(maxWidth: .infinity, alignment: .leading) + .padding(8) } } - - // Under the row it belongs to, with what is happening right now: the - // engine returns a guest command's output only when it finishes, so - // this line is the only live feedback there is. - if let progress, let progressLabel { - VStack(alignment: .leading, spacing: 3) { - ProgressView(value: progress).progressViewStyle(.linear) - HStack(spacing: 4) { - Text("\(Int(progress * 100))%") - Text(progressLabel) + } + .navigationTitle("Log") + .toolbar { + ToolbarItem(placement: .primaryAction) { + Menu { + ShareLink(item: XForgeLog.url) { + Label("Share log", systemImage: "square.and.arrow.up") } - .font(.caption2) - .foregroundStyle(.secondary) + Button(role: .destructive) { + XForgeLog.clear() + load() + } label: { + Label("Clear", systemImage: "trash") + } + } label: { + Label("Actions", systemImage: "ellipsis.circle") } - .padding(.leading, 28) } } + .task { load() } + } + + private func load() { + text = XForgeLog.text() + size = XForgeLog.byteCount() } } diff --git a/App/Views/Terminal/GuestFileBrowserView.swift b/App/Views/Terminal/GuestFileBrowserView.swift deleted file mode 100644 index d1d993c..0000000 --- a/App/Views/Terminal/GuestFileBrowserView.swift +++ /dev/null @@ -1,216 +0,0 @@ -import SwiftUI -import UIKit - -private final class GuestListingBox: @unchecked Sendable { - private let lock = NSLock() - private var value = "" - func append(_ chunk: String) { lock.lock(); value += chunk; lock.unlock() } - var text: String { lock.lock(); defer { lock.unlock() }; return value } -} - -/// File explorer for the actual Alpine filesystem. -struct GuestFileBrowserView: View { - @State private var path: String - @State private var entries: [Entry] = [] - @State private var loading = false - @State private var error: String? - @State private var vm: (any LinuxVM)? - @State private var showHidden = false - @State private var pendingAction: FileAction? - @State private var destination = "" - @State private var deleteTarget: Entry? - @Environment(\.dismiss) private var dismiss - - struct Entry: Identifiable, Hashable { - let name: String - let path: String - let isDirectory: Bool - let size: Int64 - var id: String { path } - } - - enum FileAction: Identifiable { - case copy(Entry), move(Entry) - var id: String { - switch self { case .copy(let e): return "copy:" + e.path; case .move(let e): return "move:" + e.path } - } - var title: String { switch self { case .copy: return "Copy"; case .move: return "Move" } } - var entry: Entry { switch self { case .copy(let e), .move(let e): return e } } - } - - init(path: String = "/") { _path = State(initialValue: path) } - - var body: some View { - NavigationStack { - List { - if path != "/" { - Button { navigate(to: parent(path)) } label: { - Label("..", systemImage: "arrow.turn.up.left") - } - } - if loading { ProgressView("Reading \(path)…") } - if let error { - ContentUnavailableViewCompat(title: "Could Not Read Folder", - systemImage: "exclamationmark.triangle", - message: error) - } - ForEach(entries) { entry in row(entry) } - } - .navigationTitle(path) - .navigationBarTitleDisplayMode(.inline) - .refreshable { await load() } - .toolbar { - ToolbarItem(placement: .navigationBarLeading) { - Menu { - Toggle("Show Hidden Files", isOn: $showHidden) - Button("Copy Current Path", systemImage: "doc.on.doc") { - UIPasteboard.general.string = path - } - Button("Refresh", systemImage: "arrow.clockwise") { Task { await load() } } - } label: { Image(systemName: "ellipsis.circle") } - } - ToolbarItem(placement: .confirmationAction) { Button("Done") { dismiss() } } - } - .onChange(of: showHidden) { _ in Task { await load() } } - .task { await load() } - .alert(pendingAction?.title ?? "File Action", - isPresented: Binding(get: { pendingAction != nil }, set: { if !$0 { pendingAction = nil } })) { - TextField("Destination path", text: $destination) - .textInputAutocapitalization(.never) - .autocorrectionDisabled() - Button("Cancel", role: .cancel) { pendingAction = nil } - Button(pendingAction?.title ?? "Apply") { - guard let action = pendingAction else { return } - Task { await perform(action) } - } - } message: { - Text("Enter an absolute Alpine path or a destination directory.") - } - .confirmationDialog("Delete \(deleteTarget?.name ?? "item")?", - isPresented: Binding(get: { deleteTarget != nil }, set: { if !$0 { deleteTarget = nil } }), - titleVisibility: .visible) { - Button("Delete", role: .destructive) { - guard let target = deleteTarget else { return } - Task { await remove(target) } - } - Button("Cancel", role: .cancel) { deleteTarget = nil } - } - } - } - - @ViewBuilder private func row(_ entry: Entry) -> some View { - Button { - if entry.isDirectory { navigate(to: entry.path) } - } label: { - HStack(spacing: 10) { - Image(systemName: entry.isDirectory ? "folder.fill" : icon(entry.name)) - .foregroundStyle(entry.isDirectory ? Color.accentColor : Color.secondary) - VStack(alignment: .leading) { - Text(entry.name).lineLimit(1) - if !entry.isDirectory { - Text(ByteCountFormatter.string(fromByteCount: entry.size, countStyle: .file)) - .font(.caption2).foregroundStyle(.secondary) - } - } - Spacer() - if entry.isDirectory { Image(systemName: "chevron.right").foregroundStyle(.tertiary) } - } - .contentShape(Rectangle()) - } - .buttonStyle(.plain) - .contextMenu { - Button("Copy Path", systemImage: "doc.on.doc") { UIPasteboard.general.string = entry.path } - Button("Copy…", systemImage: "plus.square.on.square") { begin(.copy(entry)) } - Button("Move…", systemImage: "folder") { begin(.move(entry)) } - Divider() - Button(role: .destructive) { deleteTarget = entry } label: { Label("Delete", systemImage: "trash") } - } - .swipeActions(edge: .trailing) { - Button(role: .destructive) { deleteTarget = entry } label: { Label("Delete", systemImage: "trash") } - } - } - - private func begin(_ action: FileAction) { - pendingAction = action - destination = path - } - - private func navigate(to value: String) { - path = value.hasPrefix("/") ? value : "/" + value - Task { await load() } - } - - private func load() async { - loading = true; error = nil - do { - let guest = vm ?? XForgeEnvironment.makeVM() - vm = guest - try await guest.boot() - let box = GuestListingBox() - let hiddenGlob = showHidden ? " \(GuestShell.quote(path))/.[!.]*" : "" - let listing = """ - for item in \(GuestShell.quote(path))/*\(hiddenGlob); do - [ -e "$item" ] || [ -L "$item" ] || continue - if [ -d "$item" ]; then kind=d; else kind=f; fi - size=$(stat -c %s "$item" 2>/dev/null || echo 0) - printf '%s\\t%s\\t%s\\n' "$kind" "$item" "$size" - done | sort -k2 - """ - let status = try await guest.run(listing, environment: nil) { box.append($0) } - guard status == 0 else { throw BrowserError.operationFailed("The guest could not list this folder.") } - entries = box.text.split(separator: "\n").compactMap { line in - let parts = line.split(separator: "\t", maxSplits: 2).map(String.init) - guard parts.count == 3 else { return nil } - let full = parts[1] - return Entry(name: URL(fileURLWithPath: full).lastPathComponent, - path: full, isDirectory: parts[0] == "d", size: Int64(parts[2]) ?? 0) - } - } catch { self.error = error.localizedDescription } - loading = false - } - - private func perform(_ action: FileAction) async { - let target = destination.trimmingCharacters(in: .whitespacesAndNewlines) - guard target.hasPrefix("/") else { - error = "Destination must be an absolute Alpine path."; pendingAction = nil; return - } - do { - let guest = vm ?? XForgeEnvironment.makeVM() - vm = guest - let verb = action.title == "Copy" ? "cp -R" : "mv" - let command = "\(verb) -- \(GuestShell.quote(action.entry.path)) \(GuestShell.quote(target))" - let status = try await guest.run(command, environment: nil) { _ in } - guard status == 0 else { throw BrowserError.operationFailed("\(action.title) failed.") } - pendingAction = nil - await load() - } catch { self.error = error.localizedDescription; pendingAction = nil } - } - - private func remove(_ entry: Entry) async { - do { - let guest = vm ?? XForgeEnvironment.makeVM() - vm = guest - let status = try await guest.run("rm -rf -- \(GuestShell.quote(entry.path))", environment: nil) { _ in } - guard status == 0 else { throw BrowserError.operationFailed("Delete failed.") } - deleteTarget = nil - await load() - } catch { self.error = error.localizedDescription; deleteTarget = nil } - } - - private func parent(_ value: String) -> String { - let p = (value as NSString).deletingLastPathComponent - return p.isEmpty ? "/" : p - } - - private func icon(_ name: String) -> String { - if name.hasSuffix(".ipa") { return "app.badge" } - if name.hasSuffix(".zip") || name.hasSuffix(".tar.gz") { return "archivebox" } - if name.hasSuffix(".dylib") { return "shippingbox" } - return "doc" - } - - enum BrowserError: LocalizedError { - case operationFailed(String) - var errorDescription: String? { if case .operationFailed(let message) = self { return message }; return nil } - } -} diff --git a/App/Views/Terminal/TerminalKeys.swift b/App/Views/Terminal/TerminalKeys.swift deleted file mode 100644 index 9d0d4c7..0000000 --- a/App/Views/Terminal/TerminalKeys.swift +++ /dev/null @@ -1,152 +0,0 @@ -import Foundation -import SwiftUI - -/// One key on the terminal's extra-keys bar. -/// -/// The bar exists to supply the characters and actions a phone keyboard cannot -/// produce, so which keys are useful depends on what the user is doing — someone -/// running `apk` wants different keys from someone editing a script. Each key is -/// therefore a named, individually hideable item rather than a hard-coded row. -enum TerminalKey: String, CaseIterable, Identifiable, Codable { - case interrupt - case escape - case tab - case previous - case next - case dash - case dot - case slash - case colon - case bang - case pipe - case components - case files - case paste - case hideKeyboard - - var id: String { rawValue } - - /// The title shown in the configuration list. - var title: String { - switch self { - case .interrupt: return "Interrupt (Ctrl-C)" - case .escape: return "Escape" - case .tab: return "Tab" - case .previous: return "Previous command" - case .next: return "Next command" - case .dash: return "Minus" - case .dot: return "Period" - case .slash: return "Slash" - case .colon: return "Colon" - case .bang: return "Exclamation mark" - case .pipe: return "Pipe" - case .components: return "Components" - case .files: return "List files" - case .paste: return "Paste" - case .hideKeyboard: return "Hide keyboard" - } - } - - /// What the key shows. Punctuation and the history arrows are their own glyph; - /// the rest are SF Symbols. - var label: String { - switch self { - case .previous: return "↑" - case .next: return "↓" - case .dash: return "-" - case .dot: return "." - case .slash: return "/" - case .colon: return ":" - case .bang: return "!" - case .pipe: return "|" - default: return "" - } - } - - var symbol: String? { - switch self { - case .interrupt: return "control" - case .escape: return "escape" - case .tab: return "arrow.right.to.line.alt" - case .components: return "wrench.and.screwdriver" - case .files: return "folder" - case .paste: return "doc.on.clipboard" - case .hideKeyboard: return "keyboard.chevron.compact.down" - default: return nil - } - } - - /// A key that cannot be turned off. - /// - /// Hide Keyboard is pinned: without it there is no way back to the terminal's - /// screen once the keyboard covers it, so a mis-tap in the configuration list - /// could leave the terminal unusable. Every other key is a preference. - var isPinned: Bool { self == .hideKeyboard } - - /// Grouping for the configuration list, so related keys read together. - static let groups: [(title: String, keys: [TerminalKey])] = [ - ("Control", [.interrupt, .escape, .tab]), - ("History", [.previous, .next]), - ("Punctuation", [.dash, .dot, .slash, .colon, .bang, .pipe]), - ("Actions", [.components, .files, .paste]), - ("Always shown", [.hideKeyboard]), - ] -} - -/// Which keys the terminal shows, persisted between launches. -/// -/// Stored as the set of *hidden* keys rather than the visible ones, so a key added -/// in a future version shows up by default instead of being silently absent from -/// an existing user's bar. -@MainActor -final class TerminalKeyConfiguration: ObservableObject { - @Published private(set) var hidden: Set = [] - @Published private(set) var order: [TerminalKey] = [] - - private static let defaultsKey = "org.xforge.terminal.hiddenKeys" - private static let orderKey = "org.xforge.terminal.keyOrder" - - init() { - let raw = UserDefaults.standard.stringArray(forKey: Self.defaultsKey) ?? [] - hidden = Set(raw.compactMap(TerminalKey.init(rawValue:))) - let saved = (UserDefaults.standard.stringArray(forKey: Self.orderKey) ?? []) - .compactMap(TerminalKey.init(rawValue:)) - let defaults = TerminalKey.groups.flatMap(\.keys) - order = saved + defaults.filter { !saved.contains($0) } - } - - /// Whether a key is shown. Pinned keys ignore the stored preference. - func isVisible(_ key: TerminalKey) -> Bool { - if key.isPinned { return true } - return !hidden.contains(key) - } - - func setVisible(_ visible: Bool, for key: TerminalKey) { - guard !key.isPinned else { return } - if visible { hidden.remove(key) } else { hidden.insert(key) } - save() - } - - func toggle(_ key: TerminalKey) { - setVisible(!isVisible(key), for: key) - } - - /// Restore every key. - func resetToDefaults() { - hidden = [] - order = TerminalKey.groups.flatMap(\.keys) - save() - } - - func move(from source: IndexSet, to destination: Int) { - order.move(fromOffsets: source, toOffset: destination) - save() - } - - var visibleKeys: [TerminalKey] { order.filter(isVisible) } - - private func save() { - UserDefaults.standard.set(hidden.map(\.rawValue).sorted(), forKey: Self.defaultsKey) - UserDefaults.standard.set(order.map(\.rawValue), forKey: Self.orderKey) - } -} diff --git a/App/Views/Terminal/TerminalSession.swift b/App/Views/Terminal/TerminalSession.swift deleted file mode 100644 index d3008ad..0000000 --- a/App/Views/Terminal/TerminalSession.swift +++ /dev/null @@ -1,296 +0,0 @@ -import Foundation -import SwiftUI -import UIKit - -/// The app's one terminal into the embedded Alpine Linux. -/// -/// This is the guest's own console: `/sbin/init` owns it, and it respawns -/// `/sbin/xforge-login root` on it, so the shell you type into is a real login -/// session with a real tty — the login shell root is configured with in -/// `/etc/passwd`. See `InteractiveShellSession` for what that buys. -/// -/// Consequences of it being a real console, which are the point of the change: -/// - the working directory is the shell's own, so `cd` persists; -/// - the *guest's* line discipline supplies echo, backspace, `Ctrl-D` and -/// editing — the host does not synthesise any of it; -/// - `Ctrl-C` is a byte again, so the tty turns it into `SIGINT` for whatever is -/// running in the foreground, and `Ctrl-Z` suspends it; -/// - logging out (or killing the shell) gives a fresh login, because init -/// respawns it rather than leaving a dead screen. -@MainActor -final class TerminalSession: ObservableObject { - /// The screen model. Views render `buffer.lines` and re-render when - /// `revision` changes. - let buffer = TerminalBuffer() - - /// Bumped whenever the screen changes. - @Published private(set) var revision = 0 - /// True while the guest console is attached and input is being forwarded to it. - @Published private(set) var running = false - @Published private(set) var booting = false - /// Commands handed over by other screens, run through the same console. - @Published private(set) var pending: [QueuedLine] = [] - @Published private(set) var activeLabel: String? - @Published private(set) var isBooted = false - @Published private(set) var problem: String? - - /// A line queued for the shell, optionally with the label of the screen that - /// asked for it. - struct QueuedLine: Identifiable, Equatable { - let id = UUID() - let text: String - var label: String? - } - - private var shell: (any InteractiveShellSession)? - private var bootTask: Task? - private var didAttemptBoot = false - - /// How the session obtains a guest. Injectable so the terminal's logic can be - /// tested against a stub instead of a real emulator. - private let makeVM: @MainActor () -> any LinuxVM - - init(makeVM: @escaping @MainActor () -> any LinuxVM = { XForgeEnvironment.makeVM() }) { - self.makeVM = makeVM - let saved = Self.read(Self.transcriptURL) - if let saved, !saved.isEmpty { - if !saved.hasSuffix("\n") { buffer.feed(saved + "\n") } - else { buffer.feed(saved) } - } - if buffer.isAtStart { banner() } - } - - /// Whether the exit key bar should send to a live shell. - var acceptsInput: Bool { !booting && shell?.isRunning == true } - - // MARK: - Boot - - /// Start the shell, booting the guest first. Safe to call from several - /// screens; only the first call does work. - func boot() async { - guard !didAttemptBoot else { - await bootTask?.value - return - } - didAttemptBoot = true - booting = true - problem = nil - buffer.appendLine("[starting the embedded Linux…]", - style: TerminalStyle(foreground: .index(8))) - revision += 1 - - let task = Task { [weak self] in - guard let self else { return } - do { - let vm = self.makeVM() - await vm.prepareRootfs() - try await vm.boot() - let session = try await vm.startInteractiveShell( - onOutput: { [weak self] chunk in - Task { @MainActor in self?.consume(chunk) } - } - ) - self.shell = session - self.isBooted = true - self.running = true - self.buffer.appendLine("[guest is up — Alpine aarch64 Linux]", - style: TerminalStyle(foreground: .index(2))) - // The console's size is the screen's, but layout may have run - // before the shell existed. - self.reportSize() - } catch { - self.problem = error.localizedDescription - self.buffer.appendLine("[error] \(error.localizedDescription)", - style: TerminalStyle(foreground: .index(1))) - } - self.booting = false - self.revision += 1 - self.save() - self.drainQueue() - } - bootTask = task - await task.value - } - - /// Stop the shell. The console belongs to the guest's init, which keeps - /// running; this only stops this screen from reading and writing to it. - func shutdown() { - shell?.stop() - shell = nil - running = false - revision += 1 - } - - // MARK: - Screen size - - /// The size the terminal is drawn at, in characters. The guest needs it or its - /// programs wrap to nothing — `ls` prints one name per line at width 0. - private var screenCols = 0 - private var screenRows = 0 - - /// Called by the terminal surface whenever its geometry changes. - func consoleResized(cols: Int, rows: Int) { - guard cols > 0, rows > 0 else { return } - guard cols != screenCols || rows != screenRows else { return } - screenCols = cols - screenRows = rows - shell?.resize(cols: cols, rows: rows) - } - - private func reportSize() { - guard screenCols > 0, screenRows > 0 else { return } - shell?.resize(cols: screenCols, rows: screenRows) - } - - // MARK: - Input - - /// Send raw keyboard/terminal input straight to the console. - /// - /// Nothing is interpreted here: the console is a tty, so the guest's line - /// discipline and the shell's own line editor are what echo, erase, complete - /// and recall. That is why there is no command field and no host-side prompt. - func sendRaw(_ text: String) { - guard !text.isEmpty else { return } - sendToShell(text, label: nil) - } - - /// Hand a line to the console from another screen. It is written like anything - /// else, so it runs in the same session — with the same environment, the same - /// directory, and after whatever is already queued. - /// - /// Booting this session is part of the job. A screen that queues a command - /// boots the *guest* first, and that is not the same as this console being - /// attached to it: only the Terminal tab called `boot()`. So a command queued - /// before the Terminal tab had ever been opened — which is every install from - /// the Toolchain, Downloads and Settings screens on a fresh launch, since - /// SwiftUI does not build an unselected tab — was reported as undeliverable to - /// a buffer nobody was looking at, while the screen that queued it said it was - /// running in the Terminal. - func enqueue(_ line: String, label: String? = nil) { - enqueue(QueuedLine(text: line, label: label)) - } - - func enqueue(_ line: QueuedLine) { - pending.append(line) - guard shell?.isRunning != true else { - drainQueue() - return - } - Task { [weak self] in - guard let self else { return } - await self.boot() - // If the boot failed the queue is still drained, which reports the - // line as undelivered — the honest outcome, and the reason the drop - // path stays. - self.drainQueue() - } - } - - private func drainQueue() { - guard !pending.isEmpty else { return } - guard running, shell?.isRunning == true else { - // No shell to write to. Take the line out of the queue and report it, - // rather than leaving it pending forever and looking like it will run: - // a command that silently never executes is the worst of the options. - let dropped = pending.removeFirst() - sendToShell(dropped.text, label: dropped.label) - return - } - let next = pending.removeFirst() - // A command from another screen is announced, because the user did not - // type it and needs to know where it came from. The echo of the command - // itself comes from the shell. - if let label = next.label, !label.isEmpty { - buffer.appendLine("↑ requested by \(label)", - style: TerminalStyle(foreground: .index(8))) - revision += 1 - } - sendToShell(next.text + "\n", label: next.label) - } - - private func sendToShell(_ text: String, label: String?) { - guard let shell, shell.isRunning else { - // Nothing to write to. Report it rather than dropping the line, since - // a queued install that silently never runs is worse than an error. - buffer.appendLine("[error] the shell is not running; \(text.count) character(s) not sent", - style: TerminalStyle(foreground: .index(1))) - revision += 1 - return - } - activeLabel = label - if !shell.send(text) { - buffer.appendLine("[error] the guest did not accept input", - style: TerminalStyle(foreground: .index(1))) - revision += 1 - } - } - - /// Interrupt the foreground program with Ctrl-C. - /// - /// The byte, not a host-side `kill`: the console is a real tty, so the guest's - /// line discipline turns `0x03` into `SIGINT` for the foreground process group - /// — which reaches whatever is running, and leaves the shell itself alone. - func interrupt() { - guard let shell, shell.isRunning else { return } - Task { [weak self] in - await shell.interruptForeground() - self?.activeLabel = nil - self?.revision += 1 - } - } - - func pasteFromClipboard() { - guard let text = UIPasteboard.general.string, !text.isEmpty else { return } - // Hand it straight to the shell: it may be multi-line, or contain control - // characters the key bar cannot produce. - sendToShell(text, label: nil) - } - - // MARK: - Output - - private func consume(_ chunk: String) { - // Straight to the screen: the shell prints its own prompt, echoes what it - // reads, and emits its own escape sequences — and the buffer understands - // carriage returns, backspaces and SGR colour, so there is nothing to - // interpret here. - buffer.feed(chunk) - revision += 1 - } - - // MARK: - Screen - - func clear() { - buffer.clear() - banner() - revision += 1 - save() - } - - private func banner() { - buffer.appendLine("XForge terminal — the guest's own console.", - style: TerminalStyle(foreground: .index(11))) - buffer.appendLine("Alpine boots /sbin/init as pid 1, which starts root's login shell on this", - style: TerminalStyle(foreground: .index(8))) - buffer.appendLine("terminal — the shell /etc/passwd names for root. Ctrl-C interrupts the", - style: TerminalStyle(foreground: .index(8))) - buffer.appendLine("foreground program; `exit` ends the session and init starts a fresh one.", - style: TerminalStyle(foreground: .index(8))) - buffer.appendLine("") - revision += 1 - } - - // MARK: - Persistence - - private static var transcriptURL: URL { - XForgeEnvironment.documentDirectory.appendingPathComponent("terminal-transcript.txt") - } - - private static func read(_ url: URL) -> String? { - try? String(contentsOf: url, encoding: .utf8) - } - - private func save() { - try? String(buffer.plainText.suffix(200_000)) - .write(to: Self.transcriptURL, atomically: true, encoding: .utf8) - } -} diff --git a/App/Views/Terminal/TerminalText.swift b/App/Views/Terminal/TerminalText.swift deleted file mode 100644 index bb3f558..0000000 --- a/App/Views/Terminal/TerminalText.swift +++ /dev/null @@ -1,446 +0,0 @@ -import SwiftUI - -// MARK: - Colour - -/// One terminal colour: an indexed palette entry (0–255) or a direct RGB value. -/// -/// The guest is a real Linux system, so its programs emit whatever SGR codes -/// they like — `xtool`, `swift` and `apk` all colourise their output. Parsing -/// them here is what lets the terminal show a build the way it looks in a -/// desktop shell instead of a wall of escape sequences. -enum TerminalColor: Equatable, Hashable { - case index(Int) - case rgb(Int, Int, Int) - - /// The 16 ANSI colours, tuned for a dark terminal background. - private static let palette: [Color] = [ - Color(red: 0.13, green: 0.13, blue: 0.15), // 0 black - Color(red: 0.87, green: 0.29, blue: 0.29), // 1 red - Color(red: 0.35, green: 0.82, blue: 0.44), // 2 green - Color(red: 0.93, green: 0.78, blue: 0.32), // 3 yellow - Color(red: 0.33, green: 0.58, blue: 0.96), // 4 blue - Color(red: 0.78, green: 0.47, blue: 0.96), // 5 magenta - Color(red: 0.32, green: 0.82, blue: 0.86), // 6 cyan - Color(red: 0.86, green: 0.88, blue: 0.91), // 7 white - Color(red: 0.38, green: 0.39, blue: 0.44), // 8 bright black - Color(red: 0.96, green: 0.42, blue: 0.42), // 9 bright red - Color(red: 0.47, green: 0.92, blue: 0.56), // 10 bright green - Color(red: 0.99, green: 0.86, blue: 0.42), // 11 bright yellow - Color(red: 0.47, green: 0.69, blue: 1.00), // 12 bright blue - Color(red: 0.87, green: 0.62, blue: 1.00), // 13 bright magenta - Color(red: 0.47, green: 0.92, blue: 0.96), // 14 bright cyan - Color(red: 0.99, green: 0.99, blue: 0.99), // 15 bright white - ] - - var color: Color { - switch self { - case .rgb(let r, let g, let b): - return Color(red: Self.clamp(r), green: Self.clamp(g), blue: Self.clamp(b)) - case .index(let value): - let n = max(0, min(value, 255)) - if n < 16 { return Self.palette[n] } - if n < 232 { - // The 6×6×6 colour cube. - let i = n - 16 - func level(_ v: Int) -> Double { - v == 0 ? 0 : Double(55 + v * 40) / 255 - } - return Color(red: level(i / 36), - green: level((i / 6) % 6), - blue: level(i % 6)) - } - // The 24-step grey ramp. - let v = Double(8 + (n - 232) * 10) / 255 - return Color(red: v, green: v, blue: v) - } - } - - private static func clamp(_ value: Int) -> Double { - Double(max(0, min(value, 255))) / 255 - } -} - -/// The attributes an SGR sequence can turn on. -struct TerminalStyle: Equatable { - var foreground: TerminalColor? - var background: TerminalColor? - var bold = false - var dim = false - var italic = false - var underline = false - var inverse = false - - static let plain = TerminalStyle() - - /// The colour text is drawn in, with `inverse` resolved. - var textColor: Color { - if inverse { return (background ?? .index(0)).color } - return (foreground ?? Self.defaultForeground).color - } - - /// The colour behind the run, or nil for "the terminal's own background". - var backgroundColor: Color? { - if inverse { return (foreground ?? .index(15)).color } - guard let background else { return nil } - return background.color - } - - /// Slightly warm off-white — the same default the iSH terminal uses. - static let defaultForeground = TerminalColor.rgb(0xE6, 0xE9, 0xE6) -} - -// MARK: - Cells, lines, and the buffer - -/// One screen cell: a character and the style it was written with. -struct TerminalCell: Equatable { - var character: Character - var style: TerminalStyle -} - -/// One completed line of the scrollback. -struct TerminalLine: Equatable { - var cells: [TerminalCell] = [] - - var isEmpty: Bool { cells.isEmpty } - - var text: String { String(cells.map(\.character)) } - - /// The line as an `AttributedString`, with runs of equal style merged. - func attributedString(fontSize: CGFloat) -> AttributedString { - guard !cells.isEmpty else { return AttributedString(" ") } - - var result = AttributedString() - var index = 0 - while index < cells.count { - let style = cells[index].style - var text = "" - while index < cells.count, cells[index].style == style { - text.append(cells[index].character) - index += 1 - } - var run = AttributedString(text) - run.foregroundColor = style.textColor - if let background = style.backgroundColor { - run.backgroundColor = background - } - if style.underline { - run.underlineStyle = .single - } - if style.bold || style.italic || style.dim { - var font = Font.system(size: fontSize, design: .monospaced) - if style.bold { font = font.weight(.bold) } - if style.italic { font = font.italic() } - run.font = font - if style.dim, let foreground = style.foreground { - // "Dim" has no monospaced-font equivalent; approximate it by - // blending the colour toward the background. - run.foregroundColor = foreground.color.opacity(0.65) - } else if style.dim { - run.foregroundColor = style.textColor.opacity(0.65) - } - } - result.append(run) - } - return result - } -} - -/// A very small terminal screen model. -/// -/// It exists because the guest *is* a terminal program: a command that only -/// returns its output when it finishes still returns carriage-return rewrites -/// (curl's progress bar), backspaces and SGR colour. Feeding those through a -/// screen model instead of appending raw bytes is the difference between a log -/// view and a terminal. -/// -/// It is a line/screen model, not a full emulator: cursor addressing, alternate -/// screens and scrolling regions are recognised and either honoured (`clear`, -/// `\r`, `\b`) or dropped, because XForge's bridge runs one command at a time -/// rather than hosting a live pty. -@MainActor -final class TerminalBuffer { - /// Completed lines, oldest first. - private(set) var lines: [TerminalLine] = [] - - /// The line still being written to. - private(set) var current = TerminalLine() - private var cursor = 0 - - /// Scrollback cap. The transcript is also persisted, so this is about the - /// cost of keeping it on screen, not about losing history. - var maxLines = 2000 - private let maxColumns = 512 - - private var style = TerminalStyle() - - private enum EscapeState { - case ground - case escape - case csi - case osc - case oscEscape - } - - private var state: EscapeState = .ground - private var csiParameters = "" - private var pendingCarriageReturn = false - - var isAtStart: Bool { lines.isEmpty && current.isEmpty } - - /// Everything on screen plus the scrollback, as plain text. - var plainText: String { - (lines.map(\.text) + [current.text]).joined(separator: "\n") - } - - func clear() { - lines.removeAll() - current = TerminalLine() - cursor = 0 - style = .plain - state = .ground - csiParameters = "" - pendingCarriageReturn = false - } - - /// Feed a chunk of guest output through the screen model. - func feed(_ text: String) { - for character in text { - switch state { - case .ground: - handleGround(character) - case .escape: - handleEscape(character) - case .csi: - handleCSI(character) - case .osc: - // OSC payload: skipped until BEL or ST. - if character == "\u{07}" { - state = .ground - } else if character == "\u{1b}" { - state = .oscEscape - } - case .oscEscape: - state = character == "\\" ? .ground : .osc - } - } - } - - /// Put a line on screen immediately (used for the terminal's own prompt and - /// status lines, which are not guest output). - func appendLine(_ text: String, style: TerminalStyle = .plain) { - let saved = self.style - self.style = style - for character in text { write(character) } - newline() - self.style = saved - } - - // MARK: - Ground state - - private func handleGround(_ character: Character) { - switch character { - case "\u{1b}": - pendingCarriageReturn = false - state = .escape - case "\n", "\r\n": - // Swift treats CR+LF as a *single* grapheme cluster, so a guest - // program that emits Windows-style line endings delivers "\r\n" as - // ONE Character — not as "\r" followed by "\n". Matching only "\n" - // here would write that character into a cell and the line would - // never break, turning a whole command's output into one line. - pendingCarriageReturn = false - newline() - case "\r": - // Held until the next character: "\r\n" is a newline, a bare "\r" - // rewinds to column 0 so a progress bar can redraw its line. - pendingCarriageReturn = true - case "\u{08}": - backspace() - case "\t": - applyPendingCarriageReturn() - tab() - default: - // Remaining control characters (BEL, NUL, …) are not printable. A - // grapheme can hold more than one scalar, so test all of them rather - // than `asciiValue`, which is nil for anything multi-scalar. - if character.unicodeScalars.allSatisfy({ $0.value < 0x20 }) { return } - applyPendingCarriageReturn() - write(character) - } - } - - private func applyPendingCarriageReturn() { - guard pendingCarriageReturn else { return } - pendingCarriageReturn = false - cursor = 0 - } - - private func handleEscape(_ character: Character) { - switch character { - case "[": - csiParameters = "" - state = .csi - case "]": - state = .osc - case "(", ")", "#", "%": - // Two-byte sequences with a designator byte; stay in escape state - // for one more character so the designator is swallowed too. - state = .escape - default: - state = .ground - } - } - - private func handleCSI(_ character: Character) { - // Parameter bytes, then a final byte in the range 0x40-0x7E. - guard let ascii = character.asciiValue, ascii >= 0x40, ascii <= 0x7E else { - if csiParameters.count < 32 { csiParameters.append(character) } - return - } - state = .ground - applyCSI(final: character, parameters: csiParameters) - csiParameters = "" - } - - private func applyCSI(final: Character, parameters: String) { - switch final { - case "m": - applySGR(parameters) - case "J": - // Erase in display. Only the "whole screen" form is meaningful here - // (the bridge has no cursor addressing), so `clear` and - // `printf '\033[2J'` wipe the scrollback while the partial forms just - // clear the line being written. - if parameters.isEmpty || parameters == "2" { - lines.removeAll() - current = TerminalLine() - cursor = 0 - } else if cursor < current.cells.count { - current.cells.removeSubrange(cursor.. Int? { - guard index + offset < parts.count else { return nil } - return Int(parts[index + offset]) - } - while index < parts.count { - let value = Int(parts[index]) ?? 0 - switch value { - case 0: - style = .plain - case 1: - style.bold = true - case 2: - style.dim = true - case 3: - style.italic = true - case 4: - style.underline = true - case 7: - style.inverse = true - case 21, 22: - style.bold = false - style.dim = false - case 23: - style.italic = false - case 24: - style.underline = false - case 27: - style.inverse = false - case 30...37: - style.foreground = .index(value - 30) - case 39: - style.foreground = nil - case 40...47: - style.background = .index(value - 40) - case 49: - style.background = nil - case 90...97: - style.foreground = .index(value - 90 + 8) - case 100...107: - style.background = .index(value - 100 + 8) - case 38, 48: - let isForeground = value == 38 - if let mode = parameter(1), mode == 5, let n = parameter(2) { - let colour = TerminalColor.index(n) - if isForeground { style.foreground = colour } else { style.background = colour } - index += 2 - } else if let mode = parameter(1), mode == 2, - let r = parameter(2), let g = parameter(3), let b = parameter(4) { - let colour = TerminalColor.rgb(r, g, b) - if isForeground { style.foreground = colour } else { style.background = colour } - index += 4 - } - default: - break - } - index += 1 - } - } - - // MARK: - Writing - - private func write(_ character: Character) { - guard cellsAllowMore() else { return } - let cell = TerminalCell(character: character, style: style) - if cursor < current.cells.count { - current.cells[cursor] = cell - } else { - current.cells.append(cell) - } - cursor += 1 - } - - private func cellsAllowMore() -> Bool { - cursor < maxColumns || cursor < current.cells.count - } - - private func tab() { - let next = (cursor / 8 + 1) * 8 - while cursor < next { write(" ") } - } - - private func backspace() { - pendingCarriageReturn = false - guard cursor > 0 else { return } - cursor -= 1 - if cursor < current.cells.count { - current.cells.remove(at: cursor) - } - } - - private func newline() { - lines.append(current) - if lines.count > maxLines { - lines.removeFirst(lines.count - maxLines) - } - current = TerminalLine() - cursor = 0 - } -} diff --git a/App/Views/Terminal/TerminalView.swift b/App/Views/Terminal/TerminalView.swift deleted file mode 100644 index a780048..0000000 --- a/App/Views/Terminal/TerminalView.swift +++ /dev/null @@ -1,533 +0,0 @@ -import SwiftUI -import UIKit -import UniformTypeIdentifiers - -/// The Terminal tab: the guest's console. -/// -/// The screen *is* the terminal. Alpine boots `/sbin/init` as pid 1, which starts -/// root's login shell on the console, and the console is this view: what you type goes -/// into the guest's tty, and what the tty produces is drawn here. The shell prints -/// its own prompt and echoes its own input, so there is nothing host-side to draw -/// or interpret. -struct TerminalView: View { - @EnvironmentObject private var session: TerminalSession - - @State private var fontSize: Double = 12 - @State private var showGuestFiles = false - @State private var showEngineLog = false - @State private var importingXIP = false - /// Which keys the extra-keys bar shows. Shared with the configuration sheet. - @StateObject private var keyConfiguration = TerminalKeyConfiguration() - @State private var showKeyConfiguration = false - - var body: some View { - VStack(spacing: 0) { - canvas - if !session.pending.isEmpty || session.activeLabel != nil { - activityStrip - } - TerminalKeyBar(session: session, - configuration: keyConfiguration, - onFiles: { showGuestFiles = true }, - onComponents: { component in install(component) }, - onHideKeyboard: { UIApplication.shared.sendAction(#selector(UIResponder.resignFirstResponder), to: nil, from: nil, for: nil) }, - onConfigure: { showKeyConfiguration = true }) - } - .background(Color.black) - .navigationTitle("Alpine Linux") - .navigationBarTitleDisplayMode(.inline) - .toolbar { toolbarContent } - .task { - await session.boot() - } - .fileImporter( - isPresented: $importingXIP, - allowedContentTypes: [UTType(filenameExtension: "xip") ?? .data], - allowsMultipleSelection: false - ) { result in - guard case .success(let urls) = result, let url = urls.first else { return } - stageXIP(url) - } - .sheet(isPresented: $showEngineLog) { - NavigationStack { EngineLogView() } - } - .sheet(isPresented: $showGuestFiles) { - GuestFileBrowserView() - } - .sheet(isPresented: $showKeyConfiguration) { - TerminalKeyConfigurationView() - .environmentObject(keyConfiguration) - } - } - - // MARK: - Screen - - private var canvas: some View { - InteractiveTerminalSurface(session: session, fontSize: fontSize) - .frame(maxWidth: .infinity, maxHeight: .infinity) - .background(Color.black) - } - - private static let bottomID = "org.xforge.terminal.bottom" - - // MARK: - Status strip - - /// Shown while something is outstanding. There is no "running" indicator for - /// an ordinary command any more: the shell is always there, so the absence of - /// a prompt is the indication that a program is still working. - private var activityStrip: some View { - HStack(spacing: 8) { - if !session.pending.isEmpty { - ProgressView().controlSize(.mini).tint(.white) - Text("\(session.pending.count) command(s) queued") - .font(.caption2) - .foregroundStyle(.white.opacity(0.85)) - } else if let label = session.activeLabel { - Text("requested by \(label)") - .font(.caption2) - .foregroundStyle(.white.opacity(0.85)) - } - Spacer() - } - .padding(.horizontal, 12) - .padding(.vertical, 5) - .background(Color(white: 0.13)) - } - - // MARK: - Toolbar - - @ToolbarContentBuilder - private var toolbarContent: some ToolbarContent { - ToolbarItem(placement: .principal) { - // No working-directory line: the shell owns its cwd and nothing here - // can see it, so a hardcoded "/root" was wrong the moment anyone ran - // `cd`. (Asking the guest for it would mean a command per keystroke; - // if it is wanted, that is what it costs.) - Text("Alpine Linux").font(.footnote.bold()) - } - ToolbarItem(placement: .primaryAction) { - Menu { - Section("Components") { - ForEach(SystemComponents.Component.allCases) { component in - Button { - install(component) - } label: { - Label("Install the \(component.title)", systemImage: icon(for: component)) - } - } - Button { - importingXIP = true - } label: { - Label("Install the Darwin SDK from an Xcode.xip…", - systemImage: "doc.badge.plus") - } - } - - Section("Session") { - Button { UIPasteboard.general.string = session.buffer.plainText } label: { - Label("Copy transcript", systemImage: "doc.on.doc") - } - Button { session.clear() } label: { - Label("Clear", systemImage: "eraser") - } - Button { session.sendRaw("\u{1b}[A") } label: { - Label("Previous command", systemImage: "chevron.up") - } - } - - Section("Keys") { - Button { showKeyConfiguration = true } label: { - Label("Configure terminal keys…", systemImage: "slider.horizontal.3") - } - } - - Section("Font size") { - Button { fontSize = max(9, fontSize - 1) } label: { - Label("Smaller", systemImage: "textformat.size.smaller") - } - Button { fontSize = min(20, fontSize + 1) } label: { - Label("Larger", systemImage: "textformat.size.larger") - } - } - - Section { - Button { showEngineLog = true } label: { - Label("Engine log", systemImage: "doc.text.magnifyingglass") - } - } - } label: { - Label("Terminal options", systemImage: "ellipsis.circle") - } - } - } - - private func icon(for component: SystemComponents.Component) -> String { - switch component { - case .glibc: return "shippingbox" - case .xtool: return "hammer" - case .swift: return "swift" - case .darwinSDK: return "externaldrive.connected.to.line.below" - } - } - - // MARK: - Actions - - /// Queue a component install in the terminal so its output is visible here. - private func install(_ component: SystemComponents.Component) { - switch component { - case .glibc: - session.enqueue(SystemComponents.scriptCommand(.glibc), label: "Components") - case .xtool: - Task { await prepareInstallerScript(); session.enqueue(SystemComponents.xtoolInstallCommand, label: "Components") } - case .swift: - Task { await prepareInstallerScript(); session.enqueue(SystemComponents.swiftInstallCommand, label: "Components") } - case .darwinSDK: - importingXIP = true - } - } - - private func prepareInstallerScript() async { - do { - let vm = XForgeEnvironment.makeVM() - try await vm.boot() - try await SystemComponents.ensureInstallerScript(in: vm) - } catch { - session.enqueue("echo \(GuestShell.quote("install-toolchain.sh: \(error.localizedDescription)"))", - label: "Components") - } - } - - /// Copy the chosen `.xip` into the guest's own filesystem, then hand xtool - /// the install command. - private func stageXIP(_ url: URL) { - session.enqueue("echo \(GuestShell.quote("staging \(url.lastPathComponent) into the guest…"))", - label: "Components") - Task { - do { - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - try await vm.boot() - let guestPath = try await SystemComponents.stageXIP(url, in: vm) { line in - XForgeLog.note("components: \(line)") - } - session.enqueue(SystemComponents.darwinSDKInstallCommand(guestXIPPath: guestPath), - label: "Components") - } catch { - session.enqueue("echo \(GuestShell.quote("could not stage the xip: \(error.localizedDescription)"))", - label: "Components") - } - } - } -} - -/// A native iOS terminal surface. The view itself becomes first responder, so -/// software/hardware keyboard events go directly to Alpine instead of through a -/// separate command field. Touch controls scrolling and selects/copies terminal -/// text; tapping the terminal summons the iOS keyboard. -private struct InteractiveTerminalSurface: UIViewRepresentable { - @ObservedObject var session: TerminalSession - let fontSize: Double - - func makeUIView(context: Context) -> TerminalTextView { - let view = TerminalTextView() - view.session = session - view.font = .monospacedSystemFont(ofSize: fontSize, weight: .regular) - view.refresh(from: session) - DispatchQueue.main.async { view.becomeFirstResponder() } - return view - } - - func updateUIView(_ view: TerminalTextView, context: Context) { - view.session = session - view.font = .monospacedSystemFont(ofSize: fontSize, weight: .regular) - view.refresh(from: session) - } -} - -private final class TerminalTextView: UITextView { - weak var session: TerminalSession? - private var lastRevision = -1 - - override var canBecomeFirstResponder: Bool { true } - override init(frame: CGRect, textContainer: NSTextContainer?) { - super.init(frame: frame, textContainer: textContainer) - backgroundColor = .black - textColor = UIColor(white: 0.92, alpha: 1) - tintColor = .systemGreen - isEditable = true - isSelectable = true - keyboardType = .asciiCapable - autocorrectionType = .no - autocapitalizationType = .none - spellCheckingType = .no - smartQuotesType = .no - smartDashesType = .no - smartInsertDeleteType = .no - alwaysBounceVertical = true - keyboardDismissMode = .interactive - textContainerInset = UIEdgeInsets(top: 6, left: 8, bottom: 8, right: 8) - // `self.` is required: inside this initializer the parameter named - // `textContainer` shadows the property of the same name, and the - // parameter is optional — `textContainer.lineFragmentPadding` reads as - // an optional member access and does not compile. - self.textContainer.lineFragmentPadding = 0 - addGestureRecognizer(UITapGestureRecognizer(target: self, action: #selector(focusTerminal))) - } - - required init?(coder: NSCoder) { fatalError("init(coder:) has not been implemented") } - - @objc private func focusTerminal() { becomeFirstResponder() } - - /// Report the terminal's size in characters to the guest. - /// - /// Without this the guest believes its terminal is 0×0 and every program that - /// lays out by width breaks: `ls` prints one name per line, and anything - /// full-screen draws in a corner. The cell size comes from the same - /// monospaced font the text is drawn in. - override func layoutSubviews() { - super.layoutSubviews() - reportSize() - } - - private func reportSize() { - guard let session, let font, - bounds.width > 0, bounds.height > 0 else { return } - let advance = ("W" as NSString).size(withAttributes: [.font: font]).width - guard advance > 0 else { return } - let usableWidth = bounds.width - textContainerInset.left - textContainerInset.right - let usableHeight = bounds.height - textContainerInset.top - textContainerInset.bottom - let cols = Int(usableWidth / advance) - let rows = Int(usableHeight / font.lineHeight) - guard cols > 0, rows > 0 else { return } - session.consoleResized(cols: cols, rows: rows) - } - - func refresh(from session: TerminalSession) { - guard lastRevision != session.revision else { return } - lastRevision = session.revision - let wasNearBottom = contentOffset.y + bounds.height >= contentSize.height - 44 - let rendered = session.buffer.plainText - if text != rendered { text = rendered } - selectedRange = NSRange(location: (text as NSString).length, length: 0) - if wasNearBottom || !isTracking { - let end = NSRange(location: (text as NSString).length, length: 0) - scrollRangeToVisible(end) - } - } - - // UIKeyInput is intentionally implemented even though the UITextView is not - // editable: UIKit still presents the keyboard, while every keystroke is sent - // to the guest shell instead of mutating host-side text. - override var hasText: Bool { true } - override func insertText(_ text: String) { - session?.sendRaw(text) - } - override func deleteBackward() { - session?.sendRaw("\u{7f}") - } - - override var keyCommands: [UIKeyCommand]? { - [ - UIKeyCommand(input: UIKeyCommand.inputUpArrow, modifierFlags: [], action: #selector(upArrow)), - UIKeyCommand(input: UIKeyCommand.inputDownArrow, modifierFlags: [], action: #selector(downArrow)), - UIKeyCommand(input: UIKeyCommand.inputLeftArrow, modifierFlags: [], action: #selector(leftArrow)), - UIKeyCommand(input: UIKeyCommand.inputRightArrow, modifierFlags: [], action: #selector(rightArrow)), - UIKeyCommand(input: "c", modifierFlags: .control, action: #selector(controlC)), - UIKeyCommand(input: "\t", modifierFlags: [], action: #selector(tabKey)), - UIKeyCommand(input: UIKeyCommand.inputEscape, modifierFlags: [], action: #selector(escapeKey)) - ] - } - - @objc private func upArrow() { session?.sendRaw("\u{1b}[A") } - @objc private func downArrow() { session?.sendRaw("\u{1b}[B") } - @objc private func leftArrow() { session?.sendRaw("\u{1b}[D") } - @objc private func rightArrow() { session?.sendRaw("\u{1b}[C") } - @objc private func controlC() { session?.interrupt() } - @objc private func tabKey() { session?.sendRaw("\t") } - @objc private func escapeKey() { session?.sendRaw("\u{1b}") } -} - -/// the iSH terminal's extra-keys bar: the characters a phone keyboard cannot produce, -/// between the terminal and the keyboard. -private struct TerminalKeyBar: View { - @ObservedObject var session: TerminalSession - @ObservedObject var configuration: TerminalKeyConfiguration - var onFiles: () -> Void - var onComponents: (SystemComponents.Component) -> Void - var onHideKeyboard: () -> Void - var onConfigure: () -> Void - - /// Key metrics, defined once. The keys are deliberately small — they sit - /// under the terminal and exist to supply characters a phone keyboard cannot - /// produce, not to be a primary control surface. - private static let keyWidth: CGFloat = 15 - private static let keyHeight: CGFloat = 14 - private static let keySpacing: CGFloat = 3 - /// The glyph is inset from the key so a symbol never touches the border. - private static var glyphSize: CGFloat { 9 } - - var body: some View { - ScrollView(.horizontal, showsIndicators: false) { - HStack(spacing: Self.keySpacing) { - ForEach(configuration.visibleKeys) { key in - self.key(key) - } - } - .padding(.horizontal, 6) - .padding(.vertical, 4) - } - .background(Color(white: 0.16)) - .accessibilityLabel("Terminal keyboard") - } - - /// One key, rendered from its declaration rather than from a hard-coded row, - /// so the configuration and the bar cannot disagree about what exists. - @ViewBuilder - private func key(_ key: TerminalKey) -> some View { - switch key { - case .components: - componentsMenu - default: - Button { - activate(key) - } label: { - if let symbol = key.symbol { - Image(systemName: symbol) - .font(.system(size: Self.glyphSize)) - .frame(width: Self.keyWidth, height: Self.keyHeight) - } else { - Text(key.label) - .font(.system(size: Self.glyphSize + 1, design: .monospaced)) - .frame(width: Self.keyWidth, height: Self.keyHeight) - } - } - .buttonStyle(.bordered) - .tint(.white) - .accessibilityLabel(key.title) - .contextMenu { - // Turning a key off from the key itself is the fastest way to - // trim the bar, but the configuration sheet is where keys that - // are already hidden can be brought back. - if !key.isPinned { - Button("Hide “\(key.title)”", systemImage: "eye.slash") { - configuration.setVisible(false, for: key) - } - } - Button("Configure keys…", systemImage: "slider.horizontal.3") { - onConfigure() - } - } - } - } - - private func activate(_ key: TerminalKey) { - switch key { - case .interrupt: session.interrupt() - // Everything else is a raw byte sequence for the console. The guest owns - // the editing — its line discipline and the shell's own line editor — so - // the key bar only has to produce the characters a phone keyboard cannot, - // and the shell's history is reached with the arrow keys it understands. - case .escape: session.sendRaw("\u{1b}") - case .tab: session.sendRaw("\t") - case .previous: session.sendRaw("\u{1b}[A") - case .next: session.sendRaw("\u{1b}[B") - case .dash: session.sendRaw("-") - case .dot: session.sendRaw(".") - case .slash: session.sendRaw("/") - case .colon: session.sendRaw(":") - case .bang: session.sendRaw("!") - case .pipe: session.sendRaw("|") - case .files: onFiles() - case .paste: session.pasteFromClipboard() - case .hideKeyboard: onHideKeyboard() - case .components: break // rendered as a menu, never reaches here - } - } - - /// The gear/wrench key: the system components, installed by command. - private var componentsMenu: some View { - Menu { - ForEach(SystemComponents.Component.allCases) { component in - Button(component.title) { onComponents(component) } - } - Divider() - Button("Configure keys…", systemImage: "slider.horizontal.3") { - onConfigure() - } - } label: { - Image(systemName: "wrench.and.screwdriver") - .font(.system(size: Self.glyphSize)) - .frame(width: Self.keyWidth, height: Self.keyHeight) - } - .buttonStyle(.bordered) - .tint(.white) - .accessibilityLabel("Components") - } -} - -/// Chooses which keys the extra-keys bar shows. -/// -/// Hide Keyboard is listed but not switchable, with the reason stated: without it -/// there is no way back to the screen once the keyboard covers it. -private struct TerminalKeyConfigurationView: View { - @EnvironmentObject private var configuration: TerminalKeyConfiguration - @Environment(\.dismiss) private var dismiss - - var body: some View { - NavigationStack { - List { - Section { - ForEach(configuration.order) { key in - row(for: key) - } - .onMove(perform: configuration.move) - } header: { - Text("Command bar") - } footer: { - Text("Drag to reorganize buttons. Hidden buttons remain here so they can be restored.") - } - - Section { - Button("Restore default buttons and order", systemImage: "arrow.counterclockwise") { - configuration.resetToDefaults() - } - } - } - .navigationTitle("Terminal keys") - .navigationBarTitleDisplayMode(.inline) - .toolbar { - ToolbarItem(placement: .navigationBarLeading) { EditButton() } - ToolbarItem(placement: .confirmationAction) { - Button("Done") { dismiss() } - } - } - } - } - - @ViewBuilder - private func row(for key: TerminalKey) -> some View { - if key.isPinned { - HStack { - Text(key.title) - Spacer() - Text("Always shown") - .font(.footnote) - .foregroundStyle(.secondary) - } - } else { - Toggle(key.title, isOn: Binding( - get: { configuration.isVisible(key) }, - set: { configuration.setVisible($0, for: key) } - )) - } - } -} - -/// Tab wrapper, so the root view can hand the terminal its session. -struct TerminalTab: View { - var body: some View { - NavigationStack { - TerminalView() - } - } -} diff --git a/App/Views/Toolchain/ToolchainView.swift b/App/Views/Toolchain/ToolchainView.swift index 218ba2c..0aed3ec 100644 --- a/App/Views/Toolchain/ToolchainView.swift +++ b/App/Views/Toolchain/ToolchainView.swift @@ -1,291 +1,216 @@ import SwiftUI import UniformTypeIdentifiers -/// Manage the on-device build infrastructure. +/// Manage the build toolchain. /// -/// Every component lives in the embedded Alpine system, and every one of them is -/// installed *by running a command inside it*. That is what this screen sets up: -/// it puts the user's files where the guest can reach them and hands the right -/// command to the Terminal, where the install runs and reports what it is doing. +/// Two halves, and the screen is honest about which one is missing: /// -/// - **Alpine rootfs** — bundled in the app; "installing" it is a local import. -/// - **Swift / xtool** — the commands swift.org and xtool document, run in the -/// guest (see `SystemComponents`). -/// - **Darwin SDK** — either your own `Xcode.xip`, copied into the guest's -/// storage and installed there with `xtool sdk install`, or XForge's prebuilt -/// bundle, downloaded and installed inside the guest. +/// - **the compiler**, linked into this build of the app +/// (`Support/NativeToolchain.generated.xcconfig`, written by +/// `NativeToolchain/prepare-xcode.sh`). It cannot be installed at runtime — +/// either the libraries are in the binary or they are not — so this reports +/// what is there; +/// - **the Darwin SDK**, a bundle in the app's container, downloaded or imported +/// here. +/// +/// The smoke test compiles a real file through the linked compiler and links it, +/// which is the only way to know the toolchain works before a build depends on it. struct ToolchainView: View { - @EnvironmentObject private var terminal: TerminalSession - @StateObject private var toolchain = ToolchainManager() - @State private var importingXIP = false - @State private var importingRootfs = false + @State private var sdkPath: String? + @State private var sdkError: String? + @State private var importing = false + @State private var working = false + @State private var message: String? + @State private var smokeResult: String? @State private var confirmingDownload = false - @State private var preparing: String? + + private var capabilities: NativeToolchainCapabilities { .current } var body: some View { Form { Section { - ForEach(ToolchainManager.Component.allCases) { component in - row(component) - } + row("clang", present: capabilities.hasClang) + row("ld64.lld (Mach-O linker)", present: capabilities.hasLLDMachO) + row("swift-frontend", present: capabilities.hasSwiftFrontend) } header: { - Text("Components") + Text("Compiler") } footer: { - Text("Installs run in the Terminal tab, so you can watch them — and " - + "answer anything they ask — while they work.") + Text(capabilities.hasClang + ? "Linked into this build: \(capabilities.backendDescription)" + : "Not linked into this build. The native compiler has to be in the binary, " + + "so install the toolchain bundle and rebuild (NativeToolchain/install-bundle.sh).") } Section { - Button { - Task { await toolchain.refresh(probeGuest: true) } - } label: { - Label("Check the embedded Linux", systemImage: "arrow.clockwise") + if sdkPath != nil { + LabeledContent("SDK", value: "installed") + LabeledContent("Path", value: sdkPath ?? "") + } else { + Text("Not installed. A build cannot compile anything without it.") + .foregroundStyle(.secondary) + } + if let sdkError { + Text(sdkError).font(.footnote).foregroundStyle(.red) } - .disabled(toolchain.activity != nil || preparing != nil) + } header: { + Text("Darwin SDK") } footer: { - Text("Green = present. Guest components are verified inside the " - + "embedded Linux, which this boots if it is not running.") + Text("The SDK is xtool's darwin.artifactbundle: the iPhoneOS headers, the " + + "tbd stubs and the static Swift runtime, read straight out of the " + + "app's container. About 460 MB.") } Section { Button { - importingXIP = true + confirmingDownload = true } label: { - Label("Install the Darwin SDK from an Xcode.xip…", - systemImage: "doc.badge.plus") + Label(working ? "Working…" : "Download the Darwin SDK", systemImage: "arrow.down.circle") } + .disabled(working) + Button { - confirmingDownload = true + importing = true } label: { - Label("Install the prebuilt Darwin SDK…", systemImage: "arrow.down.circle") + Label("Install from a bundle in Files…", systemImage: "doc.badge.plus") + } + .disabled(working) + + if sdkPath != nil { + Button(role: .destructive) { + removeSDK() + } label: { + Label("Remove the SDK", systemImage: "trash") + } + .disabled(working) } } header: { - Text("Darwin SDK") - } footer: { - Text("With an Xcode.xip: the file is copied into the Alpine system's own " - + "storage and then installed there with " - + "`xtool sdk install \"path/to/xip\"`. Without one, XForge's " - + "prebuilt darwin.artifactbundle is fetched and installed inside " - + "the guest instead. Either way the SDK already in the guest is " - + "removed first — SwiftPM will not install a second bundle " - + "carrying the same artifact ID, and the bundled rootfs ships one.") + Text("Actions") } Section { Button { - importingRootfs = true + Task { await runSmokeTest() } } label: { - Label("Import an Alpine rootfs archive…", - systemImage: "shippingbox.and.arrow.backward") + Label("Compile and link a test file", systemImage: "checkmark.seal") } - .disabled(toolchain.isInstalling != nil || toolchain.isGuestBooted) - } header: { - Text("Offline Imports") - } footer: { - Text("Choose a rootfs archive to replace the bundled one. A plain Alpine " - + ".tar.gz is imported on the host, so it will not have XForge's " - + "root configuration. Quit and reopen XForge first if Linux is running.") - } + .disabled(working || !capabilities.canCompile || sdkPath == nil) - if let activity = toolchain.activity ?? preparing { - Section { - HStack(spacing: 8) { - ProgressView().controlSize(.small) - Text(activity).font(.footnote).foregroundStyle(.secondary) - } + if let smokeResult { + Text(smokeResult) + .font(.footnote) + .foregroundStyle(smokeResult.hasPrefix("✓") ? .green : .red) } + } header: { + Text("Verify") + } footer: { + Text("Compiles a C file with clang and links it with ld64.lld, in this process, " + + "against the installed SDK. This is what a real build does.") } - if let message = toolchain.message { + if let message { Section { Label(message, systemImage: "info.circle") - .font(.footnote).foregroundStyle(.secondary) - } - } - - Section(footer: Text("Reset removes the imported rootfs, the staged darwin SDK " - + "and any downloads. The app re-imports the bundled rootfs " - + "on the next boot.")) { - Button(role: .destructive) { - Task { await toolchain.reset() } - } label: { - Label("Reset Toolchain", systemImage: "trash") - } - .disabled(toolchain.isInstalling != nil) - } - - Section(footer: Text("If something dies without an explanation, share the log: " - + "the engine's own messages and XForge's install " - + "breadcrumbs are both in it.")) { - NavigationLink { - EngineLogView() - } label: { - Label("Engine log", systemImage: "doc.text.magnifyingglass") + .font(.footnote) + .foregroundStyle(.secondary) } } } .navigationTitle("Toolchain") - .task { await toolchain.refresh() } - .fileImporter( - isPresented: $importingXIP, - allowedContentTypes: [UTType(filenameExtension: "xip") ?? .data], - allowsMultipleSelection: false - ) { result in - guard case .success(let urls) = result, let url = urls.first else { return } - Task { await installFromXIP(url) } - } + .task { refresh() } .fileImporter( - isPresented: $importingRootfs, - allowedContentTypes: [.archive, .gzip, .data], + isPresented: $importing, + allowedContentTypes: [.folder], allowsMultipleSelection: false ) { result in guard case .success(let urls) = result, let url = urls.first else { return } - Task { await toolchain.importRootfs(from: url) } + install(from: url) } - .confirmationDialog("Install the prebuilt Darwin SDK?", - isPresented: $confirmingDownload) { - Button("Download and install in the Terminal") { - Task { await installPrebuiltSDK() } - } + .confirmationDialog( + "Download the Darwin SDK?", + isPresented: $confirmingDownload + ) { + Button("Download") { Task { await downloadSDK() } } } message: { - Text("About 457 MB is downloaded inside the guest and installed there. " - + "It runs in the Terminal tab.") + Text("About 460 MB, downloaded into the app's container.") } } // MARK: - Rows - @ViewBuilder - private func row(_ component: ToolchainManager.Component) -> some View { - VStack(alignment: .leading, spacing: 6) { - HStack(spacing: 10) { - Image(systemName: component.icon) - .foregroundStyle(.tint) - .frame(width: 22) - VStack(alignment: .leading, spacing: 2) { - Text(component.rawValue).font(.subheadline.weight(.semibold)) - Text(component.blurb).font(.caption).foregroundStyle(.secondary) - } - Spacer() - statusBadge(component) - } - - HStack(spacing: 12) { - ForEach(actions(for: component), id: \.title) { action in - Button(action.title) { action.run() } - .font(.footnote) - .disabled(toolchain.isInstalling != nil || preparing != nil) - } - } - .padding(.leading, 32) + private func row(_ title: String, present: Bool) -> some View { + HStack { + Image(systemName: present ? "checkmark.circle.fill" : "circle.dashed") + .foregroundStyle(present ? .green : .secondary) + Text(title) + Spacer() + Text(present ? "linked" : "missing") + .font(.caption) + .foregroundStyle(present ? .green : .secondary) } - .padding(.vertical, 2) } - private struct RowAction { - let title: String - let run: () -> Void - } - - private func actions(for component: ToolchainManager.Component) -> [RowAction] { - switch component { - case .rootfs: - // The bundled rootfs installs itself; a custom archive is the - // "Offline Imports" section below. - return [] - case .swift: - return [RowAction(title: "Install in Terminal") { - installInTerminal(.swift) - }] - case .xtool: - return [RowAction(title: "Install in Terminal") { - installInTerminal(.xtool) - }] - case .sdk: - return [ - RowAction(title: "From an Xcode.xip…") { importingXIP = true }, - RowAction(title: "Prebuilt download…") { confirmingDownload = true }, - ] - } - } + // MARK: - Actions - @ViewBuilder - private func statusBadge(_ component: ToolchainManager.Component) -> some View { - if toolchain.isInstalled(component) { - Label("Installed", systemImage: "checkmark") - .labelStyle(.iconOnly) - .font(.footnote.bold()) - .foregroundStyle(.green) - } else if component.livesInGuest && !toolchain.guestChecked { - Text("?").font(.footnote.bold()).foregroundStyle(.secondary) - } else { - Image(systemName: "circle.dashed") - .font(.footnote.bold()) - .foregroundStyle(.orange) + private func refresh() { + do { + let layout = try NativeSDK.layout() + sdkPath = layout.sdkRoot.path + sdkError = nil + } catch { + sdkPath = nil + sdkError = NativeSDK.isInstalled ? error.localizedDescription : nil } } - // MARK: - Actions - - /// Prepare the guest and hand the component's install command to the Terminal. - private func installInTerminal(_ component: SystemComponents.Component) { - Task { - preparing = "Preparing the guest for \(component.title)…" - defer { preparing = nil } - do { - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - try await vm.boot() - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(command(for: component), label: "Toolchain") - toolchain.message = "\(component.title): running in the Terminal tab." - } catch { - toolchain.message = error.localizedDescription - } + private func downloadSDK() async { + working = true + message = nil + defer { working = false } + do { + try await NativeSDK.installLatestPrebuilt() + refresh() + message = "Darwin SDK installed." + } catch { + sdkError = error.localizedDescription } } - private func command(for component: SystemComponents.Component) -> String { - switch component { - case .glibc: - return SystemComponents.scriptCommand(.glibc) - case .xtool: - return SystemComponents.xtoolInstallCommand - case .swift: - return SystemComponents.swiftInstallCommand - case .darwinSDK: - // Always driven by a file or a download, never by a bare tap. - return SystemComponents.scriptCommand(.xtool) + private func install(from url: URL) { + working = true + message = nil + defer { working = false } + do { + try NativeSDK.install(from: url) + refresh() + message = "Darwin SDK installed from \(url.lastPathComponent)." + } catch { + sdkError = error.localizedDescription } } - /// Copy the user's Xcode.xip into the guest's storage, then install it there. - private func installFromXIP(_ url: URL) async { - preparing = "Copying \(url.lastPathComponent) into the guest…" - defer { preparing = nil } + private func removeSDK() { do { - let command = try await toolchain.installSDKFromXcode(xip: url) - terminal.enqueue(command, label: "Toolchain") + try NativeSDK.remove() + refresh() + message = "Darwin SDK removed." } catch { - toolchain.message = error.localizedDescription + sdkError = error.localizedDescription } } - /// Download XForge's prebuilt Darwin SDK bundle inside the guest and install it. - private func installPrebuiltSDK() async { - preparing = "Resolving the latest Darwin SDK release…" - defer { preparing = nil } + private func runSmokeTest() async { + working = true + smokeResult = nil + defer { working = false } do { - let url = try await XForgeReleases.darwinSDKURL() - let vm = XForgeEnvironment.makeVM() - await vm.prepareRootfs() - try await vm.boot() - try await SystemComponents.ensureInstallerScript(in: vm) - terminal.enqueue(SystemComponents.darwinSDKDownloadCommand(from: url), - label: "Toolchain") - toolchain.message = "Downloading and installing the Darwin SDK in the Terminal tab." + let layout = try NativeSDK.layout() + let object = try await Task.detached(priority: .userInitiated) { + try NativeToolchain.smokeCompile(sdk: layout.sdkRoot) + }.value + let size = (try? object.resourceValues(forKeys: [.fileSizeKey]))?.fileSize ?? 0 + smokeResult = "✓ \(object.lastPathComponent) (\(size) bytes)" } catch { - toolchain.message = error.localizedDescription + smokeResult = error.localizedDescription } } } diff --git a/AppTests/ProjectTests.swift b/AppTests/ProjectTests.swift index e4b22bc..fdeaa5a 100644 --- a/AppTests/ProjectTests.swift +++ b/AppTests/ProjectTests.swift @@ -3,10 +3,10 @@ import XCTest final class ProjectTests: XCTestCase { func testProjectDefaults() { - let p = Project(name: "Demo", rootPath: "/root/projects/Demo") + let p = Project(name: "Demo", rootPath: "projects/Demo") XCTAssertEqual(p.organizationIdentifier, "com.example") - XCTAssertEqual(p.packageManifestPath, "/root/projects/Demo/Package.swift") - XCTAssertEqual(p.ipaOutputPath, "/root/projects/Demo/.build/xforge-Demo.ipa") + XCTAssertEqual(p.packageManifestPath, "projects/Demo/Package.swift") + XCTAssertEqual(p.ipaOutputPath, "projects/Demo/.build/xforge-Demo.ipa") } func testBuildConfigurationRawValues() { @@ -14,11 +14,6 @@ final class ProjectTests: XCTestCase { XCTAssertEqual(BuildConfiguration.release.rawValue, "release") } - func testGuestShellQuoteEscapesSingleQuotes() { - XCTAssertEqual(GuestShell.quote("one'two"), "'one'\\''two'") - XCTAssertEqual(GuestShell.environment(["TOKEN": "a b'c"]), "TOKEN='a b'\\''c' ") - } - func testProjectNameValidationAndSafePath() throws { let name = try Project.validatedName("Demo-App_2") XCTAssertEqual(name, "Demo-App_2") diff --git a/AppTests/ReleaseResolutionTests.swift b/AppTests/ReleaseResolutionTests.swift index 1984eab..878247e 100644 --- a/AppTests/ReleaseResolutionTests.swift +++ b/AppTests/ReleaseResolutionTests.swift @@ -6,6 +6,9 @@ import XCTest /// The darwin SDK is published under its own `darwin-sdk-*` series, so /// `releases/latest/download/` 404s whenever the newest release is an /// XForge IPA. These tests pin the behaviour that replaced it. +/// +/// The tests that lived beside these — the guest rootfs naming, the in-guest +/// toolchain manager, the embedded Linux executor — went with the guest. @MainActor final class ReleaseResolutionTests: XCTestCase { @@ -65,245 +68,10 @@ final class ReleaseResolutionTests: XCTestCase { in: fixtures, tagPrefix: "alpine-rootfs-", assetName: "alpine-rootfs.tar.xz") - XCTAssertNil(url, "the rootfs is bundled in the app, not published as an asset") + XCTAssertNil(url, "the guest rootfs is gone; nothing should resolve it") } func testUnwrapsTheCorrectXForgeRepo() { XCTAssertEqual(XForgeReleases.repository, "R0GUEEE/XForge") } - - func testOnlyTheBundledAlpineRootfsIsEligibleForBoot() { - // The bundled root is a ZIP of an already-converted fakefs, not a tarball - // the app imports: the conversion happens at build time - // (EmbeddedLinux/build-rootfs.sh), so first launch is an unzip. - XCTAssertEqual(RootfsInstaller.archiveName, "alpine-rootfs") - XCTAssertEqual(RootfsInstaller.archiveExtension, "zip") - XCTAssertEqual(RootfsInstaller.rootName, "alpine-3.21") - // Roots from the earlier layouts must not be reused. - XCTAssertTrue(RootfsInstaller.legacyRootNames.contains("alpine")) - XCTAssertTrue(RootfsInstaller.legacyRootNames.contains("alpine-3.24.2")) - XCTAssertFalse(RootfsInstaller.legacyRootNames.contains(RootfsInstaller.rootName)) - } - - func testCurrentRootRemovesTheLegacyAlpineInstall() throws { - let roots = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-roots-\(UUID().uuidString)", isDirectory: true) - defer { try? FileManager.default.removeItem(at: roots) } - - let current = RootfsInstaller.installedRoot(in: roots) - try FileManager.default.createDirectory( - at: current.appendingPathComponent("data", isDirectory: true), - withIntermediateDirectories: true - ) - try Data([1]).write(to: current.appendingPathComponent("meta.db")) - - let legacy = roots.appendingPathComponent("alpine", isDirectory: true) - try FileManager.default.createDirectory(at: legacy, withIntermediateDirectories: true) - - XCTAssertEqual(try RootfsInstaller.installIfNeeded(into: roots), current) - XCTAssertFalse(FileManager.default.fileExists(atPath: legacy.path)) - } -} - -/// A `LinuxVM` that answers from a script, so the toolchain probes can be tested -/// without booting the real guest. -@MainActor -final class StubLinuxVM: LinuxVM { - var isBooted = true - var bootCount = 0 - /// Commands that should report success (exit 0). - var succeeding: [String] = [] - private(set) var ranCommands: [String] = [] - - func boot() async throws { bootCount += 1; isBooted = true } - - func prepareRootfs() async {} - - func run( - _ command: String, - environment: [String: String]?, - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> Int32 { - ranCommands.append(command) - return succeeding.contains(where: { command.contains($0) }) ? 0 : 1 - } - - func copyOut(guestPath: String, to hostURL: URL) async throws {} - func copyIn(hostURL: URL, to guestPath: String) async throws {} - - // MARK: - Interactive shell - - /// A fake console session that records what is sent to it and can be made to - /// emit output, so the terminal's behaviour is testable without a guest. - func startInteractiveShell( - onOutput: @Sendable @escaping (String) -> Void - ) async throws -> any InteractiveShellSession { - let shell = StubShellSession(onOutput: onOutput) - startedShells.append(shell) - return shell - } - - private(set) var startedShells: [StubShellSession] = [] -} - -/// Records what the terminal writes to the console, and can emit output back. -@MainActor -final class StubShellSession: InteractiveShellSession { - /// init owns the console the fake session speaks for. - let pid: Int32 = 4242 - private(set) var isRunning = false - /// Everything written to the console, in order. - private(set) var received: [String] = [] - private(set) var interrupts = 0 - private(set) var stopped = false - private(set) var sizes: [(cols: Int, rows: Int)] = [] - - private let onOutput: @Sendable (String) -> Void - - init(onOutput: @escaping @Sendable (String) -> Void) { - self.onOutput = onOutput - self.isRunning = true - } - - @discardableResult - func send(_ text: String) -> Bool { - guard isRunning else { return false } - received.append(text) - return true - } - - /// Ctrl-C is the byte again: the guest's tty is what turns it into SIGINT, so - /// the session records it as a byte and counts the intent. - func interruptForeground() async { - interrupts += 1 - received.append(String(UnicodeScalar(0x03))) - } - - @discardableResult - func sendControl(_ scalar: UInt8) -> Bool { - guard isRunning else { return false } - received.append(String(UnicodeScalar(scalar))) - return true - } - - func resize(cols: Int, rows: Int) { - sizes.append((cols, rows)) - } - - func stop() { - isRunning = false - stopped = true - } - - /// Simulate the guest producing output. - func emit(_ text: String) { onOutput(text) } -} - -@MainActor -final class ToolchainManagerTests: XCTestCase { - - func testProbeFindsInstalledComponentsInTheGuest() async { - let vm = StubLinuxVM() - vm.succeeding = ["command -v swift", "command -v xtool", "swift sdk list"] - let manager = ToolchainManager(vm: vm) - - await manager.refresh(probeGuest: true) - - XCTAssertTrue(manager.guestChecked) - XCTAssertTrue(manager.isInstalled(.swift)) - XCTAssertTrue(manager.isInstalled(.xtool)) - XCTAssertTrue(manager.isInstalled(.sdk)) - XCTAssertEqual(vm.bootCount, 1, "probing should boot the guest exactly once") - } - - func testProbeReportsMissingComponents() async { - let vm = StubLinuxVM() - vm.succeeding = [] - let manager = ToolchainManager(vm: vm) - - await manager.refresh(probeGuest: true) - - XCTAssertTrue(manager.guestChecked) - XCTAssertFalse(manager.isInstalled(.swift)) - XCTAssertFalse(manager.isInstalled(.xtool)) - XCTAssertFalse(manager.isInstalled(.sdk)) - } - - func testRefreshWithoutProbeDoesNotBootOrInventGuestState() async { - let vm = StubLinuxVM() - vm.isBooted = false - let manager = ToolchainManager(vm: vm) - - await manager.refresh() - - XCTAssertEqual(vm.bootCount, 0, "a plain refresh must not boot the guest") - XCTAssertFalse(manager.guestChecked) - XCTAssertFalse(manager.isInstalled(.swift)) - } - - func testEveryComponentIsAccountedFor() { - // A component that is neither bundled nor guest-side would silently never - // be checked; make sure the enum and the split stay in step. - for component in ToolchainManager.Component.allCases { - XCTAssertFalse(component.rawValue.isEmpty) - XCTAssertFalse(component.blurb.isEmpty) - XCTAssertFalse(component.icon.isEmpty) - } - XCTAssertTrue(ToolchainManager.Component.rootfs.livesInGuest) - XCTAssertTrue(ToolchainManager.Component.swift.livesInGuest) - XCTAssertTrue(ToolchainManager.Component.sdk.livesInGuest) - } - - func testDownloadErrorsDescribeTheFailure() { - let http = DownloadError.http(status: 404, url: URL(string: "https://example.com/a.zip")!) - XCTAssertTrue(http.localizedDescription.contains("404")) - XCTAssertTrue(http.localizedDescription.contains("a.zip")) - - let missing = DownloadError.assetNotFound("darwin.artifactbundle.zip in any darwin-sdk-* release") - XCTAssertTrue(missing.localizedDescription.contains("darwin.artifactbundle.zip")) - } - - func testXcodeImportStagesTheXIPInTheGuestAndHandsBackTheInstallCommand() async throws { - let xip = FileManager.default.temporaryDirectory - .appendingPathComponent("xforge-sdk-test-\(UUID().uuidString).xip") - try Data().write(to: xip) - defer { try? FileManager.default.removeItem(at: xip) } - - let vm = StubLinuxVM() - vm.succeeding = ["mkdir -p", "wc -c"] - let manager = ToolchainManager(vm: vm) - - let command = try await manager.installSDKFromXcode(xip: xip) - - // The xip goes into the guest's own storage, and xtool is pointed at that - // path: current xtool installs an Xcode.xip directly, so there is no - // host-side unpack step and no obsolete artifactbundle hand-off. - XCTAssertTrue(vm.ranCommands.contains { $0.contains("mkdir -p '/root/xforge/xip'") }, - "the guest storage directory should be created: \(vm.ranCommands)") - XCTAssertTrue(command.contains("xtool sdk install"), - "the guest should install the xip with xtool: \(command)") - XCTAssertFalse(command.contains("xtool sdk build")) - XCTAssertTrue(command.contains("/root/xforge/xip/"), - "the command must point at the guest's own copy: \(command)") - XCTAssertTrue(command.contains("swift sdk list"), - "the install should verify itself: \(command)") - } - - func testBuildBootstrapDoesNotProvisionAMissingToolchain() async throws { - let vm = StubLinuxVM() - let executor = EmbeddedLinuxExecutor( - vm: vm, - stagingDir: FileManager.default.temporaryDirectory - ) - let stream = try await executor.bootstrap() - - do { - for try await _ in stream {} - } catch { - // The current implementation throws after it tries the installer; - // the replacement reports a missing toolchain as a failed event. - } - - XCTAssertFalse(vm.ranCommands.contains { $0.contains("install-toolchain.sh") }) - } } diff --git a/AppTests/TerminalBufferTests.swift b/AppTests/TerminalBufferTests.swift deleted file mode 100644 index f964f63..0000000 --- a/AppTests/TerminalBufferTests.swift +++ /dev/null @@ -1,139 +0,0 @@ -import XCTest -@testable import XForge - -/// The terminal's screen model is the part of the Terminal tab that cannot be -/// checked by looking at the app: the guest is a real Linux system, so its output -/// arrives as carriage returns, backspaces and SGR colour that have to be turned -/// into a screen. These tests feed it what the guest actually sends. -@MainActor -final class TerminalBufferTests: XCTestCase { - - func testPlainOutputIsSplitIntoScreenLines() { - let buffer = TerminalBuffer() - buffer.feed("hello\nworld") - - XCTAssertEqual(buffer.lines.map(\.text), ["hello"]) - XCTAssertEqual(buffer.current.text, "world") - } - - func testCarriageReturnOverwritesTheCurrentLine() { - let buffer = TerminalBuffer() - buffer.feed("100%\r50%") - - // What a terminal shows: the new text written over the old one, so the - // column the old text occupied but the new one does not keeps its - // character. - XCTAssertEqual(buffer.current.text, "50%%") - XCTAssertTrue(buffer.lines.isEmpty) - } - - func testCarriageReturnNewlineIsOneNewline() { - let buffer = TerminalBuffer() - buffer.feed("abc\r\n") - - XCTAssertEqual(buffer.lines.map(\.text), ["abc"]) - XCTAssertTrue(buffer.current.isEmpty) - } - - func testCarriageReturnAndNewlineSplitAcrossChunksIsStillOneNewline() { - // The other half of the same problem: CR and LF can also arrive in - // separate chunks, where CR is held until the next character arrives. - let buffer = TerminalBuffer() - buffer.feed("abc\r") - buffer.feed("\nnext") - - XCTAssertEqual(buffer.lines.map(\.text), ["abc"]) - XCTAssertEqual(buffer.current.text, "next") - } - - func testBackspaceErasesThePreviousCharacter() { - let buffer = TerminalBuffer() - buffer.feed("ab\u{08}c") - - XCTAssertEqual(buffer.current.text, "ac") - } - - func testTabAdvancesToTheNextTabStop() { - let buffer = TerminalBuffer() - buffer.feed("a\tb") - - XCTAssertEqual(buffer.current.text, "a b") - } - - func testColourIsKeptAsStyleAndNotAsText() { - let buffer = TerminalBuffer() - buffer.feed("\u{1b}[1;31mred\u{1b}[0m plain") - - XCTAssertEqual(buffer.current.text, "red plain") - XCTAssertEqual(buffer.current.cells.first?.style.foreground, .index(1)) - XCTAssertEqual(buffer.current.cells.first?.style.bold, true) - // The reset returns the rest of the line to the default style. - XCTAssertEqual(buffer.current.cells.last?.style, .plain) - } - - func testTrueColourAndExtendedPalette() { - let buffer = TerminalBuffer() - buffer.feed("\u{1b}[38;2;10;20;30mtruecolour") - XCTAssertEqual(buffer.current.cells.first?.style.foreground, .rgb(10, 20, 30)) - - buffer.feed("\u{1b}[0m\u{1b}[38;5;208mindexed") - XCTAssertEqual(buffer.current.cells.last?.style.foreground, .index(208)) - } - - func testClearScreenRemovesTheScrollback() { - let buffer = TerminalBuffer() - buffer.feed("one\ntwo\n") - buffer.feed("\u{1b}[2J") - - XCTAssertTrue(buffer.lines.isEmpty) - XCTAssertTrue(buffer.current.isEmpty) - XCTAssertTrue(buffer.isAtStart) - } - - func testEraseInLineLeavesTheScrollbackAlone() { - let buffer = TerminalBuffer() - buffer.feed("kept\npartly") - buffer.feed("\u{1b}[K") - - XCTAssertEqual(buffer.lines.map(\.text), ["kept"]) - // The cursor sits at the end of the text, so there is nothing after it to - // erase — the line is unchanged and the scrollback is untouched. - XCTAssertEqual(buffer.current.text, "partly") - } - - func testAnEscapeSequenceSplitAcrossChunksStillApplies() { - // Output arrives in whatever size the transport hands over; a sequence - // must survive being cut in half. - let buffer = TerminalBuffer() - buffer.feed("\u{1b}[3") - buffer.feed("1mgreen") - - XCTAssertEqual(buffer.current.text, "green") - XCTAssertEqual(buffer.current.cells.first?.style.foreground, .index(1)) - } - - func testUnsupportedSequencesAreSwallowedWithoutLosingText() { - let buffer = TerminalBuffer() - buffer.feed("\u{1b}[1;1Hmove\u{1b}]0;window title\u{07}after") - - XCTAssertEqual(buffer.current.text, "moveafter") - } - - func testScrollbackIsCapped() { - let buffer = TerminalBuffer() - buffer.maxLines = 10 - for index in 0..<50 { buffer.feed("line \(index)\n") } - - XCTAssertEqual(buffer.lines.count, 10) - XCTAssertEqual(buffer.lines.last?.text, "line 49") - } - - func testHostLinesAreStyledWithoutLeakingIntoGuestOutput() { - let buffer = TerminalBuffer() - buffer.appendLine("$ ls", style: TerminalStyle(foreground: .index(10))) - buffer.feed("guest output") - - XCTAssertEqual(buffer.lines.first?.text, "$ ls") - XCTAssertEqual(buffer.current.cells.first?.style, .plain) - } -} diff --git a/AppTests/TerminalKeyConfigurationTests.swift b/AppTests/TerminalKeyConfigurationTests.swift deleted file mode 100644 index 205c1d8..0000000 --- a/AppTests/TerminalKeyConfigurationTests.swift +++ /dev/null @@ -1,91 +0,0 @@ -import XCTest -@testable import XForge - -/// The extra-keys bar is configurable, with one key that deliberately is not. -@MainActor -final class TerminalKeyConfigurationTests: XCTestCase { - - private func makeConfiguration() -> TerminalKeyConfiguration { - // Start from a clean slate: the configuration persists in UserDefaults, - // so a test that ran before this one would otherwise leak into it. - UserDefaults.standard.removeObject(forKey: "org.xforge.terminal.hiddenKeys") - return TerminalKeyConfiguration() - } - - func testHideKeyboardIsAlwaysVisible() { - let configuration = makeConfiguration() - XCTAssertTrue(configuration.isVisible(.hideKeyboard)) - - // Even if something tries to switch it off: without it there is no way - // back to the screen once the keyboard covers it. - configuration.setVisible(false, for: .hideKeyboard) - XCTAssertTrue(configuration.isVisible(.hideKeyboard), - "the hide-keyboard key is pinned on purpose") - XCTAssertTrue(configuration.visibleKeys.contains(.hideKeyboard)) - } - - func testKeysStartVisible() { - let configuration = makeConfiguration() - for key in TerminalKey.allCases { - XCTAssertTrue(configuration.isVisible(key), "\(key.rawValue) should start shown") - } - } - - func testHidingAKeyRemovesItFromTheBar() { - let configuration = makeConfiguration() - configuration.setVisible(false, for: .pipe) - - XCTAssertFalse(configuration.isVisible(.pipe)) - XCTAssertFalse(configuration.visibleKeys.contains(.pipe)) - XCTAssertTrue(configuration.visibleKeys.contains(.slash), "others are unaffected") - } - - func testHidingPersistsBetweenInstances() { - let configuration = makeConfiguration() - configuration.setVisible(false, for: .colon) - - // A new instance reads what was stored — otherwise the choice would be - // lost every time the terminal is reopened. - let reloaded = TerminalKeyConfiguration() - XCTAssertFalse(reloaded.isVisible(.colon)) - } - - func testResetShowsEverything() { - let configuration = makeConfiguration() - configuration.setVisible(false, for: .bang) - configuration.setVisible(false, for: .previous) - configuration.resetToDefaults() - - XCTAssertEqual(configuration.visibleKeys.count, TerminalKey.allCases.count) - } - - func testBarOrderIsStable() { - // The keys keep their declared order regardless of what is hidden, so - // hiding one does not shuffle the rest. - let configuration = makeConfiguration() - let before = configuration.visibleKeys - configuration.setVisible(false, for: .tab) - let after = configuration.visibleKeys - - XCTAssertEqual(after, before.filter { $0 != .tab }) - } - - func testEveryKeyHasALabelOrASymbol() { - // A key that renders as neither would be an invisible button. - for key in TerminalKey.allCases { - let described = !key.label.isEmpty || key.symbol != nil - XCTAssertTrue(described, "\(key.rawValue) has nothing to draw") - } - } - - func testPunctuationKeysInsertTheirOwnCharacter() { - // The label and the inserted text are declared separately, so they can - // drift; this pins the ones where drift would be a real bug. - for key in [TerminalKey.dash, .dot, .slash, .colon, .bang, .pipe] { - XCTAssertEqual(key.label.count, 1, "\(key.rawValue) should be one character") - } - XCTAssertEqual(TerminalKey.pipe.label, "|") - XCTAssertEqual(TerminalKey.bang.label, "!") - XCTAssertEqual(TerminalKey.dash.label, "-") - } -} diff --git a/AppTests/TerminalSessionTests.swift b/AppTests/TerminalSessionTests.swift deleted file mode 100644 index 1ce3f2d..0000000 --- a/AppTests/TerminalSessionTests.swift +++ /dev/null @@ -1,165 +0,0 @@ -import XCTest -@testable import XForge - -/// The terminal is the guest's console: what is typed goes into the guest's tty, -/// the guest's own line discipline and shell do the editing and echoing, and what -/// the tty produces is the screen. -/// -/// These tests pin the parts that are easy to get subtly wrong — that a typed -/// line reaches the console verbatim (no host-side shell wrapping, no host-side -/// echoing), that an interrupt is the Ctrl-C *byte* for the tty to turn into a -/// signal, and that the guest is told how big the screen is. -@MainActor -final class TerminalSessionTests: XCTestCase { - - /// A session wired to a stub guest, with its transcript kept out of the way. - private func makeSession() async -> (TerminalSession, StubLinuxVM) { - let vm = StubLinuxVM() - let session = TerminalSession(makeVM: { vm }) - await session.boot() - return (session, vm) - } - - private func shell(of vm: StubLinuxVM) throws -> StubShellSession { - try XCTUnwrap(vm.startedShells.first, "the terminal should have attached to a console") - } - - func testBootingAttachesToTheConsoleOnce() async throws { - let (session, vm) = await makeSession() - await session.boot() - - XCTAssertEqual(vm.startedShells.count, 1, "booting twice should reuse the session") - XCTAssertTrue(session.isBooted) - XCTAssertTrue(try shell(of: vm).isRunning) - } - - func testTypedLineGoesToTheConsoleVerbatim() async throws { - let (session, vm) = await makeSession() - session.sendRaw("echo hello\n") - - // Exactly what was typed, newline included: the tty and the shell are what - // interpret it. - XCTAssertEqual(try shell(of: vm).received, ["echo hello\n"]) - } - - func testTheLineIsNotWrappedInAShellInvocation() async throws { - // The old design composed `sh -c ` per line. Nothing is wrapped - // now: the console receives exactly what was typed, which is what lets - // shell state (cd, variables) persist and lets a program read stdin. - let (session, vm) = await makeSession() - session.sendRaw("cd /tmp && pwd\n") - - let sent = try XCTUnwrap(try shell(of: vm).received.first) - XCTAssertEqual(sent, "cd /tmp && pwd\n") - XCTAssertFalse(sent.contains("/bin/sh"), "the host must not wrap the line in a shell call") - XCTAssertFalse(sent.contains("__XFORGE_PWD__"), - "the host no longer needs to ask the shell where it is") - } - - func testKeystrokesReachTheConsoleOneChunkAtATime() async throws { - // A terminal sends what was typed, not whole lines: the guest has to see - // each keystroke so its line discipline can echo it, and so `read` in a - // running program can consume it character by character. - let (session, vm) = await makeSession() - for text in ["l", "s", "\n"] { - session.sendRaw(text) - } - - XCTAssertEqual(try shell(of: vm).received, ["l", "s", "\n"]) - } - - func testMultipleLinesAccumulateInOneSession() async throws { - // The console is persistent: each line joins the same shell session, so the - // second command could depend on the first. - let (session, vm) = await makeSession() - for line in ["cd /etc", "cat hostname"] { - session.sendRaw(line + "\n") - } - - XCTAssertEqual(try shell(of: vm).received, ["cd /etc\n", "cat hostname\n"]) - } - - func testOutputFromTheConsoleReachesTheScreen() async throws { - let (session, vm) = await makeSession() - let shell = try shell(of: vm) - - // The guest writes from its own thread, and TerminalSession hops that onto - // the main actor before touching the screen — so the assertion has to let - // that hop run. That asynchrony is deliberate: the reader must never block - // on the UI. - shell.emit("total 8\ndrwxr-xr-x\n") - try await settle() - XCTAssertTrue(session.buffer.plainText.contains("total 8")) - - let before = session.revision - shell.emit("more\n") - try await settle() - XCTAssertGreaterThan(session.revision, before, "the view must be told to redraw") - } - - func testOutputThatDoesNotEndInANewlineIsShown() async throws { - // A prompt, or a program waiting for input, writes without a trailing - // newline. Dropping that would leave the screen looking dead while the - // guest waits for an answer. - let (session, vm) = await makeSession() - try shell(of: vm).emit("Password: ") - try await settle() - - XCTAssertTrue(session.buffer.plainText.contains("Password: ")) - } - - /// Let queued main-actor hops from the guest run. - private func settle() async { - for _ in 0..<5 { await Task.yield() } - try? await Task.sleep(for: .milliseconds(20)) - } - - func testInterruptIsTheCtrlCByte() async throws { - // Ctrl-C is a byte again, because the console is a real tty: the guest's - // line discipline is what turns 0x03 into SIGINT for the foreground - // process group. - let (session, vm) = await makeSession() - session.sendRaw("sleep 100\n") - let shell = try shell(of: vm) - let before = shell.received.count - - session.interrupt() - try await settle() - - XCTAssertEqual(shell.interrupts, 1) - XCTAssertEqual(shell.received.count, before + 1, "the interrupt is one byte") - XCTAssertEqual(shell.received.last, "\u{3}", "and it is Ctrl-C") - } - - func testCommandsFromOtherScreensJoinTheSameConsole() async throws { - let (session, vm) = await makeSession() - session.enqueue("apk add --no-cache git", label: "Toolchain") - try await settle() - - XCTAssertEqual(try shell(of: vm).received, ["apk add --no-cache git\n"]) - XCTAssertEqual(try shell(of: vm).pid, 4242, "it runs on the console the terminal is on") - } - - func testTheGuestIsToldHowBigTheScreenIs() async throws { - // A guest that believes its terminal is 0×0 wraps everything to one column, - // so the size has to reach it — and only when it actually changes, since - // each report raises SIGWINCH in the guest. - let (session, vm) = await makeSession() - session.consoleResized(cols: 80, rows: 24) - session.consoleResized(cols: 80, rows: 24) - session.consoleResized(cols: 120, rows: 40) - - XCTAssertEqual(try shell(of: vm).sizes.map { "\($0.cols)x\($0.rows)" }, - ["80x24", "120x40"]) - } - - func testSendingWithNoConsoleReportsRatherThanDroppingTheLine() async throws { - let (session, _) = await makeSession() - session.shutdown() - session.enqueue("echo lost") - try await settle() - - XCTAssertTrue(session.buffer.plainText.contains("the shell is not running"), - "a queued line that cannot run must be visible") - } -} diff --git a/Support/XForge-Bridging-Header.h b/Support/XForge-Bridging-Header.h index 7fdd522..e5ba439 100644 --- a/Support/XForge-Bridging-Header.h +++ b/Support/XForge-Bridging-Header.h @@ -2,9 +2,8 @@ // XForge-Bridging-Header.h // XForge // -// Exposes the C shims to Swift: the embedded Linux engine bridge (ish-arm64) -// and the host DNS reader (which keeps the guest's /etc/resolv.conf correct). +// Exposes the C shim to Swift: the native LLVM/Clang/LLD compiler bridge. +// The embedded Linux engine (ISHBridge.h) and the guest DNS reader +// (HostDNS.h) are gone with the guest. // -#import "ISHBridge.h" -#import "HostDNS.h" #import "NativeToolchainBridge.h" diff --git a/project.yml b/project.yml index 1dd6750..31f0127 100644 --- a/project.yml +++ b/project.yml @@ -28,9 +28,10 @@ packages: # `Support/Info.plist` is generated by XcodeGen from the `info.properties` of # the XForge target — do not hand-edit it (it is gitignored). # -# ish-arm64 (Vendor/ish-arm64, a git submodule) provides the embedded Linux engine. -# Its core static libraries are built by EmbeddedLinux/build-ish-core.sh into -# Vendor/ish-arm64-build/lib, where the linker finds them. +# The on-device toolchain is linked in from Vendor/NativeToolchain (written by +# NativeToolchain/prepare-xcode.sh); there is no embedded Linux and no engine to +# build. A clone without the toolchain bundle still builds: the bridge compiles +# to a "not available" stub and the app reports that on the Toolchain screen. # --------------------------------------------------------------------------- settings: base: @@ -39,9 +40,6 @@ settings: XFORGE_DEVELOPMENT_TEAM: V5GXMU3HC2 XFORGE_MARKETING_VERSION: "0.5.0" XFORGE_BUILD_NUMBER: "8" - ISH_ROOT: $(SRCROOT)/Vendor/ish-arm64 - ISH_BUILD_DIR: $(SRCROOT)/Vendor/ish-arm64-build - ISH_LIB_DIR: $(SRCROOT)/Vendor/ish-arm64-build/lib targets: XForge: @@ -56,15 +54,6 @@ targets: - path: Support/Resources buildPhase: resources optional: true - # In-guest provisioning scripts. The terminal can run the toolchain - # installer and the Build tab can install zsign into the guest before - # configuring/signing an IPA. - - path: EmbeddedLinux/install-toolchain.sh - buildPhase: resources - - path: EmbeddedLinux/install-zsign.sh - buildPhase: resources - - path: EmbeddedLinux/patch-zsign-password-file.py - buildPhase: resources dependencies: - package: xtool product: XKit @@ -72,42 +61,6 @@ targets: product: ZIPFoundation - sdk: WebKit.framework - sdk: SystemConfiguration.framework - preBuildScripts: - # The engine is a prerequisite of *compiling* this target, not just of - # linking it: ISHBridge.c includes the engine's headers, and a device build - # links libish.a / libish_emu.a / libfakefs.a out of $(ISH_LIB_DIR). So this - # runs before Sources rather than after them. - # - # It builds the engine when it is missing or out of date and does nothing - # otherwise (the engine build records a stamp; see build-ish-core.sh), and it - # skips itself entirely for simulator builds, which compile the stub branch - # of the bridge instead. Without it, a plain `xcodebuild` — or any pipeline - # that does not know to run EmbeddedLinux/build-ish-core.sh first — fails at - # ISHBridge.c with "'kernel/init.h' file not found". - - name: Build the embedded Linux engine - # `bash ` rather than the bare path: the phase must not depend on the - # file's executable bit surviving a checkout. - script: 'bash "$SRCROOT/EmbeddedLinux/build-engine-for-xcode.sh"' - basedOnDependencyAnalysis: false - # The other half of a working app is the guest rootfs, and it cannot be - # fetched here: it is a resource, so XcodeGen had to see it when it walked the - # directory (that is why EmbeddedLinux/fetch-rootfs.sh exists and why CI runs - # it before generating the project). A device build without it produces an app - # that cannot boot Linux, so this says so instead of shipping it quietly. - - name: Check the bundled guest rootfs - script: | - if [ "${PLATFORM_NAME:-}" != "iphoneos" ] || [ "${XFORGE_SKIP_ROOTFS:-0}" = "1" ]; then - exit 0 - fi - if [ -f "${SRCROOT}/Support/Resources/alpine-rootfs.zip" ]; then - exit 0 - fi - echo "error: no guest rootfs is staged at Support/Resources/alpine-rootfs.zip, so this app would boot no Linux." >&2 - echo " Stage the pinned one (about 165 MB) and build again:" >&2 - echo " bash EmbeddedLinux/fetch-rootfs.sh" >&2 - echo " Set XFORGE_SKIP_ROOTFS=1 to build without it anyway." >&2 - exit 1 - basedOnDependencyAnalysis: false settings: base: PRODUCT_BUNDLE_IDENTIFIER: $(XFORGE_BUNDLE_ID) @@ -136,49 +89,34 @@ targets: # generates that file from `properties`, and with none declared it wrote # an empty plist over it, which silently dropped every entitlement. CODE_SIGN_ENTITLEMENTS: $(SRCROOT)/Support/XForge.entitlements - # The engine's build phase above runs git, Homebrew and meson/ninja, and - # writes into Vendor/ish-arm64-build — all outside what a sandboxed script # phase may touch. Xcode 15+ defaults this to YES for new projects, which # would turn the phase into "Operation not permitted". ENABLE_USER_SCRIPT_SANDBOXING: NO ASSETCATALOG_COMPILER_APPICON_NAME: AppIcon ASSETCATALOG_COMPILER_GLOBAL_ACCENT_COLOR_NAME: AccentColor - # --- embedded ish-arm64 engine (device builds only) ----------------- - # The engine is built for the iOS device (arm64). Simulator builds - # compile the bridge's stub instead and link nothing extra. + # --- native compiler bridge ------------------------------------------- + # Hidden for the *guest* headers that used to be here; what remains is the + # bridge header and whatever toolchain bundle the xcconfig points at. SWIFT_OBJC_BRIDGING_HEADER: Support/XForge-Bridging-Header.h HEADER_SEARCH_PATHS: - $(inherited) - - $(SRCROOT)/App/EmbeddedVM - $(SRCROOT)/App/Services - $(SRCROOT)/App/NativeToolchain - $(XFORGE_NATIVE_HEADER_SEARCH_PATHS) "HEADER_SEARCH_PATHS[sdk=iphoneos*]": - $(inherited) - - $(SRCROOT)/App/EmbeddedVM - $(SRCROOT)/App/Services - $(SRCROOT)/App/NativeToolchain - - $(ISH_ROOT) - $(XFORGE_NATIVE_HEADER_SEARCH_PATHS) "LIBRARY_SEARCH_PATHS[sdk=iphoneos*]": - $(inherited) - - $(ISH_LIB_DIR) - $(XFORGE_NATIVE_LIBRARY_SEARCH_PATHS) - # Must match the guest-arch configuration the core was built with - # (-Dguest_arch=arm64); struct layouts differ per enabled arch. "OTHER_CFLAGS[sdk=iphoneos*]": - $(inherited) - - -DGUEST_ARM64=1 - $(XFORGE_NATIVE_CFLAGS) "OTHER_LDFLAGS[sdk=iphoneos*]": - $(inherited) - $(XFORGE_NATIVE_LDFLAGS) - - -lish - - -lish_emu - - -lfakefs - - -lz - - -liconv - - -lsqlite3 info: path: Support/Info.plist properties: @@ -193,15 +131,6 @@ targets: - arm64 UIApplicationSupportsIndirectInputEvents: true ITSAppUsesNonExemptEncryption: false - # The guest makes its own DNS and socket calls in this process, and a - # home router's resolver is a *local* address. iOS refuses connections - # to those unless the app declares why -- without this key the user - # never even gets the prompt, and the failure shows up inside the guest - # as "DNS: transient error" with a perfectly correct resolv.conf. - NSLocalNetworkUsageDescription: >- - XForge runs a Linux guest. Programs in it (apk, wget, git) make their - own network connections, including to servers on your local network such - as your router's DNS resolver. # The app builds, stores and exports .ipa artifacts in its Documents # directory; make that visible in the Files app. UIFileSharingEnabled: true From 077492a07b46887fb5ddb174215f48d78e8799a4 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 05:08:12 -0400 Subject: [PATCH 03/54] native: drop the engine, the rootfs and the guest workflows MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The code removal in the previous commit left the CI still provisioning the guest it no longer uses: a 1.6 GB rootfs download, a meson build of the ish-arm64 engine, and a validation step that checks the guest's zsign installer — which failed as soon as the installer was deleted, which is how this was found. `Build XForge IPA` is now fourteen steps instead of twenty-two, and builds the app rather than the guest: - no rootfs download or verification, no engine checkout/cache/build, no guest tooling validation; - the build tools step installs `xcodegen` and nothing else (meson, ninja, llvm, lld and shellcheck were all for the engine); - the "built app" and "final IPA" checks keep their real content — arm64 binary, identity, entitlements wiring, `Assets.car`, one `.app` in `Payload/`, no `__MACOSX`, the executable is byte-identical after packaging — and lose the rootfs assertions that only described the guest; - the "app must be self-contained" check now greps for the executable rather than `alpine-rootfs.zip`. Also removed: - `EmbeddedLinux/**` — the engine build, the rootfs builder/verifier, the guest provisioning and login scripts, the zsign installer and its patch. - `Tools/engine-smoke/**` — the harness that compiled programs inside the guest. - `Vendor/ish-arm64` (and `.gitmodules`): a submodule whose sources nothing reads any more. `make bootstrap` no longer inits it; `make toolchain` installs the compiler bundle the app actually links. - `.github/workflows/build-rootfs.yml` (publishes the rootfs) and `.github/workflows/ios-build.yml`: the latter staged `EmbeddedLinux/fetch-rootfs.sh` and would fail now, and `build-ipa.yml` replaced it in September. `ios-share.yml` stays — it is the simulator-sharing capability for another app and never referenced the guest. Two Swift 6 fixes found while re-reading the new code, both of which the release build would not have reported: the signer needs `import Security` for `SecPKCS12Import`, and a static `ISO8601DateFormatter` in `XForgeLog` is a non-`Sendable` global that strict concurrency rejects — the formatter is built per line now. The key material is also labelled PKCS#1 in its PEM envelope, because that is what `SecKeyCopyExternalRepresentation` returns. --- .github/workflows/build-ipa.yml | 316 +---- .github/workflows/build-rootfs.yml | 248 ---- .github/workflows/ios-build.yml | 1357 -------------------- .gitmodules | 5 - App/Services/AppBundleSigner.swift | 8 +- App/Services/XForgeLog.swift | 20 +- EmbeddedLinux/build-engine-for-xcode.sh | 110 -- EmbeddedLinux/build-ish-core.sh | 225 ---- EmbeddedLinux/build-rootfs.sh | 973 -------------- EmbeddedLinux/fetch-rootfs.sh | 117 -- EmbeddedLinux/install-toolchain.sh | 920 ------------- EmbeddedLinux/install-zsign.sh | 33 - EmbeddedLinux/patch-zsign-password-file.py | 40 - EmbeddedLinux/verify-rootfs.sh | 522 -------- EmbeddedLinux/xforge-login | 74 -- Makefile | 60 +- Tools/engine-smoke/engine-smoke.c | 444 ------- Tools/engine-smoke/futex-pi-probe.c | 174 --- Tools/engine-smoke/swift-mutex-probe.swift | 37 - Vendor/ish-arm64 | 1 - 20 files changed, 66 insertions(+), 5618 deletions(-) delete mode 100644 .github/workflows/build-rootfs.yml delete mode 100644 .github/workflows/ios-build.yml delete mode 100644 .gitmodules delete mode 100755 EmbeddedLinux/build-engine-for-xcode.sh delete mode 100644 EmbeddedLinux/build-ish-core.sh delete mode 100644 EmbeddedLinux/build-rootfs.sh delete mode 100755 EmbeddedLinux/fetch-rootfs.sh delete mode 100644 EmbeddedLinux/install-toolchain.sh delete mode 100644 EmbeddedLinux/install-zsign.sh delete mode 100644 EmbeddedLinux/patch-zsign-password-file.py delete mode 100755 EmbeddedLinux/verify-rootfs.sh delete mode 100755 EmbeddedLinux/xforge-login delete mode 100644 Tools/engine-smoke/engine-smoke.c delete mode 100644 Tools/engine-smoke/futex-pi-probe.c delete mode 100644 Tools/engine-smoke/swift-mutex-probe.swift delete mode 160000 Vendor/ish-arm64 diff --git a/.github/workflows/build-ipa.yml b/.github/workflows/build-ipa.yml index ec04950..26cbed2 100644 --- a/.github/workflows/build-ipa.yml +++ b/.github/workflows/build-ipa.yml @@ -2,19 +2,11 @@ name: Build XForge IPA # Builds the unsigned XForge IPA. # -# There is no rootfs-provisioning job here. The Alpine root — including the build -# toolchain, when the published root carries one — is built once by the `rootfs` -# workflow, published as a pinned release asset and downloaded here; see -# EmbeddedLinux/build-rootfs.sh. Provisioning a Swift toolchain into every CI run -# is what used to make this job slow and its artifact enormous. What arrives now -# is a fakefs ZIP the app only unzips: a guest that can build an iOS app on first -# launch, with nothing to install on the device. -# -# The engine (OpenMinis/ish-arm64) is built here, on macOS, because it can only -# be: the aarch64 gadget sources alias registers with `.req` and only clang's -# integrated assembler accepts that, so a GNU binutils toolchain cannot assemble -# them at all. `EmbeddedLinux/build-ish-core.sh` probes for the capability and -# fails with an explanation rather than a wall of assembler errors. +# There is no guest to provision and no engine to build: the compiler is linked +# into the app from the toolchain bundle (Vendor/NativeToolchain), which +# `native-toolchain.yml` builds in its own workflow. A checkout without the bundle +# still builds — the bridge compiles to a "not available" stub — so the archive +# here is the app itself, and the Toolchain screen reports what is missing. on: push: @@ -48,249 +40,33 @@ jobs: env: APP_PATH: build/XForge.xcarchive/Products/Applications/XForge.app DERIVED_DATA_PATH: build/DerivedData - ENGINE_ROOT: Vendor/ish-arm64 - ENGINE_LIB_DIR: Vendor/ish-arm64-build/lib # Must match XFORGE_MARKETING_VERSION in project.yml; asserted below so a # version bump cannot silently produce a mislabelled release. EXPECTED_VERSION: "0.5.0" - # The pinned rootfs. Changing this tag is how the bundled Alpine root is - # upgraded; `make rootfs` (or the `rootfs` workflow) builds and publishes - # a new one. The sha256 is asserted below, so a tag that is moved — or an - # upload that was truncated — fails the build instead of shipping silently. - # - # rootfs-v5 is the *provisioned* root: plain Alpine plus xtool, swiftly, the - # Swift toolchain and the Darwin SDK, so the guest can build without first - # installing anything. Its manifest says how it was built, and the checks - # below hold it to that. - ROOTFS_TAG: rootfs-v5 - ROOTFS_ASSET: alpine-rootfs.zip SPM_CLONES_DIR: .spm steps: - name: Check out the repository uses: actions/checkout@v4 - - name: Download the pinned Alpine rootfs - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - # The same script any other build calls (and that Xcode users run by - # hand): it reads the pinned tag from this file, downloads the asset, - # verifies the published sha256 and checks it is a fakefs ZIP. - bash EmbeddedLinux/fetch-rootfs.sh - - - name: Verify the downloaded rootfs - run: | - set -euo pipefail - cd Support/Resources - # Read the checksum file and verify, rather than piping a listing into - # grep: `producer | grep -q` closes the pipe early, the producer dies - # of SIGPIPE, and under `set -o pipefail` a check that passed then - # reports failure. - shasum -a 256 -c "$ROOTFS_ASSET.sha256" - - unzip -Z1 "$ROOTFS_ASSET" > /tmp/rootfs-entries.txt - grep -q '^alpine-rootfs/meta\.db$' /tmp/rootfs-entries.txt \ - || { echo '::error::the rootfs archive has no meta.db'; exit 1; } - grep -q '^alpine-rootfs/data/$' /tmp/rootfs-entries.txt \ - || { echo '::error::the rootfs archive has no data/ directory'; exit 1; } - grep -q '^alpine-rootfs/data/bin/sh$' /tmp/rootfs-entries.txt \ - || { echo '::error::the rootfs archive has no /bin/sh'; exit 1; } - # SQLite sidecars are transient; the builder excludes them, and a - # regression there would put a stale WAL into every install. - ! grep -qE '\.db-(wal|shm)$' /tmp/rootfs-entries.txt - - # The glibc compatibility layer must be in the root *and indexed in - # meta.db*: it is installed before the fakefs conversion precisely so - # the engine can resolve it. Checking the ZIP listing alone would pass - # for a root where the files exist on disk but the database has no rows - # for them, which the guest reads as "not there at all". - grep -q '^alpine-rootfs/data/opt/glibc/usr/lib/aarch64-linux-gnu/ld-linux-aarch64\.so\.1$' \ - /tmp/rootfs-entries.txt \ - || { echo '::error::the rootfs has no glibc layer'; exit 1; } - grep -q '^alpine-rootfs/data/usr/local/share/xforge/glibc\.env$' \ - /tmp/rootfs-entries.txt \ - || { echo '::error::the rootfs has no glibc.env marker'; exit 1; } - unzip -p "$ROOTFS_ASSET" alpine-rootfs/meta.db > /tmp/rootfs-meta.db - indexed="$(sqlite3 /tmp/rootfs-meta.db \ - "SELECT COUNT(*) FROM paths WHERE CAST(path AS TEXT) LIKE '%aarch64-linux-gnu%';")" - [ "${indexed:-0}" -gt 0 ] \ - || { echo "::error::the glibc layer is not indexed in meta.db ($indexed paths)"; exit 1; } - echo "glibc layers paths indexed: $indexed" - - # The guest boots /sbin/init, which reads /etc/inittab and respawns - # /sbin/xforge-login root on the console — the script that starts root's - # login shell in /etc/passwd. That is the whole Terminal tab, and the - # shell it names is part of the root's contract, so both are checked - # here: a root pinned to a tag that predates them fails the build - # instead of shipping a terminal that cannot start a shell. - unzip -p "$ROOTFS_ASSET" alpine-rootfs/data/etc/inittab > /tmp/rootfs-inittab.txt - grep -qE '^tty1::respawn:/sbin/xforge-login root$' /tmp/rootfs-inittab.txt \ - || { echo '::error::the rootfs does not respawn xforge-login on the console'; exit 1; } - unzip -p "$ROOTFS_ASSET" alpine-rootfs/data/usr/local/share/xforge/rootfs-manifest.txt \ - > /tmp/rootfs-manifest.txt - ROOTFS_SHELL="$(awk '/^[[:space:]]*shell:/ { print $2 }' /tmp/rootfs-manifest.txt)" - [ -n "$ROOTFS_SHELL" ] \ - || { echo '::error::the rootfs manifest does not name a login shell'; exit 1; } - unzip -Z1 "$ROOTFS_ASSET" > /tmp/rootfs-files.txt - for path in \ - alpine-rootfs/data/sbin/init \ - alpine-rootfs/data/bin/login \ - alpine-rootfs/data/etc/inittab \ - alpine-rootfs/data/etc/init.d/rcS \ - alpine-rootfs/data/sbin/xforge-login \ - "alpine-rootfs/data${ROOTFS_SHELL}" \ - alpine-rootfs/data/usr/share/terminfo/x/xterm-256color ; do - grep -qx "$path" /tmp/rootfs-files.txt \ - || { echo "::error::the rootfs has no ${path#alpine-rootfs/data}"; exit 1; } - done - echo "console: /sbin/xforge-login root → $ROOTFS_SHELL" - - # The root's own manifest also says whether the build toolchain is baked - # in, and this is the other half of what the root is for: a guest that - # can build an iOS app without installing anything first. The checks are - # held to the manifest's claim, so the *plain* root still passes and a - # provisioned root that lost its toolchain does not — and the SDK is - # checked through meta.db as well as the listing, because a file the - # engine cannot see is a file the guest does not have. - TOOLCHAIN_CLAIM="$(awk '/^[[:space:]]*toolchain:/ { sub(/^[[:space:]]*toolchain:[[:space:]]*/, ""); print; exit }' /tmp/rootfs-manifest.txt)" - echo "toolchain: ${TOOLCHAIN_CLAIM:-}" - case "$TOOLCHAIN_CLAIM" in - ""|"not provisioned"*) - echo "::error::the pinned rootfs ($ROOTFS_TAG) ships no build toolchain — - the guest would have to install Swift and xtool itself on the device. Bump - ROOTFS_TAG to a provisioned root, or say so deliberately here." - exit 1 - ;; - *) - for path in \ - alpine-rootfs/data/usr/local/bin/xtool \ - alpine-rootfs/data/usr/local/bin/swift \ - alpine-rootfs/data/usr/local/bin/swiftc \ - alpine-rootfs/data/usr/local/bin/swiftly \ - alpine-rootfs/data/opt/xtool/usr/bin/xtool ; do - grep -qx "$path" /tmp/rootfs-entries.txt \ - || { echo "::error::the rootfs claims a toolchain but has no ${path#alpine-rootfs/data}"; exit 1; } - done - grep -qE '^alpine-rootfs/data/root/\.local/share/swiftly/toolchains/[^/]+/usr/bin/swift-frontend$' \ - /tmp/rootfs-entries.txt \ - || { echo '::error::the rootfs has no Swift toolchain'; exit 1; } - # By shape, and *unanchored* at the end, for two reasons that each - # cost a build: Swift 6.4 ships linux and linux-musl variants, so - # naming one exact directory fails a good root; and a `.swiftmodule` - # is a *directory* in Swift 6, whose entry ends in a slash — a - # pattern anchored with `$` finds nothing. verify-rootfs.sh asks the - # same two questions the same way. - grep -qE '^alpine-rootfs/data/root/\.local/share/swiftly/toolchains/[^/]+/usr/lib/swift/.*/libswiftCore\.so' \ - /tmp/rootfs-entries.txt \ - || { echo '::error::the Swift toolchain has no Linux stdlib'; exit 1; } - grep -qE '^alpine-rootfs/data/root/\.local/share/swiftly/toolchains/[^/]+/usr/lib/swift/.*/Swift\.swiftmodule' \ - /tmp/rootfs-entries.txt \ - || { echo '::error::the Swift toolchain has no Linux stdlib interface'; exit 1; } - - SDK_PATH="$(awk '/^[[:space:]]*darwin-sdk-path:/ { print $2; exit }' /tmp/rootfs-manifest.txt)" - [ -n "$SDK_PATH" ] \ - || { echo '::error::the rootfs claims a toolchain but names no Darwin SDK path'; exit 1; } - grep -qE "^alpine-rootfs/data${SDK_PATH}/" /tmp/rootfs-entries.txt \ - || { echo "::error::the rootfs has no Darwin SDK at $SDK_PATH"; exit 1; } - sdk_indexed="$(sqlite3 /tmp/rootfs-meta.db \ - "SELECT COUNT(*) FROM paths WHERE CAST(path AS TEXT) LIKE '${SDK_PATH%/}/%';")" - [ "${sdk_indexed:-0}" -gt 0 ] \ - || { echo "::error::the Darwin SDK is in the zip but not indexed in meta.db ($sdk_indexed paths)"; exit 1; } - echo "toolchain: xtool, swift, and the Darwin SDK at $SDK_PATH ($sdk_indexed indexed paths)" - ;; - esac - - echo "rootfs: $(du -h "$ROOTFS_ASSET" | cut -f1), $(wc -l < /tmp/rootfs-entries.txt) entries" - - - name: Check out the embedded Linux engine - run: | - set -euo pipefail - git submodule update --init --depth 1 "$ENGINE_ROOT" - # The engine's own submodules are deliberately skipped: deps/libarchive - # and deps/libapps feed tools this app does not link, and deps/linux is - # only used with -Dkernel=linux, which XForge does not build. - git -C "$ENGINE_ROOT" log --oneline -1 - - name: Install the build tools run: | set -euo pipefail - # xcodegen generates the Xcode project from project.yml; meson/ninja - # build the engine. libarchive is not needed at all: the app unpacks the - # fakefs ZIP with ZIPFoundation instead of importing an archive. - # llvm/lld provide the aarch64-linux cross-compiler the guest VDSO - # needs (`clang -target aarch64-linux-gnu -fuse-ld=lld`) — Apple's - # clang cannot link that. shellcheck validates the pinned zsign - # installer, hence it is installed with the build tools. - required=(xcodegen meson ninja llvm lld shellcheck) - missing=() - for formula in "${required[@]}"; do - if ! brew list --versions "$formula" >/dev/null 2>&1; then - missing+=("$formula") - fi - done - if [ "${#missing[@]}" -gt 0 ]; then - brew install "${missing[@]}" + # xcodegen generates the Xcode project from project.yml. Nothing else is + # needed: the compiler comes from the toolchain bundle, and the app + # unpacks archives with ZIPFoundation. + if ! brew list --versions xcodegen >/dev/null 2>&1; then + brew install xcodegen else echo "Homebrew tools already installed" fi - - name: Identify engine cache inputs - id: engine-revision - run: echo "value=$(git -C "$ENGINE_ROOT" rev-parse HEAD)" >> "$GITHUB_OUTPUT" - - name: Identify the iOS compiler toolchain id: toolchain-versions run: | { - echo "value=$(xcodebuild -version | tr '\n' ' ')-$(clang --version | head -n 1)-$(meson --version)" + echo "value=$(xcodebuild -version | tr '\n' ' ')-$(clang --version | head -n 1)" } >> "$GITHUB_OUTPUT" - - name: Cache the embedded Linux engine - id: engine-cache - uses: actions/cache@v4 - with: - path: ${{ env.ENGINE_LIB_DIR }} - key: ${{ runner.os }}-${{ runner.arch }}-engine-${{ hashFiles('.gitmodules', 'EmbeddedLinux/build-ish-core.sh', 'project.yml') }}-${{ steps.engine-revision.outputs.value }}-${{ steps.toolchain-versions.outputs.value }} - - - name: Build the embedded Linux engine (ish-arm64, iOS arm64) - if: steps.engine-cache.outputs.cache-hit != 'true' - run: | - set -euo pipefail - bash EmbeddedLinux/build-ish-core.sh - # Prove the engine is really there before spending 20 minutes on an - # archive that would fail to link. - for lib in libish.a libish_emu.a libfakefs.a; do - test -f "$ENGINE_LIB_DIR/$lib" \ - || { echo "::error::the engine build did not produce $lib"; exit 1; } - done - # The arm64 backend must be the one that was built: app and engine - # share struct layouts, so the wrong guest arch links but misbehaves. - nm -gU "$ENGINE_LIB_DIR/libish_emu.a" > /tmp/engine-symbols.txt 2>/dev/null || true - if grep -qi 'guest_arm64\|gadget' /tmp/engine-symbols.txt; then - echo "arm64 gadget backend present in libish_emu.a" - else - echo "::error::libish_emu.a has no arm64 gadget symbols — wrong guest_arch?" - exit 1 - fi - - - name: Validate the bundled guest tools - run: | - set -euo pipefail - # zsign is built inside Alpine on first use. These two files are the - # guest-side installer and the tiny patch that adds -Q/--password-file - # to the pinned upstream CLI (so a P12 password never appears in argv). - bash -n EmbeddedLinux/install-zsign.sh - shellcheck -S error EmbeddedLinux/install-zsign.sh - python3 -m py_compile EmbeddedLinux/patch-zsign-password-file.py - rm -rf EmbeddedLinux/__pycache__ - grep -q '^ZSIGN_COMMIT=614caa8d1ca949e260e5746144aa52d27a4b08d6$' \ - EmbeddedLinux/install-zsign.sh - grep -q '^ZSIGN_SHA256=a373dc5ddbf81ba5c435a48c14b4a2857506c3dde02ec62e0136b6968797384b$' \ - EmbeddedLinux/install-zsign.sh - grep -q 'install-zsign.sh' project.yml - grep -q 'patch-zsign-password-file.py' project.yml - - name: Prepare the native toolchain configuration run: | set -euo pipefail @@ -353,52 +129,22 @@ jobs: *arm64*) ;; *) echo "::error::the app binary is not arm64: $binary_info"; exit 1 ;; esac - # NOTE: do not try to prove the engine is linked by scanning the app - # binary for its symbols. Xcode's Release configuration strips, and on - # a stripped binary `nm` reports "no symbols" and finds nothing — so a - # symbol check fails for a perfectly good build. The engine's presence - # is proven where it can be observed instead: the engine libraries are - # asserted after they are built, and the guest is exercised by the - # smoke harness. What is worth checking here is that the app is - # self-consistent — the rootfs it needs is inside it. + # NOTE: do not try to prove the compiler is linked by scanning the app + # binary for LLVM symbols. Xcode's Release configuration strips, and on a + # stripped binary `nm` reports "no symbols" and finds nothing — so a + # symbol check fails for a perfectly good build. What the toolchain can do + # is checked where it is observable: the Toolchain screen's smoke test, on + # a device. size_mb=$(( $(stat -f%z "$APP_PATH/XForge") / 1048576 )) echo "app binary: ${binary_info#*: } (${size_mb} MB)" test "$size_mb" -gt 1 \ || { echo '::error::the app binary is implausibly small'; exit 1; } - # The rootfs must be inside the app, and must be the fakefs ZIP the app - # knows how to unpack. - ROOTFS="$APP_PATH/alpine-rootfs.zip" - test -f "$ROOTFS" \ - || { echo '::error::the app does not bundle alpine-rootfs.zip'; exit 1; } - unzip -Z1 "$ROOTFS" > /tmp/bundled-rootfs.txt - grep -q '^alpine-rootfs/meta\.db$' /tmp/bundled-rootfs.txt \ - || { echo '::error::the bundled rootfs has no meta.db'; exit 1; } - grep -q '^alpine-rootfs/data/bin/sh$' /tmp/bundled-rootfs.txt \ - || { echo '::error::the bundled rootfs has no /bin/sh'; exit 1; } - - # No engine build leftovers in the bundle. + # No build leftovers in the bundle. strays="$(find "$APP_PATH" \( -name '*.a' -o -name 'meson-*' -o -name 'build.ninja' \) -print -quit)" [ -z "$strays" ] \ || { echo "::error::build leftovers in the app bundle: $strays"; exit 1; } - # Exactly one rootfs, and it is the one the app reads. This is a real - # regression: the app used to bundle a tarball it imported at runtime, - # that file stayed committed after the switch to a fakefs ZIP, and the - # IPA shipped both — 4 MB of dead weight nothing ever opened. - rootfs_files="$(find "$APP_PATH" -maxdepth 1 \ - \( -name '*.tar.gz' -o -name '*.tar.xz' -o -name 'alpine-rootfs*.zip' \) -print)" - count="$(printf '%s\n' "$rootfs_files" | grep -c . || true)" - [ "$count" = "1" ] || { - echo "::error::expected exactly one rootfs in the bundle, found $count:" - printf ' %s\n' $rootfs_files - exit 1 - } - case "$rootfs_files" in - */alpine-rootfs.zip) ;; - *) echo "::error::the bundled rootfs is not alpine-rootfs.zip: $rootfs_files"; exit 1 ;; - esac - # Identity and the icon wiring that makes the app installable. PLIST="$APP_PATH/Info.plist" test -f "$PLIST" @@ -433,9 +179,6 @@ jobs: echo '### Built app' echo echo "- binary: ${binary_info#*: } (${size_mb} MB)" - echo "- bundled rootfs: $ROOTFS_TAG ($(du -h "$ROOTFS" | cut -f1))" - echo "- guest toolchain: $(unzip -p "$ROOTFS" alpine-rootfs/data/usr/local/share/xforge/rootfs-manifest.txt \ - | awk '/^[[:space:]]*toolchain:/ { sub(/^[[:space:]]*toolchain:[[:space:]]*/, ""); print; exit }')" echo "- version: $(get CFBundleShortVersionString)" } >> "$GITHUB_STEP_SUMMARY" @@ -454,11 +197,9 @@ jobs: ditto -c -k --keepParent dist/Payload "$IPA" unzip -t "$IPA" >/dev/null - # The rootfs is the difference between an app that works on first - # launch and one that has nothing to boot. unzip -Z1 "$IPA" > /tmp/ipa-contents.txt - grep -q 'Payload/XForge.app/alpine-rootfs\.zip$' /tmp/ipa-contents.txt \ - || { echo '::error::the IPA does not contain the rootfs'; exit 1; } + grep -q '^Payload/XForge\.app/XForge$' /tmp/ipa-contents.txt \ + || { echo '::error::the IPA does not contain the app executable'; exit 1; } # Exactly one IPA, and it is the version the app says it is. count="$(find dist -maxdepth 1 -name '*.ipa' | wc -l | tr -d ' ')" @@ -467,12 +208,11 @@ jobs: test -f "$IPA" || { echo "::error::$IPA was not produced"; exit 1; } echo "IPA: $IPA ($(du -h "$IPA" | cut -f1))" - # A provisioned root makes this a ~1.6 GB IPA, and GitHub refuses a - # release asset over 2 GiB. Failing here — with the number — is better - # than a publish step that errors out after the whole build. + # GitHub refuses a release asset over 2 GiB. The app is tens of MB, so + # this only catches a runaway bundle — before the publish step does. ipa_bytes="$(stat -f%z "$IPA")" - if [ "$ipa_bytes" -gt 1900000000 ]; then - echo "::error::the IPA is $((ipa_bytes / 1048576)) MB; GitHub refuses a release asset over 2 GiB" + if [ "$ipa_bytes" -gt 2000000000 ]; then + echo "::error::the IPA is $((ipa_bytes / 1048576)) MB, over GitHub's 2 GiB release limit" exit 1 fi echo "ipa=$IPA" >> "$GITHUB_ENV" @@ -500,7 +240,6 @@ jobs: test -f "$FINAL_PLIST" || { echo '::error::IPA app has no Info.plist'; exit 1; } test -x "$FINAL_APP/XForge" || { echo '::error::IPA app has no executable XForge binary'; exit 1; } test -f "$FINAL_APP/Assets.car" || { echo '::error::IPA app has no compiled asset catalog'; exit 1; } - test -f "$FINAL_APP/alpine-rootfs.zip" || { echo '::error::IPA app has no bundled rootfs'; exit 1; } get_final() { /usr/libexec/PlistBuddy -c "Print :$1" "$FINAL_PLIST"; } [ "$(get_final CFBundleIdentifier)" = "com.r0gueee.xforge" ] || { echo "::error::wrong bundle identifier in final IPA"; exit 1; } @@ -521,11 +260,8 @@ jobs: exit 1 } - # Check the rootfs inside the final IPA, not only the archive source. - unzip -t "$FINAL_APP/alpine-rootfs.zip" >/dev/null - unzip -Z1 "$FINAL_APP/alpine-rootfs.zip" > /tmp/final-rootfs-entries.txt - grep -qx 'alpine-rootfs/meta.db' /tmp/final-rootfs-entries.txt || { echo '::error::final IPA rootfs has no meta.db'; exit 1; } - grep -qx 'alpine-rootfs/data/bin/sh' /tmp/final-rootfs-entries.txt || { echo '::error::final IPA rootfs has no /bin/sh'; exit 1; } + # The app must be self-contained; a missing executable or Info.plist is + # what makes an IPA uninstallable, and both were checked above. # Reject common malformed packaging layouts. [ ! -d "$VERIFY_DIR/Payload/Payload" ] || { echo '::error::IPA contains a nested Payload/Payload directory'; exit 1; } diff --git a/.github/workflows/build-rootfs.yml b/.github/workflows/build-rootfs.yml deleted file mode 100644 index ac4e7a1..0000000 --- a/.github/workflows/build-rootfs.yml +++ /dev/null @@ -1,248 +0,0 @@ -name: Publish the Alpine rootfs - -# Builds the Alpine root filesystem that XForge bundles, and publishes it as a -# pinned release asset for `build-ipa.yml` to download. -# -# Run this by hand when the root changes (a new Alpine release, a change to the -# root's configuration, a new Swift or xtool release). It is deliberately NOT part -# of the IPA build: rebuilding the root on every IPA run is what used to make that -# job slow, and the root changes far less often than the app. -# -# What goes in it is `provision`: -# -# all (default) plain Alpine *plus the build toolchain* — xtool, swiftly, the -# Swift toolchain and the Darwin Swift SDK — installed in a chroot of the -# root being built, so the guest boots ready to build an iOS app. ~1.6 GB. -# none plain Alpine only; the guest installs the toolchain itself on demand. -# -# The output is a ZIP of an *already-converted* fakefs — a `data/` tree plus a -# `meta.db` — produced by the engine's own tools/fakefsify. The app only unzips -# it, so first launch costs an unzip rather than a multi-minute import. -# -# Builds on Linux rather than macOS on purpose: the conversion tool must run on -# the build machine (it is not cross-compiled), and provisioning runs the guest's -# own tools (arm64 Linux binaries) — both need an aarch64 Linux host, which the -# arm64 runner is natively. - -on: - workflow_dispatch: - inputs: - tag: - description: Release tag to publish under (e.g. rootfs-v5) - type: string - required: true - alpine_version: - description: Alpine release series - type: string - default: "3.21" - provision: - description: Bake the build toolchain (xtool, Swift, Darwin SDK) into the root - type: choice - options: - - all - - none - default: all - provision_sdk: - description: Include the Darwin Swift SDK (~400 MB download, ~1.3 GB installed) - type: boolean - default: true - darwin_sdk_tag: - description: Release the Darwin Swift SDK comes from - type: string - default: darwin-sdk-7 - default_shell: - description: Login shell root's console starts (must be installed by console_packages) - type: string - default: "/bin/bash" - console_packages: - description: Space-separated packages baked in for the console session - type: string - default: "bash coreutils less ncurses-terminfo" - -permissions: - contents: write - -jobs: - rootfs: - name: Build and publish the rootfs - runs-on: ubuntu-24.04-arm - # Provisioning downloads the Swift toolchain, unpacks the Darwin SDK and - # installs both inside a chroot, then converts ~5 GB of filesystem to fakefs. - # The unprovisioned root takes a couple of minutes; this takes the rest. - timeout-minutes: 240 - steps: - - name: Check out the repository - uses: actions/checkout@v4 - - - name: Make sure there is room for a provisioned root - if: inputs.provision == 'all' - run: | - set -euo pipefail - # A provisioned root is ~5 GB on disk before it is packed, and the - # conversion used to want another 11 GB: the tree, the tar of it, and - # the fakefs root it converts to. The builder no longer holds the tree - # and the fakefs root at the same time (the tree is deleted once it is - # packed), which brings the peak to ~7 GB — but it is worth both saying - # so and taking back the ~2 GB of preinstalled SDKs a runner has and - # this job does not use. - df -h / - sudo apt-get clean || true - for dir in /usr/share/dotnet /opt/ghc /usr/local/share/boost /usr/lib/jvm \ - /usr/local/lib/android /opt/hostedtoolcache; do - sudo rm -rf "$dir" || true - done - df -h / - - - name: Check out the engine (for tools/fakefsify) - run: | - set -euo pipefail - git submodule update --init --depth 1 Vendor/ish-arm64 - # fakefsify links libarchive, so its submodule is required too. - git -C Vendor/ish-arm64 submodule update --init --depth 1 deps/libarchive - - - name: Install the build tools - run: | - set -euo pipefail - # meson/ninja build fakefsify; libarchive-dev is what it links - # against (the engine's meson looks for it via pkg-config); zip packs - # the result; shellcheck guards the builder scripts; sqlite3 reads the - # meta.db of the packed root, which is where the published root's paths - # and modes are actually checked. - sudo apt-get update -qq - sudo apt-get install -y -qq meson ninja-build libarchive-dev zip unzip shellcheck sqlite3 - - - name: Check the rootfs builder - run: | - set -euo pipefail - bash -n EmbeddedLinux/build-rootfs.sh - bash -n EmbeddedLinux/verify-rootfs.sh - shellcheck -S error EmbeddedLinux/build-rootfs.sh EmbeddedLinux/verify-rootfs.sh - # The console program is /bin/sh (busybox ash), and it is installed into - # the guest rather than run here — parse it and lint it, but as POSIX sh, - # which is the dialect it is written in. install-toolchain.sh is in the - # same dialect: it is the guest's own installer, and this build runs it - # in a chroot of the root being built. - sh -n EmbeddedLinux/xforge-login - shellcheck -s sh -S error EmbeddedLinux/xforge-login - sh -n EmbeddedLinux/install-toolchain.sh - shellcheck -s sh -S error EmbeddedLinux/install-toolchain.sh - # The toolchain half of the verification: a packed fakefs root cannot be - # chrooted and run, so "the toolchain is really in the artifact" is a - # structural check — and a structural check that looks for the wrong - # path, or that believes the manifest, passes everything. This drives - # verify-rootfs.sh against roots that are complete except for one thing - # and asserts it fails, naming that thing. - bash -n Tools/test-verify-toolchain.sh - shellcheck -S error Tools/test-verify-toolchain.sh - bash Tools/test-verify-toolchain.sh - - - name: Build the rootfs - env: - XFORGE_ALPINE_VERSION: ${{ inputs.alpine_version }} - XFORGE_PROVISION: ${{ inputs.provision }} - XFORGE_PROVISION_SDK: ${{ inputs.provision_sdk && '1' || '0' }} - XFORGE_DARWIN_SDK_TAG: ${{ inputs.darwin_sdk_tag }} - XFORGE_DEFAULT_SHELL: ${{ inputs.default_shell }} - XFORGE_CONSOLE_PACKAGES: ${{ inputs.console_packages }} - run: | - set -euo pipefail - # As root: the guest's installer and apk run in a chroot of the tree - # being built (installing the glibc layer unpacks Ubuntu packages with - # ar/zstd and writes to /lib, /usr/lib and /opt/glibc; provisioning runs - # install-toolchain.sh itself), and a chroot needs mounts. Keep the work - # directory and the output separate so the resulting files are owned by - # the runner, not root. - sudo -E env "PATH=$PATH" \ - XFORGE_ALPINE_VERSION="$XFORGE_ALPINE_VERSION" \ - XFORGE_PROVISION="$XFORGE_PROVISION" \ - XFORGE_PROVISION_SDK="$XFORGE_PROVISION_SDK" \ - XFORGE_DARWIN_SDK_TAG="$XFORGE_DARWIN_SDK_TAG" \ - XFORGE_DEFAULT_SHELL="$XFORGE_DEFAULT_SHELL" \ - XFORGE_CONSOLE_PACKAGES="$XFORGE_CONSOLE_PACKAGES" \ - bash EmbeddedLinux/build-rootfs.sh "$PWD/dist/rootfs" - sudo chown -R "$(id -u):$(id -g)" dist - ls -la dist/rootfs - - - name: Verify the root boots what XForge expects - run: | - set -euo pipefail - # XForge boots pid 1 as /sbin/init and drives the guest's single console, - # where /etc/inittab respawns /sbin/xforge-login root — the script that - # execs root's login shell from /etc/passwd. A root that cannot do that - # has no usable Terminal, and nothing else would notice: it fails here - # rather than on a device, and on the *published artifact*, not on the - # tree it was made from. - # - # For a provisioned root the same run also checks the toolchain the - # manifest claims: xtool, a Swift toolchain with a stdlib, and the Darwin - # SDK at the path it recorded — including that the engine can *see* the - # SDK, i.e. that it is indexed in meta.db. A fakefs root cannot be - # chrooted, so those checks are structural; the "it compiles" check ran - # on the tree during the build, where install-toolchain.sh verify could - # run a real compilation. - # - # The checker unpacks the ZIP and runs the guest's own console program in - # a chroot, so "root's shell starts" is answered by the root itself. - # Needs root for the chroot (the runner is aarch64, same as the root). - sudo -E env "PATH=$PATH" \ - bash EmbeddedLinux/verify-rootfs.sh "$PWD/dist/rootfs/alpine-rootfs.zip" - - - name: Report the artifact - run: | - set -euo pipefail - ZIP=dist/rootfs/alpine-rootfs.zip - SIZE="$(stat -c %s "$ZIP")" - # GitHub refuses a release asset over 2 GiB, and the IPA that carries - # this root is a few MB larger than the root itself. A provisioned root - # is ~1.6 GB, so the margin exists — but it is not large, and finding out - # at the upload is worse than finding out here. - if [ "$SIZE" -gt 1900000000 ]; then - echo "::error::$ZIP is $((SIZE / 1048576)) MB; an IPA bundling it would not fit in a 2 GiB release asset" - exit 1 - fi - # `data/` because the archive is a fakefs: its tree is under - # alpine-rootfs/data/, and the manifest is a file in that tree. - unzip -p "$ZIP" alpine-rootfs/data/usr/local/share/xforge/rootfs-manifest.txt \ - > /tmp/rootfs-manifest.txt - { - echo "### Rootfs: $((SIZE / 1048576)) MB" - echo - echo '```' - cat /tmp/rootfs-manifest.txt - echo '```' - } >> "$GITHUB_STEP_SUMMARY" - - - name: Publish the release - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euo pipefail - TAG="${{ inputs.tag }}" - # Create the release if it is new; uploading with --clobber makes a - # re-run of the same tag replace the assets rather than fail. - gh release view "$TAG" --repo "${{ github.repository }}" >/dev/null 2>&1 || \ - gh release create "$TAG" \ - --repo "${{ github.repository }}" \ - --title "XForge Alpine rootfs (${{ inputs.alpine_version }})" \ - --notes "Alpine aarch64 fakefs for the XForge embedded Linux engine, built by EmbeddedLinux/build-rootfs.sh. Tools: ${{ inputs.provision == 'all' && format('xtool, swiftly, Swift{0}', inputs.provision_sdk && format(' and the {0} Darwin SDK', inputs.darwin_sdk_tag) || '') || 'plain Alpine only (the guest provisions itself)' }}. Pinned by build-ipa.yml via ROOTFS_TAG." - - gh release upload "$TAG" \ - --repo "${{ github.repository }}" \ - dist/rootfs/alpine-rootfs.zip \ - dist/rootfs/alpine-rootfs.zip.sha256 \ - --clobber - - - name: Summary - run: | - set -euo pipefail - { - echo "### Published \`${{ inputs.tag }}\`" - echo - echo "- asset: \`alpine-rootfs.zip\`" - echo "- size: $(du -h dist/rootfs/alpine-rootfs.zip | cut -f1)" - echo "- toolchain: ${{ inputs.provision == 'all' && 'provisioned' || 'none' }}" - echo "- darwin SDK: ${{ inputs.provision == 'all' && inputs.provision_sdk && inputs.darwin_sdk_tag || 'not included' }}" - echo "- sha256: $(cut -d' ' -f1 < dist/rootfs/alpine-rootfs.zip.sha256)" - echo - echo "Set \`ROOTFS_TAG\` in .github/workflows/build-ipa.yml to this tag." - } >> "$GITHUB_STEP_SUMMARY" diff --git a/.github/workflows/ios-build.yml b/.github/workflows/ios-build.yml deleted file mode 100644 index 4bf9b19..0000000 --- a/.github/workflows/ios-build.yml +++ /dev/null @@ -1,1357 +0,0 @@ -name: iOS Build -run-name: iOS Build ${{ inputs.build_id || github.ref_name }} - -on: - # Pushing a tag named ios-build/ starts the same job without the - # GitHub API, for environments where only git access is available. A push - # event runs the workflow file from the tagged commit, and the parameters - # come from builder.json there. The tag is deleted when the job ends. - push: - tags: - - 'ios-build/*' - workflow_dispatch: - inputs: - build_id: - description: 'Unique build identifier' - required: true - type: string - snapshot_ref: - description: 'Ref holding the working-tree snapshot to build (empty builds this branch)' - required: false - type: string - default: '' - ios_path: - description: 'Path to iOS project (e.g., "ios" for React Native)' - required: false - type: string - default: '.' - scheme: - description: 'Xcode scheme to build (auto-detected if empty)' - required: false - type: string - default: '' - use_signing: - description: 'Enable code signing (requires secrets to be configured)' - required: false - type: boolean - default: false - configuration: - description: 'Build configuration (Debug is faster, Release for production)' - required: false - type: string - default: 'Debug' - flutter_version: - description: 'Flutter version to use (e.g., 3.24.0). Empty means latest stable.' - required: false - type: string - default: '' - jdk_version: - description: 'JDK version for Kotlin Multiplatform Gradle builds (e.g., 17).' - required: false - type: string - default: '17' - # One input for the profile fields that are not inputs of their own, so - # the workflow stays under the ten-input limit of workflow_dispatch. - profile: - description: 'Selected builder.json profile as JSON: {"name": "...", "env": {...}, "distribution": "..."}' - required: false - type: string - default: '{}' - # The tenth and last input workflow_dispatch allows; combine before adding. - build_number: - description: 'CFBundleVersion to stamp on the build: N, or X.Y.Z+N to set the marketing version too. Empty leaves the project as is.' - required: false - type: string - default: '' - -jobs: - build: - runs-on: macos-latest - permissions: - contents: write # deletes the trigger tag - timeout-minutes: 90 # a cold React Native/Expo build on the free runner takes 30-60 minutes - - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - # The dispatch always runs the workflow from the default branch, so the - # sources come from a snapshot ref pushed by the CLI instead. - - name: Checkout working-tree snapshot - if: inputs.snapshot_ref != '' - env: - SNAPSHOT_REF: ${{ inputs.snapshot_ref }} - run: | - set -e - git fetch --depth=1 origin "+$SNAPSHOT_REF:refs/heads/snapshot" - git checkout --force snapshot - echo "Building $(git rev-parse --short HEAD)" - - # Dispatch inputs arrive with their declared defaults; a tag push has none, - # so the values come from builder.json in the tagged commit (defaultProfile - # included) and the build id is the tag name after the prefix. - - name: Resolve parameters - id: params - env: - IN_BUILD_ID: ${{ inputs.build_id }} - IN_IOS_PATH: ${{ inputs.ios_path }} - IN_SCHEME: ${{ inputs.scheme }} - IN_USE_SIGNING: ${{ inputs.use_signing }} - IN_CONFIGURATION: ${{ inputs.configuration }} - IN_FLUTTER_VERSION: ${{ inputs.flutter_version }} - IN_JDK_VERSION: ${{ inputs.jdk_version }} - IN_PROFILE: ${{ inputs.profile }} - IN_BUILD_NUMBER: ${{ inputs.build_number }} - run: | - set -e - PROFILE="" - if [ "$GITHUB_EVENT_NAME" != "workflow_dispatch" ] && [ -f builder.json ]; then - PROFILE=$(jq -r '.defaultProfile // empty' builder.json) - if [ -n "$PROFILE" ] && [ "$(jq -r --arg p "$PROFILE" '.profiles[$p] != null' builder.json)" != "true" ]; then - echo "::error::defaultProfile \"$PROFILE\" is not defined under profiles in builder.json" - exit 1 - fi - fi - param() { # name dispatch-value jq-path default - local v="" - if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - v="$2" - elif [ -f builder.json ]; then - v=$(jq -r --arg p "$PROFILE" "$3 // empty" builder.json) - fi - v="${v:-$4}" - echo "$1=$v" >> "$GITHUB_OUTPUT" - echo "$1=$v" - } - # Profile fields override ios.*. A selected profile signs exactly - # when it has a distribution (ios.signing is the no-profile path), - # and its configuration defaults to Debug for development and - # Release for every other distribution. - param build_id "$IN_BUILD_ID" '""' "${GITHUB_REF_NAME##*/}" - param ios_path "$IN_IOS_PATH" '.ios.path' '.' - param scheme "$IN_SCHEME" '(.profiles[$p].scheme // .ios.scheme)' '' - param use_signing "$IN_USE_SIGNING" '(if $p != "" then ((.profiles[$p].distribution // "") != "") else .ios.signing end)' 'false' - param configuration "$IN_CONFIGURATION" '(.profiles[$p].configuration // (if $p != "" and (.profiles[$p].distribution // "") != "" then (if .profiles[$p].distribution == "development" then "Debug" else "Release" end) else .ios.configuration end))' 'Debug' - param flutter_version "$IN_FLUTTER_VERSION" '.flutter.version' '' - param jdk_version "$IN_JDK_VERSION" '.kmp.jdkVersion' '17' - # A build number only comes from `builder ios release`, never builder.json. - param build_number "$IN_BUILD_NUMBER" '""' '' - - # The rest of the profile: name, distribution (selects the signing - # set) and env, exported from here on so dependency installs see it. - if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - PROFILE_JSON="$IN_PROFILE" - elif [ -f builder.json ]; then - PROFILE_JSON=$(jq -c --arg p "$PROFILE" '{name: $p, env: (.profiles[$p].env // {}), distribution: (.profiles[$p].distribution // "")}' builder.json) - fi - [ -n "${PROFILE_JSON:-}" ] || PROFILE_JSON='{}' - PROFILE=$(jq -r '.name // ""' <<< "$PROFILE_JSON") - DISTRIBUTION=$(jq -r '.distribution // ""' <<< "$PROFILE_JSON") - # The CLI sends the canonical name; a tag build reads whatever builder.json says. - case "$DISTRIBUTION" in internal) DISTRIBUTION=ad-hoc ;; esac - # The suffix of the IOS_* secrets a distribution is signed with; no - # distribution means the legacy unsuffixed secrets. Same table in runner.sh. - signing_set() { - case "$1" in - '') echo '' ;; - development) echo DEVELOPMENT ;; - ad-hoc|internal) echo AD_HOC ;; - store) echo STORE ;; - enterprise) echo ENTERPRISE ;; - *) return 1 ;; - esac - } - if ! SIGNING_SET=$(signing_set "$DISTRIBUTION"); then - echo "::error::distribution \"$DISTRIBUTION\" must be development, ad-hoc (or internal), store or enterprise"; exit 1 - fi - echo "profile=$PROFILE" >> "$GITHUB_OUTPUT" - echo "distribution=$DISTRIBUTION" >> "$GITHUB_OUTPUT" - echo "signing_set=$SIGNING_SET" >> "$GITHUB_OUTPUT" - echo "profile=${PROFILE:-(none)}" - echo "distribution=$DISTRIBUTION" - echo "signing_set=$SIGNING_SET" - # Values are base64 per entry so newlines and quotes survive, and the - # GITHUB_ENV heredoc gets a random delimiter so no value line can end - # it early. Names are checked so a value cannot smuggle in a second variable. - if [ "$(jq -r '.env // {} | type' <<< "$PROFILE_JSON")" != "object" ]; then - echo "::error::the profile's env must be a JSON object of variable names to string values"; exit 1 - fi - while IFS=' ' read -r key encoded; do - name=$(printf '%s' "$key" | base64 --decode) - if ! [[ "$name" =~ ^[A-Za-z_][A-Za-z0-9_]*$ ]]; then - echo "::error::env name \"$name\" is not a valid environment variable name"; exit 1 - fi - delim="BUILDER_ENV_${RANDOM}${RANDOM}${RANDOM}" - { - echo "$name<<$delim" - printf '%s' "$encoded" | base64 --decode - echo - echo "$delim" - } >> "$GITHUB_ENV" - echo "env: $name" - done < <(jq -r '.env // {} | to_entries[] | "\(.key | @base64) \(.value | tostring | @base64)"' <<< "$PROFILE_JSON") - - - name: Setup Xcode - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: latest-stable - - - name: Restore DerivedData cache - uses: actions/cache/restore@v6 - id: cache-deriveddata - with: - path: DerivedData - key: deriveddata-${{ github.run_id }} - restore-keys: | - deriveddata- - - - name: Detect project type - id: detect - run: | - if [ -f "pubspec.yaml" ]; then - echo "type=flutter" >> $GITHUB_OUTPUT - elif [ -f "package.json" ] && grep -q '"expo"' package.json; then - echo "type=expo" >> $GITHUB_OUTPUT - elif [ -f "package.json" ] && grep -q '"react-native"' package.json; then - echo "type=reactnative" >> $GITHUB_OUTPUT - elif grep -rqlE 'kotlin\("multiplatform"\)|org\.jetbrains\.kotlin\.multiplatform|id\("org.jetbrains.kotlin.multiplatform"\)' --include='*.gradle.kts' --include='*.gradle' --include='libs.versions.toml' . 2>/dev/null; then - echo "type=kmp" >> $GITHUB_OUTPUT - else - echo "type=native" >> $GITHUB_OUTPUT - fi - - # A gitignored *.xcconfig next to a committed template is a common iOS - # convention. The snapshot then carries the template but not the config, - # and xcodebuild fails with "Unable to open base configuration reference - # file", which says nothing about which file is missing. - - name: Materialize missing xcconfig files - run: | - set -e - find . -path ./DerivedData -prune -o -type f \ - \( -name '*.xcconfig.template' -o -name '*.xcconfig.example' \ - -o -name '*.xcconfig.sample' -o -name '*.template.xcconfig' \ - -o -name '*.example.xcconfig' -o -name '*.sample.xcconfig' \) -print \ - | while IFS= read -r template; do - case "$template" in - *.xcconfig.template|*.xcconfig.example|*.xcconfig.sample) target="${template%.*}" ;; - *) target="$(dirname "$template")/$(basename "$template" | sed -E 's/\.(template|example|sample)\.xcconfig$/.xcconfig/')" ;; - esac - if [ ! -f "$target" ]; then - cp "$template" "$target" - echo "::warning::$target was missing (gitignored?); copied $template over it. Any placeholder values in it are what the build sees." - fi - done - - - name: Stage the guest rootfs (XForge) - # XForge bundles an Alpine rootfs (1.6 GB with the build toolchain in it) - # that is published as a release asset rather than committed, and the app - # target lists it as a resource — - # so it has to be on disk *before* XcodeGen walks the project, or the app - # is built without a Linux guest. The script reads the pinned tag from - # .github/workflows/build-ipa.yml and verifies the sha256. It is a no-op in - # a repository that has no such script. - run: | - set -e - # -f, not -x: a script that lost its executable bit must still run rather - # than make this step a silent no-op (which is how an app with no Linux - # guest got built and shipped to an artifact once already). - if [ -f EmbeddedLinux/fetch-rootfs.sh ]; then - bash EmbeddedLinux/fetch-rootfs.sh - fi - - - name: Generate Xcode project (XcodeGen) - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - run: | - set -e - # A managed Expo project has no iOS directory at all until the - # prebuild step below creates it. - if [ ! -d "$IOS_PATH" ]; then - echo "No '$IOS_PATH' directory yet — skipping." - exit 0 - fi - cd "$IOS_PATH" - - # Projects generated by XcodeGen usually keep the .xcodeproj out of git, - # so the checkout has a project.yml manifest but nothing to build. - MANIFEST="" - for f in project.yml project.yaml; do - if [ -f "$f" ] && grep -qE '^targets:' "$f"; then - MANIFEST="$f" - break - fi - done - - if [ -z "$MANIFEST" ]; then - echo "No XcodeGen manifest — skipping." - elif [ -n "$(find . -maxdepth 1 -name '*.xcodeproj' -print -quit)" ]; then - echo "Found a committed .xcodeproj — skipping generation." - else - echo "Found $MANIFEST and no .xcodeproj — generating with XcodeGen." - brew install xcodegen - xcodegen generate - fi - - # Whatever the previous step could not fill in is named here, instead of - # surfacing as xcodebuild's "Unable to open base configuration reference". - - name: Check base configuration files - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - run: | - set -e - if [ ! -d "$IOS_PATH" ]; then - echo "No '$IOS_PATH' directory yet — skipping." - exit 0 - fi - MISSING="" - for pbx in $(find "$IOS_PATH" -maxdepth 2 -name project.pbxproj -not -path '*/Pods.xcodeproj/*'); do - for id in $(grep -o 'baseConfigurationReference = [A-Za-z0-9]*' "$pbx" | awk '{print $3}' | sort -u); do - ref=$(grep -m1 "^[[:space:]]*$id .*isa = PBXFileReference" "$pbx" || true) - name=$(printf '%s' "$ref" | sed -n 's/.*[ \t]path = "\{0,1\}\([^";]*\)"\{0,1\};.*/\1/p') - [ -n "$name" ] || continue - base=$(basename "$name") - # CocoaPods writes its xcconfigs during pod install and Flutter - # writes Generated.xcconfig during pub get — both happen later in - # this job, so they are legitimately absent right now. - case "$name" in *Pods/*|*"Target Support Files"*) continue ;; esac - case "$base" in Pods-*|Generated.xcconfig) continue ;; esac - if [ -z "$(find . -name "$base" -print -quit)" ]; then - MISSING="$MISSING $base" - fi - done - done - if [ -n "$MISSING" ]; then - echo "::error::The project has base configuration files that are not in the repository:$MISSING. They are probably gitignored — commit them, or commit a .xcconfig.template next to each so the build can fill them in." - exit 1 - fi - - - name: Setup Flutter - if: steps.detect.outputs.type == 'flutter' - uses: subosito/flutter-action@v2 - with: - flutter-version: ${{ steps.params.outputs.flutter_version || '' }} - channel: stable - cache: true - - - name: Flutter pub get - if: steps.detect.outputs.type == 'flutter' - run: flutter pub get - - # setup-node cannot make `with:` keys conditional, so the version and the - # version file are both computed here; it ignores an empty version file. - - name: Resolve JS toolchain - id: js - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - run: | - set -e - # >>> js toolchain (keep identical across ios-build.yml, ios-share.yml, runner.sh) - # The package manager the project actually uses: the packageManager field - # first, then the lockfile, then npm. Guessing npm is not safe — `npm install` - # in a pnpm or yarn workspace fails with - # 'EUNSUPPORTEDPROTOCOL Unsupported URL Type "workspace:"'. - js_detect_manager() { - if [ -z "${JS_MANAGER:-}" ]; then - case "$(jq -r '.packageManager // empty' package.json 2>/dev/null || true)" in - pnpm|pnpm@*) JS_MANAGER=pnpm ;; - yarn|yarn@*) JS_MANAGER=yarn ;; - bun|bun@*) JS_MANAGER=bun ;; - npm|npm@*) JS_MANAGER=npm ;; - *) - if [ -f pnpm-lock.yaml ]; then JS_MANAGER=pnpm - elif [ -f yarn.lock ]; then JS_MANAGER=yarn - elif [ -f bun.lock ] || [ -f bun.lockb ]; then JS_MANAGER=bun - else JS_MANAGER=npm # package-lock.json, or no lockfile at all - fi - ;; - esac - fi - case "$JS_MANAGER" in - pnpm) JS_EXEC='pnpm exec' ;; - yarn) JS_EXEC=yarn ;; - bun) JS_EXEC=bunx ;; - *) JS_EXEC=npx ;; - esac - export JS_MANAGER JS_EXEC - echo "Package manager: $JS_MANAGER" - } - - # The Node version the project asks for: .nvmrc or .node-version, else the - # first comparator of engines.node, else 22. A leading "v" or range prefix - # is dropped: nvm, n and setup-node all take "20.11.1" and "24.x" as they are. - js_node_version() { - js_version="" - for js_file in .nvmrc .node-version; do - if [ -f "$js_file" ]; then - js_version=$(tr -d ' \t\r' < "$js_file" | head -n1) - break - fi - done - if [ -z "$js_version" ]; then - js_version=$(jq -r '.engines.node // empty' package.json 2>/dev/null || true) - js_version=$(printf '%s' "${js_version%%|*}" | awk '{print $1}') - fi - js_version=$(printf '%s' "$js_version" | sed -E 's/^(>=|<=|>|<|=|\^|~|v)+//') - case "$js_version" in - [0-9]*|lts/*) ;; - *) js_version=22 ;; - esac - printf '%s\n' "$js_version" - } - - # Put the manager on PATH first, then install — unless a cache already restored - # node_modules, in which case the manager is still wanted for `expo prebuild` - # and for the React Native build phases that shell out to it. - js_install() { - js_detect_manager - case "$JS_MANAGER" in - pnpm|yarn) - # corepack installs the version packageManager pins. Node 16.10-24 - # bundle it; Node 25+ and some provider images do not. - command -v corepack >/dev/null 2>&1 || npm install -g corepack || true - corepack enable || true - command -v "$JS_MANAGER" >/dev/null 2>&1 || npm install -g "$JS_MANAGER" - ;; - bun) - if ! command -v bun >/dev/null 2>&1; then - curl -fsSL https://bun.sh/install | bash - export BUN_INSTALL="${BUN_INSTALL:-$HOME/.bun}" - export PATH="$BUN_INSTALL/bin:$PATH" - # A step's PATH does not survive into the next GitHub Actions step. - if [ -n "${GITHUB_PATH:-}" ]; then echo "$BUN_INSTALL/bin" >> "$GITHUB_PATH"; fi - fi - ;; - esac - if [ "${JS_DEPS_CACHED:-}" = "true" ]; then - echo "node_modules restored from cache; skipping install" - return 0 - fi - case "$JS_MANAGER" in - pnpm) - if [ -f pnpm-lock.yaml ]; then pnpm install --frozen-lockfile; else pnpm install; fi - ;; - yarn) - if [ -f .yarnrc.yml ]; then yarn install --immutable - elif [ -f yarn.lock ]; then yarn install --frozen-lockfile - else yarn install - fi - ;; - bun) - if [ -f bun.lock ] || [ -f bun.lockb ]; then bun install --frozen-lockfile; else bun install; fi - ;; - *) - if [ -f package-lock.json ]; then npm ci; else npm install; fi - ;; - esac - } - # <<< js toolchain - js_detect_manager - NODE_VERSION_FILE="" - for f in .nvmrc .node-version; do - if [ -f "$f" ]; then NODE_VERSION_FILE="$f"; break; fi - done - NODE_VERSION="" - if [ -n "$NODE_VERSION_FILE" ]; then - echo "Node version: from $NODE_VERSION_FILE" - else - NODE_VERSION=$(js_node_version) - echo "Node version: $NODE_VERSION" - fi - { - echo "manager=$JS_MANAGER" - echo "exec=$JS_EXEC" - echo "node_version=$NODE_VERSION" - echo "node_version_file=$NODE_VERSION_FILE" - } >> "$GITHUB_OUTPUT" - - - name: Setup Node.js - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - uses: actions/setup-node@v7 - with: - node-version: ${{ steps.js.outputs.node_version }} - node-version-file: ${{ steps.js.outputs.node_version_file }} - - # pnpm links its store into node_modules/.pnpm, so caching node_modules - # stays correct for every manager. The key carries the manager name - # because the same lockfiles can be installed by different ones. - - name: Restore node_modules cache - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - uses: actions/cache@v6 - id: node-modules-cache - with: - path: node_modules - key: node-modules-${{ runner.os }}-${{ steps.js.outputs.manager }}-${{ hashFiles('package-lock.json', 'yarn.lock', 'pnpm-lock.yaml', 'bun.lock', 'bun.lockb') }} - restore-keys: | - node-modules-${{ runner.os }}-${{ steps.js.outputs.manager }}- - - # Runs even on a cache hit: the manager itself still has to be on PATH for - # the prebuild and build steps that follow. - - name: Install JS dependencies - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - env: - JS_MANAGER: ${{ steps.js.outputs.manager }} - JS_DEPS_CACHED: ${{ steps.node-modules-cache.outputs.cache-hit }} - run: | - set -e - # >>> js toolchain (keep identical across ios-build.yml, ios-share.yml, runner.sh) - # The package manager the project actually uses: the packageManager field - # first, then the lockfile, then npm. Guessing npm is not safe — `npm install` - # in a pnpm or yarn workspace fails with - # 'EUNSUPPORTEDPROTOCOL Unsupported URL Type "workspace:"'. - js_detect_manager() { - if [ -z "${JS_MANAGER:-}" ]; then - case "$(jq -r '.packageManager // empty' package.json 2>/dev/null || true)" in - pnpm|pnpm@*) JS_MANAGER=pnpm ;; - yarn|yarn@*) JS_MANAGER=yarn ;; - bun|bun@*) JS_MANAGER=bun ;; - npm|npm@*) JS_MANAGER=npm ;; - *) - if [ -f pnpm-lock.yaml ]; then JS_MANAGER=pnpm - elif [ -f yarn.lock ]; then JS_MANAGER=yarn - elif [ -f bun.lock ] || [ -f bun.lockb ]; then JS_MANAGER=bun - else JS_MANAGER=npm # package-lock.json, or no lockfile at all - fi - ;; - esac - fi - case "$JS_MANAGER" in - pnpm) JS_EXEC='pnpm exec' ;; - yarn) JS_EXEC=yarn ;; - bun) JS_EXEC=bunx ;; - *) JS_EXEC=npx ;; - esac - export JS_MANAGER JS_EXEC - echo "Package manager: $JS_MANAGER" - } - - # The Node version the project asks for: .nvmrc or .node-version, else the - # first comparator of engines.node, else 22. A leading "v" or range prefix - # is dropped: nvm, n and setup-node all take "20.11.1" and "24.x" as they are. - js_node_version() { - js_version="" - for js_file in .nvmrc .node-version; do - if [ -f "$js_file" ]; then - js_version=$(tr -d ' \t\r' < "$js_file" | head -n1) - break - fi - done - if [ -z "$js_version" ]; then - js_version=$(jq -r '.engines.node // empty' package.json 2>/dev/null || true) - js_version=$(printf '%s' "${js_version%%|*}" | awk '{print $1}') - fi - js_version=$(printf '%s' "$js_version" | sed -E 's/^(>=|<=|>|<|=|\^|~|v)+//') - case "$js_version" in - [0-9]*|lts/*) ;; - *) js_version=22 ;; - esac - printf '%s\n' "$js_version" - } - - # Put the manager on PATH first, then install — unless a cache already restored - # node_modules, in which case the manager is still wanted for `expo prebuild` - # and for the React Native build phases that shell out to it. - js_install() { - js_detect_manager - case "$JS_MANAGER" in - pnpm|yarn) - # corepack installs the version packageManager pins. Node 16.10-24 - # bundle it; Node 25+ and some provider images do not. - command -v corepack >/dev/null 2>&1 || npm install -g corepack || true - corepack enable || true - command -v "$JS_MANAGER" >/dev/null 2>&1 || npm install -g "$JS_MANAGER" - ;; - bun) - if ! command -v bun >/dev/null 2>&1; then - curl -fsSL https://bun.sh/install | bash - export BUN_INSTALL="${BUN_INSTALL:-$HOME/.bun}" - export PATH="$BUN_INSTALL/bin:$PATH" - # A step's PATH does not survive into the next GitHub Actions step. - if [ -n "${GITHUB_PATH:-}" ]; then echo "$BUN_INSTALL/bin" >> "$GITHUB_PATH"; fi - fi - ;; - esac - if [ "${JS_DEPS_CACHED:-}" = "true" ]; then - echo "node_modules restored from cache; skipping install" - return 0 - fi - case "$JS_MANAGER" in - pnpm) - if [ -f pnpm-lock.yaml ]; then pnpm install --frozen-lockfile; else pnpm install; fi - ;; - yarn) - if [ -f .yarnrc.yml ]; then yarn install --immutable - elif [ -f yarn.lock ]; then yarn install --frozen-lockfile - else yarn install - fi - ;; - bun) - if [ -f bun.lock ] || [ -f bun.lockb ]; then bun install --frozen-lockfile; else bun install; fi - ;; - *) - if [ -f package-lock.json ]; then npm ci; else npm install; fi - ;; - esac - } - # <<< js toolchain - js_install - - # A managed Expo project keeps no ios/ directory in git — `expo prebuild` - # generates it from app.json / app.config.js. It needs node_modules, so it - # runs here, and it must run before the Pods steps: the Podfile it writes - # is what `pod install` later reads. - - name: Expo prebuild - if: steps.detect.outputs.type == 'expo' - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - JS_EXEC: ${{ steps.js.outputs.exec }} - CI: '1' # prebuild answers its own prompts instead of hanging - run: | - set -e - if [ -n "$(find "$IOS_PATH" -maxdepth 1 \( -name '*.xcodeproj' -o -name '*.xcworkspace' \) -print -quit 2>/dev/null)" ]; then - echo "'$IOS_PATH' already holds an Xcode project (ejected Expo) — skipping prebuild." - exit 0 - fi - - # Without a bundle identifier prebuild asks for one interactively, - # which on a runner means waiting for the job timeout. - BUNDLE_ID=$(${JS_EXEC:-npx} expo config --type public --json 2>/dev/null | jq -r '.ios.bundleIdentifier // empty' 2>/dev/null || true) - if [ -z "$BUNDLE_ID" ]; then - for f in app.json app.config.json; do - [ -f "$f" ] || continue - BUNDLE_ID=$(jq -r '(.expo.ios.bundleIdentifier // .ios.bundleIdentifier) // empty' "$f" 2>/dev/null || true) - if [ -n "$BUNDLE_ID" ]; then break; fi - done - fi - if [ -z "$BUNDLE_ID" ]; then - echo "::error::This managed Expo project has no ios.bundleIdentifier. Set it in app.json (expo.ios.bundleIdentifier) or app.config.js, commit that, and build again — 'expo prebuild' cannot run unattended without it." - exit 1 - fi - echo "Prebuilding $BUNDLE_ID into '$IOS_PATH'..." - - ${JS_EXEC:-npx} expo prebuild --platform ios --no-install - if [ -z "$(find "$IOS_PATH" -maxdepth 1 -name '*.xcodeproj' -print -quit 2>/dev/null)" ]; then - echo "::error::'expo prebuild' produced no Xcode project in '$IOS_PATH'. If the project generates it elsewhere, set ios.path in builder.json." - exit 1 - fi - - - name: Setup JDK - if: steps.detect.outputs.type == 'kmp' - uses: actions/setup-java@v6 - with: - distribution: temurin - java-version: ${{ steps.params.outputs.jdk_version || '17' }} - - - name: Setup Gradle - if: steps.detect.outputs.type == 'kmp' - uses: gradle/actions/setup-gradle@v6 - - # A managed Expo project has no Podfile.lock before the prebuild above, so - # hashFiles returns an empty string and the key degrades to the prefix. - # That still restores and saves a Pods directory, and the `pod install` in - # the build step syncs it, so an empty hash is not an error here. - - name: Restore Pods cache - uses: actions/cache@v6 - id: pods-cache - with: - path: | - ${{ steps.params.outputs.ios_path }}/Pods - ~/.cocoapods/repos - key: pods-${{ runner.os }}-${{ hashFiles(format('{0}/Podfile.lock', steps.params.outputs.ios_path)) }} - restore-keys: | - pods-${{ runner.os }}- - - # One set of secrets per distribution, IOS_*_; the unsuffixed names - # serve builds without a profile. A secret that does not exist arrives empty. - - name: Install certificate and provisioning profile - if: steps.params.outputs.use_signing == 'true' - env: - SIGNING_SET: ${{ steps.params.outputs.signing_set }} - DISTRIBUTION: ${{ steps.params.outputs.distribution }} - CONFIGURATION: ${{ steps.params.outputs.configuration }} - IOS_CERTIFICATE: ${{ secrets.IOS_CERTIFICATE }} - IOS_CERTIFICATE_PASSWORD: ${{ secrets.IOS_CERTIFICATE_PASSWORD }} - IOS_PROVISIONING_PROFILE: ${{ secrets.IOS_PROVISIONING_PROFILE }} - IOS_EXTENSION_PROFILES: ${{ secrets.IOS_EXTENSION_PROFILES }} - IOS_CERTIFICATE_DEVELOPMENT: ${{ secrets.IOS_CERTIFICATE_DEVELOPMENT }} - IOS_CERTIFICATE_PASSWORD_DEVELOPMENT: ${{ secrets.IOS_CERTIFICATE_PASSWORD_DEVELOPMENT }} - IOS_PROVISIONING_PROFILE_DEVELOPMENT: ${{ secrets.IOS_PROVISIONING_PROFILE_DEVELOPMENT }} - IOS_EXTENSION_PROFILES_DEVELOPMENT: ${{ secrets.IOS_EXTENSION_PROFILES_DEVELOPMENT }} - IOS_CERTIFICATE_AD_HOC: ${{ secrets.IOS_CERTIFICATE_AD_HOC }} - IOS_CERTIFICATE_PASSWORD_AD_HOC: ${{ secrets.IOS_CERTIFICATE_PASSWORD_AD_HOC }} - IOS_PROVISIONING_PROFILE_AD_HOC: ${{ secrets.IOS_PROVISIONING_PROFILE_AD_HOC }} - IOS_EXTENSION_PROFILES_AD_HOC: ${{ secrets.IOS_EXTENSION_PROFILES_AD_HOC }} - IOS_CERTIFICATE_STORE: ${{ secrets.IOS_CERTIFICATE_STORE }} - IOS_CERTIFICATE_PASSWORD_STORE: ${{ secrets.IOS_CERTIFICATE_PASSWORD_STORE }} - IOS_PROVISIONING_PROFILE_STORE: ${{ secrets.IOS_PROVISIONING_PROFILE_STORE }} - IOS_EXTENSION_PROFILES_STORE: ${{ secrets.IOS_EXTENSION_PROFILES_STORE }} - IOS_CERTIFICATE_ENTERPRISE: ${{ secrets.IOS_CERTIFICATE_ENTERPRISE }} - IOS_CERTIFICATE_PASSWORD_ENTERPRISE: ${{ secrets.IOS_CERTIFICATE_PASSWORD_ENTERPRISE }} - IOS_PROVISIONING_PROFILE_ENTERPRISE: ${{ secrets.IOS_PROVISIONING_PROFILE_ENTERPRISE }} - IOS_EXTENSION_PROFILES_ENTERPRISE: ${{ secrets.IOS_EXTENSION_PROFILES_ENTERPRISE }} - run: | - set -e - fail() { echo "::error::$*" >&2; exit 1; } - - # Picks the set's IOS_*_ secrets into the unsuffixed names, or with - # no distribution takes the unsuffixed ones as they are (password - # optional). A suffixed set needs all three, since builder signing setup - # always writes a password. - select_signing_set() { - if [ -z "$SIGNING_SET" ]; then - if [ -z "${IOS_CERTIFICATE:-}" ] || [ -z "${IOS_PROVISIONING_PROFILE:-}" ]; then - fail "No signing secrets: ios.signing needs IOS_CERTIFICATE, IOS_CERTIFICATE_PASSWORD and IOS_PROVISIONING_PROFILE; a build profile with a distribution reads its own IOS_*_ secrets instead (builder signing setup --distribution writes them)." - fi - IOS_CERTIFICATE_PASSWORD="${IOS_CERTIFICATE_PASSWORD:-}" - IOS_EXTENSION_PROFILES="${IOS_EXTENSION_PROFILES:-}" - SIGNING_SET_USED=legacy - else - local cert="IOS_CERTIFICATE_$SIGNING_SET" pass="IOS_CERTIFICATE_PASSWORD_$SIGNING_SET" prof="IOS_PROVISIONING_PROFILE_$SIGNING_SET" ext="IOS_EXTENSION_PROFILES_$SIGNING_SET" - local missing="" name - for name in "$cert" "$pass" "$prof"; do - [ -n "${!name:-}" ] || missing="${missing:+$missing, }$name" - done - [ -z "$missing" ] || fail "Signing set $SIGNING_SET for distribution $DISTRIBUTION is missing $missing. Run builder signing setup --distribution $DISTRIBUTION (builder ios build does it too when an App Store Connect key is configured)." - IOS_CERTIFICATE="${!cert}" - IOS_CERTIFICATE_PASSWORD="${!pass}" - IOS_PROVISIONING_PROFILE="${!prof}" - IOS_EXTENSION_PROFILES="${!ext:-}" - SIGNING_SET_USED="$SIGNING_SET" - fi - echo "Signing set: $SIGNING_SET_USED" - } - - # Decodes the set's extension profiles (IOS_EXTENSION_PROFILES, a JSON object - # of extension bundle id to base64 .mobileprovision) into $1, installs them - # next to the app's, and prints a JSON object of bundle id to profile name - # for apply_signing_to_app_target and write_export_options. - install_extension_profiles() { - local dir="$1" json="${IOS_EXTENSION_PROFILES:-}" i=0 id encoded path uuid name map='{}' - [ -n "$json" ] || json='{}' - if [ "$(jq -r 'type' <<< "$json" 2>/dev/null)" != "object" ]; then - fail "IOS_EXTENSION_PROFILES${SIGNING_SET:+_$SIGNING_SET} must be a JSON object of extension bundle id to base64 .mobileprovision, as builder signing setup writes it." - fi - mkdir -p "$dir" "$HOME/Library/MobileDevice/Provisioning Profiles" - while IFS=' ' read -r id encoded; do - id=$(printf '%s' "$id" | base64 --decode) - i=$((i + 1)) - path="$dir/extension-$i.mobileprovision" - printf '%s' "$encoded" | base64 --decode > "$path" - security cms -D -i "$path" > "$path.plist" - uuid=$(plutil -extract UUID raw -o - "$path.plist") - name=$(plutil -extract Name raw -o - "$path.plist") - cp "$path" "$HOME/Library/MobileDevice/Provisioning Profiles/$uuid.mobileprovision" - echo "$uuid" >> "$dir/installed" - map=$(jq -c --arg id "$id" --arg name "$name" '. + {($id): $name}' <<< "$map") - echo " extension: $id -> '$name' ($uuid)" >&2 - done < <(jq -r 'to_entries[] | "\(.key | @base64) \(.value)"' <<< "$json") - printf '%s' "$map" - } - - # The profile in the set must be the type the build profile asked for, - # or the IPA would not be what the profile promised. Compared - # canonically: the export method says app-store for store, and a tag - # build may say internal for ad-hoc. - check_signing_set() { - [ "$SIGNING_SET_USED" != legacy ] || return 0 - local have="$1" want="$DISTRIBUTION" - case "$have" in app-store) have=store ;; esac - case "$want" in internal) want=ad-hoc ;; esac - if [ "$have" != "$want" ]; then - fail "IOS_PROVISIONING_PROFILE_$SIGNING_SET holds a $1 provisioning profile, but the build profile asks for distribution $want. Run builder signing setup --distribution $want, or set the profile's distribution to $have." - fi - } - - # The export method has to match the profile, or -exportArchive fails - # and App Store Connect rejects the IPA. These legacy names are the only - # ones older Xcodes (pinned or self-hosted runners) understand, and - # Xcode 16 still takes them. - detect_export_method() { - if [ "$(plutil -extract ProvisionsAllDevices raw -o - "$1" 2>/dev/null)" = "true" ]; then - echo enterprise - elif plutil -extract ProvisionedDevices xml1 -o /dev/null "$1" >/dev/null 2>&1; then - if [ "$(plutil -extract Entitlements.get-task-allow raw -o - "$1" 2>/dev/null)" = "true" ]; then - echo development - else - echo ad-hoc - fi - else - echo app-store - fi - } - - # The certificate names that can sign for a profile of this type, the - # current one first: keychains still hold pre-2021 iPhone Developer / - # iPhone Distribution certificates that sign the same profiles. - signing_identities() { - case "$1" in - development) printf '%s\n' "Apple Development" "iPhone Developer" ;; - ad-hoc|app-store|enterprise) printf '%s\n' "Apple Distribution" "iPhone Distribution" ;; - *) return 1 ;; - esac - } - - # The identity to archive with: the first of those names in security - # find-identity's output ($2). Without an explicit CODE_SIGN_IDENTITY the - # archive keeps the project's default, which Xcode refuses to pair with a - # distribution profile ("No signing certificate iOS Development found"). - signing_identity() { - local names name - names=$(signing_identities "$1") || return 1 - while IFS= read -r name; do - case "$2" in *"$name: "*) echo "$name"; return 0 ;; esac - done <<< "$names" - return 2 - } - - select_signing_set - - # Read the profile first: its type is checked against the build - # profile before any keychain exists or the certificate is imported. - PROFILE_PATH=$RUNNER_TEMP/profile.mobileprovision - echo "$IOS_PROVISIONING_PROFILE" | base64 --decode > "$PROFILE_PATH" - PROFILE_PLIST=$RUNNER_TEMP/profile.plist - security cms -D -i "$PROFILE_PATH" > "$PROFILE_PLIST" - PROFILE_UUID=$(plutil -extract UUID raw -o - "$PROFILE_PLIST") - - # xcodebuild will not pick a team on its own on a CI machine, so read - # the team and profile name out of the profile and pass them to the - # build as manual signing settings. - TEAM_ID=$(plutil -extract TeamIdentifier.0 raw -o - "$PROFILE_PLIST") - PROFILE_NAME=$(plutil -extract Name raw -o - "$PROFILE_PLIST") - APP_ID=$(plutil -extract Entitlements.application-identifier raw -o - "$PROFILE_PLIST") - PROFILE_BUNDLE_ID=${APP_ID#"$TEAM_ID".} - - EXPORT_METHOD=$(detect_export_method "$PROFILE_PLIST") - check_signing_set "$EXPORT_METHOD" - - # A Debug archive carries get-task-allow=true, which no distribution - # profile grants: the export fails, or an IPA that App Store Connect - # rejects comes out. Say so now instead of after the whole build. - if [ "$EXPORT_METHOD" != "development" ] && [ "$CONFIGURATION" = "Debug" ]; then - echo "::error::The provisioning profile is an $EXPORT_METHOD profile, but the build configuration is Debug. A Debug build is signed with get-task-allow, which distribution profiles do not allow and App Store Connect rejects. Drop the profile's \"configuration\" (a distribution build defaults to Release) or set \"configuration\": \"Release\", or build with a development profile." - exit 1 - fi - - # Create temporary keychain - KEYCHAIN_PATH=$RUNNER_TEMP/app-signing.keychain-db - KEYCHAIN_PASSWORD=$(openssl rand -base64 32) - - security create-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH" - security set-keychain-settings -lut 21600 "$KEYCHAIN_PATH" - security unlock-keychain -p "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH" - - # Import certificate - CERTIFICATE_PATH=$RUNNER_TEMP/certificate.p12 - echo "$IOS_CERTIFICATE" | base64 --decode > "$CERTIFICATE_PATH" - security import "$CERTIFICATE_PATH" -P "$IOS_CERTIFICATE_PASSWORD" -A -t cert -f pkcs12 -k "$KEYCHAIN_PATH" - security set-key-partition-list -S apple-tool:,apple: -k "$KEYCHAIN_PASSWORD" "$KEYCHAIN_PATH" - security list-keychain -d user -s "$KEYCHAIN_PATH" - - # The certificate has to be the kind the profile asks for, under - # whichever of its names it carries. Say so here instead of letting - # xcodebuild discover it after the whole archive. - IDENTITIES=$(security find-identity -v -p codesigning "$KEYCHAIN_PATH") - echo "$IDENTITIES" - CODE_SIGN_IDENTITY=$(signing_identity "$EXPORT_METHOD" "$IDENTITIES") || fail "IOS_CERTIFICATE${SIGNING_SET:+_$SIGNING_SET} holds no $(signing_identities "$EXPORT_METHOD" | paste -sd '/' -) certificate, which an $EXPORT_METHOD profile must be signed with. Run builder signing setup --distribution ${DISTRIBUTION:-} to issue the right one." - - # Install the provisioning profiles: the app's, then its extensions' - mkdir -p ~/Library/MobileDevice/Provisioning\ Profiles - cp "$PROFILE_PATH" ~/Library/MobileDevice/Provisioning\ Profiles/"$PROFILE_UUID".mobileprovision - EXTENSION_PROFILES=$(install_extension_profiles "$RUNNER_TEMP/extensions") - - echo "KEYCHAIN_PATH=$KEYCHAIN_PATH" >> $GITHUB_ENV - echo "DEVELOPMENT_TEAM=$TEAM_ID" >> $GITHUB_ENV - echo "PROVISIONING_PROFILE_NAME=$PROFILE_NAME" >> $GITHUB_ENV - echo "PROFILE_BUNDLE_ID=$PROFILE_BUNDLE_ID" >> $GITHUB_ENV - echo "EXTENSION_PROFILES=$EXTENSION_PROFILES" >> $GITHUB_ENV - echo "EXPORT_METHOD=$EXPORT_METHOD" >> $GITHUB_ENV - echo "CODE_SIGN_IDENTITY=$CODE_SIGN_IDENTITY" >> $GITHUB_ENV - echo "SIGNING_SET_USED=$SIGNING_SET_USED" >> $GITHUB_ENV - echo "Certificate and provisioning profile installed" - echo " team: $TEAM_ID" - echo " profile: $PROFILE_NAME ($PROFILE_UUID)" - echo " app id: $PROFILE_BUNDLE_ID" - echo " set: $SIGNING_SET_USED" - echo " export: $EXPORT_METHOD" - echo " identity: $CODE_SIGN_IDENTITY" - echo "The build writes these into the app target's build settings; its PRODUCT_BUNDLE_IDENTIFIER must match the app id above. Extension targets get the profiles listed above by bundle id." - - - name: Build IPA - env: - BUILD_ID: ${{ steps.params.outputs.build_id }} - IOS_PATH: ${{ steps.params.outputs.ios_path }} - SCHEME: ${{ steps.params.outputs.scheme }} - PROJECT_TYPE: ${{ steps.detect.outputs.type }} - USE_SIGNING: ${{ steps.params.outputs.use_signing }} - CONFIGURATION: ${{ steps.params.outputs.configuration }} - BUILD_NUMBER: ${{ steps.params.outputs.build_number }} - DISTRIBUTION: ${{ steps.params.outputs.distribution }} - run: | - set -e - fail() { echo "::error::$*" >&2; exit 1; } - - # Writes the manual signing settings (team, profile and identity from the - # environment) into the application targets of ./*.xcodeproj that the - # profile's app id ($1, "*" wildcards) covers, and into every extension target - # with the EXTENSION_PROFILES entry (bundle id to name) covering its bundle - # id, as an XML plist Xcode reads. On the xcodebuild command line they would - # apply to every target, and a CocoaPods framework target refuses a profile. - apply_signing_to_app_target() { - local projects=(*.xcodeproj) out - [ -d "${projects[0]}" ] || fail "No .xcodeproj in $PWD to apply the signing settings to" - out=$(python3 - "$1" "${projects[@]}" 2>&1 <<'PY' - import json, os, subprocess, sys - app_id, projects = sys.argv[1], sys.argv[2:] - settings = {'CODE_SIGN_STYLE': 'Manual', 'DEVELOPMENT_TEAM': os.environ['DEVELOPMENT_TEAM'], - 'PROVISIONING_PROFILE_SPECIFIER': os.environ['PROVISIONING_PROFILE_NAME'], - 'CODE_SIGN_IDENTITY': os.environ['CODE_SIGN_IDENTITY']} - extension_profiles = json.loads(os.environ.get('EXTENSION_PROFILES') or '{}') - # The same list as ExtensionProductTypes in internal/xcodeproj. - extension_types = {'com.apple.product-type.app-extension', 'com.apple.product-type.app-extension.messages', - 'com.apple.product-type.extensionkit-extension', 'com.apple.product-type.application.watchapp2', - 'com.apple.product-type.watchkit2-extension', 'com.apple.product-type.application.on-demand-install-capable'} - - def covers(pattern, bundle_id): - if pattern.endswith('*'): - return bundle_id.startswith(pattern[:-1]) - return bundle_id == pattern - - apps, extensions, plists = [], [], {} - for project in projects: - path = os.path.join(project, 'project.pbxproj') - plists[project] = json.loads(subprocess.check_output(['plutil', '-convert', 'json', '-o', '-', path])) - objects = plists[project]['objects'] - for target in objects.values(): - kind = target.get('productType') - if target.get('isa') != 'PBXNativeTarget' or (kind != 'com.apple.product-type.application' and kind not in extension_types): - continue - configs = [objects[c] for c in objects[target['buildConfigurationList']]['buildConfigurations']] - ids = sorted({c.setdefault('buildSettings', {}).get('PRODUCT_BUNDLE_IDENTIFIER', '') for c in configs}) - (apps if kind == 'com.apple.product-type.application' else extensions).append((project, target['name'], configs, ids)) - if not apps: - sys.exit('No application target in %s to apply the signing settings to' % ', '.join(projects)) - chosen = apps if len(apps) == 1 else [a for a in apps if any(covers(app_id, i) for i in a[3])] - if not chosen: - found = '; '.join('%s in %s (%s)' % (name, project, ', '.join(i or '?' for i in ids)) for project, name, _, ids in apps) - sys.exit('No application target has the PRODUCT_BUNDLE_IDENTIFIER the provisioning profile covers (%s): %s' % (app_id, found)) - chosen = [a + (settings['PROVISIONING_PROFILE_SPECIFIER'],) for a in chosen] - # An extension is signed with the most specific profile covering it; the app's never does. - missing = [] - for project, name, configs, ids in extensions: - names = [extension_profiles[p] for p in sorted(extension_profiles, key=len, reverse=True) if any(covers(p, i) for i in ids)] - if names: - chosen.append((project, name, configs, ids, names[0])) - else: - missing.append('%s in %s (%s)' % (name, project, ', '.join(i or '?' for i in ids))) - if missing: - sys.exit('No provisioning profile for extension target %s. Add each bundle id to ios.extensions in builder.json and run builder signing setup --distribution %s.' % ('; '.join(missing), os.environ.get('DISTRIBUTION') or '')) - for project, name, configs, _, profile in chosen: - for config in configs: - build = config['buildSettings'] - # A conditional setting (CODE_SIGN_IDENTITY[sdk=iphoneos*]) would win over the plain one. - for key in [k for k in build if k.split('[')[0] in settings]: - del build[key] - build.update(settings) - build['PROVISIONING_PROFILE_SPECIFIER'] = profile - print('Signing settings applied to target %s in %s: %s (profile %s)' % (name, project, ', '.join(c['name'] for c in configs), profile)) - for project in sorted({a[0] for a in chosen}): - subprocess.run(['plutil', '-convert', 'xml1', '-o', os.path.join(project, 'project.pbxproj'), '-'], - input=json.dumps(plists[project]).encode(), check=True) - PY - ) || fail "$out" - echo "$out" - } - - # BUILD_NUMBER is "N" or "X.Y.Z+N" (pubspec convention); sets build_number, - # build_name and version_settings for xcodebuild. Given a target and scheme - # it also rewrites an Info.plist that hardcodes CFBundleVersion, which - # CURRENT_PROJECT_VERSION never reaches; identical in ios-build.yml and runner.sh. - apply_build_number() { - build_number="" build_name="" version_settings="" - [ -n "${BUILD_NUMBER:-}" ] || return 0 - if ! [[ "$BUILD_NUMBER" =~ ^([0-9]+(\.[0-9]+)*\+)?[0-9]+(\.[0-9]+)*$ ]]; then - echo "BUILD_NUMBER must be N or X.Y.Z+N, got '$BUILD_NUMBER'" >&2 - return 1 - fi - build_number="${BUILD_NUMBER##*+}" - case "$BUILD_NUMBER" in *+*) build_name="${BUILD_NUMBER%+*}" ;; esac - version_settings="CURRENT_PROJECT_VERSION=$build_number${build_name:+ MARKETING_VERSION=$build_name}" - echo "Build number: $build_number${build_name:+ (version $build_name)}" - [ $# -gt 0 ] || return 0 - plist=$(xcodebuild "$@" -showBuildSettings -json 2>/dev/null \ - | jq -r 'map(.buildSettings) - | map(select(.PRODUCT_TYPE == "com.apple.product-type.application" and .INFOPLIST_FILE != null and .INFOPLIST_FILE != "")) - | map(.SRCROOT + "/" + .INFOPLIST_FILE) | first // empty' || true) - if [ -z "$plist" ] || [ ! -f "$plist" ]; then - echo "No Info.plist file in the app target; CFBundleVersion comes from CURRENT_PROJECT_VERSION" - return 0 - fi - current=$(plutil -extract CFBundleVersion raw -o - "$plist" 2>/dev/null || true) - case "$current" in - '$(CURRENT_PROJECT_VERSION)'|'${CURRENT_PROJECT_VERSION}'|'$(FLUTTER_BUILD_NUMBER)'|'${FLUTTER_BUILD_NUMBER}') - echo "$plist reads CFBundleVersion from $current" ;; - *) - echo "$plist hardcodes CFBundleVersion '$current'; setting $build_number in the plist" - plutil -replace CFBundleVersion -string "$build_number" "$plist" ;; - esac - [ -n "$build_name" ] || return 0 - current=$(plutil -extract CFBundleShortVersionString raw -o - "$plist" 2>/dev/null || true) - case "$current" in - '$(MARKETING_VERSION)'|'${MARKETING_VERSION}'|'$(FLUTTER_BUILD_NAME)'|'${FLUTTER_BUILD_NAME}') ;; - *) - echo "$plist hardcodes CFBundleShortVersionString '$current'; setting $build_name in the plist" - plutil -replace CFBundleShortVersionString -string "$build_name" "$plist" ;; - esac - } - - # Writes the export options ($1) with the same manual signing as the archive: - # the app's real bundle id (APP_BUNDLE_ID) maps to its profile and every - # extension keeps its own from EXTENSION_PROFILES. - write_export_options() { - python3 - "$1" <<'PY' - import json, os, plistlib, sys - options = {'method': os.environ['EXPORT_METHOD'], 'signingStyle': 'manual', - 'teamID': os.environ['DEVELOPMENT_TEAM'], - 'provisioningProfiles': {os.environ['APP_BUNDLE_ID']: os.environ['PROVISIONING_PROFILE_NAME']}} - options['provisioningProfiles'].update(json.loads(os.environ.get('EXTENSION_PROFILES') or '{}')) - # Distribution exports keep the version numbers the archive was built with; - # Xcode would otherwise renumber the build on export. - if options['method'] != 'development': - options['manageAppVersionAndBuildNumber'] = False - with open(sys.argv[1], 'wb') as out: - plistlib.dump(options, out) - PY - } - - # Navigate to iOS project - cd "$IOS_PATH" - - # Find workspace or project - WORKSPACE=$(find . -maxdepth 1 -name "*.xcworkspace" | head -1) - PROJECT=$(find . -maxdepth 1 -name "*.xcodeproj" | head -1) - - # Install pods if Podfile exists (always run to ensure sync with Podfile.lock) - if [ -f "Podfile" ]; then - echo "Running pod install..." - pod install - WORKSPACE=$(find . -maxdepth 1 -name "*.xcworkspace" | head -1) - fi - - if [ -n "$WORKSPACE" ]; then - TARGET_FLAG=-workspace - TARGET_PATH=$WORKSPACE - else - TARGET_FLAG=-project - TARGET_PATH=$PROJECT - fi - - # Determine scheme (only needed for native projects) - if [ -z "$SCHEME" ] && [ "$PROJECT_TYPE" = "native" ]; then - if [ -n "$WORKSPACE" ]; then - BASE=$(basename "$WORKSPACE" .xcworkspace) - else - BASE=$(basename "$PROJECT" .xcodeproj) - fi - - SCHEMES=$(xcodebuild "$TARGET_FLAG" "$TARGET_PATH" -list -json 2>/dev/null | jq -r '(.workspace // .project).schemes[]?' || true) - if [ -z "$SCHEMES" ]; then - SCHEMES=$(xcodebuild "$TARGET_FLAG" "$TARGET_PATH" -list 2>/dev/null | sed -n '/Schemes:/,$p' | tail -n +2 | sed 's/^[[:space:]]*//' | sed '/^$/d' || true) - fi - echo "Schemes:" - printf '%s\n' "$SCHEMES" | sed 's/^/ /' - - # In a workspace, the scheme list also holds package and pod - # schemes, and they sort ahead of the app as often as not. The - # scheme named after the project is the app in nearly every repo. - SCHEME=$(printf '%s\n' "$SCHEMES" | grep -Fx "$BASE" | head -1 || true) - if [ -z "$SCHEME" ]; then - SCHEME=$(printf '%s\n' "$SCHEMES" | grep -v '^Pods-' | while IFS= read -r s; do - [ -n "$s" ] || continue - # A scheme that belongs to a local Swift package is not the app. - [ -z "$(find . -maxdepth 4 -path "*/$s/Package.swift" -print -quit)" ] || continue - printf '%s\n' "$s" - break - done) - fi - echo "Building with scheme: $SCHEME (configuration: $CONFIGURATION)" - fi - - if [ "$PROJECT_TYPE" = "native" ]; then - if [ -z "$WORKSPACE" ] && [ -z "$PROJECT" ]; then - echo "::error::No .xcworkspace or .xcodeproj found in '$IOS_PATH'. If the project is generated (XcodeGen/Tuist), commit it or add a generation step." - exit 1 - fi - if [ -z "$SCHEME" ]; then - echo "::error::Could not determine a scheme. Pass one via the 'scheme' input, or make sure the project has a shared scheme." - exit 1 - fi - fi - - # Use custom DerivedData path for caching - DERIVED_DATA_PATH="${GITHUB_WORKSPACE}/DerivedData" - - if [ "$PROJECT_TYPE" = "flutter" ]; then - cd "$GITHUB_WORKSPACE" - apply_build_number - FLUTTER_VERSION_FLAGS="" - if [ -n "$build_number" ]; then - FLUTTER_VERSION_FLAGS="--build-number=$build_number${build_name:+ --build-name=$build_name}" - fi - if [ "$CONFIGURATION" = "Debug" ]; then - echo "Building Flutter iOS Debug..." - flutter build ios --debug --no-codesign $FLUTTER_VERSION_FLAGS - else - echo "Building Flutter iOS Release..." - flutter build ios --release --no-codesign $FLUTTER_VERSION_FLAGS - fi - cd "$IOS_PATH" - - if [ "$USE_SIGNING" = "true" ]; then - # 'flutter build ios' never signs, so archive the Runner project it - # generated. The IPA step below exports build/App.xcarchive - # whenever signing is on. - ARCHIVE_CMD="xcodebuild" - if [ -n "$WORKSPACE" ]; then - ARCHIVE_CMD="$ARCHIVE_CMD -workspace '$WORKSPACE'" - elif [ -n "$PROJECT" ]; then - ARCHIVE_CMD="$ARCHIVE_CMD -project '$PROJECT'" - else - echo "::error::Signing requested but no Runner workspace/project found in '$IOS_PATH'." - exit 1 - fi - - # Flutter wrote the build number into Generated.xcconfig; this - # only catches a Runner Info.plist that hardcodes it. - apply_build_number "$TARGET_FLAG" "$TARGET_PATH" -scheme "${SCHEME:-Runner}" - ARCHIVE_CMD="$ARCHIVE_CMD -scheme '${SCHEME:-Runner}'" - ARCHIVE_CMD="$ARCHIVE_CMD -configuration '$CONFIGURATION'" - ARCHIVE_CMD="$ARCHIVE_CMD -destination 'generic/platform=iOS'" - ARCHIVE_CMD="$ARCHIVE_CMD -derivedDataPath '$DERIVED_DATA_PATH'" - ARCHIVE_CMD="$ARCHIVE_CMD COMPILER_INDEX_STORE_ENABLE=NO $version_settings" - ARCHIVE_CMD="$ARCHIVE_CMD -quiet" - ARCHIVE_CMD="$ARCHIVE_CMD -archivePath '$GITHUB_WORKSPACE/build/App.xcarchive' archive" - - # The manual signing settings go on the Runner target, after - # flutter build ios has regenerated the project. - apply_signing_to_app_target "$PROFILE_BUNDLE_ID" - echo "Running: $ARCHIVE_CMD" - eval $ARCHIVE_CMD - fi - elif [ "$PROJECT_TYPE" = "reactnative" ] || [ "$PROJECT_TYPE" = "expo" ] || [ "$PROJECT_TYPE" = "kmp" ]; then - # Use xcodebuild directly for React Native/Expo and Kotlin Multiplatform. - # For KMP, the Xcode build phase (or CocoaPods) invokes Gradle to - # compile the shared Kotlin framework, so the JDK must be on PATH. - if [ "$PROJECT_TYPE" = "kmp" ]; then - echo "Building Kotlin Multiplatform iOS ($CONFIGURATION)..." - if [ -f "${GITHUB_WORKSPACE}/gradlew" ]; then - chmod +x "${GITHUB_WORKSPACE}/gradlew" - fi - else - echo "Building React Native iOS ($CONFIGURATION)..." - fi - BUILD_CMD="xcodebuild" - if [ -n "$WORKSPACE" ]; then - BUILD_CMD="$BUILD_CMD -workspace $WORKSPACE" - elif [ -n "$PROJECT" ]; then - BUILD_CMD="$BUILD_CMD -project $PROJECT" - fi - - # Auto-detect scheme from workspace/project name (e.g., MyApp.xcworkspace -> MyApp) - # unless an explicit scheme was provided. - if [ -z "$SCHEME" ]; then - if [ -n "$WORKSPACE" ]; then - SCHEME=$(basename "$WORKSPACE" .xcworkspace) - elif [ -n "$PROJECT" ]; then - SCHEME=$(basename "$PROJECT" .xcodeproj) - fi - fi - - apply_build_number "$TARGET_FLAG" "$TARGET_PATH" -scheme "$SCHEME" - BUILD_CMD="$BUILD_CMD -scheme '$SCHEME'" - BUILD_CMD="$BUILD_CMD -configuration '$CONFIGURATION'" - BUILD_CMD="$BUILD_CMD -destination 'generic/platform=iOS'" - BUILD_CMD="$BUILD_CMD -derivedDataPath '$DERIVED_DATA_PATH'" - BUILD_CMD="$BUILD_CMD COMPILER_INDEX_STORE_ENABLE=NO $version_settings" - BUILD_CMD="$BUILD_CMD -quiet" - - if [ "$USE_SIGNING" = "true" ]; then - # Must archive, not build: the IPA step below exports - # build/App.xcarchive whenever signing is on. The signing settings - # go on the app target now that pod install has generated the project. - apply_signing_to_app_target "$PROFILE_BUNDLE_ID" - BUILD_CMD="$BUILD_CMD -archivePath '$GITHUB_WORKSPACE/build/App.xcarchive' archive" - else - BUILD_CMD="$BUILD_CMD CODE_SIGNING_ALLOWED=NO build" - fi - - echo "Running: $BUILD_CMD" - eval $BUILD_CMD - else - BUILD_CMD="xcodebuild" - if [ -n "$WORKSPACE" ]; then - BUILD_CMD="$BUILD_CMD -workspace $WORKSPACE" - elif [ -n "$PROJECT" ]; then - BUILD_CMD="$BUILD_CMD -project $PROJECT" - fi - - apply_build_number "$TARGET_FLAG" "$TARGET_PATH" -scheme "$SCHEME" - BUILD_CMD="$BUILD_CMD -scheme '$SCHEME'" - BUILD_CMD="$BUILD_CMD -configuration '$CONFIGURATION'" - BUILD_CMD="$BUILD_CMD -destination 'generic/platform=iOS'" - BUILD_CMD="$BUILD_CMD -derivedDataPath '$DERIVED_DATA_PATH'" - BUILD_CMD="$BUILD_CMD $version_settings" - - # Speed optimization flags - BUILD_CMD="$BUILD_CMD COMPILER_INDEX_STORE_ENABLE=NO" - BUILD_CMD="$BUILD_CMD DEBUG_INFORMATION_FORMAT=dwarf" - BUILD_CMD="$BUILD_CMD ONLY_ACTIVE_ARCH=YES" - BUILD_CMD="$BUILD_CMD -quiet" - - # Xcode 16+ native compilation caching - BUILD_CMD="$BUILD_CMD SWIFT_ENABLE_COMPILE_CACHE=YES" - BUILD_CMD="$BUILD_CMD CLANG_ENABLE_COMPILE_CACHE=YES" - - if [ "$USE_SIGNING" = "true" ]; then - # For signed builds, use archive. Automatic signing cannot resolve a - # team on a runner (no Xcode account), so sign the app target - # manually against the profile installed above. - apply_signing_to_app_target "$PROFILE_BUNDLE_ID" - BUILD_CMD="$BUILD_CMD -archivePath '$GITHUB_WORKSPACE/build/App.xcarchive' archive" - else - # For unsigned builds, use faster 'build' action - BUILD_CMD="$BUILD_CMD CODE_SIGNING_ALLOWED=NO build" - fi - - echo "Running: $BUILD_CMD" - eval $BUILD_CMD - fi - - # Create IPA - if [ "$USE_SIGNING" = "true" ]; then - # For signed builds, use xcodebuild -exportArchive. - # Everything below is relative to the repo root, which is also what - # the upload-artifact step ("build/*.ipa") resolves against — do not - # use "cd .." here, it breaks when ios_path is "." (native at root). - cd "$GITHUB_WORKSPACE" - - # The export has to use the same manual signing as the archive, and - # it needs the app's real bundle id to map it to the profile. - APP_BUNDLE_ID=$(plutil -extract ApplicationProperties.CFBundleIdentifier raw -o - build/App.xcarchive/Info.plist) - echo "Exporting $APP_BUNDLE_ID with profile '$PROVISIONING_PROFILE_NAME' (team $DEVELOPMENT_TEAM), method $EXPORT_METHOD" - export APP_BUNDLE_ID - write_export_options ExportOptions.plist - - xcodebuild -exportArchive \ - -archivePath build/App.xcarchive \ - -exportOptionsPlist ExportOptions.plist \ - -exportPath build/export - - # Move IPA to expected location - mv build/export/*.ipa "build/$BUILD_ID.ipa" - echo "Created signed IPA: build/$BUILD_ID.ipa" - else - # For unsigned builds, find .app and create IPA - if [ "$PROJECT_TYPE" = "flutter" ]; then - # Flutter outputs to build/ios/iphoneos/ - APP_PATH=$(find "${GITHUB_WORKSPACE}/build/ios/iphoneos" -name "*.app" -type d | head -1) - else - # Native and React Native/Expo use DerivedData. Ask the project - # which bundle this scheme produces: the first .app under - # Build/Products is the wrong one whenever the repo has more than - # one app target. - APP_PATH=$(xcodebuild "$TARGET_FLAG" "$TARGET_PATH" \ - -scheme "$SCHEME" \ - -configuration "$CONFIGURATION" \ - -destination 'generic/platform=iOS' \ - -derivedDataPath "$DERIVED_DATA_PATH" \ - -showBuildSettings -json 2>/dev/null \ - | jq -r 'map(.buildSettings) - | map(select(.PRODUCT_TYPE == "com.apple.product-type.application" - and (.TARGET_BUILD_DIR | contains("-iphoneos")))) - | map(.TARGET_BUILD_DIR + "/" + .FULL_PRODUCT_NAME) - | first // empty' || true) - if [ ! -d "$APP_PATH" ]; then - APP_PATH=$(find "$DERIVED_DATA_PATH/Build/Products/${CONFIGURATION}-iphoneos" -maxdepth 1 -name "*.app" -type d -print -quit 2>/dev/null || true) - fi - fi - - if [ -z "$APP_PATH" ]; then - echo "Error: Could not find .app" - exit 1 - fi - echo "Found app at: $APP_PATH" - cd "$GITHUB_WORKSPACE" - mkdir -p build/Payload - cp -R "$APP_PATH" build/Payload/ - cd build - zip -rq "$BUILD_ID.ipa" Payload - rm -rf Payload - echo "Created unsigned IPA: build/$BUILD_ID.ipa" - fi - - - name: Upload IPA artifact - uses: actions/upload-artifact@v7 - with: - name: ipa - path: build/*.ipa - retention-days: 7 - if-no-files-found: error - - # Flutter builds through its own tree, so DerivedData can be absent and a - # save would fail on a path that never existed. - - name: Check for DerivedData - id: dd - if: always() - run: | - if [ -d DerivedData ]; then echo "exists=yes" >> $GITHUB_OUTPUT; fi - - - name: Save DerivedData cache - uses: actions/cache/save@v6 - if: always() && steps.dd.outputs.exists == 'yes' - with: - path: DerivedData - key: deriveddata-${{ github.run_id }} - - - name: Cleanup signing - if: always() && steps.params.outputs.use_signing == 'true' - run: | - if [ -n "$KEYCHAIN_PATH" ] && [ -f "$KEYCHAIN_PATH" ]; then - security delete-keychain "$KEYCHAIN_PATH" || true - fi - - - name: Build summary - if: always() - env: - USE_SIGNING: ${{ steps.params.outputs.use_signing }} - CONFIGURATION: ${{ steps.params.outputs.configuration }} - PROFILE: ${{ steps.params.outputs.profile }} - BUILD_NUMBER: ${{ steps.params.outputs.build_number }} - run: | - echo "## Build Summary" >> $GITHUB_STEP_SUMMARY - echo "" >> $GITHUB_STEP_SUMMARY - echo "- **Build ID:** ${{ steps.params.outputs.build_id }}" >> $GITHUB_STEP_SUMMARY - echo "- **Status:** ${{ job.status }}" >> $GITHUB_STEP_SUMMARY - if [ -n "$PROFILE" ]; then - echo "- **Profile:** $PROFILE" >> $GITHUB_STEP_SUMMARY - fi - echo "- **Configuration:** $CONFIGURATION" >> $GITHUB_STEP_SUMMARY - if [ -n "$BUILD_NUMBER" ]; then - echo "- **Build number:** $BUILD_NUMBER" >> $GITHUB_STEP_SUMMARY - fi - echo "- **DerivedData Cache:** ${{ steps.cache-deriveddata.outputs.cache-hit == 'true' && 'Hit' || 'Miss' }}" >> $GITHUB_STEP_SUMMARY - echo "- **Pods Cache:** ${{ steps.pods-cache.outputs.cache-hit == 'true' && 'Hit' || 'Miss' }}" >> $GITHUB_STEP_SUMMARY - echo "- **Node Modules Cache:** ${{ steps.node-modules-cache.outputs.cache-hit == 'true' && 'Hit' || 'N/A' }}" >> $GITHUB_STEP_SUMMARY - if [ "$USE_SIGNING" = "true" ]; then - echo "- **Signing:** Signed (${EXPORT_METHOD:-unknown}, set ${SIGNING_SET_USED:-unknown})" >> $GITHUB_STEP_SUMMARY - else - echo "- **Signing:** Unsigned (sign locally with AltStore/Sideloadly)" >> $GITHUB_STEP_SUMMARY - fi - if [ -f build/*.ipa ]; then - IPA_PATH=$(find build -name "*.ipa" -type f | head -1) - echo "- **IPA Size:** $(ls -lh "$IPA_PATH" | awk '{print $5}')" >> $GITHUB_STEP_SUMMARY - fi - - # The tag exists only to start this run, so it leaves with it. Deleting - # a tag fires no push event, so this cannot start another run. - - name: Delete trigger tag - if: always() && github.ref_type == 'tag' - run: git push origin ":refs/tags/$GITHUB_REF_NAME" diff --git a/.gitmodules b/.gitmodules deleted file mode 100644 index ab22d80..0000000 --- a/.gitmodules +++ /dev/null @@ -1,5 +0,0 @@ -[submodule "Vendor/ish-arm64"] - path = Vendor/ish-arm64 - url = https://github.com/OpenMinis/ish-arm64.git - branch = master - shallow = true diff --git a/App/Services/AppBundleSigner.swift b/App/Services/AppBundleSigner.swift index 85c74b8..91875ef 100644 --- a/App/Services/AppBundleSigner.swift +++ b/App/Services/AppBundleSigner.swift @@ -1,4 +1,5 @@ import Foundation +import Security import XKit /// Signing an app bundle with XKit, in this process. @@ -150,9 +151,14 @@ enum PKCS12Identity { } /// Wrap raw private-key bytes in a PEM envelope. + /// + /// `SecKeyCopyExternalRepresentation` returns the key in its *natural* form: + /// PKCS#1 for RSA. The label therefore says RSA rather than the PKCS#8 + /// "PRIVATE KEY" — a parser that trusts the label would read PKCS#1 bytes as + /// PKCS#8 and fail on the wrapper it expects to find there. private static func pem(_ der: Data) -> Data { let base64 = der.base64EncodedString(options: [.lineLength64Characters, .endLineWithLineFeed]) - let text = "-----BEGIN PRIVATE KEY-----\n\(base64)\n-----END PRIVATE KEY-----\n" + let text = "-----BEGIN RSA PRIVATE KEY-----\n\(base64)\n-----END RSA PRIVATE KEY-----\n" return Data(text.utf8) } } diff --git a/App/Services/XForgeLog.swift b/App/Services/XForgeLog.swift index a468930..41fd4a3 100644 --- a/App/Services/XForgeLog.swift +++ b/App/Services/XForgeLog.swift @@ -20,12 +20,6 @@ enum XForgeLog { /// Rotate past this size so the file stays shareable. static let maxBytes = 512 * 1024 - private static let timestamp: ISO8601DateFormatter = { - let formatter = ISO8601DateFormatter() - formatter.formatOptions = [.withInternetDateTime] - return formatter - }() - /// Create the log directory and rotate an oversized log. /// /// Kept as a separate step from `note(_:)` so the directory exists before the @@ -53,12 +47,24 @@ enum XForgeLog { return true } + /// ISO-8601 UTC, built per call. + /// + /// `ISO8601DateFormatter` is not `Sendable`, so it cannot live in a static + /// that `note(_:)` — a nonisolated function — reaches: Swift 6 rejects that + /// outright. Constructing one per line costs a few microseconds on a call that + /// always writes a file anyway. + private static func timestamp() -> String { + let formatter = ISO8601DateFormatter() + formatter.formatOptions = [.withInternetDateTime] + return formatter.string(from: Date()) + } + /// Append one timestamped breadcrumb line. /// /// Best effort by design: a diagnostic that can fail a build is worse than a /// diagnostic that is missing a line. static func note(_ line: String) { - let entry = "\(timestamp.string(from: Date())) \(line)\n" + let entry = "\(Self.timestamp()) \(line)\n" let fm = FileManager.default if !fm.fileExists(atPath: url.path) { _ = prepare() diff --git a/EmbeddedLinux/build-engine-for-xcode.sh b/EmbeddedLinux/build-engine-for-xcode.sh deleted file mode 100755 index d6479fa..0000000 --- a/EmbeddedLinux/build-engine-for-xcode.sh +++ /dev/null @@ -1,110 +0,0 @@ -#!/bin/bash -# -# build-engine-for-xcode.sh — the XForge target's first build phase. -# -# The app cannot compile without the embedded Linux engine: ISHBridge.c includes -# the engine's headers, and a device build links libish.a / libish_emu.a / -# libfakefs.a out of Vendor/ish-arm64-build/lib. Both come from -# EmbeddedLinux/build-ish-core.sh, and until now *only* the IPA workflow ran it — -# so any other way of building the project (Xcode itself, another CI pipeline) -# failed part-way through compiling the bridge with -# -# App/EmbeddedVM/ISHBridge.c:123:10: error: 'kernel/init.h' file not found -# -# which says nothing about the engine being missing. This phase closes that gap: -# it fetches the engine, builds it once per engine revision, and skips itself for -# builds that do not link the engine at all. -# -# Builds that link the engine: iphoneos (`-lish` in OTHER_LDFLAGS[sdk=iphoneos*]) -# Builds that do not: the simulator, which compiles the stub branch -# in ISHBridge.c (the unit tests use it) -# -# Environment: -# XFORGE_REBUILD_ENGINE=1 force a rebuild even if the engine is up to date -# XFORGE_SKIP_ENGINE=1 skip the engine entirely (a fast check that the -# rest of the app compiles for the simulator) -# -set -euo pipefail - -if [ "${XFORGE_SKIP_ENGINE:-0}" = "1" ]; then - echo "note: XFORGE_SKIP_ENGINE=1 — not building the embedded Linux engine" - exit 0 -fi - -# The engine is device-only. A simulator build compiles the stub branch instead, -# so there is nothing to fetch or build, and doing it anyway would make every unit -# test build spend minutes on an engine it will not link. -if [ "${PLATFORM_NAME:-}" != "iphoneos" ]; then - echo "note: ${PLATFORM_NAME:-unknown} build — the engine is not linked, nothing to do" - exit 0 -fi - -cd "${SRCROOT:-$(cd "$(dirname "$0")/.." && pwd)}" - -# Xcode hands a script phase a bare PATH (`/usr/bin:/bin:/usr/sbin:/sbin`), which -# contains neither Homebrew nor anything Homebrew installs — so `command -v meson` -# and even `command -v brew` would come back empty on a machine that has both. Put -# the usual prefixes on PATH first. -for prefix in /opt/homebrew/bin /usr/local/bin /opt/local/bin; do - if [ -d "$prefix" ]; then - case ":$PATH:" in - *":$prefix:"*) ;; - *) PATH="$prefix:$PATH" ;; - esac - fi -done -export PATH - -# --------------------------------------------------------------------------- -# The engine's sources -# -# A fresh clone has the submodule empty (`git submodule update --init` is a -# separate step in every CI pipeline, and easy to forget in a new one). Fetching -# it here is cheap — one shallow clone — and turns "error: 'kernel/init.h' file -# not found" into a build that just works. -# --------------------------------------------------------------------------- -if [ ! -f Vendor/ish-arm64/meson.build ]; then - echo "note: Vendor/ish-arm64 is empty — fetching the engine submodule" - # Only the engine itself: its own submodules (libarchive, libapps, linux) are - # not read by this build — see EmbeddedLinux/build-ish-core.sh. - git submodule update --init --depth 1 Vendor/ish-arm64 -fi - -# --------------------------------------------------------------------------- -# The build tools -# -# meson/ninja build it, libarchive is what the engine links against, and llvm -# provides the clang+lld the arm64 VDSO needs (Apple's clang cannot link a Linux -# target). On a machine with Homebrew, install what is missing rather than failing -# a build the user cannot act on from inside Xcode. -# --------------------------------------------------------------------------- -missing="" -for tool in meson ninja; do - command -v "$tool" >/dev/null 2>&1 || missing="$missing $tool" -done - -if [ -n "$missing" ]; then - if command -v brew >/dev/null 2>&1; then - echo "note: installing the engine's build tools:$missing (plus llvm/lld/libarchive) with Homebrew" - for formula in meson ninja llvm lld libarchive; do - brew list "$formula" >/dev/null 2>&1 || brew install "$formula" - done - else - printf 'error: the embedded Linux engine needs meson and ninja, and this machine\n' >&2 - printf ' has neither (missing:%s) nor Homebrew to install them.\n' "$missing" >&2 - printf ' Install them and build the engine once:\n' >&2 - printf ' brew install meson ninja llvm lld libarchive\n' >&2 - printf ' EmbeddedLinux/build-ish-core.sh\n' >&2 - exit 1 - fi -fi -command -v meson >/dev/null 2>&1 || { - printf 'error: meson is installed but not on this build phase'"'"'s PATH.\n' >&2 - printf ' Add %s/bin to it (Xcode does not read your shell profile).\n' "$(brew --prefix 2>/dev/null)" >&2 - exit 1 -} - -# --------------------------------------------------------------------------- -# Build (the script is a no-op when the engine is already up to date) -# --------------------------------------------------------------------------- -exec bash EmbeddedLinux/build-ish-core.sh diff --git a/EmbeddedLinux/build-ish-core.sh b/EmbeddedLinux/build-ish-core.sh deleted file mode 100644 index c35a49a..0000000 --- a/EmbeddedLinux/build-ish-core.sh +++ /dev/null @@ -1,225 +0,0 @@ -#!/usr/bin/env bash -# -# build-ish-core.sh — build the embedded Linux engine (OpenMinis/ish-arm64) for -# iOS and stage the static libraries where XForge's linker finds them. -# -# XForge embeds **ish-arm64** (Vendor/ish-arm64, a git submodule) as its -# in-process Linux runtime. It is a fork of ish-app/ish that adds a native -# AArch64 guest backend to the threaded-code interpreter, so the Alpine aarch64 -# rootfs runs as a same-architecture guest instead of being cross-translated. -# -# This script produces, for the iOS device (arm64): -# -# libish.a the emulator + kernel -# libish_emu.a per-arch instruction translation -# libfakefs.a the SQLite-backed filesystem -# -# into Vendor/ish-arm64-build/lib. -# -# Nothing else is built here. In particular the app does *not* link -# tools/fakefs.c (`fakefs_import`) or libarchive: the rootfs it installs is already -# a fakefs ZIP, unpacked with ZIPFoundation, so there is no import step at runtime -# (see App/EmbeddedVM/ISHBridge.h). The `fakefsify` the *build machine* needs to -# make that ZIP is produced by the engine's own meson build in -# EmbeddedLinux/build-rootfs.sh, on a Linux host. -# -# Requires: macOS, Xcode command line tools, meson, ninja, python3. -# brew install meson ninja llvm lld -# -# Usage: EmbeddedLinux/build-ish-core.sh -# -set -euo pipefail - -HERE="$(cd "$(dirname "$0")" && pwd)" -REPO="$(cd "$HERE/.." && pwd)" - -ISH="${ISH_ROOT:-$REPO/Vendor/ish-arm64}" -BUILD="${ISH_BUILD_DIR:-$REPO/Vendor/ish-arm64-build}" -OUT="${ISH_LIB_DIR:-$BUILD/lib}" - -# The guest is aarch64 (the bundled Alpine rootfs), so only that backend is -# built. Must match the -DGUEST_ARM64=1 in project.yml: the guest-arch defines -# select struct layouts, so engine and app have to agree. -GUEST_ARCH="${ISH_GUEST_ARCH:-arm64}" -MIN_SDK="${IPHONEOS_DEPLOYMENT_TARGET:-17.0}" - -log() { printf '\033[1m==> %s\033[0m\n' "$*"; } -note() { printf ' %s\n' "$*"; } -die() { printf 'error: %s\n' "$*" >&2; exit 1; } - -[ -f "$ISH/meson.build" ] || die "ish-arm64 sources not found at $ISH - run: git submodule update --init --depth 1 Vendor/ish-arm64" - -# ish-arm64's own submodules are deliberately not required. deps/libarchive only -# feeds the fakefs tools (which this app does not link — see the note at the top), -# deps/libapps is for the native host shells XForge does not embed, and deps/linux -# is for -Dkernel=linux, which XForge does not build. Requiring them would mean -# fetching something the build never reads. - -for tool in meson ninja xcrun python3; do - command -v "$tool" >/dev/null 2>&1 || die "$tool not found in PATH" -done - -# --------------------------------------------------------------------------- -# Already built? -# -# The libraries are a function of the engine's sources and of the flags this -# script passes, so the engine revision (the submodule commit) plus a hash of this -# script and of project.yml — which carries -DGUEST_ARM64=1, and has to agree with -# the engine or struct layouts differ — is enough to decide. That matters because -# this script is run from two places now: the IPA workflow, and the app target's -# first build phase (EmbeddedLinux/build-engine-for-xcode.sh), which runs on every -# Xcode build. A cold engine build takes minutes; the stamp makes the common case -# a couple of file reads. -# -# XFORGE_REBUILD_ENGINE=1 forces the build. So does deleting the lib directory. -# --------------------------------------------------------------------------- -stamp_value() { - local revision script_hash project_hash - revision="$(git -C "$ISH" rev-parse HEAD 2>/dev/null \ - || git -C "$REPO" rev-parse HEAD:Vendor/ish-arm64 2>/dev/null \ - || echo unknown)" - script_hash="$(shasum -a 256 "$HERE/build-ish-core.sh" | cut -d' ' -f1)" - project_hash="$(shasum -a 256 "$REPO/project.yml" 2>/dev/null | cut -d' ' -f1)" - printf '%s %s %s %s %s' "$revision" "$script_hash" "${project_hash:-none}" "$GUEST_ARCH" "$MIN_SDK" -} - -STAMP="$OUT/.engine-stamp" -WANT_STAMP="$(stamp_value)" -if [ "${XFORGE_REBUILD_ENGINE:-0}" != "1" ] && [ -f "$STAMP" ] \ - && [ "$(cat "$STAMP")" = "$WANT_STAMP" ]; then - complete=1 - for lib in libish.a libish_emu.a libfakefs.a; do - [ -f "$OUT/$lib" ] || complete="" - done - if [ -n "$complete" ]; then - log "The engine is already built ($OUT) — nothing to do" - note "engine revision: $(printf '%s' "$WANT_STAMP" | cut -d' ' -f1)" - note "force a rebuild with XFORGE_REBUILD_ENGINE=1" - exit 0 - fi - note "the stamp matches but libraries are missing — rebuilding" -fi - -# The VDSO step compiles a 32-bit i386-linux object with `-fuse-ld=lld`, which -# Apple's clang cannot do; it needs an LLVM clang with lld. Put Homebrew's LLVM -# first and make sure ld.lld is reachable (newer `llvm` bottles ship clang but -# not ld.lld — the separate `lld` formula provides it). -if command -v brew >/dev/null 2>&1; then - LLVM_BIN="$(brew --prefix llvm 2>/dev/null)/bin" - [ -d "$LLVM_BIN" ] && { export PATH="$LLVM_BIN:$PATH"; log "Using Homebrew LLVM: $LLVM_BIN"; } - LLD_BIN="$(brew --prefix lld 2>/dev/null)/bin" - [ -d "$LLD_BIN" ] && export PATH="$LLD_BIN:$PATH" -fi -if ! clang -target i386-linux -fuse-ld=lld -shared -nostdlib -x c /dev/null -o /dev/null 2>/dev/null; then - printf 'warning: clang cannot build the VDSO (needs LLVM + lld). Install: brew install llvm lld\n' >&2 -fi - -SDK="$(xcrun --sdk iphoneos --show-sdk-path)" -TRIPLE="arm64-apple-ios${MIN_SDK}" -MESON_BUILD="$BUILD/meson" - -rm -rf "$MESON_BUILD" -mkdir -p "$MESON_BUILD" "$OUT" - -# --- meson cross file for iOS ------------------------------------------------ -CROSS="$MESON_BUILD/ios-arm64.txt" -cat > "$CROSS" < "$MESON_BUILD/req-probe.S" 2>/dev/null; then - die "could not write the assembler capability probe" -fi -if ! clang -c "$MESON_BUILD/req-probe.S" -o "$MESON_BUILD/req-probe.o" 2>/dev/null; then - rm -f "$MESON_BUILD/req-probe.S" "$MESON_BUILD/req-probe.o" - die "this clang cannot assemble the engine's aarch64 gadgets (no .req register - alias support). Build the engine on macOS with Xcode's clang — the IPA - workflow does exactly that — rather than with a GNU binutils toolchain." -fi -rm -f "$MESON_BUILD/req-probe.S" "$MESON_BUILD/req-probe.o" - -log "Building libish / libish_emu / libfakefs" -ninja -C "$MESON_BUILD" libish.a libish_emu.a libfakefs.a - -# The VDSO is not a separate artifact to ship: kernel/vdso.c `.incbin`s -# libvdso.so.elf straight into libish.a, so building libish already built it. -# But the build degrades silently — if meson cannot find an aarch64-linux -# cross-compiler it substitutes a target that just `touch`es an empty file, and -# the guest then gets a bogus VDSO with nothing in the log to say so. Verify the -# file is a real aarch64 ELF shared object rather than trusting the build. -VDSO="$MESON_BUILD/vdso/arm64/libvdso.so.elf" -if [ ! -s "$VDSO" ]; then - die "the arm64 VDSO was not built: $VDSO is missing or empty. - meson falls back to an EMPTY placeholder when it cannot find an - aarch64-linux cross-compiler, so this is a silent failure otherwise. - Install one (brew install llvm lld) — clang must support - '-target aarch64-linux-gnu -fuse-ld=lld'." -fi -if command -v file >/dev/null 2>&1; then - file "$VDSO" | grep -q 'ELF 64-bit.*aarch64' || die \ - "$VDSO is not an aarch64 ELF (got: $(file -b "$VDSO"))" -fi -note "VDSO: $(du -h "$VDSO" | cut -f1) (embedded into libish.a)" - -for lib in libish.a libish_emu.a libfakefs.a; do - [ -f "$MESON_BUILD/$lib" ] || die "$lib was not produced" - cp "$MESON_BUILD/$lib" "$OUT/$lib" -done - -log "Staged into $OUT" -ls -lh "$OUT" - -# Record what was built, so the next run (Xcode's build phase runs this on every -# build) can tell in a moment that there is nothing to do. -printf '%s\n' "$WANT_STAMP" > "$STAMP" diff --git a/EmbeddedLinux/build-rootfs.sh b/EmbeddedLinux/build-rootfs.sh deleted file mode 100644 index 6f37f4f..0000000 --- a/EmbeddedLinux/build-rootfs.sh +++ /dev/null @@ -1,973 +0,0 @@ -#!/usr/bin/env bash -# -# build-rootfs.sh — build the Alpine aarch64 root filesystem that XForge boots, -# and pack it as a ZIP of an already-converted fakefs root. -# -# This follows OpenMinis's own recipe (deps/prepare_alpine_rootfs.sh in -# github.com/OpenMinis/OpenMinis), which is the reference implementation for -# this engine: -# -# 1. download the *plain* Alpine aarch64 minirootfs -# 2. build the engine's own `tools/fakefsify` for the build machine -# (NOT cross-compiled: it runs here, on this host) -# 3. unpack it as a plain tree, install the glibc layer into it, and install the -# packages the console session needs (bash/coreutils/less/terminfo) -# 4. provision the build toolchain into it — xtool, swiftly, Swift and the -# Darwin Swift SDK — by running EmbeddedLinux/install-toolchain.sh in the -# chroot, so the guest arrives ready to build iOS apps -# 5. configure the root in place (passwd, profile, motd, inittab, DNS, apk repos) -# — including the console program, EmbeddedLinux/xforge-login -# 6. convert the configured tree to fakefs — a `data/` tree + `meta.db` -# 7. `zip -r` the result, excluding the SQLite WAL/SHM sidecars -# -# Why a ZIP of a fakefs and not a tarball the app imports at runtime: the -# conversion is the expensive part (thousands of files into SQLite), and doing it -# here means the app's first launch is a plain unzip — no multi-minute import on -# a phone. This is the same reason OpenMinis ships `alpine-rootfs.zip` and its -# app only calls unzip + mount_root. -# -# What is *in* the root is XFORGE_PROVISION's decision, and it is the whole -# difference between a small root and a complete one: -# -# XFORGE_PROVISION=all plain Alpine plus the whole build toolchain: -# xtool, swiftly, the Swift toolchain and the Darwin -# Swift SDK, all installed before the conversion and -# therefore visible to the engine. The app then boots -# into a guest that can build an iOS app immediately, -# at the cost of ~1.6 GB of artifact. -# XFORGE_PROVISION=none plain Alpine only. The guest installs the toolchain -# itself on demand (`sh /root/install-toolchain.sh -# all`), which is what a few-MB root buys. -# -# Both are the same script, the same installer and the same layout: this decides -# only *when* install-toolchain.sh runs, on the build machine or in the guest. The -# small root exists so the choice is available; the provisioned one is what the -# release ships. -# -# Output: -# dist/rootfs/alpine-rootfs.zip the root (data/ + meta.db) -# dist/rootfs/alpine-rootfs.sha256 checksum, for pinning -# -# Usage: -# EmbeddedLinux/build-rootfs.sh [output-dir] # default: dist/rootfs -# -# Environment: -# XFORGE_ALPINE_VERSION Alpine release (default: 3.21, matching the -# reference; the engine is built for this guest) -# XFORGE_ALPINE_MINOR minor version (default: 0) -# XFORGE_ISH_ROOT engine checkout (default: /Vendor/ish-arm64) -# XFORGE_WORK_DIR scratch dir (default: /.rootfs-work) -# XFORGE_KEEP_WORK 1 to keep the scratch dir -# XFORGE_SKIP_GLIBC 1 to build a root without the glibc layer -# XFORGE_PROVISION all (default) to bake the build toolchain into the -# root; none to ship plain Alpine and let the guest -# install it. See above. -# XFORGE_PROVISION_SDK 1 (default) to include the Darwin Swift SDK in a -# provisioned root; 0 to stop after the Swift -# toolchain. The SDK is ~400 MB of download and -# ~1.3 GB installed, so this is the size lever. -# XFORGE_DARWIN_SDK_TAG release tag the SDK comes from (default: -# darwin-sdk-7). Pinned, not resolved: the published -# root has to name the SDK it contains. -# XFORGE_DARWIN_SDK_URL the SDK asset itself, overriding repo/tag/asset -# XFORGE_DARWIN_SDK_SHA256 -# refuse to use a downloaded SDK with any other hash -# XFORGE_SLIM_TOOLCHAIN 1 (default) to drop the parts of the Swift -# toolchain an iOS build never loads (the static -# Linux stdlib, lldb, the editor tooling). The -# provisioning verification runs *after* this, so a -# toolchain that stopped working fails the build. -# XFORGE_DEFAULT_SHELL login shell root's console starts (default: -# /bin/bash). Must be a shell the package list below -# installs; if it is not, the build falls back to -# /bin/sh and says so. -# XFORGE_CONSOLE_PACKAGES Alpine packages baked into the root for the console -# session (default: bash coreutils less -# ncurses-terminfo). Space-separated. -# -set -euo pipefail - -HERE="$(cd "$(dirname "$0")" && pwd)" -REPO="$(cd "$HERE/.." && pwd)" - -# Resolve the output directory to an absolute path up front. Packing happens -# from inside the work directory (so the ZIP holds relative paths), and a -# relative OUT_DIR would then resolve against the wrong directory — which is -# exactly how a CI run failed with "Could not create output file" while the same -# script worked locally with an absolute path. -OUT_DIR="${1:-$REPO/dist/rootfs}" -mkdir -p "$OUT_DIR" -OUT_DIR="$(cd "$OUT_DIR" && pwd)" -ALPINE_VERSION="${XFORGE_ALPINE_VERSION:-3.21}" -ALPINE_MINOR="${XFORGE_ALPINE_MINOR:-0}" -ALPINE_ARCH="aarch64" -ALPINE_MIRROR="https://dl-cdn.alpinelinux.org/alpine" -ISH="${XFORGE_ISH_ROOT:-$REPO/Vendor/ish-arm64}" -WORK="${XFORGE_WORK_DIR:-$REPO/.rootfs-work}" - -# The guest's console session, and what it needs from the root. -# -# XFORGE_DEFAULT_SHELL is the shell root's console starts in: it is written into -# /etc/passwd, and /etc/inittab runs xforge-login, which reads that field — so -# this is the one place the shell is chosen, and the guest can change it at -# runtime (`apk add zsh` + the passwd field). -# -# The package list is the shell's own environment, and every entry earns its -# place: -# bash the default shell (pulls readline, ncurses-libs) -# coreutils the tools a shell session expects to exist -# less a pager for them -# ncurses-terminfo terminfo entries — xterm/xterm-256color are NOT in -# ncurses-terminfo-base, and /etc/profile exports -# TERM=xterm-256color, so without this every curses -# program (less, top, vim) runs against an unknown -# terminal -DEFAULT_SHELL="${XFORGE_DEFAULT_SHELL:-/bin/bash}" -CONSOLE_PACKAGES="${XFORGE_CONSOLE_PACKAGES:-bash coreutils less ncurses-terminfo}" - -# Whether the build toolchain is provisioned into the root (see the header). The -# guest's installer is the only implementation of that, and it is run here in a -# chroot of the tree, so a provisioned root and a hand-provisioned guest differ -# in nothing but when the work happened. -PROVISION="${XFORGE_PROVISION:-all}" -PROVISION_SDK="${XFORGE_PROVISION_SDK:-1}" -SLIM_TOOLCHAIN="${XFORGE_SLIM_TOOLCHAIN:-1}" - -# The Darwin Swift SDK: which release it comes from, and what the asset is called. -# Kept in step with the app's XForgeReleases (the Toolchain screen downloads the -# same asset) — the tag is pinned here, not resolved through the API, because the -# published root has to name the exact SDK a device is booting. -DARWIN_SDK_REPO="${XFORGE_DARWIN_SDK_REPO:-R0GUEEE/XForge}" -DARWIN_SDK_TAG="${XFORGE_DARWIN_SDK_TAG:-darwin-sdk-7}" -DARWIN_SDK_ASSET="${XFORGE_DARWIN_SDK_ASSET:-darwin.artifactbundle.zip}" -DARWIN_SDK_URL="${XFORGE_DARWIN_SDK_URL:-https://github.com/$DARWIN_SDK_REPO/releases/download/$DARWIN_SDK_TAG/$DARWIN_SDK_ASSET}" -DARWIN_SDK_SHA256="${XFORGE_DARWIN_SDK_SHA256:-}" - -ROOTFS_NAME="alpine-rootfs" -ZIP_NAME="$ROOTFS_NAME.zip" - -log() { printf '\n==> %s\n' "$*"; } -note() { printf ' %s\n' "$*"; } -die() { printf '\nerror: %s\n' "$*" >&2; exit 1; } - -# Size of a tree in KiB, as a number, always. -# -# Measurement must not be able to fail the build, and it must not be able to -# confuse itself either. `du` over a tree with /proc or /dev mounted inside it -# walks entries that vanish mid-traversal — the guest's own mounts are still up -# when this runs — so it can exit non-zero after printing a perfectly good number; -# and a `|| echo 0` on top of that hands the caller two lines, which arithmetic -# then reads as "6256892\n0: syntax error in expression". Taking the first field -# of whatever was printed is what makes both impossible. -size_kib() { - { du -sk "$1" 2>/dev/null || true; } | awk 'NF { print $1; exit }' -} - -# --------------------------------------------------------------------------- -# Running things *inside* the root being built -# -# Two steps need this: installing the glibc layer (step 4) and installing the -# guest's own packages (step 5). Both run the guest's own apk in a chroot of the -# tree, which needs the kernel's /proc and a /dev with a working /dev/null — apk -# opens it, and so does every shell — plus /etc/resolv.conf, because the -# minirootfs ships no nameservers at all and every one of those commands -# downloads something. -# -# The mounts are recorded so the exit trap can undo them: a failed build must not -# leave a mount behind, or the next run's `rm -rf $WORK` walks into a live /proc. -# --------------------------------------------------------------------------- -GUEST_MOUNTS="" - -guest_mount() { - [ "$(id -u)" -eq 0 ] || die "building the root needs root: the guest's own - installer and apk are run in a chroot of the tree being built, and a chroot - needs mounts. Re-run with sudo." - - mount -t proc none "$DATA/proc" || die "could not mount /proc in $DATA" - GUEST_MOUNTS="$DATA/proc" - mount --rbind /dev "$DATA/dev" 2>/dev/null || true - GUEST_MOUNTS="$DATA/dev $GUEST_MOUNTS" - mount --rbind /sys "$DATA/sys" 2>/dev/null || true - GUEST_MOUNTS="$DATA/sys $GUEST_MOUNTS" - - # Name resolution for anything the chroot runs: the minirootfs ships no - # nameservers at all, and both steps below download (apk indexes, packages). - # - # Written with awk rather than `grep … | sed …`, and with the fallback keyed on - # the *result* rather than on the host file's contents: a pipeline whose output - # is redirected leaves an empty file behind if it produces nothing, and a - # truncated /etc/resolv.conf inside the chroot turns a network failure into a - # name-resolution failure three steps away. (Seen for real: iSH wedged the - # pipeline and the root being built got a 0-byte resolv.conf.) - awk '/^nameserver/ { print; if (++n == 3) exit }' /etc/resolv.conf \ - > "$DATA/etc/resolv.conf" 2>/dev/null || true - if ! [ -s "$DATA/etc/resolv.conf" ]; then - printf 'nameserver 1.1.1.1\nnameserver 8.8.8.8\n' > "$DATA/etc/resolv.conf" - fi -} - -guest_umount() { - for m in $GUEST_MOUNTS; do - umount -l "$m" 2>/dev/null || true - done - GUEST_MOUNTS="" -} - -# The environment every chrooted step starts from. -# -# HOME is what makes swiftly and SwiftPM install into /root — the tree's own -# /root, inside the root being built — and the XDG directories are *unset* on top -# of it, because SwiftPM's SDK store follows them: with the runner's -# XDG_CONFIG_HOME inherited, `swift sdk install` put the SDK in -# /swiftpm/swift-sdks inside the root being built, where the -# guest's own SwiftPM (which has no XDG set) would never look for it. Both the -# provisioning steps and the verification use this, so they agree on where things -# are. -GUEST_ENV="unset XDG_CONFIG_HOME XDG_DATA_HOME XDG_CACHE_HOME XDG_STATE_HOME; \ - export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin \ - HOME=/root SWIFTLY_HOME_DIR=/root/.local/share/swiftly" - -cleanup() { - local status=$? - guest_umount - if [ "$status" -ne 0 ]; then - printf '\nFailed (exit %d). Work directory kept at %s\n' "$status" "$WORK" >&2 - elif [ "${XFORGE_KEEP_WORK:-0}" != "1" ]; then - rm -rf "$WORK" - fi - return $status -} -trap cleanup EXIT - -# --------------------------------------------------------------------------- -# Preconditions -# --------------------------------------------------------------------------- -[ -f "$ISH/meson.build" ] || die "the engine is not checked out at $ISH - run: git submodule update --init --depth 1 Vendor/ish-arm64" - -# fakefsify needs libarchive; it is a submodule of the engine. -[ -d "$ISH/deps/libarchive" ] && [ -n "$(ls -A "$ISH/deps/libarchive" 2>/dev/null)" ] || die \ - "the engine's deps/libarchive submodule is missing (fakefsify needs it) - run: git -C Vendor/ish-arm64 submodule update --init --depth 1 deps/libarchive" - -case "$PROVISION" in - all|none) ;; - *) die "XFORGE_PROVISION must be 'all' or 'none', not '$PROVISION'" ;; -esac - -for tool in curl meson ninja python3 zip; do - command -v "$tool" >/dev/null 2>&1 || die "$tool is required" -done -if [ "$PROVISION" = "all" ] && [ "$PROVISION_SDK" = "1" ]; then - for tool in unzip sha256sum; do - command -v "$tool" >/dev/null 2>&1 \ - || die "$tool is required to stage the Darwin Swift SDK" - done -fi - -mkdir -p "$WORK" - -# --------------------------------------------------------------------------- -# 1. Download the plain minirootfs -# --------------------------------------------------------------------------- -ROOTFS_FILE="alpine-minirootfs-${ALPINE_VERSION}.${ALPINE_MINOR}-${ALPINE_ARCH}.tar.gz" -ROOTFS_PATH="$WORK/$ROOTFS_FILE" -ROOTFS_URL="$ALPINE_MIRROR/v${ALPINE_VERSION}/releases/${ALPINE_ARCH}/${ROOTFS_FILE}" - -log "Fetching $ROOTFS_FILE" -if [ -f "$ROOTFS_PATH" ] && tar -tzf "$ROOTFS_PATH" >/dev/null 2>&1; then - note "using the cached copy" -else - curl -fL --retry 3 --retry-delay 2 -o "$ROOTFS_PATH.partial" "$ROOTFS_URL" - mv "$ROOTFS_PATH.partial" "$ROOTFS_PATH" -fi -tar -tzf "$ROOTFS_PATH" >/dev/null || die "$ROOTFS_FILE is not a readable .tar.gz" -note "$ROOTFS_FILE: $(du -h "$ROOTFS_PATH" | cut -f1)" - -# --------------------------------------------------------------------------- -# 2. Build fakefsify for the build machine -# -# A *native* build, deliberately: this tool only ever runs here, never on a -# device, and it is what converts the tarball into fakefs. Note the absent -# `-Db_ndebug` — unlike the device build, asserts stay ENABLED, so a broken -# assumption fails loudly at build time instead of quietly producing a wrong -# rootfs. -# --------------------------------------------------------------------------- -FAKEFSIFY_BUILD="$WORK/build-native" -FAKEFSIFY="$FAKEFSIFY_BUILD/tools/fakefsify" - -log "Building tools/fakefsify for this machine" -if [ -x "$FAKEFSIFY" ]; then - note "already built" -else - if [ ! -f "$FAKEFSIFY_BUILD/build.ninja" ]; then - meson setup "$FAKEFSIFY_BUILD" "$ISH" \ - --buildtype=release \ - -Dlog='' \ - -Dkernel=ish \ - -Dengine=asbestos \ - -Dguest_arch=arm64 - fi - ninja -C "$FAKEFSIFY_BUILD" tools/fakefsify -fi -[ -x "$FAKEFSIFY" ] || die "fakefsify was not produced at $FAKEFSIFY" -note "$FAKEFSIFY" - -# --------------------------------------------------------------------------- -# 3. Unpack the plain rootfs we will shape -# -# Everything happens on this tree — configuration and the glibc layer both — -# and it is converted to fakefs exactly once, at the end. That ordering is not -# cosmetic: `fakefsify` builds `meta.db` as an index of the files present when -# it runs, and the engine reads *the database*, not the directory. Files copied -# into `data/` after conversion exist on disk and are invisible in the guest — -# a first attempt installed the glibc layer that way and the layer did not exist -# as far as the engine was concerned (568 files on disk, 0 rows in meta.db). -# --------------------------------------------------------------------------- -DATA="$WORK/rootfs-tree" - -log "Unpacking $ROOTFS_FILE into the root tree" -rm -rf "$DATA" -mkdir -p "$DATA" -tar -xzf "$ROOTFS_PATH" -C "$DATA" -[ -x "$DATA/bin/sh" ] || die "the unpacked root has no /bin/sh" - -# Mount points the guest expects. The engine mounts /proc and /dev/pts itself; -# these directories have to exist first. -for dir in dev proc sys tmp run root home; do - mkdir -p "$DATA/$dir" -done - -# --------------------------------------------------------------------------- -# 4. Install the glibc compatibility layer -# -# Alpine is musl-based, but every tool XForge builds with is a *glibc* binary: -# xtool is a Swift program built on Ubuntu, and the Swift toolchain is too. -# `gcompat` is not enough for them (they die on strptime_l, fts_*, fcntl64 …), so -# the layer takes Ubuntu's own glibc and its libraries and points the loader at -# them. -# -# Baking it in rather than letting the guest install it removes the most -# failure-prone step of an on-device provision: a package renamed between Ubuntu -# releases produces a layer that loads but cannot resolve a symbol, which -# surfaces much later as a tool dying on an undefined symbol. It costs ~95 MB -# compressed, so the root stays small. -# -# This runs the *same* `install-toolchain.sh glibc` step the guest would run, in -# a chroot of this very tree, so there is only ever one implementation of the -# layer. The chroot is what makes it work: the step unpacks Ubuntu packages with -# `ar`/`zstd` and writes to /lib, /usr/lib and /opt/glibc, and those paths have -# to mean the guest root. -# --------------------------------------------------------------------------- -if [ "${XFORGE_SKIP_GLIBC:-0}" = "1" ]; then - log "Skipping the glibc layer (XFORGE_SKIP_GLIBC=1)" -else - log "Installing the glibc compatibility layer (this is the slow part)" - - # The step runs the guest's installer in a chroot, which needs /proc, /dev and - # DNS. guest_mount() sets those up (and refuses to run unless we are root: - # otherwise this surfaces as "mount: must be superuser to use mount" from - # somewhere in the middle of a long install). - guest_mount - - # The Alpine-side prerequisites for the step itself (it fetches packages with - # curl and unpacks them with ar/zstd), and its copy of the installer. - install -m 0755 "$HERE/install-toolchain.sh" "$DATA/root/install-toolchain.sh" - chroot "$DATA" /bin/sh -c \ - "apk add --no-cache curl tar xz zstd binutils ca-certificates" \ - || die "could not install the glibc step's own prerequisites in the chroot" - - chroot "$DATA" /bin/sh -c \ - "export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin HOME=/root; \ - sh /root/install-toolchain.sh glibc" \ - || die "installing the glibc layer in the chroot failed" - - # The layer is only useful if the guest can resolve it, so check the entry - # points a Swift or xtool binary loads rather than trusting the step's status. - # - # `-L` rather than `-e` on the symlinks, deliberately: they are absolute and - # point into the *guest's* /opt/glibc, which does not exist on the build - # machine, so `-e` (which follows the link) reports them missing here even - # though they resolve perfectly inside the guest. What has to exist at build - # time is the link itself. - for required in \ - opt/glibc/usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1 \ - usr/local/share/xforge/glibc.env; do - [ -e "$DATA/$required" ] || die "the glibc layer is incomplete: $required is missing" - done - for link in lib/ld-linux-aarch64.so.1 lib/aarch64-linux-gnu usr/lib/aarch64-linux-gnu; do - [ -L "$DATA/$link" ] || die "the glibc layer is not wired in: $link is missing" - done - - guest_umount - - # The layer's own prerequisite packages are no longer needed: they were - # installed to run the step, not to run the guest. - chroot "$DATA" /bin/sh -c "apk del --no-cache zstd binutils" >/dev/null 2>&1 || true - - note "glibc layer installed at /opt/glibc" -fi - -# --------------------------------------------------------------------------- -# 5. Install the guest's own packages -# -# The shell session XForge's Terminal *is* has dependencies, and they belong in -# the root rather than in an on-device install: the console is the first thing a -# new install shows, and a shell whose pager is missing (or whose TERM is unknown -# to curses) is a broken-looking guest, not a hint to run `apk add`. -# -# Installed with the guest's own apk, in a chroot of this tree, so the packages -# land in the root's own database (/lib/apk/db) and `apk del`/`apk upgrade` in the -# guest stay consistent with them. This is also why it runs *before* the fakefs -# conversion below: the conversion indexes the tree, and anything installed -# afterwards would be on disk and invisible to the engine. -# --------------------------------------------------------------------------- -log "Installing the guest's console packages: $CONSOLE_PACKAGES" -guest_mount -chroot "$DATA" /bin/sh -c "apk add --no-cache $CONSOLE_PACKAGES" \ - || die "installing the guest's own packages in the chroot failed" -guest_umount - -for package in $CONSOLE_PACKAGES; do - chroot "$DATA" /bin/sh -c "apk info -e $package" >/dev/null 2>&1 \ - || die "$package is not installed in the built root" -done -note "$(chroot "$DATA" /bin/sh -c 'apk info 2>/dev/null | wc -l' | tr -d ' ') packages in the root" - -# --------------------------------------------------------------------------- -# 5b. Provision the build toolchain into the root -# -# This is the difference between a root that can build an iOS app and one that -# first has to provision itself. The work is the guest's own installer, run in a -# chroot of this very tree — the same steps, the same order, the same result a -# user would get from `sh /root/install-toolchain.sh all` in the Terminal, except -# that it happens on a build machine with a fast network and a real CPU, once, -# instead of on a phone under emulation. -# -# Order matters and is not the installer's own: -# * glibc before anything else (xtool and the Swift toolchain are glibc -# binaries), which the build already did in step 4 — the step is idempotent -# and returns immediately; -# * `deps` before the tools that need it (xtool unpacks with `tar`, the SDK -# installs with `unzip`); -# * the SDK *after* Swift, because `swift sdk install` is a Swift program; -# * the slimming pass after the SDK and *before* verify, so what the -# verification checks is what ships; -# * verify last, and its verdict is load-bearing (see below). -# -# Everything is installed before the fakefs conversion in step 7: the conversion -# indexes the tree, and anything added afterwards would be on disk and invisible -# to the engine. That is not a detail — it is how a previous attempt shipped a -# glibc layer the guest could not see. -# --------------------------------------------------------------------------- -SDK_SHA256="" -if [ "$PROVISION" = "none" ]; then - log "Skipping the build toolchain (XFORGE_PROVISION=none)" - note "the root will provision itself: sh /root/install-toolchain.sh all" -else - [ "${XFORGE_SKIP_GLIBC:-0}" != "1" ] || die \ - "XFORGE_PROVISION=all needs the glibc layer (xtool and the Swift - toolchain are glibc binaries), but XFORGE_SKIP_GLIBC=1. Drop one of the two." - - log "Provisioning the build toolchain into the root (long: ~2 GB of downloads)" - - install -m 0755 "$HERE/install-toolchain.sh" "$DATA/root/install-toolchain.sh" - - # The Darwin SDK is the largest download by far, and it is fetched *here* - # rather than inside the chroot: on the build machine a failure is a line in - # the log, the archive is cached between runs, and the hash of what went in - # can be written into the published root's manifest — which is otherwise the - # only way to answer "which SDK is in this root" from the artifact alone. - SDK_GUEST_PATH="" - if [ "$PROVISION_SDK" = "1" ]; then - SDK_ARCHIVE="$WORK/$DARWIN_SDK_ASSET" - log "Fetching the Darwin Swift SDK: $DARWIN_SDK_TAG/$DARWIN_SDK_ASSET" - if [ -s "$SDK_ARCHIVE" ]; then - note "using the cached copy" - else - rm -f "$SDK_ARCHIVE.partial" - curl -fL --retry 3 --retry-delay 2 --no-progress-meter \ - -o "$SDK_ARCHIVE.partial" "$DARWIN_SDK_URL" \ - || die "could not download $DARWIN_SDK_URL - (set XFORGE_DARWIN_SDK_URL, or XFORGE_PROVISION_SDK=0 to build a root - with the Swift toolchain but no Darwin SDK)" - mv "$SDK_ARCHIVE.partial" "$SDK_ARCHIVE" - fi - [ -s "$SDK_ARCHIVE" ] || die "$SDK_ARCHIVE is empty" - SDK_SHA256="$(sha256sum "$SDK_ARCHIVE" | cut -d' ' -f1)" - if [ -n "$DARWIN_SDK_SHA256" ] && [ "$DARWIN_SDK_SHA256" != "$SDK_SHA256" ]; then - die "the SDK archive does not match XFORGE_DARWIN_SDK_SHA256 - expected $DARWIN_SDK_SHA256 - got $SDK_SHA256 - (remove $SDK_ARCHIVE to download it again)" - fi - note "$DARWIN_SDK_ASSET: $(du -h "$SDK_ARCHIVE" | cut -f1), sha256 $SDK_SHA256" - # Staged inside the tree, where the chroot can see it. The sdk step - # removes it again once SwiftPM has the SDK. - SDK_GUEST_PATH="/root/.cache/xforge-sdk/$DARWIN_SDK_ASSET" - mkdir -p "$DATA/root/.cache/xforge-sdk" - install -m 0644 "$SDK_ARCHIVE" "$DATA$SDK_GUEST_PATH" - else - note "the Darwin SDK is not included (XFORGE_PROVISION_SDK=0)" - fi - - guest_mount - - # One guest process per provisioning step, through the guest's own installer. - # - # `HOME=/root` is what makes swiftly and SwiftPM install into /root (the - # tree's own /root, so it is inside the root being built) rather than into - # whatever the build machine's environment has in mind — and the XDG - # directories are *unset* on top of it, because SwiftPM's SDK store follows - # them: with the runner's XDG_CONFIG_HOME inherited, `swift sdk install` put - # the SDK in /swiftpm/swift-sdks inside the root being built, - # where the guest's own SwiftPM (which has no XDG set) would never look. - provision() { - local step="$1" - shift - note "install-toolchain.sh $step" - chroot "$DATA" /bin/sh -c \ - "$GUEST_ENV $*; sh /root/install-toolchain.sh $step" \ - || die "the '$step' provisioning step failed in the chroot" - } - - provision deps - provision glibc - provision xtool - provision swiftly - provision swift - if [ -n "$SDK_GUEST_PATH" ]; then - provision sdk \ - "XFORGE_DARWIN_SDK_ARCHIVE=$SDK_GUEST_PATH" \ - "XFORGE_DARWIN_SDK_SHA256=$SDK_SHA256" \ - "XFORGE_DARWIN_SDK_TAG=$DARWIN_SDK_TAG" \ - "XFORGE_DARWIN_SDK_ASSET=$DARWIN_SDK_ASSET" \ - "XFORGE_DARWIN_SDK_URL=$DARWIN_SDK_URL" - fi - - # ----------------------------------------------------------------------- - # Slimming. - # - # Everything removed here is either a download cache (the archive just - # installed, the Ubuntu .deb pile the glibc layer was built from, the apk - # index) or a part of the Swift toolchain an iOS build never loads: the - # *static Linux* stdlib, the debugger and the editor tooling. Nothing here is - # guesswork in the sense that matters — the verification below runs after it, - # so a toolchain this broke would fail the build rather than a user's first - # build on a device. - # ----------------------------------------------------------------------- - log "Slimming the root" - before_kib="$(size_kib "$DATA")" - before_kib="${before_kib:-0}" - - rm -rf "$DATA/var/cache/apk" "$DATA/tmp/xforge-glibc" "$DATA/tmp/xforge-silent."* \ - "$DATA/root/.cache/xforge-sdk" - # /tmp is scratch, but the directory itself stays: apk, the shells and the - # engine's own tools all expect it to exist. - find "$DATA/tmp" -mindepth 1 -maxdepth 1 -exec rm -rf {} + 2>/dev/null || true - - if [ "$SLIM_TOOLCHAIN" = "1" ]; then - removed=0 - for toolchain in "$DATA"/root/.local/share/swiftly/toolchains/*/; do - [ -d "$toolchain" ] || continue - note "slim: $(basename "$toolchain")" - # The static Linux stdlib: only linked when a *Linux* program asks to - # be statically linked against it. xtool builds iOS binaries, whose - # stdlib comes from the Darwin SDK. - rm -rf "${toolchain}usr/lib/swift_static" - # The debugger and the editor tooling: lldb, sourcekit-lsp and the - # index stores exist to serve an IDE, not to compile. - # NOT usr/lib/swift/host/plugins: those are the host-side macro - # plugin libraries, and a package that uses macros needs them. They - # are a few MB, and the compile probe below would not notice their - # absence — a build on a device would. - rm -rf "${toolchain}usr/lib/python3"* \ - "${toolchain}usr/share/doc" "${toolchain}usr/share/man" 2>/dev/null || true - for path in \ - usr/bin/lldb usr/bin/lldb-server usr/bin/sourcekit-lsp \ - usr/lib/liblldb.so usr/lib/libsourcekitdInProc.so \ - usr/lib/libIndexStore.so ; do - rm -rf "${toolchain}${path}"* - done - removed=1 - done - [ "$removed" = "1" ] || note "slim: no toolchain directory found to slim" - else - note "slim: keeping the whole Swift toolchain (XFORGE_SLIM_TOOLCHAIN=0)" - fi - - after_kib="$(size_kib "$DATA")" - after_kib="${after_kib:-0}" - note "tree: $((before_kib / 1024)) MB before slimming, $((after_kib / 1024)) MB after" - - # Verification, and this time the verdict is load-bearing. - # - # install-toolchain.sh verify reports rather than fails (its job in the - # Terminal is to explain), so its XFORGE-VERIFY lines are read here: a tool - # that is missing or not running means the root being packaged cannot do the - # thing it exists to do, and a published artifact is the wrong place to find - # that out. XFORGE_VERIFY_COMPILE adds the check that actually matters — that - # the toolchain can compile and run a program, not just print a version. - log "Verifying the provisioned toolchain" - verify_log="$WORK/verify-toolchain.log" - chroot "$DATA" /bin/sh -c \ - "$GUEST_ENV XFORGE_VERIFY_COMPILE=${XFORGE_VERIFY_COMPILE:-1}; \ - sh /root/install-toolchain.sh verify 2>&1" | tee "$verify_log" - if grep -qE 'XFORGE-VERIFY (swift|swiftly|xtool|swift-sdk|swift-compile) (missing|broken)' \ - "$verify_log"; then - die "the provisioned toolchain does not work in the root being built — - see the XFORGE-VERIFY lines above. Packaging it would ship a guest that - cannot build anything." - fi - grep -q 'XFORGE-VERIFY swift ok' "$verify_log" \ - || die "the Swift toolchain did not report ok — see $verify_log" - - guest_umount - - note "toolchain provisioned: xtool, swiftly, swift${SDK_SHA256:+, darwin SDK}" -fi - -# --------------------------------------------------------------------------- -# 6. Configure the root -# -# A bare Alpine minirootfs is not quite bootable as XForge's guest: it has no -# mount points, no DNS, no apk repositories, and root may not have a usable -# shell. These are the same adjustments the reference makes. -# --------------------------------------------------------------------------- -log "Configuring the root" - -# The shell the console starts is a property of /etc/passwd — /sbin/xforge-login -# reads this field and execs what it names, so this one line is the whole -# "default shell" setting. `chsh` is not needed and is not in the root; editing -# the field (or `apk add shadow` for chsh) is the way to change it in the guest. -if [ -f "$DATA/etc/passwd" ]; then - if [ -x "$DATA$DEFAULT_SHELL" ]; then - sed -i "s|^root:.*|root:x:0:0:root:/root:$DEFAULT_SHELL|" "$DATA/etc/passwd" - else - # Not a hard failure: a root without the shell it promised is worse than a - # root that boots into the one it certainly has. Say which it is, loudly. - printf 'warning: %s is not installed in the root (add its package to\n' "$DEFAULT_SHELL" >&2 - printf ' XFORGE_CONSOLE_PACKAGES, or set XFORGE_DEFAULT_SHELL)\n' >&2 - printf ' — falling back to /bin/sh\n' >&2 - DEFAULT_SHELL=/bin/sh - sed -i 's|^root:.*|root:x:0:0:root:/root:/bin/sh|' "$DATA/etc/passwd" - fi -fi -note "root's login shell: $DEFAULT_SHELL" - -# What the console says about the toolchain, which is the one thing a user -# notices first: whether the guest can build anything yet, or whether it has to -# provision itself. It follows XFORGE_PROVISION rather than describing one of the -# two states as if it were the only one. -if [ "$PROVISION" = "all" ]; then - MOTD_TOOLCHAIN=" The build toolchain is installed: xtool, the Swift toolchain - and the Darwin SDK are ready to use. \`xtool --version\` and - \`swift sdk list\` answer from this shell with no setup." -else - MOTD_TOOLCHAIN=" Swift and xtool are not installed yet. Install them in the guest with: - sh /root/install-toolchain.sh all" -fi - -# /etc/profile: the non-interactive environment is built by XForge's exec layer, -# but an interactive shell (the terminal tab) reads this. -cat >> "$DATA/etc/profile" <<'EOF' - -# XForge guest configuration -export PS1='\u@xforge:\w\$ ' -export TERM=xterm-256color -export HOME=/root -export LANG=C.UTF-8 -export CHARSET=UTF-8 -export PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/bin - -alias ll='ls -la' -alias la='ls -A' -alias l='ls -CF' - -cd ~ -EOF - -cat > "$DATA/etc/motd" < "$DATA/etc/inittab" <<'EOF' -# /etc/inittab — XForge -# -# pid 1 is /sbin/init. There is one console: tty1, the same terminal as -# /dev/console, and the one the app shows. -::sysinit:/etc/init.d/rcS - -# Respawned, so logging out (or a crash) gives a fresh session rather than a dead -# screen. No password, no getty: xforge-login execs the login shell configured for -# root in /etc/passwd. -tty1::respawn:/sbin/xforge-login root - -::ctrlaltdel:/sbin/reboot -::shutdown:/bin/umount -a -r -EOF -note "inittab: pid 1 is /sbin/init, tty1 respawns /sbin/xforge-login root" - -mkdir -p "$DATA/etc/init.d" -cat > "$DATA/etc/init.d/rcS" <<'EOF' -#!/bin/sh -# XForge: the sysinit step, run by init before it starts the console login. -# -# /proc and /dev/pts are already mounted by the engine before pid 1 exists, and -# this root has no openrc and no services, so there is nothing to start — rcS -# only has to finish the runtime layout a login expects. -mkdir -p /run /tmp /dev/pts /dev/shm -chmod 1777 /tmp -mount -t proc proc /proc 2>/dev/null -mount -t devpts devpts /dev/pts 2>/dev/null -hostname xforge 2>/dev/null -exit 0 -EOF -chmod 0755 "$DATA/etc/init.d/rcS" - -# apk needs to know where packages come from; this is what makes the in-guest -# toolchain install (and any `apk add`) work at all. -mkdir -p "$DATA/etc/apk" -cat > "$DATA/etc/apk/repositories" < "$DATA/etc/resolv.conf" <<'EOF' -nameserver 1.1.1.1 -nameserver 8.8.8.8 -EOF - -# The provisioning script is what the guest runs to install Swift/xtool. It ships -# inside the app and is written to /host by the host side, but a copy in the root -# makes the command in the motd work without the app. -if [ -f "$HERE/install-toolchain.sh" ]; then - install -m 0755 "$HERE/install-toolchain.sh" "$DATA/root/install-toolchain.sh" - note "install-toolchain.sh staged at /root/install-toolchain.sh" -fi - -# A stamp the app can check to know which root this is. XForge's installer reads -# it to decide whether an existing install can be reused. -mkdir -p "$DATA/usr/local/share/xforge" -MANIFEST_PATH="$DATA/usr/local/share/xforge/rootfs-manifest.txt" -{ - echo "base: alpine-minirootfs-${ALPINE_VERSION}.${ALPINE_MINOR}-${ALPINE_ARCH}.tar.gz" - echo "rootfs: ${ALPINE_VERSION}.${ALPINE_MINOR}" - echo "engine: ish-arm64" - echo "format: fakefs-zip" - echo "shell: $DEFAULT_SHELL" - echo "console: /sbin/xforge-login root (tty1, respawned by init)" - # What the guest arrives with. This is the line that tells a user (and the - # verification, and the app) whether the root is the plain one or the - # provisioned one, without booting it. - if [ "$PROVISION" = "all" ]; then - echo "toolchain: xtool, swiftly, swift" - if [ -n "$SDK_SHA256" ]; then - echo "darwin-sdk: $DARWIN_SDK_TAG ($DARWIN_SDK_ASSET)" - echo "sdk-sha256: $SDK_SHA256" - # The SDK's own record of where SwiftPM put it, passed through rather - # than recomputed: verification checks this exact path, and a path - # that is written down where it is made cannot drift from one that is - # guessed at where it is checked. - sdk_path="$(awk '/^[[:space:]]*path:/ { print $2 }' \ - "$DATA/usr/local/share/xforge/darwin-sdk.txt" 2>/dev/null || true)" - [ -n "$sdk_path" ] && echo "darwin-sdk-path: $sdk_path" - else - echo "darwin-sdk: not included" - fi - else - echo "toolchain: not provisioned (the guest installs it on demand)" - fi - echo "built-at: $(date -u +%Y-%m-%dT%H:%M:%SZ)" - # Bump this whenever anything about the guest's own setup changes — the app - # compares it with the root it already installed and replaces the root when - # they differ (see RootfsInstaller.installedRootIsStale). It must match the - # release tag the root is published under. - echo "stamp: rootfs-v5" -} > "$MANIFEST_PATH" - -# --------------------------------------------------------------------------- -# The console, exercised. -# -# Everything above is configuration, and configuration that looks right reads -# exactly like configuration that works. So the root is handed to -# EmbeddedLinux/verify-rootfs.sh, which runs the guest's own login program in a -# chroot of this tree — the way init runs it — and checks which shell ends up -# running. The same script checks the ZIP that gets published, so a device and -# this build agree on what "correct" means. -# --------------------------------------------------------------------------- -log "Checking the finished root" -guest_mount -"$HERE/verify-rootfs.sh" "$DATA" -guest_umount - -# --------------------------------------------------------------------------- -# 7. Convert to fakefs -# -# Last, after everything that adds, removes or links anything — the conversion -# builds `meta.db` as an index of the tree as it stands, and the engine reads -# the database rather than scanning the directory. Anything added afterwards -# would be on disk and invisible in the guest. -# -# `fakefs_import` reads an *archive* (libarchive, gzip+tar), not a directory, so -# the configured tree is packed once here and converted from that. -# --------------------------------------------------------------------------- -OUT_ROOTFS="$WORK/$ROOTFS_NAME" - -# The manifest is the root's own description of itself, the summary below prints -# it, and the tree it lives in is deleted by the conversion below — so it is -# copied out first. -MANIFEST_COPY="$WORK/rootfs-manifest.txt" -cp "$MANIFEST_PATH" "$MANIFEST_COPY" - -log "Converting to fakefs" -rm -rf "$OUT_ROOTFS" -STAGED_TAR="$WORK/$ROOTFS_NAME.tar.gz" -# Report what is being converted. The conversion is where a mistake in the -# ordering shows up (the engine reads meta.db, not the directory), so the tree's -# own numbers are worth having in the log next to the result. -note "tree: $(du -sh "$DATA" | cut -f1), $(find "$DATA" -mindepth 1 | wc -l) entries" -tar -czf "$STAGED_TAR" -C "$DATA" . -note "staged: $(du -h "$STAGED_TAR" | cut -f1)" - -# The tree is deleted *here*, between packing and converting, and the ordering is -# load-bearing rather than tidy. A provisioned tree is ~5 GB and the fakefs root -# it converts to is about the same size again, so holding both at once — which is -# what this did while the rootfs was small — needs ~11 GB, and a GitHub runner -# has 14 GB in total. Spending the tar instead (which fakefsify is about to read -# anyway) halves the peak, and the tar is already a complete copy of the tree. -# The cost is that a failure inside fakefsify leaves no tree to inspect; the -# packaging is what needs the disk, and the tar is kept until the conversion -# succeeds. -rm -rf "$DATA" -"$FAKEFSIFY" "$STAGED_TAR" "$OUT_ROOTFS" -rm -f "$STAGED_TAR" - -[ -d "$OUT_ROOTFS/data" ] || die "fakefsify produced no data/ directory" -[ -f "$OUT_ROOTFS/meta.db" ] || die "fakefsify produced no meta.db" -note "data: $(du -sh "$OUT_ROOTFS/data" | cut -f1)" -note "meta: $(du -h "$OUT_ROOTFS/meta.db" | cut -f1)" - -# The glibc layer must be *indexed*, not merely present. This is the check that -# would have caught installing it after conversion: the files existed on disk and -# the guest could not see them at all. -# -# `CAST(path AS TEXT)` is not decoration. fakefs stores paths as BLOB, and a -# LIKE against a BLOB only works through SQLite's implicit coercion — which is -# version-dependent, so the same query that matched every row locally matched -# zero rows on the runner's older SQLite. Comparing as text removes the -# ambiguity instead of relying on it. -if [ "${XFORGE_SKIP_GLIBC:-0}" != "1" ]; then - if command -v sqlite3 >/dev/null 2>&1; then - indexed="$(sqlite3 "$OUT_ROOTFS/meta.db" \ - "SELECT COUNT(*) FROM paths WHERE CAST(path AS TEXT) LIKE '%aarch64-linux-gnu%';" \ - 2>/dev/null || echo 0)" - total="$(sqlite3 "$OUT_ROOTFS/meta.db" \ - "SELECT COUNT(*) FROM paths;" 2>/dev/null || echo 0)" - [ "${indexed:-0}" -gt 0 ] || die \ - "the glibc layer is not indexed in meta.db ($indexed of ${total:-0} paths) - — the guest would not see it. It must be installed before the fakefs - conversion, not copied in afterwards." - note "glibc paths indexed in meta.db: $indexed of $total" - else - note "sqlite3 not available — skipping the glibc indexing check" - fi -fi - -# --------------------------------------------------------------------------- -# 8. Pack -# --------------------------------------------------------------------------- -log "Packing $ZIP_NAME" -ZIP_PATH="$OUT_DIR/$ZIP_NAME" -rm -f "$ZIP_PATH" - -# Run from the work dir so the archive contains `alpine-rootfs/...` rather than -# absolute paths — the app unzips it straight into Documents. -# -# The SQLite sidecar files are excluded on purpose: `meta.db-wal`/`-shm` are -# transient and are not part of the root. Shipping them would be harmless at -# best and a corrupt-looking database at worst. -( - cd "$WORK" - zip -qr "$ZIP_PATH" "$ROOTFS_NAME" -x "*.db-shm" -x "*.db-wal" -) - -[ -f "$ZIP_PATH" ] || die "the ZIP was not produced" -# Read the listing once: `unzip -Z1 … | grep -q` is not safe under `set -o -# pipefail` (grep exits at the first match, unzip dies of SIGPIPE, and the -# pipeline then reports failure for a check that passed). -unzip -Z1 "$ZIP_PATH" > "$WORK/zip-contents.txt" -grep -qE "^$ROOTFS_NAME/data/$" "$WORK/zip-contents.txt" \ - || die "the ZIP has no $ROOTFS_NAME/data/ entry" -grep -qE "^$ROOTFS_NAME/meta\.db$" "$WORK/zip-contents.txt" \ - || die "the ZIP has no $ROOTFS_NAME/meta.db entry" -! grep -qE '\.db-(wal|shm)$' "$WORK/zip-contents.txt" \ - || die "the ZIP contains SQLite WAL/SHM sidecars" - -# Checksum, so a consumer can pin the exact root it expects. -( cd "$OUT_DIR" && sha256sum "$ZIP_NAME" > "$ZIP_NAME.sha256" ) - -log "Done" -note "rootfs: $ZIP_PATH" -note "size: $(du -h "$ZIP_PATH" | cut -f1)" -note "sha256: $(cut -d' ' -f1 < "$ZIP_PATH.sha256")" -note "entries: $(wc -l < "$WORK/zip-contents.txt")" -echo -cat "$MANIFEST_COPY" - -if [ -n "${GITHUB_OUTPUT:-}" ]; then - { - echo "rootfs=$ZIP_PATH" - echo "rootfs_name=$ZIP_NAME" - echo "rootfs_size=$(stat -c %s "$ZIP_PATH" 2>/dev/null || stat -f %z "$ZIP_PATH")" - echo "rootfs_sha256=$(cut -d' ' -f1 < "$ZIP_PATH.sha256")" - echo "rootfs_toolchain=$([ "$PROVISION" = all ] && echo provisioned || echo none)" - } >> "$GITHUB_OUTPUT" -fi - -if [ -n "${GITHUB_STEP_SUMMARY:-}" ]; then - { - echo "### Alpine rootfs" - echo - echo '```' - cat "$MANIFEST_COPY" - echo '```' - echo - echo "\`$ZIP_NAME\` — $(du -h "$ZIP_PATH" | cut -f1)" - echo - echo "- toolchain: $([ "$PROVISION" = all ] && echo \ - "xtool, swiftly, Swift${SDK_SHA256:+ and $DARWIN_SDK_TAG}" || \ - 'not provisioned')" - echo "- entries: $(wc -l < "$WORK/zip-contents.txt")" - } >> "$GITHUB_STEP_SUMMARY" -fi diff --git a/EmbeddedLinux/fetch-rootfs.sh b/EmbeddedLinux/fetch-rootfs.sh deleted file mode 100755 index 15e5a49..0000000 --- a/EmbeddedLinux/fetch-rootfs.sh +++ /dev/null @@ -1,117 +0,0 @@ -#!/bin/bash -# -# fetch-rootfs.sh — put the pinned Alpine rootfs where the app target bundles it. -# -# The guest root is a release asset, not a committed file: `alpine-rootfs.zip` is -# an already-converted fakefs published by the rootfs workflow, and -# `.github/workflows/build-ipa.yml` pins the tag it was built from (`ROOTFS_TAG`). -# It is ~165 MB when the root is plain Alpine and ~1.6 GB when it also carries the -# build toolchain (xtool, Swift, the Darwin SDK) — which is what the current tag -# pins, and the reason the download here is worth resuming and checking. -# It has to be in place *before* the Xcode project is generated — the app target -# lists it as a resource, and a build phase cannot add a file to a resource list -# that was fixed when XcodeGen walked the directory. -# -# Anything that builds this repo therefore needs this step before XcodeGen (or -# before opening the project in Xcode): -# -# EmbeddedLinux/fetch-rootfs.sh -# -# The workflow that publishes the IPA uses the same script, so there is one -# implementation of "which rootfs, verified how". -# -# Environment: -# XFORGE_ROOTFS_TAG override the pinned tag (default: ROOTFS_TAG in -# .github/workflows/build-ipa.yml) -# XFORGE_ROOTFS_FORCE=1 re-download even if the file is already there -# XFORGE_ROOTFS_DIR where it lands (default: /Support/Resources) -# -set -euo pipefail - -HERE="$(cd "$(dirname "$0")" && pwd)" -REPO="$(cd "$HERE/.." && pwd)" -DEST="${XFORGE_ROOTFS_DIR:-$REPO/Support/Resources}" -ASSET="alpine-rootfs.zip" - -log() { printf '\n==> %s\n' "$*"; } -note() { printf ' %s\n' "$*"; } -die() { printf '\nerror: %s\n' "$*" >&2; exit 1; } - -# The pinned tag lives in the IPA workflow, which is where someone upgrading the -# root changes it. Reading it from there keeps this from becoming a second place -# that has to be edited in step. -TAG="${XFORGE_ROOTFS_TAG:-$(sed -n 's/^[[:space:]]*ROOTFS_TAG:[[:space:]]*//p' \ - "$REPO/.github/workflows/build-ipa.yml" | head -1)}" -[ -n "$TAG" ] || die "could not read ROOTFS_TAG from .github/workflows/build-ipa.yml - (set XFORGE_ROOTFS_TAG to name the release yourself)" -# Which repository publishes the rootfs: this one. Ask git, so a fork fetches its -# own — and *not* `.gitmodules`, whose first entry is the engine (that mistake -# resolved the engine's URL and 404ed). A tarball export has no remote, so fall back -# to the app's release repository; keep in step with XForgeReleases.repository. -SLUG="$(git -C "$REPO" config --get remote.origin.url 2>/dev/null \ - | sed -e 's|^.*github\.com[:/]||' -e 's|\.git$||')" -[ -n "$SLUG" ] || SLUG="R0GUEEE/XForge" - -mkdir -p "$DEST" -TARGET="$DEST/$ASSET" - -if [ -f "$TARGET" ] && [ "${XFORGE_ROOTFS_FORCE:-0}" != "1" ]; then - log "The pinned rootfs is already staged" - note "$TARGET ($(du -h "$TARGET" | cut -f1))" - exit 0 -fi - -log "Fetching the pinned Alpine rootfs: $TAG" -note "into $TARGET" - -# `gh` when it is installed *and* has a token, plain curl otherwise. These releases -# are public, so no credential is needed — and a `gh` with no token is not an error -# to report, it is a reason to use curl (which is how a pipeline that does not pass -# GH_TOKEN gets its rootfs; treating that as fatal is how this step failed once). -BASE="https://github.com/$SLUG/releases/download/$TAG" - -if command -v gh >/dev/null 2>&1 \ - && { [ -n "${GH_TOKEN:-}" ] || [ -n "${GITHUB_TOKEN:-}" ]; }; then - note "using gh (a token is in the environment)" - rm -f "$TARGET" - if ! gh release download "$TAG" --repo "$SLUG" \ - --pattern "$ASSET" --pattern "$ASSET.sha256" \ - --dir "$DEST" --clobber; then - note "gh could not fetch it — falling back to curl" - fi -fi - -if [ ! -f "$TARGET" ]; then - # A half-downloaded asset must not survive a failure: the next run (or a human) - # would find a plausible-looking file of the wrong size. - trap 'rm -f "$TARGET.partial"' EXIT - note "downloading $BASE/$ASSET" - curl -fL --retry 3 --retry-delay 2 -o "$TARGET.partial" "$BASE/$ASSET" \ - || die "could not download $BASE/$ASSET" - mv "$TARGET.partial" "$TARGET" - curl -fL --retry 3 -o "$DEST/$ASSET.sha256" "$BASE/$ASSET.sha256" 2>/dev/null || true -fi - -[ -f "$TARGET" ] || die "$ASSET was not downloaded" - -# The checksum is the point of pinning a tag: a re-uploaded or truncated asset has -# to fail here rather than inside the guest on someone's phone. -if [ -f "$TARGET.sha256" ]; then - ( cd "$DEST" && shasum -a 256 -c "$ASSET.sha256" >/dev/null 2>&1 ) \ - || die "the downloaded $ASSET does not match its published sha256" - note "sha256 verified" -else - note "warning: no .sha256 beside the asset — not verified" -fi - -# And it has to be the *shape* the app unpacks: a fakefs ZIP, not a tarball. -command -v unzip >/dev/null 2>&1 || { note "$ASSET ($(du -h "$TARGET" | cut -f1)), not inspected"; exit 0; } -unzip -Z1 "$TARGET" > /tmp/xforge-rootfs-entries.txt -for entry in "alpine-rootfs/data/" "alpine-rootfs/meta.db"; do - grep -qx "$entry" /tmp/xforge-rootfs-entries.txt \ - || die "$TARGET has no $entry — that is not a fakefs root" -done -rm -f /tmp/xforge-rootfs-entries.txt - -log "Rootfs staged" -note "$TARGET ($(du -h "$TARGET" | cut -f1))" diff --git a/EmbeddedLinux/install-toolchain.sh b/EmbeddedLinux/install-toolchain.sh deleted file mode 100644 index 27a85f0..0000000 --- a/EmbeddedLinux/install-toolchain.sh +++ /dev/null @@ -1,920 +0,0 @@ -#!/bin/sh -# -# install-toolchain.sh — provision the XForge embedded Linux (inside the guest). -# -# Runs *in the guest* (Alpine aarch64 / musl), so the same script works from -# XForge's Terminal and from the Toolchain screen, which drives the steps one at -# a time to show progress: -# -# sh /root/install-toolchain.sh # every step -# sh /root/install-toolchain.sh deps # one step -# -# Set XFORGE_INSTALL_XTOOL=0 for a base build root that leaves xtool for the -# app's on-device component installer. -# Steps: deps | glibc | xtool | swiftly | swift | sdk | sdk-remove | verify -# -# Every step is idempotent: re-running it is a no-op once it has succeeded. -# -# The tools are glibc binaries (xtool is a Swift program built on Ubuntu; the -# toolchains swiftly installs are Ubuntu builds too) and this guest is musl, so -# a real glibc runtime is the first thing to install — `gcompat` is not enough. -# `test-glibc` below exercises it. -# -set -eu - -ARCH="$(uname -m)" -# Ubuntu names the same machine two different ways and both are needed: its -# archive and pool use "arm64"/"amd64", while the directory the libraries land in -# is the GNU triple, "aarch64-linux-gnu". -case "$ARCH" in - aarch64) UBUNTU_ARCH=arm64 ;; - x86_64) UBUNTU_ARCH=amd64 ;; - *) UBUNTU_ARCH="$ARCH" ;; -esac -MULTIARCH="$ARCH-linux-gnu" -GLIBC_ROOT=/opt/glibc -GLIBC_LIB="$GLIBC_ROOT/usr/lib/$MULTIARCH" -GLIBC_LD="$GLIBC_LIB/ld-linux-aarch64.so.1" -SHARE=/usr/local/share/xforge -SWIFTLY_HOME_DIR="${SWIFTLY_HOME_DIR:-$HOME/.local/share/swiftly}" -export SWIFTLY_HOME_DIR -INSTALL_XTOOL="${XFORGE_INSTALL_XTOOL:-1}" - -# Where the Darwin Swift SDK comes from, and where it is unpacked on the way in. -# -# The SDK is the one component XForge cannot generate: it is Apple's, and either -# the user's own Xcode.xip or a bundle someone already built from one. XForge -# publishes its own (`darwin-sdk-`, built with xtool) so a guest with no .xip -# still gets a working SDK — the app's Downloads screen installs exactly this -# asset, and this step installs the same thing when the rootfs is built. -# -# The tag is pinned rather than resolved through the API: the rootfs is packaged -# once and shipped, so "which SDK is in this root" has to be answerable from the -# build log, not from whatever `releases/latest` answered that minute. Set -# XFORGE_DARWIN_SDK_URL to install a bundle from somewhere else entirely. -SDK_REPO="${XFORGE_DARWIN_SDK_REPO:-R0GUEEE/XForge}" -SDK_TAG="${XFORGE_DARWIN_SDK_TAG:-darwin-sdk-7}" -SDK_ASSET="${XFORGE_DARWIN_SDK_ASSET:-darwin.artifactbundle.zip}" -SDK_URL="${XFORGE_DARWIN_SDK_URL:-https://github.com/$SDK_REPO/releases/download/$SDK_TAG/$SDK_ASSET}" -SDK_CACHE="${XFORGE_SDK_CACHE:-/root/.cache/xforge-sdk}" - -# Ubuntu release whose glibc the Swift toolchains are built against. -UBUNTU_SUITE="${XFORGE_UBUNTU_SUITE:-noble}" - -log() { echo "==> $*"; } - -# --------------------------------------------------------------------------- -# Output sinks -# -# Never send a *guest process*'s output to /dev/null here. the engine's arm64 -# engine has been observed to SIGKILL a forked guest program whose stdout/stderr -# points at /dev/null — `swift --version >/dev/null 2>&1` died where the same -# command without the redirect ran fine, and `apk info -e … >/dev/null` died -# intermittently. Pipes and real files are safe, and a shell *builtin* -# (`command -v x >/dev/null`) is not affected, so silence goes to a file. -# --------------------------------------------------------------------------- -SILENT="${TMPDIR:-/tmp}/xforge-silent.$$" -: > "$SILENT" - -# --------------------------------------------------------------------------- -# glibc compatibility layer -# --------------------------------------------------------------------------- -# -# Binaries built on Ubuntu need Ubuntu's glibc. Rather than compiling one (an -# hour in the emulator), take the runtime out of the distribution's own packages -# and run them through *its* loader: -# -# $GLIBC_LD --library-path $GLIBC_LIB: -# -# The loaders mount the necessary packages under $GLIBC_ROOT. The package list -# is what a Swift/xtool binary actually needs beyond libc itself: the toolchains -# link against libxml2, curl, krb5, ldap and ICU (Foundation), which the Ubuntu -# AppImage does not bundle. - -# The development halves are needed to *link*, not just to run: glibc's -# crt1.o/crti.o/crtn.o/libc.so and libc_nonshared.a come from libc6-dev, and -# GCC's crtbeginS.o/crtendS.o/libgcc.a from libgcc--dev. Without them a link -# dies with -# /usr/bin/ld: cannot find crtbeginS.o / cannot find -lgcc -# which is what compiling a Swift program in this rootfs hit — `swift --version` -# was fine, because printing a version does not link anything. -# -# One name per line, NO comments: the caller reads this list with word splitting. -glibc_packages() { - cat <<'EOF' -libc6 -libc6-dev -libgcc-s1 -libgcc-13-dev -libgcc-14-dev -libstdc++6 -libstdc++-13-dev -libstdc++-14-dev -zlib1g -libcom-err2 -libkeyutils1 -libkrb5-3 -libk5crypto3 -libkrb5support0 -libgssapi-krb5-2 -libsasl2-2 -libldap2 -libgnutls30t64 -libp11-kit0 -libtasn1-6 -libunistring5 -libidn2-0 -libffi8 -libbrotli1 -libicu74 -libxml2 -libpng16-16t64 -libselinux1 -libpcre2-8-0 -libacl1 -libattr1 -libgmp10 -libnettle8t64 -libhogweed6t64 -libcurl4t64 -libnghttp2-14 -libpsl5t64 -libssh-4 -librtmp1 -libssl3t64 -libzstd1 -liblzma5 -libgcrypt20 -libgpg-error0 -libuuid1 -libblkid1 -libcap2 -libedit2 -libpython3.12t64 -libsqlite3-0 -libncurses6 -libncursesw6 -libtinfo6 -libz3-4 -EOF -} - -# Resolve package names to pool URLs with the distribution's own indexes, so the -# script does not carry pinned filenames that go stale. -glibc_package_url() { - for index in /tmp/xforge-glibc/Packages*; do - [ -f "$index" ] || continue - url="$(awk -v want="$1" ' - /^Package: / { pkg = $2 } - /^Filename: / { if (pkg == want) { print "https://ports.ubuntu.com/ubuntu-ports/" $2; exit } } - ' "$index")" - [ -n "$url" ] && { echo "$url"; return 0; } - done - return 1 -} - -glibc_fetch_indexes() { - for component in main universe; do - target="/tmp/xforge-glibc/Packages-$component" - [ -s "$target" ] && continue - url="https://ports.ubuntu.com/ubuntu-ports/dists/$UBUNTU_SUITE/$component/binary-$UBUNTU_ARCH/Packages.gz" - if curl -fL --retry 3 -o "$target.gz" "$url" 2>>"$SILENT"; then - gzip -dc "$target.gz" > "$target" - fi - done -} - -step_glibc() { - # "Already present" has to mean the layer is *usable*, not merely that the - # loader exists. The two halves can diverge: the loader lives under - # /opt/glibc, while the wiring that makes glibc binaries actually run is the - # symlink farm in /lib and /usr/lib. Checking only the loader reports a - # half-built layer as complete, and the failure then appears much later as a - # tool dying on an undefined symbol or an unloadable interpreter. - # - # This is why the check is on the wiring too: the loader a Swift binary is - # loaded by, and the unversioned link names a Swift link resolves against. - if [ -x "$GLIBC_LD" ] && [ -f "$GLIBC_LIB/libc.so.6" ] \ - && [ -e "/lib/ld-linux-$ARCH.so.1" ] && [ -e "/usr/lib/$MULTIARCH" ]; then - log "glibc runtime already present at $GLIBC_ROOT" - else - log "Installing the glibc runtime (Ubuntu $UBUNTU_SUITE, $UBUNTU_ARCH)" - mkdir -p "$GLIBC_LIB" "$GLIBC_ROOT/usr/lib" /tmp/xforge-glibc - glibc_fetch_indexes - - missing_packages="" - for pkg in $(glibc_packages); do - url="$(glibc_package_url "$pkg" || true)" - if [ -z "$url" ]; then - # Not something to skip quietly. A name the suite does not have - # means the layer will be missing a library, and that surfaces - # much later as a symbol lookup error inside one of the tools: - # `libgnutls30` became `libgnutls30t64` in noble, and the miss - # only showed up as "swift-sdk: undefined symbol - # nettle_rsa_oaep_sha512_decrypt, version HOGWEED_6" — while - # `swift --version` kept working, so nothing looked wrong. - echo " $pkg: NOT FOUND in the $UBUNTU_SUITE indexes" >&2 - missing_packages="$missing_packages $pkg" - continue - fi - deb="/tmp/xforge-glibc/$pkg.deb" - [ -s "$deb" ] || curl -fL --retry 3 -o "$deb" "$url" - ( - cd /tmp/xforge-glibc - rm -rf unpack && mkdir unpack && cd unpack - ar x "$deb" - # Ubuntu ships the payload as zstd; older packages use xz/gz. - if [ -f data.tar.zst ]; then - zstd -d -c data.tar.zst | tar -xf - - elif [ -f data.tar.xz ]; then - tar -xJf data.tar.xz - else - tar -xzf data.tar.gz - fi - # Merged-/usr layout: the libraries live under usr/lib//. - if [ -d "usr/lib/$MULTIARCH" ]; then - cp -a "usr/lib/$MULTIARCH/." "$GLIBC_LIB/" - fi - # GCC's own directory: crtbeginS.o/crtendS.o/libgcc.a live under - # usr/lib/gcc///, which is where clang looks for - # the GNU toolchain it links with. - if [ -d "usr/lib/gcc/$MULTIARCH" ]; then - mkdir -p "$GLIBC_ROOT/usr/lib/gcc/$MULTIARCH" - cp -a "usr/lib/gcc/$MULTIARCH/." "$GLIBC_ROOT/usr/lib/gcc/$MULTIARCH/" - fi - if [ -e "usr/lib/ld-linux-$ARCH.so.1" ]; then - cp -a "usr/lib/ld-linux-$ARCH.so.1" "$GLIBC_ROOT/usr/lib/" 2>>"$SILENT" || true - fi - if [ -d usr/share/icu ]; then - mkdir -p "$GLIBC_ROOT/usr/share" - cp -a usr/share/icu "$GLIBC_ROOT/usr/share/" 2>>"$SILENT" || true - fi - ) - echo " $pkg" - done - if [ -n "$missing_packages" ]; then - echo "the glibc layer cannot be built — no such package:$missing_packages" >&2 - echo "fix glibc_packages() for Ubuntu $UBUNTU_SUITE ($UBUNTU_ARCH)" >&2 - exit 1 - fi - - # A name that resolved is not the same as a library that arrived: the - # layer is only useful if the files a Swift or xtool binary loads are - # actually present, so the files are what gets checked. - glibc_have() { - for candidate in "$GLIBC_LIB/$1"*; do - [ -e "$candidate" ] && return 0 - done - return 1 - } - incomplete="" - for lib in libc.so.6 libm.so.6 libpthread.so.0 libdl.so.2 libstdc++.so libstdc++.so.6 \ - libgcc_s.so.1 libz.so.1 libzstd.so.1 liblzma.so.5 libxml2.so.2 libcurl.so.4 \ - libssl.so.3 libcrypto.so.3 libgnutls.so.30 libhogweed.so.6 \ - libnettle.so.8 libgmp.so.10 libicuuc.so.74 liblber.so.2 \ - libldap.so.2 libpng16.so.16 libpsl.so.5 libz3.so.4 libsqlite3.so.0; do - glibc_have "$lib" || incomplete="$incomplete $lib" - done - # Startup and compiler objects, which are files rather than shared - # libraries: glibc's crt1.o/crti.o/crtn.o/libc.so land in the multiarch - # directory, GCC's crtbeginS.o/crtendS.o/libgcc.a in its versioned one. - for obj in crt1.o crti.o crtn.o libc_nonshared.a; do - glibc_have "$obj" || incomplete="$incomplete $obj" - done - for obj in crtbeginS.o crtendS.o libgcc.a; do - found="" - for candidate in "$GLIBC_ROOT/usr/lib/gcc/$MULTIARCH"/*/"$obj"; do - [ -e "$candidate" ] && found=1 - done - [ -n "$found" ] || incomplete="$incomplete $obj" - done - if [ -n "$incomplete" ]; then - echo "the glibc layer is incomplete — nothing provides:$incomplete" >&2 - echo "a package in glibc_packages() was renamed in $UBUNTU_SUITE; the" >&2 - echo "names above are what the Swift/xtool binaries load and link against." >&2 - exit 1 - fi - log "glibc at $GLIBC_LD" - fi - - mkdir -p "$GLIBC_ROOT/usr/lib" "$SHARE" - # A compiler finds its GNU toolchain by path, not through a search path: - # clang looks in /usr/lib/gcc//, so the extracted copy has - # to be visible there (the same trick the multiarch directory below uses). - if [ -d "$GLIBC_ROOT/usr/lib/gcc/$MULTIARCH" ]; then - mkdir -p /usr/lib/gcc - ln -sfn "$GLIBC_ROOT/usr/lib/gcc/$MULTIARCH" "/usr/lib/gcc/$MULTIARCH" - fi - if [ ! -e "$GLIBC_ROOT/usr/lib/ld-linux-$ARCH.so.1" ]; then - ln -sf "$GLIBC_LD" "$GLIBC_ROOT/usr/lib/ld-linux-$ARCH.so.1" - fi - - # Wire this glibc in as the system's glibc, not just a directory to point a - # wrapper at. Running a tool through the loader only covers the *first* - # process: Swift's driver execs swift-frontend and swift-build as children, - # and a child starts from its own ELF interpreter. With gcompat's stub - # loader at /lib/ld-linux-.so.1 those children went back to musl and - # died on glibc symbols (__isoc23_strtol, mallinfo2, pthread_cond_clockwait) - # and on libraries musl does not have (libncurses.so.6, libuuid.so.1). - # - # Alpine's own programs are unaffected: they use /lib/ld-musl-.so.1 - # and libc.musl-.so.1, never these names. - rm -f "/lib/ld-linux-$ARCH.so.1" - ln -s "$GLIBC_LD" "/lib/ld-linux-$ARCH.so.1" - ln -sfn "$GLIBC_LIB" "/usr/lib/$MULTIARCH" - ln -sfn "$GLIBC_LIB" "/lib/$MULTIARCH" - for lib in libc.so.6 libm.so.6 libpthread.so.0 librt.so.1 libdl.so.2 \ - libresolv.so.2 libutil.so.1 libcrypt.so.1; do - if [ -e "$GLIBC_LIB/$lib" ]; then - rm -f "/lib/$lib" - ln -s "$GLIBC_LIB/$lib" "/lib/$lib" - fi - done - - # What a link searches by default is the guest's /usr/lib, where Alpine's musl - # copies of the *unversioned* names (libc.so, libstdc++.so, libm.so …) come - # first. A Swift link asks for -lstdc++ and gets the musl build, which carries - # no symbol versions, so it fails with - # undefined reference to `std::__throw_logic_error(char const*)@GLIBCXX_3.4' - # Passing -L/-Xlinker -L from the wrapper did not change that (the driver does - # not always hand those through), so the link names themselves are pointed at - # the layer. Only the unversioned files are touched: the runtime sonames - # (libstdc++.so.6, libc.so.6) are left to Alpine's for Alpine's own binaries, - # and glibc binaries resolve those through /lib/aarch64-linux-gnu as before. - for link_name in libc.so libm.so libpthread.so librt.so libdl.so libutil.so \ - libresolv.so libcrypt.so libstdc++.so libgcc_s.so; do - source="" - [ -e "$GLIBC_LIB/$link_name" ] && source="$GLIBC_LIB/$link_name" - if [ -z "$source" ]; then - # libstdc++.so and libgcc_s.so live in GCC's versioned directory, not - # in the multiarch one — the first version of this loop looked only in - # the multiarch directory and silently skipped exactly the name that - # matters most. - for candidate in "$GLIBC_ROOT"/usr/lib/gcc/$MULTIARCH/*/"$link_name"; do - [ -e "$candidate" ] && source="$candidate" - done - fi - [ -n "$source" ] || continue - rm -f "/usr/lib/$link_name" - ln -s "$source" "/usr/lib/$link_name" - done - - cat > "$SHARE/glibc.env" <"$SILENT" 2>&1 || missing="$missing $package" - done - - if [ -n "$missing" ]; then - total=$(printf '%s\n' "$missing" | xargs -n1 | sed '/^[[:space:]]*$/d' | wc -l | tr -d ' ') - current=0 - for package in $missing; do - current=$((current + 1)) - log "Installing missing Alpine build dependency $current/$total: $package" - apk add --no-cache "$package" - done - update-ca-certificates >/dev/null 2>&1 || true - log "Alpine build dependencies installed" - else - log "Alpine build dependencies already installed" - fi -} -xtool_wrapper() { - cat > /usr/local/bin/xtool <<'EOF' -#!/bin/sh -# xtool is an Ubuntu glibc binary; run it through the glibc loader installed by -# install-toolchain.sh, with the libraries it ships alongside it. -. /usr/local/share/xforge/glibc.env -# The AppImage's own libraries come first: Ubuntu's libcurl (which Swift needs) -# is newer than the nghttp2 the AppImage bundles, and mixing them the other way -# round gives "undefined symbol: nghttp2_option_set_no_rfc9113_...". -exec "$XFORGE_GLIBC_LD" --library-path "/opt/xtool/usr/lib:$XFORGE_GLIBC_LIB" \ - /opt/xtool/usr/bin/xtool "$@" -EOF - chmod +x /usr/local/bin/xtool -} - -step_xtool() { - log "Installing xtool ($ARCH)" - if [ -x /opt/xtool/usr/bin/xtool ]; then - log "xtool already unpacked at /opt/xtool" - else - xtool_asset="" - for candidate in "$ARCH" "$UBUNTU_ARCH"; do - [ -n "$candidate" ] || continue - [ "$candidate" = "$xtool_asset" ] && continue - if curl -fsIL --retry 3 \ - "https://github.com/xtool-org/xtool/releases/latest/download/xtool-$candidate.AppImage" \ - >"$SILENT" 2>&1; then - xtool_asset="$candidate" - break - fi - done - [ -n "$xtool_asset" ] || { - echo "could not resolve an xtool AppImage for architecture '$ARCH' (also tried '$UBUNTU_ARCH')" >&2 - exit 1 - } - # Exactly the release asset xtool documents, unpacked once instead of - # every run (APPIMAGE_EXTRACT_AND_RUN extracts ~50 MB per invocation). - curl -fL --retry 3 \ - "https://github.com/xtool-org/xtool/releases/latest/download/xtool-$xtool_asset.AppImage" \ - -o /tmp/xtool.AppImage - chmod +x /tmp/xtool.AppImage - ( cd /opt && rm -rf squashfs-root && /tmp/xtool.AppImage --appimage-extract >"$SILENT" 2>&1 ) - rm -rf /opt/xtool && mv /opt/squashfs-root /opt/xtool - rm -f /tmp/xtool.AppImage - fi - xtool_wrapper - log "xtool installed at /usr/local/bin/xtool (unpacked in /opt/xtool)" -} - -step_swiftly() { - log "Installing swiftly (the Swift installer)" - if [ -x /usr/local/bin/swiftly ]; then - log "swiftly already present" - return 0 - fi - cd /tmp - curl -fL --retry 3 -O "https://download.swift.org/swiftly/linux/swiftly-$ARCH.tar.gz" - tar zxf "swiftly-$ARCH.tar.gz" - rm -f "swiftly-$ARCH.tar.gz" - # --assume-yes/--quiet-shell-followup: there is no terminal to answer its - # prompts from, and its profile edits are pointless (XForge runs - # non-interactive /bin/sh -c). - ./swiftly init --assume-yes --quiet-shell-followup --skip-install --platform "ubuntu24.04" - rm -f ./swiftly - ln -sf "$SWIFTLY_HOME_DIR/bin/swiftly" /usr/local/bin/swiftly - log "swiftly installed at /usr/local/bin/swiftly" -} - -# swift and swiftc are glibc binaries too, and they live under the swiftly home -# rather than on any PATH a `sh -c` command would search. The toolchain is looked -# up directly first: swiftly's own shims only work in a shell that has sourced its -# environment file, which is exactly what XForge's command runner is not. -swift_wrapper() { - name="$1" - # The linker's own search path is the guest's (musl) one, where Alpine's - # libstdc++ — which carries no symbol versions — shadows Ubuntu's, and - # linking anything against libswiftCore fails with - # undefined reference to \`std::__throw_logic_error(char const*)@GLIBCXX_3.4' - # So the layer's directories go on the link line explicitly, ahead of - # anything the driver adds. (-L only reorders the search; it changes no - # runtime path, so Alpine's own binaries are untouched.) - # -rpath-link is the part that actually matters, and it is not the same thing - # as -L: when the linker pulls in a *shared* library, that library's own - # NEEDED entries (libswiftCore.so wants libstdc++.so.6 and libc.so.6) are - # resolved through the rpath-link path, not the -L path. Without it the - # linker falls back to the guest's /lib and /usr/lib, where Alpine's - # libc6-compat stubs and musl's libstdc++ sit — which is why every attempt - # with -L alone still failed: - # ld: /lib/libc.musl-aarch64.so.1: warning: the `gets' ... - # undefined reference to `std::__throw_logic_error(char const*)@GLIBCXX_3.4' - # Verified before shipping: with -rpath-link pointing at the layer, a link - # against Ubuntu's libxml2.so.2 is clean and the binary runs in a musl guest. - link_dirs="" - for dir in "$GLIBC_LIB" "$GLIBC_ROOT/usr/lib/gcc/$MULTIARCH" "$GLIBC_ROOT"/usr/lib/gcc/$MULTIARCH/*/; do - [ -d "$dir" ] || continue - link_dirs="$link_dirs -Xlinker -L${dir%/} -Xlinker -rpath-link -Xlinker ${dir%/}" - done - - cat > "/usr/local/bin/$name" <:0: error: error opening input file 'sdk' (No such file or directory) - # That is exactly how a rootfs build failed after installing a toolchain that - # worked, and it is why the list below exists rather than a blanket exec: the - # SwiftPM subcommands reach the linker through their own \`swiftc\` (which is - # this same script), so they are not the invocations that need the flags. - # A driver subcommand this list misses fails loudly on its first run rather - # than quietly linking the wrong libraries. - case "\${1:-}" in - sdk|experimental-sdk|package|build|test|run|demangle) - exec "\$candidate" "\$@" ;; - esac - exec "\$candidate" $link_dirs "\$@" -done - -. "\$home/env.sh" >/dev/null 2>&1 || true # sourcing, not a forked program: the only safe /dev/null here -for dir in \${PATH}; do - case "\$dir" in /usr/local/bin) continue;; esac - if [ -x "\$dir/$name" ]; then - exec "\$dir/$name" "\$@" - fi -done - -echo "xforge: $name is not installed yet (run: swiftly install latest --use)" >&2 -echo " looked in \$home/toolchains/*/usr/bin and \$home/bin" >&2 -exit 127 -EOF - chmod +x "/usr/local/bin/$name" -} - -step_swift() { - log "Installing the Swift toolchain" - [ -e "$SHARE/glibc.env" ] || { echo "run the glibc step first" >&2; exit 1; } - . "$SHARE/glibc.env" - - # The wrappers go in first, so `swift --version` always answers something - # useful — a version, or exactly why there is none yet. - swift_wrapper swift - swift_wrapper swiftc - - . "$SWIFTLY_HOME_DIR/env.sh" - # `swiftly list` exits 0 and prints a separator even with nothing installed, - # so ask the directory that actually holds toolchains. - if [ -n "$(ls -A "$SWIFTLY_HOME_DIR/toolchains" 2>>"$SILENT")" ]; then - log "a toolchain is already installed: $(ls "$SWIFTLY_HOME_DIR/toolchains" | tail -1)" - return 0 - fi - - # swiftly verifies the download's signature with gpg, and refuses without it - # ("gpg is not installed ... To skip signature verification, specify - # --no-verify"). Try to verify; fall back to skipping it, loudly. - # swiftly exits non-zero for its Ubuntu "these dependencies should be - # installed" advice even when the install worked, so the toolchain directory - # is what decides whether to retry -- not the exit status. - if command -v gpg >/dev/null 2>&1; then - swiftly install latest --use --assume-yes || true - else - echo " gpg is not installed in the guest: skipping signature verification" - swiftly install latest --use --assume-yes --no-verify || true - fi - if [ -z "$(ls -A "$SWIFTLY_HOME_DIR/toolchains" 2>>"$SILENT")" ]; then - echo " no toolchain after the first attempt; retrying without signature verification" - swiftly install latest --use --assume-yes --no-verify - fi - - # `swiftly init --skip-install` leaves it unlinked; make it manage the - # toolchain we just installed. - swiftly link >"$SILENT" 2>&1 || true - - # Say what actually resolved. A guest whose swift cannot run is much easier to - # diagnose from these four lines than from a wrapper's refusal. - log "swift wrapper installed at /usr/local/bin/swift" - echo " toolchains: $(ls "$SWIFTLY_HOME_DIR/toolchains" 2>>"$SILENT" | tr '\n' ' ')" - echo " swiftly bin: $(ls "$SWIFTLY_HOME_DIR/bin" 2>>"$SILENT" | tr '\n' ' ')" - if out="$(swift --version 2>&1)"; then - echo " swift --version: $(printf '%s' "$out" | head -1)" - else - echo " swift --version FAILED: $(printf '%s' "$out" | head -3 | tr '\n' ' ')" - fi -} - -# --------------------------------------------------------------------------- -# The Darwin SDK -# -# The last piece of the toolchain, and the only one that is not on swift.org: -# the arm64-apple-ios Swift SDK xtool compiles iOS apps against. XForge publishes -# its own bundle (`darwin-sdk-`), built with xtool from an Xcode.xip, and this -# installs that bundle with SwiftPM's own `swift sdk install` — the same command -# the Toolchain screen runs, so a root provisioned here and a guest provisioned by -# hand end up with the SDK in the same place (`~/.swiftpm/swift-sdks`). -# -# This step is deliberately NOT part of `all`: it is a 400 MB download and a -# 1.3 GB install, and a user with their own Xcode.xip is better served by -# `xtool sdk install `. The rootfs build runs it explicitly. -# --------------------------------------------------------------------------- -step_sdk() { - log "Installing the Darwin Swift SDK ($SDK_TAG)" - if ! command -v swift >/dev/null 2>&1; then - echo "the Swift toolchain is required first: sh $0 swift" >&2 - exit 1 - fi - - # An explicit replacement removes what is installed first — SwiftPM will not - # install a second bundle carrying the same artifact ID, and the bundled rootfs - # already carries one. The default is the idempotent path: a root that has an - # SDK keeps it. - if [ "${XFORGE_DARWIN_SDK_REPLACE:-0}" = "1" ]; then - step_sdk_remove - fi - - # Idempotence, answered by SwiftPM rather than by a marker file: an SDK that - # was installed and then removed must be reinstalled, and only SwiftPM knows. - sdk_listing="/tmp/xforge-sdk-list.$$" - if timeout 180 swift sdk list >"$sdk_listing" 2>&1 \ - && grep -qi 'darwin' "$sdk_listing"; then - log "a Darwin SDK is already installed: $(grep -i darwin "$sdk_listing" | head -1)" - rm -f "$sdk_listing" - return 0 - fi - rm -f "$sdk_listing" - - command -v unzip >/dev/null 2>&1 || { - echo "unzip is required to install the SDK (run: sh $0 deps)" >&2 - exit 1 - } - - mkdir -p "$SDK_CACHE" - archive="$SDK_CACHE/$SDK_ASSET" - # A caller that already has the archive (the rootfs build downloads it on the - # host, where it can be checked) stages it and points this at it, so the - # chroot never needs the network for the largest download in the chain. - if [ -n "${XFORGE_DARWIN_SDK_ARCHIVE:-}" ] && [ -f "$XFORGE_DARWIN_SDK_ARCHIVE" ]; then - log "Using the staged SDK archive at $XFORGE_DARWIN_SDK_ARCHIVE" - # The caller may well have staged it *at* this path — the rootfs build - # does, because that is where this step looks for its cache — and `cp a a` - # is an error ("are the same file"), which is how a build failed after - # every download and install in it had succeeded. - if [ "$XFORGE_DARWIN_SDK_ARCHIVE" != "$archive" ]; then - cp -f "$XFORGE_DARWIN_SDK_ARCHIVE" "$archive" - fi - else - log "Downloading $SDK_ASSET ($SDK_URL)" - rm -f "$archive.partial" - curl -fL --retry 3 --retry-delay 2 --no-progress-meter \ - -o "$archive.partial" "$SDK_URL" \ - || { echo "could not download $SDK_URL" >&2; exit 1; } - mv "$archive.partial" "$archive" - fi - [ -s "$archive" ] || { echo "the SDK archive is empty: $archive" >&2; exit 1; } - - # Recorded here, written into the root's manifest by the caller, and worth - # having: "which SDK is in this root" is otherwise unanswerable from a - # published artifact. - sdk_sha="$(sha256sum "$archive" | cut -d' ' -f1)" - if [ -n "${XFORGE_DARWIN_SDK_SHA256:-}" ] && [ "$XFORGE_DARWIN_SDK_SHA256" != "$sdk_sha" ]; then - echo "the SDK archive does not match XFORGE_DARWIN_SDK_SHA256" >&2 - printf ' expected %s\n got %s\n' "$XFORGE_DARWIN_SDK_SHA256" "$sdk_sha" >&2 - exit 1 - fi - - bundle="$SDK_CACHE/darwin.artifactbundle" - rm -rf "$bundle" "$SDK_CACHE/__MACOSX" - unzip -q "$archive" -d "$SDK_CACHE" - [ -f "$bundle/info.json" ] || { - echo "$archive did not unpack to $bundle/info.json" >&2 - exit 1 - } - - log "Installing the SDK with swift sdk install" - swift sdk install "$bundle" - - # 400 MB of archive, 1.3 GB of unpacked bundle and whatever `unzip` left - # beside them are no longer needed once SwiftPM has the SDK, and in a packaged - # rootfs they would be dead weight in the app bundle. (The zip carries a macOS - # `__MACOSX` sidecar, which unpacks into a directory that *contains* a - # `.artifactbundle` — that is what a build once recorded as the - # installed SDK, so it is removed here and excluded from the search below.) - rm -f "$archive" - rm -rf "$bundle" "$SDK_CACHE/__MACOSX" - - if ! swift sdk list 2>&1 | grep -qi darwin; then - echo "swift sdk install reported success but swift sdk list names no darwin SDK" >&2 - exit 1 - fi - - # Where SwiftPM put it, found rather than assumed. - # - # The documented location is `~/.swiftpm/swift-sdks`, and Swift 6.4's - # `swift sdk install` puts it elsewhere — this step originally asserted that - # path and failed a build whose install had succeeded. What is looked for is - # the bundle directory itself, with the store's own name as the fallback, and - # the answer is what gets recorded and verified. The download cache is - # excluded: the copy that was just installed *from* is a different bundle, and - # `__MACOSX` shadows it in a name search. - sdk_find() { - # The whole filesystem, pruned rather than bounded: SwiftPM's SDK store - # lives under the home directory in theory, and a `-maxdepth` guess at its - # depth searched every plausible place and found nothing — a search that - # reports "nowhere" for something `swift sdk list` can see is worse than a - # slow one. /proc, /sys, /dev and the download cache are the only places - # worth skipping, and -xdev keeps the virtual filesystems out anyway. - find / -xdev \ - \( -path /proc -o -path /sys -o -path /dev -o -path "$SDK_CACHE" \ - -o -name '__MACOSX' \) -prune -o \ - -type d -name "$1" -print 2>"$SILENT" | head -1 || true - } - sdk_install_dir="$(sdk_find '*.artifactbundle')" - [ -n "$sdk_install_dir" ] || sdk_install_dir="$(sdk_find 'swift-sdks')" - if [ -z "$sdk_install_dir" ]; then - echo "swift sdk list names a darwin SDK but no bundle is anywhere on this filesystem:" >&2 - echo " HOME=$HOME SWIFTLY_HOME_DIR=$SWIFTLY_HOME_DIR" >&2 - swift sdk list 2>&1 | sed 's/^/ /' >&2 - exit 1 - fi - - mkdir -p "$SHARE" - cat > "$SHARE/darwin-sdk.txt" </dev/null 2>&1; then - echo " $tool: not installed" - printf 'XFORGE-VERIFY\t%s\tmissing\tnot installed\n' "$tool" - failed=1 - continue - fi - # The exit status decides, not the output: a wrapper that reports "not - # installed" prints a line and exits 127, and counting that as success is - # how "installed" and "works" got confused the first time. - version_file="/tmp/xforge-version.$$" - if timeout 120 "$tool" --version >"$version_file" 2>&1; then - out="$(head -1 "$version_file")" - if [ -n "$out" ]; then - echo " $tool: ok — $out" - printf 'XFORGE-VERIFY\t%s\tok\t%s\n' "$tool" "$out" - else - echo " $tool: INSTALLED BUT SILENT (no version output)" - printf 'XFORGE-VERIFY\t%s\tbroken\tinstalled, but it prints nothing\n' "$tool" - failed=1 - fi - else - out="$(head -1 "$version_file")" - echo " $tool: INSTALLED BUT NOT RUNNING — ${out:-no output}" - printf 'XFORGE-VERIFY\t%s\tbroken\t%s\n' "$tool" "${out:-no output}" - failed=1 - fi - rm -f "$version_file" - done - # `swift sdk list` runs a *second* Swift binary — swift-sdk — with its own - # library needs, and it is what installs the darwin SDK. It fails differently - # from `swift --version` when the glibc layer is incomplete (a symbol lookup - # error naming a library rather than the tool), so it is checked on its own. - if command -v swift >/dev/null 2>&1; then - sdk_probe="/tmp/xforge-sdk-probe.$$" - if timeout 180 swift sdk list >"$sdk_probe" 2>&1; then - line="$(head -1 "$sdk_probe" 2>>"$SILENT" || true)" - echo " swift-sdk: ok${line:+ — $line}" - printf 'XFORGE-VERIFY\tswift-sdk\tok\t%s\n' "${line:-swift sdk list}" - else - line="$(head -1 "$sdk_probe" 2>>"$SILENT" || true)" - last="$(tail -1 "$sdk_probe" 2>>"$SILENT" || true)" - echo " swift-sdk: INSTALLED BUT NOT RUNNING — ${last:-${line:-no output}}" - echo " what the probe said:" - head -6 "$sdk_probe" 2>>"$SILENT" | sed 's/^/ /' - # A load failure names a library but not why, so ask the loader which - # libraries it picks and which it cannot find at all. "not found" - # here means a package is missing from the glibc layer. - if [ -f "$SHARE/glibc.env" ]; then - . "$SHARE/glibc.env" - sdk_bin="$(ls "$SWIFTLY_HOME_DIR"/toolchains/*/usr/bin/swift-sdk 2>>"$SILENT" | head -1 || true)" - if [ -n "$sdk_bin" ]; then - echo " loader resolution for $(basename "$sdk_bin"):" - "$XFORGE_GLIBC_LD" --list "$sdk_bin" 2>&1 | grep '=>' | sed 's/^/ /' - fi - fi - printf 'XFORGE-VERIFY\tswift-sdk\tbroken\t%s\n' "${last:-${line:-no output}}" - failed=1 - fi - rm -f "$sdk_probe" - fi - - if [ "$failed" -eq 0 ]; then - log "All tools are ready." - else - log "Finished. Tools marked 'not running' are installed in the rootfs but" - log "the guest's emulation faults on them — see the Engine log." - fi - return 0 -} - -case "${1:-all}" in - deps) step_deps ;; - glibc) step_glibc ;; - xtool) step_xtool ;; - swiftly) step_swiftly ;; - swift) step_swift ;; - sdk) step_sdk ;; - sdk-remove) step_sdk_remove ;; - verify) step_verify ;; - all) - step_deps - step_glibc - if [ "$INSTALL_XTOOL" = "0" ]; then - log "Skipping xtool (XFORGE_INSTALL_XTOOL=0; the app installs it on demand)" - else - step_xtool - fi - step_swiftly - step_swift - step_verify - # The build executor checks this stamp alongside the actual commands and - # package database. Bump its suffix whenever the required rootfs setup - # changes so existing guests are provisioned again exactly once. - mkdir -p "$SHARE" - touch "$SHARE/build-environment-v2" - ;; - *) - echo "usage: $0 [deps|glibc|xtool|swiftly|swift|sdk|sdk-remove|verify|all]" >&2 - exit 2 - ;; -esac diff --git a/EmbeddedLinux/install-zsign.sh b/EmbeddedLinux/install-zsign.sh deleted file mode 100644 index fd93eee..0000000 --- a/EmbeddedLinux/install-zsign.sh +++ /dev/null @@ -1,33 +0,0 @@ -#!/bin/sh -set -eu - -# Build zsign inside the embedded Alpine guest on first use. The upstream project -# is a C++ command-line signer (MIT), so it belongs in Linux where it can run, -# not as an iOS subprocess/framework. Pin the source revision so the installed -# signer is reproducible and cannot silently change as upstream master moves. -ZSIGN_COMMIT=614caa8d1ca949e260e5746144aa52d27a4b08d6 -ZSIGN_SHA256=a373dc5ddbf81ba5c435a48c14b4a2857506c3dde02ec62e0136b6968797384b -MARK=/usr/local/share/xforge/zsign-installed -BIN=/usr/local/bin/zsign -if [ -x "$BIN" ] && [ -f "$MARK" ]; then - exit 0 -fi -apk add --no-cache curl tar git make g++ pkgconf python3 openssl-dev >/tmp/xforge-zsign-apk.log 2>&1 -rm -rf /tmp/zsign-src -mkdir -p /tmp/zsign-src -cd /tmp/zsign-src -curl -fL --retry 3 "https://github.com/zhlynn/zsign/archive/$ZSIGN_COMMIT.tar.gz" -o zsign.tar.gz -printf '%s %s\n' "$ZSIGN_SHA256" zsign.tar.gz | sha256sum -c - -tar -xzf zsign.tar.gz --strip-components=1 -# Keep the PKCS#12 password out of zsign's argv/process listing. The upstream -# CLI accepts -p ; patch this pinned source with -Q , -# which reads the secret into memory and trims its newline. The job deletes the -# file immediately after signing. -python3 /root/xforge/patch-zsign-password-file.py -make -C build/linux clean all -install -m 0755 bin/zsign "$BIN" -mkdir -p "$(dirname "$MARK")" -printf 'source=https://github.com/zhlynn/zsign\ncommit=%s\ninstalled-at=%s\n' \ - "$ZSIGN_COMMIT" "$(date -u +%Y-%m-%dT%H:%M:%SZ)" > "$MARK" -rm -rf /tmp/zsign-src -zsign -v diff --git a/EmbeddedLinux/patch-zsign-password-file.py b/EmbeddedLinux/patch-zsign-password-file.py deleted file mode 100644 index da8bf5e..0000000 --- a/EmbeddedLinux/patch-zsign-password-file.py +++ /dev/null @@ -1,40 +0,0 @@ -#!/usr/bin/env python3 -"""Add a password-file option to the pinned zsign CLI. - -Upstream's -p takes the password as argv, visible in process listings while -signing. XForge adds -Q/--password_file: zsign reads it into its private string, -then the caller can unlink the file. This patch is applied only to the verified, -pinned upstream source tree during the one-time guest build. -""" -from pathlib import Path - -p = Path("src/zsign.cpp") -s = p.read_text() - -old_opt = '\t{"password", required_argument, NULL, \'p\'},' -new_opt = old_opt + '\n\t{"password_file", required_argument, NULL, \'Q\'},' -assert old_opt in s, "pinned zsign option table changed; review the patch" -s = s.replace(old_opt, new_opt, 1) - -old_flags = '"dfva2LhiqwCRSEWUPc:k:m:o:p:e:b:n:z:l:D:t:r:x:M:I:"' -new_flags = '"dfva2LhiqwCRSEWUPc:k:m:o:p:Q:e:b:n:z:l:D:t:r:x:M:I:"' -assert old_flags in s, "pinned zsign getopt string changed; review the patch" -s = s.replace(old_flags, new_flags, 1) - -old_case = "\t\tcase 'p':\n\t\t\tstrPassword = optarg;\n\t\t\tbreak;" -new_case = old_case + "\n\t\tcase 'Q':\n\t\t\tif (!ZFile::ReadFile(optarg, strPassword)) {\n\t\t\t\tZLog::ErrorV(\">>> Failed to read password file! %s\\n\", optarg);\n\t\t\t\treturn -1;\n\t\t\t}\n\t\t\twhile (!strPassword.empty() && (strPassword.back() == '\\n' || strPassword.back() == '\\r')) strPassword.pop_back();\n\t\t\tbreak;" -assert old_case in s, "pinned zsign password switch changed; review the patch" -s = s.replace(old_case, new_case, 1) -p.write_text(s) - -# This pinned upstream snapshot's Linux Makefile exposes a separate header bug: -# json.h uses time_t but never includes the header that defines it. Some host -# toolchains happen to include it transitively; musl/Alpine's does not. Make the -# dependency explicit so the guest build is deterministic. -p = Path("src/common/json.h") -s = p.read_text() -include = "#include " -assert include in s, "pinned zsign JSON header layout changed; review the patch" -if "#include " not in s and "#include " not in s: - s = s.replace(include, "#include \n" + include, 1) -p.write_text(s) diff --git a/EmbeddedLinux/verify-rootfs.sh b/EmbeddedLinux/verify-rootfs.sh deleted file mode 100755 index 83a0b8f..0000000 --- a/EmbeddedLinux/verify-rootfs.sh +++ /dev/null @@ -1,522 +0,0 @@ -#!/usr/bin/env bash -# -# verify-rootfs.sh — check that a built Alpine root actually boots the console -# XForge expects, and fails loudly if it does not. -# -# The rootfs is the one thing in this repo the app's tests cannot cover: a root -# that is wrong looks exactly like a root that is right until `init` runs on a -# device, where the failure is a blank terminal. So the checks live here, and both -# callers use them: -# -# * EmbeddedLinux/build-rootfs.sh runs it on the configured *tree*, before the -# fakefs conversion — where the guest's own binaries can be executed; -# * .github/workflows/build-rootfs.yml runs it on the packed *ZIP*, so the -# artifact that is actually published is the thing that was checked. -# -# It accepts three shapes of input and says which checks apply to each: -# -# plain tree the configured root before conversion. Everything runs, -# including starting the console in a chroot of it. -# fakefs ZIP what gets published: `alpine-rootfs/data/` plus `meta.db`. The -# fakefs dir ZIP is read entry by entry (nothing is unpacked). -# -# A fakefs root is a directory tree *and* a database, and the engine reads the -# database: a file can sit in data/ and be invisible to the guest. Two consequences -# shape this script, and both cost a failed build to learn: -# -# * every path is also looked up in `paths`, and the mode that matters (is the -# console program executable?) is the one in `stats` — fakefs writes its data -# files 0666 and keeps the real mode in the database; -# * a fakefs root cannot be chrooted at all, so the console cannot be *run* -# from it. fakefs does not store symlinks as symlinks: an `AE_IFLNK` entry -# becomes a regular file whose contents are the link target, and the engine -# turns it back into a symlink from `meta.db` (see tools/fakefs.c). So -# `data/bin/sh` is a text file reading "/bin/busybox" and -# `data/lib/ld-musl-aarch64.so.1` is one reading "/lib/libc.musl-aarch64.so.1" -# — a chroot into that tree has no loader and no shell. The execution check -# therefore belongs to the tree, and the ZIP is checked through the same -# database the guest will use. -# -# Usage: -# EmbeddedLinux/verify-rootfs.sh -# -# A root built with XFORGE_PROVISION=all also carries the build toolchain (xtool, -# the Swift toolchain and the Darwin SDK), and its own manifest says so — that -# claim is checked here too, because a half-provisioned root looks exactly like a -# working one until someone tries to build with it. -# -set -euo pipefail - -TARGET="${1:-dist/rootfs/alpine-rootfs.zip}" -ROOTFS_NAME="alpine-rootfs" -MANIFEST="usr/local/share/xforge/rootfs-manifest.txt" - -log() { printf '\n==> %s\n' "$*"; } -note() { printf ' %s\n' "$*"; } -die() { printf '\nerror: %s\n' "$*" >&2; exit 1; } - -[ -e "$TARGET" ] || die "$TARGET does not exist" - -# --------------------------------------------------------------------------- -# How to read the root. -# -# Everything below asks only two questions — "is this path there?" and "what does -# it say?" — and answers them per shape, so the shape is settled once, here. -# --------------------------------------------------------------------------- -MODE="" -TREE="" -ZIP="" -BASE="" -ENTRIES="" -META_DB="" # what a path is relative to: "" for a tree, "/data/" in a ZIP - -if [ -d "$TARGET" ]; then - if [ -f "$TARGET/meta.db" ] && [ -d "$TARGET/data" ]; then - MODE="fakefs-dir" - TREE="$TARGET/data" - note "input: a fakefs root directory" - elif [ -d "$TARGET/data" ] && [ -e "$TARGET/data/etc/passwd" ]; then - MODE="tree" - TREE="$TARGET/data" - note "input: a plain root tree" - else - MODE="tree" - TREE="$TARGET" - note "input: a plain root tree" - fi -elif [ -f "$TARGET" ]; then - command -v unzip >/dev/null 2>&1 || die "unzip is required to read the ZIP" - MODE="zip" - ZIP="$TARGET" - BASE="$ROOTFS_NAME/data/" - note "input: a packed fakefs ZIP (read entry by entry, nothing unpacked)" -else - die "$TARGET is neither a directory nor a file" -fi - -# path_exists -path_exists() { - if [ "$MODE" = "zip" ]; then - grep -qx "$BASE$1" "$ENTRIES" - else - # `-e` OR `-L`, deliberately. A root tree is full of symlinks whose - # targets are absolute *guest* paths — `/usr/local/bin/swiftly` → - # `/root/.local/share/swiftly/bin/swiftly`, `/bin/sh` → `/bin/busybox` — - # and on the build machine those resolve against the build machine's `/`, - # so `-e` alone reports a symlink that is perfectly present as missing. - # This is the same trap as the glibc layer's `/lib` entry points: test the - # link itself, not where it points. - [ -e "$TREE/$1" ] || [ -L "$TREE/$1" ] - fi -} - -# path_contents -path_contents() { - if [ "$MODE" = "zip" ]; then - unzip -p "$ZIP" "$BASE$1" - else - cat "$TREE/$1" - fi -} - -# path_matches -# -# For the paths that cannot be named exactly — a Swift toolchain lives under a -# versioned directory that changes with every release. The listing is written to a -# file before it is searched on purpose: `find … | grep -q` closes the pipe at the -# first match, the producer dies of SIGPIPE, and under `set -o pipefail` the -# pipeline then reports failure for a check that passed. -path_matches() { - local listing rc - listing="$(mktemp)" - if [ "$MODE" = "zip" ]; then - grep -E "^$BASE$1" "$ENTRIES" > "$listing" || true - else - # Matching, not just listing: the first version of this wrote the listing - # out and then only asked whether it was non-empty, so in tree mode every - # pattern "matched" and a root missing its toolchain passed the check. The - # test that drives those failures is Tools/test-verify-toolchain.sh. - find "$TREE" -mindepth 1 -print 2>/dev/null | sed "s|^$TREE/||" \ - | grep -E "^$1" > "$listing" || true - fi - if [ -s "$listing" ]; then rc=0; else rc=1; fi - rm -f "$listing" - return "$rc" -} - -# path_link_target -# -# In a tree this is a symlink and `readlink` is the answer; in fakefs the target is -# the file's contents. (Reading it as a file in tree mode would dump the target's -# binary, which is why this is not just `path_contents`.) -path_link_target() { - if [ "$MODE" = "zip" ] || [ "$MODE" = "fakefs-dir" ]; then - path_contents "$1" - else - readlink "$TREE/$1" || true - fi -} - -if [ "$MODE" = "zip" ]; then - ENTRIES="$(mktemp)" # the ZIP's entry list, read once - META_DB="$(mktemp)" - trap 'rm -f "$ENTRIES" "$META_DB"' EXIT - unzip -Z1 "$ZIP" > "$ENTRIES" - for entry in "$ROOTFS_NAME/data/" "$ROOTFS_NAME/meta.db"; do - grep -qx "$entry" "$ENTRIES" || die "the ZIP has no $entry" - done - # SQLite's WAL/SHM sidecars are transient and must not be captured: a stale - # `-wal` next to a fresh meta.db is a database that looks corrupt to the guest. - ! grep -qE '\.db-(wal|shm)$' "$ENTRIES" \ - || die "the ZIP contains SQLite WAL/SHM sidecars" - unzip -p "$ZIP" "$ROOTFS_NAME/meta.db" > "$META_DB" - note "entries: $(wc -l < "$ENTRIES" | tr -d ' ')" -elif [ "$MODE" = "fakefs-dir" ]; then - META_DB="$TARGET/meta.db" -fi - -# --------------------------------------------------------------------------- -# 1. What init needs -# --------------------------------------------------------------------------- -log "Files" -for path in \ - sbin/init \ - bin/sh \ - sbin/xforge-login \ - etc/inittab \ - etc/init.d/rcS \ - etc/passwd \ - etc/shadow \ - etc/profile \ - etc/apk/repositories \ - "$MANIFEST" \ - root/install-toolchain.sh ; do - path_exists "$path" || die "the root has no /$path" -done -note "11 required paths present" - -if [ "$MODE" = "tree" ]; then - # In a tree the modes are the files' own. In a fakefs root they are not (see - # the header): that is what the meta.db section below checks. - [ -x "$TREE/sbin/xforge-login" ] || die "/sbin/xforge-login is not executable" - [ -x "$TREE/bin/sh" ] || die "/bin/sh is not executable" - note "/sbin/xforge-login and /bin/sh are executable" -fi - -# /bin/sh has to lead somewhere that exists, or nothing can start at all. It is a -# symlink to busybox in every Alpine root, and fakefs will have stored the target -# as text — either way, the target has to be in the root. -SH_TARGET="$(path_link_target bin/sh)" -case "$SH_TARGET" in - /*) path_exists "${SH_TARGET#/}" \ - || die "/bin/sh points at $SH_TARGET, which is not in the root" ;; - *) die "/bin/sh is not where it should be: expected a symlink, got '${SH_TARGET:0:40}'" ;; -esac -note "/bin/sh → $SH_TARGET" - -# The login shell the root promises, from the root's own manifest — not from this -# script, so a root built with a different XFORGE_DEFAULT_SHELL is checked against -# what it was built to be. -EXPECTED_SHELL="$(path_contents "$MANIFEST" | awk '/^[[:space:]]*shell:/ { print $2 }')" -[ -n "$EXPECTED_SHELL" ] || die "the manifest does not name a login shell" -path_exists "${EXPECTED_SHELL#/}" \ - || die "the manifest promises $EXPECTED_SHELL, which is not in the root" -note "login shell in the manifest: $EXPECTED_SHELL" - -# The session's terminal has to be one curses knows about. /etc/profile is where -# the root names it, and an unknown TERM is an error nowhere — it just makes `less`, -# `top` and every full-screen program behave as if there were no terminal, which -# reads as a broken console rather than as a missing package. -TERM_VALUE="$(path_contents etc/profile | sed -n 's/^export TERM=//p' | tail -1 | tr -d "'\"")" -if [ -n "$TERM_VALUE" ]; then - ENTRY="${TERM_VALUE%${TERM_VALUE#?}}/$TERM_VALUE" - FOUND_TERMINFO="" - for dir in usr/share/terminfo etc/terminfo; do - if path_exists "$dir/$ENTRY"; then FOUND_TERMINFO="$dir/$ENTRY"; fi - done - if [ -z "$FOUND_TERMINFO" ]; then - die "TERM=$TERM_VALUE is exported but the root has no terminfo entry for it - (looked for $ENTRY). Every curses program in the guest would run against an - unknown terminal." - fi - note "TERM=$TERM_VALUE → $FOUND_TERMINFO" -fi - -# --------------------------------------------------------------------------- -# 2. The console -# -# One session, on the one terminal that exists, started by the guest's own program. -# The stock Alpine inittab fails all three: it starts openrc (not in this root) and -# respawns six gettys on terminals the engine does not provide. -# --------------------------------------------------------------------------- -log "inittab" -INITTAB="$(path_contents etc/inittab)" -grep_q() { printf '%s\n' "$INITTAB" | grep -q "$@"; } -grep_q -x '::sysinit:/etc/init.d/rcS' \ - || die "inittab does not run the sysinit step" -grep_q -x 'tty1::respawn:/sbin/xforge-login root' \ - || die "inittab does not respawn /sbin/xforge-login root on tty1" -if grep_q 'openrc'; then - die "inittab starts openrc, which this root does not contain" -fi -if grep_q -E '^tty[2-6]::'; then - die "inittab respawns gettys on terminals this guest does not have" -fi -note "pid 1 is /sbin/init; tty1 respawns /sbin/xforge-login root" - -# --------------------------------------------------------------------------- -# 3. Root's shell, in the file the console program reads -# --------------------------------------------------------------------------- -log "Root's shell" -PASSWD_ROOT="$(path_contents etc/passwd | awk -F: '$1 == "root" { print; exit }')" -[ -n "$PASSWD_ROOT" ] || die "there is no root in /etc/passwd" -PASSWD_SHELL="$(printf '%s\n' "$PASSWD_ROOT" | cut -d: -f7)" -[ "$PASSWD_SHELL" = "$EXPECTED_SHELL" ] || die \ - "root's shell in /etc/passwd is $PASSWD_SHELL, not the $EXPECTED_SHELL the manifest promises" -note "$PASSWD_ROOT" - -# --------------------------------------------------------------------------- -# 4. What the engine will resolve -# -# The tree is only half of a fakefs root; the other half is the database, and the -# paths above existing in data/ is not the same as the guest being able to see -# them. A root once shipped with 568 glibc files on disk and zero rows in `paths` -# for them, which the guest reads as "not there at all" — so everything the console -# needs is looked up here, along with the mode it will have. -# --------------------------------------------------------------------------- -if [ -n "$META_DB" ]; then - log "meta.db" - command -v sqlite3 >/dev/null 2>&1 || die "sqlite3 is required to check a fakefs root" - - # `CAST(path AS TEXT)` is not decoration: fakefs stores paths as BLOB, and - # comparing a BLOB against a string literal is version-dependent — the same - # query that matched every row locally matched zero rows on an older SQLite, so - # a correct root failed its own check. - # fakefs stores paths in its normalized form, `( '/' path-component )*`: - # "/bin/sh", not "bin/sh" and not "/bin/sh/" (see `path_normalize` in - # tools/fakefs.c). The lookup is on that form, so this takes a root-relative - # path and looks up its absolute spelling. - indexed() { - sqlite3 "$META_DB" \ - "SELECT COUNT(*) FROM paths WHERE CAST(path AS TEXT) = '/$1';" - } - for path in \ - bin/sh \ - sbin/xforge-login \ - etc/inittab \ - etc/passwd \ - etc/profile \ - "${EXPECTED_SHELL#/}" ; do - [ "$(indexed "$path")" -gt 0 ] || die "/$path is in the tree but not indexed - in meta.db — the guest would not see it. Anything the root needs has to be in - place *before* the fakefs conversion." - done - note "6 console paths indexed in meta.db" - - # The mode the engine will apply, which is the one fakefs kept (its data files - # are all written 0666). `stats.stat` is a binary `struct ish_stat` — mode, uid, - # gid, rdev as four little-endian uint32s — so the mode is the first two bytes of - # hex(), low byte first. - mode_of() { - local hex lo hi - hex="$(sqlite3 "$META_DB" "SELECT hex(substr(stats.stat, 1, 2)) FROM paths - JOIN stats ON paths.inode = stats.inode - WHERE CAST(paths.path AS TEXT) = '/$1';")" - [ -n "$hex" ] || return 1 - lo="${hex%??}" - hi="${hex#??}" - printf '%s' "$(( 0x${hi}${lo} ))" - } - for path in sbin/xforge-login "${EXPECTED_SHELL#/}"; do - mode="$(mode_of "$path")" || die "/$path has no stats row in meta.db" - # 8#111 is 0o111: any execute bit. - [ "$(( mode & 8#111 ))" -ne 0 ] || die "/$path is not executable in - meta.db (mode $(printf '%o' "$mode")) — the console would not start." - done - note "/sbin/xforge-login and $EXPECTED_SHELL are executable in meta.db" - - total="$(sqlite3 "$META_DB" 'SELECT COUNT(*) FROM paths;')" - note "meta.db: $total paths" -fi - -# --------------------------------------------------------------------------- -# 4b. The provisioned toolchain, when the manifest says there is one -# -# A root built with XFORGE_PROVISION=all carries xtool, the Swift toolchain and -# the Darwin SDK, and its manifest says so. That claim is checked here rather than -# trusted, for the same reason as everything else in this script: the failure mode -# is quiet. A root whose toolchain is half-installed boots perfectly, gives a -# working console, reports the toolchain as "installed" in the app, and fails on -# the user's first build — which is the most expensive place to find out. -# -# What cannot be done here is *run* the tools: a fakefs root cannot be chrooted -# (its symlinks are files, and only the engine resolves them), and the tree the -# verification ran on during the build no longer exists by then. So the checks are -# structural — the paths exist, and for a fakefs root the engine can see them — -# and the "it actually compiles" check happened on the tree, in the build, where -# install-toolchain.sh verify could run a real compile. -# --------------------------------------------------------------------------- -TOOLCHAIN_CLAIM="$(path_contents "$MANIFEST" | awk '/^[[:space:]]*toolchain:/ { sub(/^[[:space:]]*toolchain:[[:space:]]*/, ""); print; exit }')" -case "$TOOLCHAIN_CLAIM" in - ""|"not provisioned"*) - note "no toolchain in this root (the guest installs it on demand)" ;; - *) - log "Toolchain: $TOOLCHAIN_CLAIM" - - for path in \ - usr/local/bin/xtool \ - usr/local/bin/swift \ - usr/local/bin/swiftc \ - usr/local/bin/swiftly \ - opt/xtool/usr/bin/xtool \ - usr/local/share/xforge/glibc.env ; do - path_exists "$path" \ - || die "the manifest promises a toolchain but /$path is missing" - done - note "xtool, swift, swiftly and the glibc layer are present" - - # The toolchain itself lives in a versioned directory - # (swiftly/toolchains//usr/bin/...), so it is looked for by shape - # rather than by a name that changes with every Swift release. - path_matches 'root/\.local/share/swiftly/toolchains/[^/]+/usr/bin/swift-frontend' \ - || die "there is no Swift toolchain under /root/.local/share/swiftly/toolchains" - # Its stdlib, wherever this release keeps it: `linux`, `linux-musl`, or a - # per-architecture directory under either, which is why the pattern has a - # `.*` where the first version of this check named one exact path and - # failed a perfectly good toolchain. The interface files (.swiftmodule) are - # the half a too-thorough slimming would take, and the shared library is - # the half that would still look present. - path_matches 'root/\.local/share/swiftly/toolchains/[^/]+/usr/lib/swift/.*/libswiftCore\.so' \ - || die "the Swift toolchain has no Linux stdlib under - /root/.local/share/swiftly/toolchains/*/usr/lib/swift" - path_matches 'root/\.local/share/swiftly/toolchains/[^/]+/usr/lib/swift/.*/Swift\.swiftmodule' \ - || die "the Swift toolchain has no Linux stdlib interface under - /root/.local/share/swiftly/toolchains/*/usr/lib/swift — without it nothing - can be compiled against the stdlib" - note "a Swift toolchain with its stdlib is installed" - - if path_exists "usr/local/share/xforge/darwin-sdk.txt"; then - SDK_PATH="$(path_contents "usr/local/share/xforge/darwin-sdk.txt" \ - | awk '/^[[:space:]]*path:/ { print $2; exit }')" - SDK_TAG_LINE="$(path_contents "usr/local/share/xforge/darwin-sdk.txt" \ - | awk '/^[[:space:]]*tag:/ { print $2; exit }')" - [ -n "$SDK_PATH" ] \ - || die "/usr/local/share/xforge/darwin-sdk.txt does not name where the SDK went" - # The record names the *directory* SwiftPM installed the bundle into, - # and a ZIP listing writes directories with a trailing slash — an - # exact-line test for the bare name finds nothing and fails a root - # that has the SDK. Ask whether anything is under it instead, which is - # the question that matters either way. - sdk_rel="${SDK_PATH#/}" - if [ "$MODE" = "zip" ]; then - sdk_rel_pattern="$(echo "$sdk_rel" | sed 's/\./\\./g')" - grep -qE "^$BASE$sdk_rel_pattern/" "$ENTRIES" \ - || die "the Darwin SDK records itself at $SDK_PATH, which is not in this root - — the guest's first build would fail on a missing SDK" - else - [ -d "$TREE/$sdk_rel" ] \ - || die "the Darwin SDK records itself at $SDK_PATH, which is not in this root - — the guest's first build would fail on a missing SDK" - fi - note "Darwin SDK $SDK_TAG_LINE at $SDK_PATH" - - # And the engine has to be able to see it: the whole reason the - # toolchain is installed before the conversion is that a file in data/ - # with no row in `paths` does not exist as far as the guest is concerned. - if [ -n "$META_DB" ]; then - sdk_indexed="$(sqlite3 "$META_DB" \ - "SELECT COUNT(*) FROM paths WHERE CAST(path AS TEXT) LIKE '${SDK_PATH%/}/%';")" - [ "${sdk_indexed:-0}" -gt 0 ] \ - || die "the Darwin SDK is on disk but not indexed in meta.db - ($sdk_indexed paths under $SDK_PATH) — the guest would not see it. It has to - be installed before the fakefs conversion." - note "Darwin SDK paths indexed in meta.db: $sdk_indexed" - fi - else - note "no Darwin SDK in this root (XFORGE_PROVISION_SDK=0)" - fi - ;; -esac - -# --------------------------------------------------------------------------- -# 5. Run it — a tree only, and say plainly why it cannot be done otherwise -# --------------------------------------------------------------------------- -log "The console path" -if [ "$MODE" != "tree" ]; then - note "not run here: a fakefs root cannot be chrooted (its symlinks are files" - note "whose contents are the link target, and only the engine resolves them)," - note "so this root's console was run for real on the tree by build-rootfs.sh," - note "before the conversion. The checks above are the database the guest reads." - log "Done" - note "$TARGET checks out (console run: on the tree, before conversion)" - exit 0 -fi - -# An explicit skip is answered before the requirements of the thing being skipped: -# "set XFORGE_VERIFY_SKIP_CONSOLE=1" is the advice this script gives a caller who -# cannot chroot (not root, or not on aarch64), and it has to work for them — -# otherwise the advice is a dead end and a check that was meant to be skipped -# fails the caller anyway. -if [ "${XFORGE_VERIFY_SKIP_CONSOLE:-0}" = "1" ]; then - note "the console is not run (XFORGE_VERIFY_SKIP_CONSOLE=1): the file checks above are all" - note "that was asked for" - log "Done" - note "$TARGET checks out" - exit 0 -fi - -[ "$(id -u)" -eq 0 ] || die "checking the console needs root (it chroots into the - root). Re-run with sudo, or set XFORGE_VERIFY_SKIP_CONSOLE=1." -case "$(uname -m)" in - aarch64|arm64) ;; - *) die "this host is $(uname -m) and the root is aarch64, so its binaries cannot - run here. Run this on an aarch64 machine, or set XFORGE_VERIFY_SKIP_CONSOLE=1 - to check only the files." ;; -esac - -# A working /dev/null is not optional: every shell opens it, and a chroot without -# /dev is a chroot where commands fail for reasons that have nothing to do with what -# is being checked. -BOUND_DEV="" -MOUNTED_PROC="" -if ! grep -qs " $TREE/dev " /proc/mounts; then - mount --bind /dev "$TREE/dev" 2>/dev/null && BOUND_DEV="$TREE/dev" -fi -if ! grep -qs " $TREE/proc " /proc/mounts; then - mount -t proc none "$TREE/proc" 2>/dev/null && MOUNTED_PROC="$TREE/proc" -fi -cleanup_mounts() { - [ -n "$MOUNTED_PROC" ] && umount -l "$MOUNTED_PROC" 2>/dev/null || true - [ -n "$BOUND_DEV" ] && umount -l "$BOUND_DEV" 2>/dev/null || true - return 0 -} -trap 'cleanup_mounts' EXIT - -# Fed from a pipe, not a terminal: the shell is non-interactive and exits, so the -# session ends on its own. What is being read is the part that can silently be -# wrong — which shell runs, as a login shell (the leading dash in $0), in which -# directory, and with which environment. -CONSOLE_OUTPUT="$(printf 'echo XFORGE-CONSOLE-OK; echo "0=$0"; echo "SHELL=$SHELL"; pwd\n' \ - | chroot "$TREE" /sbin/xforge-login root 2>&1)" || true -printf '%s\n' "$CONSOLE_OUTPUT" | sed 's/^/ /' - -case "$CONSOLE_OUTPUT" in - *XFORGE-CONSOLE-OK*) ;; - *) die "/sbin/xforge-login did not start a shell — the console would be blank" ;; -esac -case "$CONSOLE_OUTPUT" in - *"0=-${EXPECTED_SHELL##*/}"*) ;; - *) die "/sbin/xforge-login did not start $EXPECTED_SHELL as a login shell" ;; -esac -case "$CONSOLE_OUTPUT" in - *"SHELL=$EXPECTED_SHELL"*) ;; - *) die "the console session's SHELL is not $EXPECTED_SHELL" ;; -esac -case "$CONSOLE_OUTPUT" in - *"not executable"*) die "/sbin/xforge-login fell back instead of running $EXPECTED_SHELL" ;; -esac -note "ok: /sbin/xforge-login root → $EXPECTED_SHELL, as a login shell" -cleanup_mounts - -log "Done" -note "$TARGET checks out" diff --git a/EmbeddedLinux/xforge-login b/EmbeddedLinux/xforge-login deleted file mode 100755 index 5e304c1..0000000 --- a/EmbeddedLinux/xforge-login +++ /dev/null @@ -1,74 +0,0 @@ -#!/bin/sh -# -# xforge-login — the program XForge's console runs, one instance per session. -# -# This file is not a build script: it is *installed into the guest* by -# EmbeddedLinux/build-rootfs.sh, at /sbin/xforge-login, and run from /etc/inittab -# (which respawns it on tty1, the guest's only console and the terminal the app -# shows). It lives here, next to the builder that installs it, because it is the -# one guest file with logic in it worth reading on its own. -# -# It used to be `/bin/login -f root`, and not needing login is the point: -# -# * login authenticates, allocates a utmp slot and takes over a controlling -# terminal. None of that applies to this guest: it has one user, whose -# password is locked (`root:*`), and init has already given the session the -# console tty. -# * what login did *not* do is what the guest actually wants — start the shell -# configured for the user. That shell is a property of /etc/passwd, so this -# script reads it: `chsh -s /bin/zsh root` (or editing the field) changes what -# the console starts, and the console still starts *something* if that shell -# was uninstalled underneath it. -# -# The session is a login session: the shell is exec'd with a leading dash in -# argv[0] — the POSIX convention that makes a shell read /etc/profile — with the -# user's home as the working directory and the environment `login` would have set. -# -# Usage: xforge-login [user] (default: root) -# -# The leading dash is done with `exec -a`, which is busybox ash's (this script is -# /bin/sh, and /bin/sh is busybox ash here). Passing `-l`/`--login` instead would -# mean knowing which flag each shell spells it with. -# -set -u - -user="${1:-root}" -fallback=/bin/sh - -# field : one colon-separated field of /etc/passwd, or nothing. -field() { - awk -F: -v u="$user" -v n="$1" '$1 == u { print $n; exit }' /etc/passwd 2>/dev/null -} - -wanted="$(field 7)" -home="$(field 6)" -[ -n "$home" ] || home=/ -[ -n "$wanted" ] || wanted="$fallback" - -# The configured shell may have been removed since it was set. Say so and run the -# fallback: init respawns this forever, and a console that dies silently on every -# respawn leaves a blank screen with nothing to read. -if [ ! -x "$wanted" ]; then - printf 'xforge-login: %s is not executable, falling back to %s\n' "$wanted" "$fallback" >&2 - wanted="$fallback" -fi - -# What `login` would export. TERM is the terminal the app emulates; /etc/profile -# sets it too, for shells that read it. -TERM="${TERM:-xterm-256color}" -HOME="$home" -SHELL="$wanted" -USER="$user" -LOGNAME="$user" -PATH=/usr/local/sbin:/usr/local/bin:/usr/sbin:/usr/bin:/sbin:/bin:/opt/bin -export TERM HOME SHELL USER LOGNAME PATH - -cd "$home" 2>/dev/null || cd / - -# The motd `login` used to print, and only when there is a terminal to print it to. -if [ -t 0 ] && [ -s /etc/motd ]; then - cat /etc/motd -fi - -name="${wanted##*/}" -exec -a "-$name" "$wanted" diff --git a/Makefile b/Makefile index 9c86aa7..f867346 100644 --- a/Makefile +++ b/Makefile @@ -1,39 +1,27 @@ -.PHONY: bootstrap submodule rootfs rootfs-publish ish-core icon gen build test ipa init clean +.PHONY: bootstrap toolchain icon gen build test ipa clean XCODE := xcodebuild SCHEME := XForge ## Everything needed to build locally, in order. -bootstrap: submodule rootfs ish-core - -## Pull the ish-arm64 engine sources (git submodule). -submodule: - git submodule update --init --depth 1 Vendor/ish-arm64 - git -C Vendor/ish-arm64 submodule update --init --depth 1 deps/libarchive - -## Build the bundled Alpine fakefs rootfs into Support/Resources. -## Needs root (it chroots into the tree) and room for the toolchain: the default -## is the *provisioned* root — xtool, Swift and the Darwin SDK installed, ~1.6 GB. -## XFORGE_PROVISION=none builds the few-MB plain one instead, where the guest -## installs the toolchain itself. -## build-ipa.yml downloads the published copy rather than rebuilding it. -rootfs: - @bash EmbeddedLinux/build-rootfs.sh Support/Resources - -## Build the rootfs and publish it as a pinned release asset for CI. -## Usage: make rootfs-publish TAG=rootfs-v5 [ALPINE=3.21] [PROVISION=none] -rootfs-publish: - @test -n "$(TAG)" || { echo "usage: make rootfs-publish TAG=rootfs-v5"; exit 1; } - @command -v gh >/dev/null 2>&1 || { \ - echo "GitHub CLI is required: https://cli.github.com/"; exit 1; \ - } - gh workflow run build-rootfs.yml -f tag=$(TAG) -f alpine_version=$(or $(ALPINE),3.21) \ - -f provision=$(or $(PROVISION),all) - @echo "Dispatched build-rootfs.yml for $(TAG); then set ROOTFS_TAG in build-ipa.yml." - -## Build the embedded ish-arm64 Linux engine into Vendor/ish-arm64-build/lib. -ish-core: - @bash EmbeddedLinux/build-ish-core.sh +bootstrap: toolchain gen + +## Install the native toolchain bundle into Vendor/NativeToolchain. +## +## The bundle is built by the `Native iOS Toolchain` workflow (Clang + Mach-O LLD +## for iPhoneOS, cross-built on a Mac) and attached to the run as an artifact. +## Without it the app still builds and runs: the C bridge compiles to a +## "not available" stub and the Toolchain screen says so, which is what a plain +## clone should do. +toolchain: + @if [ -z "$(ARCHIVE)" ]; then \ + echo "usage: make toolchain ARCHIVE=XForgeNativeToolchain-arm64-ios.tar.gz"; \ + echo ""; \ + echo "Download the artifact from a successful 'Native iOS Toolchain' run:"; \ + echo " gh run download --repo R0GUEEE/XForge --name XForgeNativeToolchain-ios-arm64"; \ + exit 64; \ + fi + @bash NativeToolchain/install-bundle.sh "$(ARCHIVE)" ## Regenerate the app icon + accent colour asset catalog (needs Pillow) icon: @@ -61,13 +49,5 @@ ipa: gh workflow run build-ipa.yml @echo "Dispatched build-ipa.yml; monitor it with: gh run watch" -## Install Swift + xtool into the guest, on the device. The published rootfs -## already carries them, so this is a no-op there; it is what a plain root -## (XFORGE_PROVISION=none) needs. Run it inside the XForge terminal, not the host: -## sh /root/install-toolchain.sh all -init: - @echo "Run this in the XForge terminal (embedded Linux), not on the host:" - @echo " sh /root/install-toolchain.sh all" - clean: - rm -rf build dist XForge.xcodeproj Vendor/ish-arm64-build .rootfs-work + rm -rf build dist XForge.xcodeproj diff --git a/Tools/engine-smoke/engine-smoke.c b/Tools/engine-smoke/engine-smoke.c deleted file mode 100644 index d7a0767..0000000 --- a/Tools/engine-smoke/engine-smoke.c +++ /dev/null @@ -1,444 +0,0 @@ -// -// engine-smoke.c -// XForge -// -// Host harness for the embedded Linux bridge. -// -// The app's install paths (the bundled Alpine rootfs, and every guest-side -// tool) all funnel through the same three bridge calls: import a rootfs into -// the engine's fakefs, boot the guest, run a command. When that goes wrong on a -// device the only symptom is "the app crashed", and the device cannot be -// attached to a debugger from here — so this harness drives *the same* -// ISHBridge.c, in *the same* order, on a real machine where the failure is -// visible. -// -// Build: see .github/workflows/engine-smoke.yml (adds an `engine_smoke` -// target to the engine's meson build, so the bridge is compiled with exactly the -// engine's own flags and guest-arch defines). -// -// Usage: engine-smoke [workdir] -// -// The root is an already-converted fakefs (a data/ tree plus meta.db), which -// is what the app has after unzipping the bundled alpine-rootfs.zip. The -// conversion and the unzip are both host-side, so neither goes through the -// engine and neither is what this harness needs to prove — it starts at the -// first call that actually touches the kernel. -// -// Every step prints a flushed breadcrumb *before* it runs, so a crash leaves -// the last step it reached on stdout with nothing after it. -// -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#include "ISHBridge.h" - -static int failures = 0; - -static void say(const char *fmt, ...) { - va_list ap; - va_start(ap, fmt); - vfprintf(stdout, fmt, ap); - va_end(ap); - fputc('\n', stdout); - fflush(stdout); -} - -// Announce a step before running it: if the process dies inside the step, the -// breadcrumb is already on the wire and nothing follows it. -static void step(const char *what) { say("[smoke] >>> %s", what); -} - -static void result(const char *what, int ok) { - say("[smoke] %s %s", ok ? "ok " : "FAIL", what); - if (!ok) failures++; -} - -static int rm_rf(const char *path) { - char cmd[4096]; - snprintf(cmd, sizeof cmd, "rm -rf '%s'", path); - return system(cmd); -} - -static int mkdirs(const char *path) { - return mkdir(path, 0777); -} - -// One guest command, with its outcome printed in full. -// -// `expect_exit` is the exit status the guest should end with -- 0 for the -// normal cases, 127 for the probes of programs a bare rootfs does not have yet, -// 7 for the deliberate failure below. The struct is read *before* it is freed: -// xf_guest_result_free() zeroes it, which is a mistake this harness made once -// and reported as every command failing. -static void run_guest(const char *label, const char *command, int timeout_ms, int expect_exit) { - step(label); - struct xf_guest_result r; - int rc = xf_ish_run(command, NULL, timeout_ms, 1 << 20, &r); - if (rc != 0) { - say("[smoke] FAIL %s: could not start (%d): %s", label, rc, xf_ish_last_error()); - failures++; - return; - } - say("[smoke] --- %s ---", label); - say("launched=%d exited=%d exit_code=%d signal=%d timed_out=%d truncated=%d bytes=%zu", - r.launched, r.exited, r.exit_code, r.term_signal, r.timed_out, r.truncated, r.output_len); - if (r.output != NULL && r.output[0] != '\0') { - fputs(r.output, stdout); - if (r.output[r.output_len > 0 ? r.output_len - 1 : 0] != '\n') - fputc('\n', stdout); - } - fflush(stdout); - - int ok = r.launched && r.exited && r.term_signal == 0 && r.exit_code == expect_exit; - if (!ok) - say("[smoke] expected exit %d", expect_exit); - xf_guest_result_free(&r); - result(label, ok); -} - -// The host's own DNS servers, the way the app gets them from iOS and writes -// them into the guest's /etc/resolv.conf (App/Services/GuestNetwork.swift). -// Without that file the guest cannot resolve anything: the bundled minirootfs -// ships no nameservers, and resolution happens inside the guest. -static int host_dns_servers(char out[][64], int max) { - FILE *f = popen("scutil --dns 2>/dev/null | awk '/nameserver\\[[0-9]+\\]/{print $3}' | " - "sort -u | head -4", "r"); - if (f == NULL) - return 0; - int count = 0; - char line[256]; - while (count < max && fgets(line, sizeof(line), f) != NULL) { - char *nl = strchr(line, '\n'); - if (nl != NULL) *nl = '\0'; - if (line[0] == '\0') continue; - snprintf(out[count], 64, "%s", line); - count++; - } - pclose(f); - return count; -} - -// Run a command and print everything about it, but do not judge it: used for -// probes whose failure is a fact about the environment (network reachability, -// package repositories) rather than a regression in the bridge. -static void probe_guest(const char *label, const char *command, int timeout_ms) { - step(label); - struct xf_guest_result r; - int rc = xf_ish_run(command, NULL, timeout_ms, 1 << 20, &r); - if (rc != 0) { - say("[smoke] probe could not start (%d): %s", rc, xf_ish_last_error()); - return; - } - say("[smoke] --- %s ---", label); - say("launched=%d exited=%d exit_code=%d signal=%d timed_out=%d truncated=%d bytes=%zu", - r.launched, r.exited, r.exit_code, r.term_signal, r.timed_out, r.truncated, r.output_len); - if (r.output != NULL && r.output[0] != '\0') { - fputs(r.output, stdout); - if (r.output[r.output_len > 0 ? r.output_len - 1 : 0] != '\n') - fputc('\n', stdout); - } - fflush(stdout); - xf_guest_result_free(&r); -} - -// Like probe_guest, but the result is asserted: these checks exist to prove a -// command *returns*, and probe_guest only reports. A hung lock burned a probe's -// whole 15-minute timeout while the harness still printed no failed steps. -static void probe_assert(const char *label, const char *command, int timeout_ms) { - step(label); - struct xf_guest_result r; - int rc = xf_ish_run(command, NULL, timeout_ms, 1 << 20, &r); - if (rc != 0) { - say("[smoke] probe could not start (%d): %s", rc, xf_ish_last_error()); - result(label, 0); - return; - } - say("[smoke] --- %s ---", label); - say("launched=%d exited=%d exit_code=%d signal=%d timed_out=%d truncated=%d bytes=%zu", - r.launched, r.exited, r.exit_code, r.term_signal, r.timed_out, r.truncated, r.output_len); - if (r.output != NULL && r.output[0] != '\0') { - fputs(r.output, stdout); - if (r.output[r.output_len > 0 ? r.output_len - 1 : 0] != '\n') - fputc('\n', stdout); - } - fflush(stdout); - result(label, r.launched && r.exited && !r.timed_out && r.term_signal == 0 && r.exit_code == 0); - xf_guest_result_free(&r); -} - -// Copy a host file into the shared folder, so the guest can see it at /host. -static int stage_into_share(const char *source, const char *destination) { - FILE *in = fopen(source, "rb"); - if (in == NULL) - return -1; - FILE *out = fopen(destination, "wb"); - if (out == NULL) { - fclose(in); - return -1; - } - char buffer[65536]; - size_t n; - while ((n = fread(buffer, 1, sizeof buffer, in)) > 0) { - if (fwrite(buffer, 1, n, out) != n) { - fclose(in); - fclose(out); - return -1; - } - } - fclose(in); - fclose(out); - return 0; -} - -int main(int argc, char **argv) { - if (argc < 2) { - fprintf(stderr, "usage: engine-smoke [workdir]\n"); - return 2; - } - const char *src_root = argv[1]; - const char *work = argc > 2 ? argv[2] : "/tmp/xforge-smoke"; - - char root[4096], host[4096]; - snprintf(root, sizeof root, "%s", src_root); - snprintf(host, sizeof host, "%s/host", work); - - say("[smoke] === XForge engine smoke test ==="); - say("[smoke] rootfs : %s (unpacked fakefs)", src_root); - say("[smoke] hostdir : %s", host); - - if (access(src_root, R_OK) != 0) { - say("[smoke] FAIL the rootfs directory is not readable: %s", strerror(errno)); - return 2; - } - - step("prepare work directory"); - rm_rf(work); - if (mkdirs(work) != 0 || mkdirs(host) != 0) { - say("[smoke] FAIL mkdir %s: %s", work, strerror(errno)); - return 1; - } - result("prepare work directory", 1); - - // --- 1. the root ------------------------------------------------------ - // The fakefs root is mounted in place, so point the harness at the caller's - // directory rather than copying it: in the app this is `/ - // embedded-linux/roots/alpine-3.21`, the result of unzipping the bundled - // alpine-rootfs.zip. Unzipping needs no engine, so this harness only checks - // that what it was given is a usable fakefs. - step("check the unpacked rootfs (data/ + meta.db)"); - char data_dir[4096]; - snprintf(data_dir, sizeof data_dir, "%s/data", root); - char meta_db[4096]; - snprintf(meta_db, sizeof meta_db, "%s/meta.db", root); - struct stat st; - int root_ok = 1; - if (stat(data_dir, &st) != 0 || !S_ISDIR(st.st_mode)) { - say("[smoke] FAIL %s is not a directory", data_dir); - root_ok = 0; - } else if (stat(meta_db, &st) != 0 || st.st_size == 0) { - say("[smoke] FAIL %s is missing or empty", meta_db); - root_ok = 0; - } - if (!root_ok) { - result("check the unpacked rootfs", 0); - say("[smoke] === stopped: not a fakefs root, so boot was never attempted ==="); - return 1; - } - say("[smoke] rootfs : %s", root); - result("check the unpacked rootfs", 1); - - // --- 2. boot ----------------------------------------------------------- - step("boot the guest (mount_root + become_first_process)"); - int rc = xf_ish_boot(root, host); - { - const char *err = xf_ish_last_error(); - say("[smoke] boot returned %d%s%s", rc, - (err != NULL && err[0] != '\0') ? " : " : "", - (err != NULL && err[0] != '\0') ? err : ""); - } - if (rc != 0) { - result("boot the guest", 0); - say("[smoke] === stopped: the guest did not boot ==="); - return 1; - } - result("boot the guest", 1); - - // --- 3. run commands --------------------------------------------------- - // The probes ToolchainManager.guestHas() runs, then the kind of work the - // install paths do. `swift`/`xtool` are *meant* to be missing here: this is - // a bare Alpine minirootfs, before any provisioning. - run_guest("uname", "uname -a", 120000, 0); - run_guest("alpine-release", "cat /etc/alpine-release; command -v sh; echo home=$HOME path=$PATH", 120000, 0); - run_guest("probe: swift (absent in a bare rootfs)", "command -v swift >/dev/null 2>&1", 120000, 127); - run_guest("probe: xtool (absent in a bare rootfs)", "command -v xtool >/dev/null 2>&1", 120000, 127); - run_guest("a failing command reports its status", "echo one; echo two >&2; exit 7", 120000, 7); - - // --- 4. the /host share ------------------------------------------------ - // The SDK install stages hundreds of MB here and installs from it, so a - // share that does not actually reach the guest breaks every install. - step("write a file into the host share"); - { - char path[4096]; - snprintf(path, sizeof path, "%s/share-probe.txt", host); - FILE *f = fopen(path, "w"); - if (f == NULL) { - say("[smoke] FAIL could not write %s: %s", path, strerror(errno)); - failures++; - } else { - fputs("host-share-ok\n", f); - fclose(f); - } - } - result("write a file into the host share", 1); - run_guest("read it from the guest at /host", "cat /host/share-probe.txt", 120000, 0); - run_guest("guest can write into /host", "echo from-guest > /host/written-by-guest.txt && cat /host/written-by-guest.txt", 120000, 0); - step("read the guest's file back on the host"); - { - char path[4096]; - snprintf(path, sizeof path, "%s/written-by-guest.txt", host); - FILE *f = fopen(path, "r"); - char buf[256] = {0}; - int ok = f != NULL && fgets(buf, sizeof buf, f) != NULL - && strncmp(buf, "from-guest", 10) == 0; - if (f != NULL) fclose(f); - say("[smoke] host sees: %s", ok ? buf : "(missing)"); - result("host share is bidirectional", ok); - } - - // --- 5. a long-running command, as a build would be -------------------- - run_guest("a slower command (shell loop)", "i=0; while [ $i -lt 200 ]; do i=$((i+1)); done; echo counted=$i", 120000, 0); - - // --- 6. what provisioning will need -------------------------------- - // Installing the Swift toolchain runs `apk add` and then downloads a - // several-hundred-megabyte tarball *inside* the guest, so the guest needs - // the host's network. These are probes, not assertions: a failure here is - // a fact about the environment, and it is far better to see it here than - // as a silent spinner on a phone. - step("write the guest's /etc/resolv.conf from the host's DNS"); - { - char servers[4][64]; - int count = host_dns_servers(servers, 4); - if (count == 0) { - say("[smoke] host published no DNS servers; leaving the guest alone"); - } else { - // One `nameserver` line per server: musl's resolver reads - // /etc/resolv.conf line by line and anything else on the line (a - // separator, a comment) becomes part of the address. - char cmd[1024]; - int n = snprintf(cmd, sizeof cmd, "rm -f /etc/resolv.conf"); - for (int i = 0; i < count && n > 0 && (size_t) n < sizeof(cmd); i++) - n += snprintf(cmd + n, sizeof(cmd) - (size_t) n, - "; echo 'nameserver %s' >> /etc/resolv.conf", servers[i]); - snprintf(cmd + n, sizeof(cmd) - (size_t) n, "; cat /etc/resolv.conf"); - struct xf_guest_result r; - int rc = xf_ish_run(cmd, NULL, 60000, 1 << 16, &r); - if (rc != 0) { - say("[smoke] could not write resolv.conf: %s", xf_ish_last_error()); - } else { - say("[smoke] guest resolv.conf now:\n%s", r.output != NULL ? r.output : ""); - } - xf_guest_result_free(&r); - } - } - - probe_guest("network: DNS + HTTP from the guest", - "wget -q -T 30 -O /dev/null https://dl-cdn.alpinelinux.org/alpine/ && echo net-ok", 180000); - probe_guest("network: apk update (the first thing provisioning runs)", - "apk update 2>&1 | tail -3", 300000); - - // --- 7. the provisioning steps themselves (opt-in) ---------------- - // With XFORGE_SMOKE_TOOLCHAIN pointing at the app's install script, run the - // same steps the Toolchain screen runs -- base packages, the glibc layer, - // xtool -- and then try to *run* what they installed. This is the difference - // between "the install command succeeded" and "the tool works in a guest". - { - const char *script = getenv("XFORGE_SMOKE_TOOLCHAIN"); - if (script != NULL && script[0] != '\0') { - char staged[4096]; - snprintf(staged, sizeof staged, "%s/install-toolchain.sh", host); - step("stage the provisioning script into the guest"); - if (stage_into_share(script, staged) != 0) { - say("[smoke] could not copy %s into the share", script); - } else { - result("stage the provisioning script", 1); - probe_guest("provision: base packages", - "cp -f /host/install-toolchain.sh /root/ && sh /root/install-toolchain.sh deps", 900000); - probe_guest("provision: glibc layer", - "sh /root/install-toolchain.sh glibc", 1800000); - probe_guest("provision: xtool", - "sh /root/install-toolchain.sh xtool", 900000); - probe_guest("provision: swiftly (the Swift installer)", - "sh /root/install-toolchain.sh swiftly", 900000); - probe_guest("provision: Swift toolchain (this downloads a lot)", - "sh /root/install-toolchain.sh swift", 3600000); - probe_guest("provision: verify (the script's own verdict)", - "sh /root/install-toolchain.sh verify", 600000); - probe_guest("does xtool actually run in the guest?", - "xtool --version", 300000); - probe_guest("does swift actually run in the guest?", - "swift --version", 600000); - - // --- PI futexes, which is what Swift's Mutex is built on ------- - // The engine answered ENOSYS for every PI futex op, and Swift 6's - // Synchronization.Mutex turns that unexpected errno into a - // fatalError -- a SIGTRAP, "Trace/breakpoint trap", exit 133. So - // any Swift binary doing concurrent work died on its first Mutex - // while single-threaded commands kept working. The C probe checks - // the syscall itself, under contention; the Swift probe is the - // end-to-end check that the fix reaches the runtime the app uses. - { - const char *sources[] = { - "Tools/engine-smoke/futex-pi-probe.c", - "Tools/engine-smoke/swift-mutex-probe.swift", - }; - int staged = 1; - step("stage the PI futex and Swift Mutex probes"); - for (unsigned i = 0; i < sizeof sources / sizeof sources[0]; i++) { - const char *slash = strrchr(sources[i], '/'); - char destination[4096]; - snprintf(destination, sizeof destination, "%s/%s", host, - slash != NULL ? slash + 1 : sources[i]); - if (stage_into_share(sources[i], destination) != 0) { - say("[smoke] could not stage %s (run this from the repository root)", - sources[i]); - staged = 0; - } - } - result("stage the PI futex and Swift Mutex probes", staged); - if (staged) { - probe_assert("PI futexes: FUTEX_LOCK_PI / TRYLOCK_PI / UNLOCK_PI", - "cd /root && cp -f /host/futex-pi-probe.c . && " - "clang -O1 -pthread -o futex-pi-probe futex-pi-probe.c && " - "./futex-pi-probe", - 900000); - // Informational, not a gate: compiling Swift in the emulated - // guest did not finish in 30 minutes on the CI host (no output - // at all), while the payload build compiles a Swift program - // natively in seconds — so *that* is where compile capability - // is asserted. What this step still shows, when it returns, is - // the end-to-end Mutex path under libdispatch. - probe_guest("Swift Synchronization.Mutex under libdispatch workers (informational)", - "swiftc -O -o /root/swift-mutex-probe /host/swift-mutex-probe.swift " - "&& /root/swift-mutex-probe", - 600000); - } - } - } - } - } - - step("shut down"); - xf_ish_shutdown(); - result("shut down", 1); - - say("[smoke] === %s (%d failed step%s) ===", - failures == 0 ? "PASSED" : "FAILED", failures, failures == 1 ? "" : "s"); - return failures == 0 ? 0 : 1; -} diff --git a/Tools/engine-smoke/futex-pi-probe.c b/Tools/engine-smoke/futex-pi-probe.c deleted file mode 100644 index f322da8..0000000 --- a/Tools/engine-smoke/futex-pi-probe.c +++ /dev/null @@ -1,174 +0,0 @@ -/* - * futex-pi-probe.c — run *inside* the XForge guest: do PI futexes work? - * - * Swift 6's Synchronization.Mutex is built on FUTEX_LOCK_PI, and the engine - * used to answer ENOSYS for it. Swift's Linux implementation turns an - * unexpected futex errno into - * - * Synchronization/LinuxImpl.swift:194: Fatal error: - * Unknown error occurred while attempting to acquire a Mutex - * - * which is a trap (exit 133, "Trace/breakpoint trap"), so any Swift binary - * doing concurrent work — xtool, libdispatch's DispatchWorker threads — died on - * its very first Mutex while single-threaded commands kept working. - * - * This checks the syscall directly (fast, deterministic) and then through the - * C library's own PI mutexes, including under contention with real threads, - * which is the part a wrong implementation would deadlock or double-take on. - * - * Built and run by Tools/engine-smoke/engine-smoke.c in the guest. - */ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#if defined(__has_include) -# if __has_include() -# include -# endif -#endif -#ifndef FUTEX_LOCK_PI -#define FUTEX_LOCK_PI 6 -#define FUTEX_UNLOCK_PI 7 -#define FUTEX_TRYLOCK_PI 8 -#endif -#ifndef FUTEX_PRIVATE_FLAG -#define FUTEX_PRIVATE_FLAG 128 -#endif - -static int failures; - -static void check(const char *what, int ok, const char *detail) { - printf(" %-46s %s%s%s\n", what, ok ? "ok" : "FAIL", - detail != NULL && detail[0] != '\0' ? " — " : "", - detail != NULL ? detail : ""); - fflush(stdout); - if (!ok) - failures++; -} - -/* A guest thread blocked on a lock that never comes free would otherwise burn - * the harness's whole timeout with no output; this turns that into a result. - * The correct probe finishes in well under a second of guest time (the emulator - * makes it seconds, not minutes). */ -static void watchdog(int sig) { - (void) sig; - printf("PI-FUTEX TIMED OUT — a lock or a wait never returned\n"); - fflush(stdout); - _exit(2); -} - -/* FUTEX_LOCK_PI/FUTEX_UNLOCK_PI take the value argument as unused, so this - * passes 0 like glibc does; errno is the useful result. */ -static int pi_op(int *word, int op) { - return (int) syscall(SYS_futex, word, op | FUTEX_PRIVATE_FLAG, 0, NULL, NULL, 0); -} - -/* Deliberately small: these ops are microseconds natively but the guest is an - * emulated aarch64, and the point is whether lock/unlock/wake work under - * contention at all — not throughput. 4 x 250 contended pairs is ample. */ -#define THREADS 4 -#define ROUNDS 250 - -static int counter; /* guarded by the raw PI word */ - -static void *raw_worker(void *arg) { - (void) arg; - for (int i = 0; i < ROUNDS; i++) { - while (pi_op(&counter, FUTEX_LOCK_PI) != 0) { - if (errno != EAGAIN && errno != EINTR) { - printf(" raw FUTEX_LOCK_PI failed: %s\n", strerror(errno)); - return NULL; - } - } - /* The word must name us while we hold it. */ - if (counter != (int) gettid()) { - printf(" owner tid wrong while held: %d != %d\n", counter, (int) gettid()); - return NULL; - } - /* The word must NOT be written here: for a PI futex the kernel owns it, - * and FUTEX_UNLOCK_PI is what clears it and wakes a waiter. Zeroing it - * first made the unlock an EPERM no-op, so no wake was ever issued and - * the waiters slept for good — which is how this probe first timed out. */ - pi_op(&counter, FUTEX_UNLOCK_PI); - } - return NULL; -} - -int main(void) { - setvbuf(stdout, NULL, _IONBF, 0); - signal(SIGALRM, watchdog); - alarm(120); - printf("futex-pi-probe: tid=%d\n", (int) gettid()); - - /* 1. The syscall on its own: free word, lock, unlock, and the owner check. */ - int word = 0; - int rc = pi_op(&word, FUTEX_LOCK_PI); - check("FUTEX_LOCK_PI on a free word", rc == 0, rc == 0 ? "" : strerror(errno)); - check("the word records the owner", word == (int) gettid(), ""); - - rc = pi_op(&word, FUTEX_TRYLOCK_PI); - check("FUTEX_TRYLOCK_PI on our own lock is EDEADLK", - rc == -1 && errno == EDEADLK, rc == -1 ? strerror(errno) : "succeeded"); - - rc = pi_op(&word, FUTEX_UNLOCK_PI); - check("FUTEX_UNLOCK_PI by the owner", rc == 0 && word == 0, - rc == 0 ? "" : strerror(errno)); - - errno = 0; - rc = pi_op(&word, FUTEX_UNLOCK_PI); - check("FUTEX_UNLOCK_PI by a non-owner is EPERM", - rc == -1 && errno == EPERM, rc == -1 ? strerror(errno) : "succeeded"); - - /* 2. Contended: take the lock in one thread, TRYLOCK_PI from another. */ - { - word = 0; - pi_op(&word, FUTEX_LOCK_PI); - pid_t child = fork(); - if (child == 0) { - errno = 0; - int r = pi_op(&word, FUTEX_TRYLOCK_PI); - _exit((r == -1 && errno == EAGAIN) ? 0 : 1); - } - int status = 0; - waitpid(child, &status, 0); - check("FUTEX_TRYLOCK_PI from another task is EAGAIN", - WIFEXITED(status) && WEXITSTATUS(status) == 0, ""); - pi_op(&word, FUTEX_UNLOCK_PI); - } - - /* 3. Real threads through the raw syscall, one word, contended. */ - { - counter = 0; - pthread_t t[THREADS]; - for (int i = 0; i < THREADS; i++) - pthread_create(&t[i], NULL, raw_worker, NULL); - for (int i = 0; i < THREADS; i++) - pthread_join(t[i], NULL); - check("raw PI futex, 4 threads x 250 contended lock/unlock", - counter == 0, "no corruption, no deadlock"); - } - - /* The C library's own PTHREAD_PRIO_INHERIT mutexes are deliberately NOT - * tested here. musl drives those through a different pattern — an absolute - * deadline passed as FUTEX_LOCK_PI's timeout, a userspace CAS of the word - * before a plain FUTEX_WAKE on unlock — and that path hung in the emulated - * guest, which says something about musl's PI mutexes in this environment - * rather than about the PI futexes Swift uses. Swift 6's Synchronization.Mutex - * calls FUTEX_LOCK_PI/FUTEX_UNLOCK_PI directly, which is what the tests above - * cover, so that is what this probe asserts. */ - - alarm(0); - if (failures == 0) - printf("PI-FUTEX OK\n"); - else - printf("PI-FUTEX FAILED (%d)\n", failures); - return failures == 0 ? 0 : 1; -} diff --git a/Tools/engine-smoke/swift-mutex-probe.swift b/Tools/engine-smoke/swift-mutex-probe.swift deleted file mode 100644 index 4e0eb58..0000000 --- a/Tools/engine-smoke/swift-mutex-probe.swift +++ /dev/null @@ -1,37 +0,0 @@ -// swift-mutex-probe.swift — run *inside* the XForge guest: does a Swift 6 -// Synchronization.Mutex work, under real concurrency? -// -// This is the exact shape of the failure the app hit: -// -// Synchronization/LinuxImpl.swift:194: Fatal error: -// Unknown error occurred while attempting to acquire a Mutex -// ... Trace/breakpoint trap (exit 133) -// -// Swift's Mutex is built on FUTEX_LOCK_PI, and the engine answered ENOSYS for -// it, so any Swift binary doing concurrent work died on its first Mutex — -// xtool's `xtool sdk build`, which drives libdispatch worker threads, was the -// one that surfaced it. `DispatchQueue.concurrentPerform` reproduces the same -// thread pattern without needing xtool's download, so this is the fast check -// that the engine fix actually reaches Swift. -// -// Built and run by Tools/engine-smoke/engine-smoke.c in the guest. - -import Dispatch -import Synchronization - -let counter = Mutex(0) -let iterations = 64 * 1000 -let workers = 8 - -DispatchQueue.concurrentPerform(iterations: workers) { _ in - for _ in 0..<(iterations / workers) { - counter.withLock { $0 += 1 } - } -} - -let total = counter.withLock { $0 } -if total == iterations { - print("swift-mutex-ok \(total)") -} else { - print("swift-mutex-wrong \(total) of \(iterations)") -} diff --git a/Vendor/ish-arm64 b/Vendor/ish-arm64 deleted file mode 160000 index e1d5794..0000000 --- a/Vendor/ish-arm64 +++ /dev/null @@ -1 +0,0 @@ -Subproject commit e1d579480fba88e8f0428e3cf23811bcdd05421f From d1c0c32518fcc17b12c3b637928a361890b5c4f7 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 05:26:22 -0400 Subject: [PATCH 04/54] native: keep DownloadError, which two live callers still raise Deleting DownloadManager took the error type with it, but XForgeReleases and NativeSDK both throw it. Recovered from the deleted file, minus the .notDownloaded case that only the download manager could produce. --- App/Services/DownloadError.swift | 26 ++++++++++++++++++++++++++ 1 file changed, 26 insertions(+) create mode 100644 App/Services/DownloadError.swift diff --git a/App/Services/DownloadError.swift b/App/Services/DownloadError.swift new file mode 100644 index 0000000..205faa4 --- /dev/null +++ b/App/Services/DownloadError.swift @@ -0,0 +1,26 @@ +import Foundation + +/// Failures of a network download. +/// +/// This used to live in `DownloadManager`, which served the Downloads screen and +/// the Xcode.xip staging flow — both of which went with the guest. The error type +/// stayed because two things still download: `XForgeReleases` (release metadata and +/// asset URLs) and `NativeSDK` (the Darwin SDK bundle). Its cases are the ones those +/// two raise; the `.notDownloaded` case went with the download manager that could +/// report a partial file. +enum DownloadError: LocalizedError { + case http(status: Int, url: URL) + case invalidResponse(URL) + case assetNotFound(String) + + var errorDescription: String? { + switch self { + case .http(let status, let url): + return "The server returned HTTP \(status) for \(url.lastPathComponent)." + case .invalidResponse(let url): + return "Unexpected response from \(url.host ?? "the server")." + case .assetNotFound(let what): + return "No published release asset found: \(what)." + } + } +} From 484b5a36bab6c822d8d02a86652645833e30d79d Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 05:34:34 -0400 Subject: [PATCH 05/54] native: make the signing identity Sendable MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Passing AppBundleSigner.Identity into an async signer is a data-race error under strict concurrency because [String: Any] is not Sendable. The entitlements now travel as property-list bytes and are decoded inside the signer, which keeps the non-Sendable value in the scope that consumes it — and the struct is honestly Sendable rather than marked @unchecked. --- App/Services/AppBundleSigner.swift | 33 +++++++++++++++--------------- App/Services/IPASigningJob.swift | 16 ++------------- 2 files changed, 19 insertions(+), 30 deletions(-) diff --git a/App/Services/AppBundleSigner.swift b/App/Services/AppBundleSigner.swift index 91875ef..23582bc 100644 --- a/App/Services/AppBundleSigner.swift +++ b/App/Services/AppBundleSigner.swift @@ -14,16 +14,22 @@ import XKit /// `DeveloperServices` path), which is a separate, device-only piece of work. This /// signs with an identity the user already has (a `.p12` and a profile). enum AppBundleSigner { - struct Identity { + /// Everything needed to sign, as `Sendable` values. + /// + /// The entitlements travel as property-list *bytes* rather than a dictionary + /// so this stays `Sendable`: `[String: Any]` is not, and passing one into an + /// async signer is a data-race error under strict concurrency. Decoding inside + /// the signer keeps the non-`Sendable` value in the scope that consumes it. + struct Identity: Sendable { /// DER bytes of the signing certificate. var certificateDER: Data /// The private key, in the format `PrivateKey(data:)` accepts (PEM). var privateKey: Data /// `embedded.mobileprovision` contents, when the identity has one. var provisioningProfile: Data? - /// Entitlements to seal into the signature. Empty means none beyond the - /// profile's own. - var entitlements: [String: Any]? + /// A property list of entitlements to seal into the signature. Nil means + /// none beyond the profile's own. + var entitlementsPlist: Data? } enum SigningError: LocalizedError { @@ -57,30 +63,25 @@ enum AppBundleSigner { ) } - let entitlements = try entitlements(from: identity.entitlements) + let sealedEntitlements = try entitlements(from: identity.entitlementsPlist) let signer = try Signer.first() try await signer.sign( app: appBundleURL, identity: .real(certificate, privateKey), - entitlementMapping: [appBundleURL: entitlements], + entitlementMapping: [appBundleURL: sealedEntitlements], progress: { _ in } ) } - /// `Entitlements` decodes from a property list, so a dictionary has to be - /// serialized back into one. Building the type directly is not public API. - private static func entitlements(from dictionary: [String: Any]?) throws -> Entitlements { - guard let dictionary, !dictionary.isEmpty else { + /// `Entitlements` decodes from a property list. Building the type directly is + /// not public API, so the bytes are decoded here. + private static func entitlements(from plist: Data?) throws -> Entitlements { + guard let plist, !plist.isEmpty else { return try Entitlements(entitlements: []) } do { - let data = try PropertyListSerialization.data( - fromPropertyList: dictionary, - format: .xml, - options: 0 - ) - return try PropertyListDecoder().decode(Entitlements.self, from: data) + return try PropertyListDecoder().decode(Entitlements.self, from: plist) } catch { throw SigningError.entitlementsUnreadable(error.localizedDescription) } diff --git a/App/Services/IPASigningJob.swift b/App/Services/IPASigningJob.swift index c98f9ae..f71dd18 100644 --- a/App/Services/IPASigningJob.swift +++ b/App/Services/IPASigningJob.swift @@ -64,7 +64,7 @@ final class IPASigningJob: ObservableObject { ) let profileData = try provisioningProfile.map { try Data(contentsOf: $0) } - let entitlementsDictionary = try entitlements.map { try readEntitlements($0) } + let entitlementsPlist = try entitlements.map { try Data(contentsOf: $0) } status = "Unpacking \(inputIPA.lastPathComponent)…" let work = FileManager.default.temporaryDirectory @@ -88,7 +88,7 @@ final class IPASigningJob: ObservableObject { certificateDER: keyMaterial.certificate, privateKey: keyMaterial.key, provisioningProfile: profileData, - entitlements: entitlementsDictionary + entitlementsPlist: entitlementsPlist ) ) @@ -149,21 +149,11 @@ final class IPASigningJob: ObservableObject { ) try updated.write(to: plistURL, options: .atomic) } - - private func readEntitlements(_ url: URL) throws -> [String: Any] { - let data = try Data(contentsOf: url) - guard let plist = try PropertyListSerialization.propertyList(from: data, format: nil) - as? [String: Any] else { - throw SigningJobError.unreadableEntitlements - } - return plist - } } enum SigningJobError: LocalizedError { case expectedOneApp([String]) case unreadableInfoPlist - case unreadableEntitlements var errorDescription: String? { switch self { @@ -172,8 +162,6 @@ enum SigningJobError: LocalizedError { return "The IPA must contain exactly one .app in Payload/; found \(list)." case .unreadableInfoPlist: return "The app's Info.plist could not be read." - case .unreadableEntitlements: - return "The entitlements file is not a property list." } } } From 0bed6a3596c946fbdc4be95dbe396bed1d027847 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 05:44:19 -0400 Subject: [PATCH 06/54] native: walk project files synchronously MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit FileManager.DirectoryEnumerator cannot be iterated from an asynchronous context — Swift 6 rejects the for-in outright — so the walk moved into a synchronous helper. The path helper also shared its name with a parameter, which made one call site read as an attempt to call a String. --- App/Services/ProjectFiles.swift | 27 ++++++++++++++------------- 1 file changed, 14 insertions(+), 13 deletions(-) diff --git a/App/Services/ProjectFiles.swift b/App/Services/ProjectFiles.swift index 3cdb5d5..91930b4 100644 --- a/App/Services/ProjectFiles.swift +++ b/App/Services/ProjectFiles.swift @@ -23,7 +23,7 @@ enum ProjectFiles { let root = sourcesOnly ? project.rootURL.appendingPathComponent("Sources", isDirectory: true) : project.rootURL - return try await loadFiles(under: root, project: project) + return loadFiles(under: root, project: project) } static func load(relativePath: String, project: Project) async throws -> File { @@ -58,13 +58,19 @@ enum ProjectFiles { // MARK: - Internals - private static func loadFiles(under root: URL, project: Project) async throws -> [File] { + /// Synchronous on purpose. + /// + /// `FileManager.DirectoryEnumerator` cannot be iterated from an asynchronous + /// context — Swift 6 refuses the `for … in` outright ("makeIterator is + /// unavailable from asynchronous contexts") — so the walk lives here and the + /// async entry points simply call it. + private static func loadFiles(under root: URL, project: Project) -> [File] { let fileManager = FileManager.default guard fileManager.fileExists(atPath: root.path) else { return [] } guard let enumerator = fileManager.enumerator( at: root, - includingPropertiesForKeys: [.isRegularFileKey, .isDirectoryKey, .fileSizeKey], + includingPropertiesForKeys: [.isRegularFileKey, .isDirectoryKey], options: [.skipsHiddenFiles, .skipsPackageDescendants] ) else { return [] } @@ -78,7 +84,7 @@ enum ProjectFiles { continue } guard values?.isRegularFile == true else { continue } - guard let relative = relativePath(of: url, in: project) else { continue } + guard let relative = projectRelativePath(of: url, in: project) else { continue } // Text only: the editor cannot show a binary, and a project full of // build products should not be read into memory to find that out. guard let contents = try? String(contentsOf: url, encoding: .utf8) else { continue } @@ -103,7 +109,7 @@ enum ProjectFiles { } /// A project-relative path for `url`, or nil when it is outside the project. - private static func relativePath(of url: URL, in project: Project) -> String? { + private static func projectRelativePath(of url: URL, in project: Project) -> String? { let root = project.rootURL.standardizedFileURL.path let path = url.standardizedFileURL.path guard path.hasPrefix(root + "/") else { return nil } @@ -114,15 +120,13 @@ enum ProjectFiles { /// /// `..` is rejected rather than merely normalized: an editor that can write /// outside its own project is a sandbox escape with a nice UI, and the - /// standardized path check catches the symlink-free cases the textual check - /// would miss. + /// standardized-path check catches what the textual one cannot. private static func resolve(relativePath: String, in project: Project) throws -> URL { guard isSafe(relativePath: relativePath) else { throw ProjectFileError.unsafeRelativePath } - let root = project.rootURL - let url = root.appendingPathComponent(relativePath) - guard relativePath(of: url, in: project) != nil else { + let url = project.rootURL.appendingPathComponent(relativePath) + guard projectRelativePath(of: url, in: project) != nil else { throw ProjectFileError.unsafeRelativePath } return url @@ -138,14 +142,11 @@ enum ProjectFiles { enum ProjectFileError: LocalizedError { case unsafeRelativePath - case unreadable(String) var errorDescription: String? { switch self { case .unsafeRelativePath: return "That path is outside the project." - case .unreadable(let path): - return "Could not read \(path) as text." } } } From 9a128b840b8e9c8c268f8ad5509fa7feb691a593 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 06:00:36 -0400 Subject: [PATCH 07/54] docs: rewrite the documentation for a guestless XForge MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The README, the design notes and the changelog still described an app that boots Alpine, ships a rootfs, installs its toolchain inside a guest and signs with zsign. None of that is true any more, so the documentation now describes what the code does — including what it cannot do yet: - **README** — the pipeline (project → build plan → in-process compile/link → `.app` → unsigned `.ipa` → XKit signing), what works today (C/Objective-C) and what does not (Swift, SwiftPM dependencies, asset catalogs, Apple ID provisioning, on-device install), how to install the compiler bundle, and the three workflows that remain. - **CHANGELOG** — a 0.6.0 entry that opens by saying this is a breaking change: the guest and the Terminal are gone, and nothing replaces them. - **Docs/DESIGN.md** — the architecture without a guest, and a correction: projects are ordinary container directories and *are* backed up. The old text argued the opposite, because they used to live inside the guest filesystem. - **Docs/IPA-BUILD.md**, **Docs/NATIVE-TOOLCHAIN.md**, **Docs/XCODE-ALTERNATIVE.md** — the stages, the bundle (Clang and Mach-O LLD; **no Swift frontend yet**), and an honest status for each remaining gap. - **Docs/ISH-ARM64-INTEGRATION.md** — deleted with the integration it documented. Follow-ups the rewrite exposed, all fixed here: - the Toolchain screen's smoke test only compiled while its own text claimed it linked — it now compiles *and* links (the C fixture gained a `main`), and it exercises the Swift frontend too when that is linked, rather than leaving `smokeCompileSwift` unreachable; - the package stage was titled "Package & sign .ipa" but never signs; signing is the separate XKit screen; - `Makefile` named the wrong artifact (`-ios-arm64` vs `-arm64-ios`) in the `gh run download` hint; - `project.yml` pointed at a workflow file that does not exist, and still wired a `Support/Resources` directory nothing fills — removed, with its `.gitignore` rules and the engine/rootfs ignore blocks; - `Tools/test-verify-toolchain.sh` tested the deleted rootfs verifier and could not run at all; - comments that described the deleted build executor, and a settings tab labelled "Tab 5" in an app with three tabs. --- .gitignore | 12 - App/Models/BuildPipeline.swift | 6 +- App/Models/Project.swift | 4 +- App/NativeToolchain/NativeToolchain.swift | 5 +- App/Views/Settings/SettingsTab.swift | 2 +- App/Views/Toolchain/ToolchainView.swift | 49 ++- CHANGELOG.md | 85 +++++ Docs/DESIGN.md | 438 ++++++++++------------ Docs/IPA-BUILD.md | 194 +++++++--- Docs/ISH-ARM64-INTEGRATION.md | 204 ---------- Docs/NATIVE-TOOLCHAIN.md | 224 +++++++---- Docs/XCODE-ALTERNATIVE.md | 172 +++++---- Makefile | 2 +- README.md | 320 ++++++++++------ Support/Resources/.gitkeep | 1 - Tools/test-verify-toolchain.sh | 199 ---------- project.yml | 5 +- 17 files changed, 920 insertions(+), 1002 deletions(-) delete mode 100644 Docs/ISH-ARM64-INTEGRATION.md delete mode 100644 Support/Resources/.gitkeep delete mode 100644 Tools/test-verify-toolchain.sh diff --git a/.gitignore b/.gitignore index ee46c35..bb9d76c 100644 --- a/.gitignore +++ b/.gitignore @@ -19,20 +19,8 @@ XForge.xcodeproj/ .vscode/* !.vscode/extensions.json -# Embedded ish-arm64 engine -# (Vendor/ish-arm64 is a submodule/gitlink, so it is tracked; only its build -# output below is ignored.) -# Static libraries produced by EmbeddedLinux/build-ish-core.sh. -Vendor/ish-arm64-build/ # Generated by XcodeGen from project.yml's `info.properties` — the spec is the # single source of truth for the app's Info.plist. Support/Info.plist -# The bundled rootfs is a build product, not a source file: CI downloads it from -# the pinned `rootfs-v1` release and `make rootfs` builds it locally, so a clean -# checkout compiles and bundles one either way. Nothing here is tracked — an -# earlier revision committed a 4 MB tarball, and it stayed in the bundle long -# after the app stopped reading it (the IPA shipped both roots). -Support/Resources/* -!Support/Resources/.gitkeep diff --git a/App/Models/BuildPipeline.swift b/App/Models/BuildPipeline.swift index 8f36b5d..97f91ab 100644 --- a/App/Models/BuildPipeline.swift +++ b/App/Models/BuildPipeline.swift @@ -1,6 +1,10 @@ import Foundation /// The stages of the on-device IPA build pipeline. +/// +/// The last stage packages; it does not sign. Signing needs an Apple Developer +/// identity, which arrives separately (the Signing screen), so a stage that claimed +/// to sign would be reporting work nobody did. enum BuildStage: String, CaseIterable, Identifiable { case provision case sdk @@ -19,7 +23,7 @@ enum BuildStage: String, CaseIterable, Identifiable { case .configure: return "Configure app" case .resolve: return "Resolve dependencies" case .compile: return "Compile (arm64-apple-ios)" - case .package: return "Package & sign .ipa" + case .package: return "Package .ipa" case .artifact: return "Stage artifact" } } diff --git a/App/Models/Project.swift b/App/Models/Project.swift index 8131fb6..c2df697 100644 --- a/App/Models/Project.swift +++ b/App/Models/Project.swift @@ -75,7 +75,9 @@ enum BuildEvent: Sendable { case failed(String) } -/// Pluggable build backend. `Local` = embedded Linux VM, `Remote` = future build server. +/// Pluggable build backend. There is one implementation — the in-process +/// toolchain (`NativeBuildExecutor`) — and the seam is kept so a future backend +/// (a remote build server, say) does not have to rewrite the pipeline. @MainActor protocol BuildExecutor { /// Verify the user-installed base toolchain + xtool without installing it. diff --git a/App/NativeToolchain/NativeToolchain.swift b/App/NativeToolchain/NativeToolchain.swift index e2b963a..a2ee9ad 100644 --- a/App/NativeToolchain/NativeToolchain.swift +++ b/App/NativeToolchain/NativeToolchain.swift @@ -208,7 +208,10 @@ enum NativeToolchain { let source = dir.appendingPathComponent("main.c") let object = dir.appendingPathComponent("main.o") - try "int xforge_native_smoke(void) { return 42; }\n" + // A real entry point, so the object can also be *linked*: a smoke test + // that only compiles cannot tell a working toolchain from one that can + // produce an object the linker rejects. + try "int main(void) { return 0; }\nint xforge_native_smoke(void) { return 42; }\n" .write(to: source, atomically: true, encoding: .utf8) let result = try compileC(source: source, object: object, sdk: sdk) diff --git a/App/Views/Settings/SettingsTab.swift b/App/Views/Settings/SettingsTab.swift index f2a2387..f5ee26b 100644 --- a/App/Views/Settings/SettingsTab.swift +++ b/App/Views/Settings/SettingsTab.swift @@ -1,6 +1,6 @@ import SwiftUI -/// Tab 5 — settings, storage, diagnostics, about. +/// Tab 3 — toolchain, app files, build defaults, diagnostics, about. struct SettingsTab: View { @ObservedObject var preferences: AppPreferences diff --git a/App/Views/Toolchain/ToolchainView.swift b/App/Views/Toolchain/ToolchainView.swift index 0aed3ec..a7ac5d2 100644 --- a/App/Views/Toolchain/ToolchainView.swift +++ b/App/Views/Toolchain/ToolchainView.swift @@ -103,8 +103,9 @@ struct ToolchainView: View { } header: { Text("Verify") } footer: { - Text("Compiles a C file with clang and links it with ld64.lld, in this process, " - + "against the installed SDK. This is what a real build does.") + Text("Compiles a C file with clang, links it into an arm64 executable with " + + "ld64.lld, and — when the Swift frontend is linked — compiles a Swift " + + "file too. All in this process, against the installed SDK.") } if let message { @@ -198,17 +199,53 @@ struct ToolchainView: View { } } + /// Compile *and* link, in this process, against the installed SDK. + /// + /// A smoke test that only compiles cannot tell a working toolchain from one + /// whose objects the linker rejects, and the linker is the half that carries + /// the platform version and the entry point. The Swift frontend is exercised + /// too when it is linked, so the moment the frontend libraries arrive this + /// screen proves the whole chain rather than half of it. private func runSmokeTest() async { working = true smokeResult = nil defer { working = false } do { let layout = try NativeSDK.layout() - let object = try await Task.detached(priority: .userInitiated) { - try NativeToolchain.smokeCompile(sdk: layout.sdkRoot) + smokeResult = try await Task.detached(priority: .userInitiated) { () -> String in + // Matches the default target of `NativeToolchain.compileC` + // (`arm64-apple-ios17.0.0`), which is also XForge's own floor. + let minimumIOSVersion = "17.0" + var lines: [String] = [] + + let object = try NativeToolchain.smokeCompile(sdk: layout.sdkRoot) + lines.append("✓ clang: \(object.lastPathComponent)") + + let executable = object + .deletingLastPathComponent() + .appendingPathComponent("xforge-smoke") + let linked = try NativeToolchain.linkMachO(arguments: [ + "-arch", "arm64", + "-platform_version", "ios", minimumIOSVersion, minimumIOSVersion, + "-syslibroot", layout.sdkRoot.path, + "-o", executable.path, + "-lSystem", + object.path + ]) + guard linked.succeeded else { + throw NativeToolchainError.link(linked.diagnostics) + } + let size = (try? executable.resourceValues(forKeys: [.fileSizeKey]))?.fileSize ?? 0 + lines.append("✓ ld64.lld: \(executable.lastPathComponent) (\(size) bytes)") + + if NativeToolchain.isSwiftAvailable { + let swiftObject = try NativeToolchain.smokeCompileSwift(sdk: layout) + lines.append("✓ swift-frontend: \(swiftObject.lastPathComponent)") + } else { + lines.append("· swift-frontend: not linked, so not tested") + } + return lines.joined(separator: "\n") }.value - let size = (try? object.resourceValues(forKeys: [.fileSizeKey]))?.fileSize ?? 0 - smokeResult = "✓ \(object.lastPathComponent) (\(size) bytes)" } catch { smokeResult = error.localizedDescription } diff --git a/CHANGELOG.md b/CHANGELOG.md index 0643436..80252bc 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -2,6 +2,91 @@ All notable changes to **XForge** are documented here. +## [0.6.0] — 2026-09-24 — The embedded Linux is removed; the toolchain runs in the app + +**Breaking change.** XForge no longer boots a Linux guest. The emulator, the +root filesystem, the Terminal tab and the guest-side services that provisioned and +drove them are gone from the app; what builds an app now is the LLVM/Clang/LLD and +Swift libraries linked into the process, called through a C++ bridge. Anyone who +relied on the guest or on the Terminal has nothing to fall back to: the Terminal +existed to type into the guest, and with the guest gone there was nothing behind +it. Projects are no longer files inside a Linux filesystem you could `cd` into; +they are ordinary directories in the app's container. + +### Removed +- **`App/EmbeddedVM/**`** — the ish-arm64 bridge, the emulator, the rootfs + installer, the console and shell sessions. The app links no Linux kernel now. +- **`App/Build/EmbeddedLinuxExecutor.swift` and `App/Build/SDKInstaller.swift`** — + replaced by `NativeBuildExecutor` and `NativeSDK`. +- **`App/Services/{SystemComponents,GuestNetwork,ToolchainManager,IPAConfigureSignService}.swift`, + `HostDNS.{c,h}`, `DownloadManager.swift`, `DirectorySize.swift`** — guest + provisioning, guest DNS, the in-guest component installer and the `zsign` job. +- **`App/Views/Terminal/**`, `EngineLogView`, `DownloadsView`** — the Terminal, the + engine log screen, and the download screen that existed to drive `curl` inside + the guest. The app is now three tabs: Projects, Build, Settings. +- **The bundled Alpine rootfs and everything that produced it.** The artifact was + ~1.6 GB and arrived with a five-gigabyte provisioning story attached; there is no + rootfs, no `install-toolchain.sh` to run in a guest, and no pinned root asset to + download. `EmbeddedLinux/` is gone, and `project.yml` no longer links `-lish`, + `-lz` or `-liconv`, sets `GUEST_ARM64`, or requires `Vendor/ish-arm64*` to exist. + +### Added +- **`App/Build/NativeBuildPlan.swift`** — reads a project directory into a fully + resolved plan (module, sources, resources, frameworks, search paths) and refuses, + **by name**, the one thing an in-process driver cannot resolve: SwiftPM + dependencies. +- **`App/Build/NativeToolchainInvocation.swift`** — the `swift-frontend` and + `ld64.lld` argument lists as pure functions, so they are testable on a simulator + where no compiler libraries exist. +- **`App/Services/AppBundleSigner.swift` and `App/Services/IPASigningJob.swift`** — + in-process signing with XKit. The private key is read through the Security + framework and used in memory; the `.p12` password file that `zsign` needed on + disk is gone with it. +- **`App/Services/ProjectFiles.swift`** — replaces `GuestProjectFiles`, keeping the + same relative-path discipline (a `..` is rejected, not normalised) without a + guest underneath it. +- **`App/Views/Toolchain/ToolchainView.swift`** — the toolchain and SDK screen, now + native: it reports which libraries are linked in, installs or removes the Darwin + SDK in the app container, and runs a compile smoke test. + +### Changed +- **Build path.** `NativeBuildExecutor` compiles each translation unit through the + bridge, links with `ld64.lld`, assembles the `.app` and packages the unsigned + `.ipa`, streaming every step into the Build screen's console. `BuildManager`'s + stages lost their guest commands: the SDK stage installs the bundle into the app + container, and "Configure" validates the project directory instead of `mkdir`-ing + inside a guest. +- **Projects move to the app's own container**, `/projects/`, and + `Project.rootURL` is derived from the project name. They used to live inside the + guest filesystem, which is why `Project.rootPath` still carries a guest-shaped + string (`projects/`) that is now only used to check a record against the + directory it names. +- **Export is a ZIP of a directory instead of a `tar` inside the guest** + (`ProjectExporter`, ZIPFoundation), skipping `.xforge-build` and `.build`. +- **The app's log is an ordinary file the user can share.** It used to be the + engine's kernel messages through the ish bridge. +- **Importing a project copies a folder picked in Files.** An iOS process cannot + run `git`, so a clone is not something the app can offer. +- **The dependencies editor is gone.** A project that declares SwiftPM dependencies + cannot be resolved without a package manager, so `NativeBuildPlanFactory` refuses + such a project with a message that names the packages — an editor that could only + produce unbuildable projects was worse than no editor. +- **The guest filesystem is no longer the reason a project is not backed up.** See + the reversal under "Projects can leave the device" below: what that entry + described as impossible is now the default, because a project is a plain + directory in the container rather than a file inside an opaque fakefs. + +### Known gap +- **The Swift frontend is not in the toolchain artifact.** + `.github/workflows/native-toolchain.yml` builds Clang and Mach-O LLD only, so a + project whose sources are Swift still cannot be compiled. The build now fails at + the Swift compile step with `swiftFrontendMissing`, naming the count of files, + instead of silently needing a guest. C and Objective-C targets compile and link + today. The guest is not coming back as a fallback: the honest replacement for it + is a toolchain artifact that carries the frontend. +- **Asset catalogs are copied uncompiled**, with a warning: `actool` is a + macOS-only tool with no open-source replacement. + ## Projects can leave the device, and the guest filesystem stays out of backups ### Added diff --git a/Docs/DESIGN.md b/Docs/DESIGN.md index 787d9db..33e0ca8 100644 --- a/Docs/DESIGN.md +++ b/Docs/DESIGN.md @@ -1,273 +1,231 @@ # XForge — build iOS apps on-device with xtool -Working title: **XForge**. An iOS app (sideload-only) that embeds a Linux userspace -running [xtool](https://github.com/xtool-org/xtool) + a Swift toolchain, so you can -author SwiftPM packages and compile them into real iOS apps entirely on your iPhone. +Working title: **XForge**. An iOS app (sideload-only) that compiles an authored +project into a real, signed iOS app entirely on the phone, using the compiler, +linker and signing libraries linked into its own process. -Status: Architecture A (fully on-device via embedded iSH-style Linux), built from -scratch, sideload-only. +Status: fully on-device, no remote build host. The old architecture — an embedded +iSH-style Linux — is gone; this document describes what replaced it and why the +replacement takes the shape it does. --- ## 1. How the build works (the mechanism) -`xtool dev build` is just an orchestration layer over **SwiftPM cross-compilation**: +XForge does not orchestrate a build by running other programs. **iOS does not let +an app spawn a process** — there is no `fork`/`exec`/`posix_spawn` in the sandbox — +so every step that a build tool would normally run as a child is instead a library +call in XForge's own process: + +1. **Read the project into a plan.** `NativeBuildPlanFactory` walks the project + directory (`Package.swift`, `Sources//`) and produces a + `NativeBuildPlan`: the module and executable names, the Swift and C-family + translation units, the resources, the frameworks, and the `-L`/`-F` search + paths. Nothing compiles until this value exists, and nothing in the executor + re-decides any of it. +2. **Compile.** Swift sources would go through `swift::performFrontend` — but the + frontend libraries are not in the toolchain artifact yet, so a plan with Swift + files is refused at this point (§2, third bullet). C, Objective-C and + Objective-C++ sources compile through clang's `CompilerInstance` with + `EmitObjAction`. Both take an argument list built by + `NativeToolchainInvocation`, against the iPhoneOS SDK in the Darwin SDK bundle. +3. **Link.** `ld64.lld` (through `lld::lldMain`) links the objects against the SDK's + `.tbd` stubs and the static Swift runtime into an `arm64-apple-ios` executable. +4. **Assemble.** The executor writes `.app`: the executable at mode 0755, an + `Info.plist` (generated keys, over which a hand-written project template is + merged), and the resources. +5. **Package.** `IPABuilder` lays the bundle out under `Payload/` and zips it. The + result is an **unsigned** `.ipa`. +6. **Sign.** `IPASigningJob` + `AppBundleSigner` sign with `XKit`, xtool's own + library and the same code path that signs on a Mac. Signing is a separate step + from packaging on purpose: the pipeline can hand an unsigned IPA to a signing + service, and it can hand the same IPA to XKit. + +### Why the plan is a separate value + +The compiler only exists in a *device* build — the libraries are linked in by +`Support/NativeToolchain.generated.xcconfig`, which is disabled on a plain clone. +A wrong argument list would therefore be undiscoverable until a build ran on a +phone with a real SDK. Splitting "what to build" (the plan) and "how to invoke +the tools" (the pure `NativeToolchainInvocation` functions) from "call the tools" +(the executor) means the parts that can be wrong silently are the parts that unit +tests can check, on a simulator, with no compiler present. `AppTests/NativeBuildTests.swift` +is exactly that. + +### Where the plan draws the line + +An in-process driver cannot do everything a build system does. Rather than fail +somewhere in the middle, the plan factory refuses by name where it cannot resolve +what is needed, and the executor states plainly where it can only copy: + +- **SwiftPM dependencies.** Resolving them means fetching packages and *running* + their manifests, which needs a package manager process. The factory scans + `Package.swift` textually for `.package(` lines — enough to answer "are there + any", which is all it needs — and refuses the project, naming the packages. +- **Projects with no `Sources/` directory**, or no compilable sources in it: there + is no useful build to produce and no way to guess one. +- **Asset catalogs are *not* refused**, but they are not compiled either: `.xcassets` + are copied into the bundle whole and the console says why. `actool` is a + macOS-only tool, so an app whose icon lives in a catalog ships without it. Copying + rather than dropping them keeps the project round-tripping. + +## 2. Where the toolchain comes from + +The Clang/LLD libraries are **cross-built for iPhoneOS in CI** on a macOS runner +(`.github/workflows/native-toolchain.yml`), flattened into a single static archive +and published as a workflow artifact. `NativeToolchain/install-bundle.sh` unpacks +that artifact into `Vendor/NativeToolchain` and `NativeToolchain/prepare-xcode.sh` +writes the xcconfig the app target consumes. + +The consequences of linking the compiler in rather than shipping it as data: + +- **It cannot be installed at runtime.** Either the libraries are in the binary or + they are not; the Toolchain screen can only *report* them. That is why + `NativeToolchainCapabilities` exists and why the first build stage fails with an + explanation instead of a linker error. +- **A plain clone still builds.** `Support/NativeToolchain.generated.xcconfig` is + checked in with the backend disabled, so the bridge compiles to a "not available" + stub. Simulator builds and `make test` need nothing else. +- **The Swift frontend is a separate, larger port.** The workflow builds Clang and + LLD only, so `swift-frontend` is absent from the artifact and Swift sources + cannot be compiled yet. The capability is reported honestly rather than assumed: + `canCompile` requires clang and LLD, not the frontend, because refusing to start + a C build would be wrong about why it works. + +## 3. Where the `darwin` SDK comes from + +`xtool sdk build ` produces the `darwin` SDK from a real Xcode — +impossible on a phone. So the `darwin.artifactbundle` is **built once in CI** and +published as a release asset under XForge's own `darwin-sdk-` release series, +and the app resolves the newest matching release at build time. + +The bundle is installed into the app container at `/native-sdk/darwin.artifactbundle` +and read directly: `NativeSDK` parses the same `swift-sdk.json` layout xtool's +builder emits, resolving the SDK root, the Swift resource directory and the static +runtime search paths. It is about 460 MB. The Toolchain screen can also install one +from a folder picked in Files, which is how a user with their own `Xcode.xip` — or +a newer artifact — replaces it; the install is staged and validated before it +replaces what is there, so a bad bundle cannot leave the app without an SDK. -1. SwiftPM (from the Linux Swift toolchain) compiles the SwiftPM package to - `arm64-apple-ios` using a Swift SDK named **`darwin`** (installed via - `swift sdk install`, SwiftPM's native mechanism). -2. xtool's `Packer` turns the `.app` bundle into a signed `.ipa` (zsign) with the - right Info.plist, entitlements, and codesigning identity. -3. `XKit` (the library) talks to Apple Developer Services and devices natively on iOS. - -So the iOS app needs three heavyweight pieces to *compile* (all Linux-side, all -self-contained, all fetchable): -- **Swift aarch64 Linux toolchain** (swift.org Ubuntu build) — compiler/clang/lld. -- **`darwin` Swift SDK bundle** for `arm64-apple-ios` — contains the iOS SDK (headers, - .tbd stubs, module maps) + the iOS Swift stdlib. This is the big one (multi-GB). -- **`xtool` aarch64 binary** (the prebuilt `xtool-aarch64.AppImage`, 51 MB). - -## 1b. The userspace is **Alpine aarch64 with the build toolchain in it**, bundled as a fakefs ZIP +> Note: Apple also publishes official iOS Swift SDKs on swift.org, but they use the +> triple `aarch64-apple-ios` under a different bundle name. xtool hardcodes +> `darwin` + `arm64-apple-ios`, so we produce our own `darwin` bundle to match +> xtool exactly. -The embedded Linux boots the official **Alpine Linux aarch64 minirootfs**: +## 4. App architecture ``` -https://dl-cdn.alpinelinux.org/alpine/v3.21/releases/aarch64/alpine-minirootfs-3.21.0-aarch64.tar.gz +XForge.app (nothing else — no guest, no subprocesses) +├─ Projects — author a SwiftPM package; ProjectFiles reads/writes the directory +│ ├─ project directory /projects/ +│ ├─ file browser / source editor / manifest editor +│ ├─ import copy a folder picked in Files (no git: an app cannot spawn it) +│ └─ export ProjectExporter → /exports/.zip +├─ Build +│ ├─ BuildManager — the stage state machine the Build screen draws +│ ├─ NativeBuildPlan — what to build, resolved and refused up front +│ ├─ NativeBuildExecutor — the BuildExecutor; calls the libraries +│ ├─ NativeToolchain* — the C++ bridge (clang, swift-frontend, ld64.lld) +│ └─ IPABuilder — Payload/ + zip → unsigned .ipa +├─ Signing — IPASigningJob: .p12 + profile → AppBundleSigner (XKit) → signed .ipa +├─ Toolchain — capabilities, Darwin SDK install/remove, smoke test +└─ Settings — app sandbox browser, build history, log, diagnostics ``` -- `EmbeddedLinux/build-rootfs.sh` downloads that archive, installs the guest's - own packages into it with the guest's own `apk` (the shell session's: see - *the console* below), **provisions the build toolchain into it by running the - guest's own `install-toolchain.sh` in a chroot of the tree** (glibc layer, apk - build dependencies, xtool, swiftly, the Swift toolchain, the Darwin SDK), - configures the root (mount points, `/etc/passwd`, `/etc/profile`, `/etc/motd`, - `/etc/apk/repositories`, a default `/etc/resolv.conf`), converts it to the - engine's `fakefs` format with the engine's own `tools/fakefsify`, and packs it - as `alpine-rootfs.zip`. -- **Provisioning is the same code path as the guest's, not a parallel one.** The - script the build runs in the chroot is the script the app ships - (`EmbeddedLinux/install-toolchain.sh`, staged at `/root/install-toolchain.sh`), - invoked step by step: `deps`, `glibc`, `xtool`, `swiftly`, `swift`, `sdk`, then - `verify`. A root provisioned on the build machine and one provisioned by hand - in the Terminal therefore differ in nothing but where and when the work - happened, and there is one implementation to keep correct. -- **The console is a root login session, and the root is what starts it.** - `/etc/inittab` respawns `/sbin/xforge-login root` on `tty1`; that script (from - `EmbeddedLinux/xforge-login`) reads root's shell out of `/etc/passwd` and execs - it as a login shell, so the Terminal tab opens in the shell the root names — - `/bin/bash` by default — and `apk add zsh` plus that field changes it. Busybox - `login -f root` is deliberately not in this path: it authenticates, needs utmp - and takes over the terminal, none of which this guest has, and none of which it - can report failing. -- **The root therefore carries the session's dependencies**: `bash`, - `coreutils`, `less` and `ncurses-terminfo` (`XFORGE_CONSOLE_PACKAGES`). The - terminfo package is not optional detail — `/etc/profile` exports - `TERM=xterm-256color`, and the *base* terminfo package does not contain the - xterm entries, so without it every curses program in the guest runs against an - unknown terminal. -- **`EmbeddedLinux/verify-rootfs.sh` runs the console program in a chroot of the - root** and asserts which shell comes up, as a login shell, from the field in - `/etc/passwd`. `build-rootfs.sh` runs it on the tree before conversion and - `build-rootfs.yml` runs it on the published ZIP, so "the terminal works" is - checked against the guest's own binaries rather than against the intent of the - configuration that produced them. -- **The conversion happens at build time, not in the app.** `fakefsify` runs on - the build machine, so the ZIP already *is* a fakefs — a `data/` tree plus - `meta.db` — and first launch costs an unzip rather than an import of thousands - of files into SQLite on a phone. This is the same layout OpenMinis ships, and - it is why `App/EmbeddedVM/RootfsInstaller.swift` uses `unzip` and then - `mount_root` instead of `fakefs_import`. -- **The root arrives ready to build.** Swift, xtool and the Darwin SDK *are* in - it, installed before the conversion, which is what the artifact's size buys: the - guest can run `xtool new` and build on first launch instead of provisioning - itself under emulation first. `XFORGE_PROVISION=none` builds the plain - few-MB root instead, where the guest installs the same things on demand with - the same script. The **glibc compatibility layer** is included either way: - every tool XForge builds with is a glibc binary (xtool is a Swift program built - on Ubuntu, and so is the toolchain), Alpine is musl, and `gcompat` is not - enough for them. Baking that layer in removes the most failure-prone step of an - on-device provision (a package renamed between Ubuntu releases yields a layer - that loads but cannot resolve a symbol, which surfaces much later inside a - tool). -- **Everything — the layer, the toolchain, the SDK — is installed before the - fakefs conversion, and that ordering is load-bearing.** `fakefsify` writes `meta.db` as an index of the tree as it - stands, and the engine resolves files through the database rather than by - scanning `data/`. Installing the layer *after* conversion therefore produces a - root where the files are on disk and completely invisible in the guest — 568 - files present, zero rows in `meta.db`. `build-rootfs.sh` asserts the layer is - indexed before it packs. -- **Slimming, then proof.** Before packing, the build drops what an iOS build - never loads — the static *Linux* stdlib, lldb, the editor tooling (sourcekit-lsp, - the index stores), and every download cache (the SDK archive, the Ubuntu `.deb` - pile the glibc layer was built from, the apk index) — and then runs - `install-toolchain.sh verify` with `XFORGE_VERIFY_COMPILE=1`, which *compiles - and runs a Swift program* and whose verdict fails the build. Printing a version - is not compiling; this is the check that makes slimming safe. -- **The root is a pinned release asset** (`rootfs-v5`) rather than rebuilt per IPA - run. `build-ipa.yml` downloads it and verifies its sha256, and fails if the - pinned root does not carry the toolchain its manifest claims; `build-rootfs.yml` - rebuilds and republishes it when the Alpine base, the toolchain or the root's - configuration changes. Committing it to git is not an option — GitHub rejects - any file over 100 MB in a push, and this one is ~1.6 GB. -- **The Darwin SDK comes from XForge's own release** (`darwin-sdk-`, built in CI - with xtool from an Xcode.xip) for the bundled root, and it is recorded — the - manifest names the tag and the sha256 of the bundle that went in, and the path - SwiftPM installed it to. A user with their own Xcode.xip can still put a - different one in: the Toolchain screen installs `xtool sdk install ` and - replaces whatever is there. - -## 2. Where the `darwin` SDK comes from - -`xtool sdk build ` produces the `darwin` SDK from a real Xcode — impossible -on a phone. Plan: **build the `darwin.artifactbundle` once in CI on a macOS runner** -(using xtool's own `SDKBuilder` from an Xcode install), then host it as a downloadable -artifact. That is what the bundled root contains: `EmbeddedLinux/build-rootfs.sh` -downloads the pinned `darwin-sdk-` asset and installs it into the root being built. -For a guest that has none — a plain root, or a swap to a newer SDK — the Toolchain -screen resolves the newest `darwin-sdk-*` release, fetches it into the guest and runs -`swift sdk install` there. +Two structural decisions are worth stating because they are not obvious: -> Note: Apple also publishes official iOS Swift SDKs on swift.org, but they use the -> triple `aarch64-apple-ios` under a different bundle name. xtool hardcodes `darwin` + -> `arm64-apple-ios`, so we build our own `darwin` bundle to match xtool exactly. +- **The build pipeline has one executor.** There used to be a choice between a + local guest and a "remote" backend that never existed. Both the choice and the + guest are gone; `XForgeEnvironment.makeExecutor` returns one implementation. +- **Signing is not part of the build stage.** The stage named "Package & sign + .ipa" packages; the signing happens on the Signing screen, where the user + supplies the identity. Keeping them apart is what lets the same unsigned IPA be + signed by XKit or handed to SideStore. -## 3. App architecture - -``` -XForge.app -├─ Native iOS (SwiftUI, fast path) ───────────────────────────────────────── -│ Project list / editor — author SwiftPM packages -│ Package manifest editor — Package.swift + Sources -│ Git integration — clone/push to GitHub -│ XKit signing — free Apple ID, Apple Developer Services -│ .ipa management — bundle, export, hand off to SideStore/AltStore -│ BuildExecutor protocol — pluggable: Local (embedded iSH) | Remote (future) -│ Embedded Linux VM — runs the compile sandbox -│ -└─ Embedded Linux userspace (aarch64, inside the VM) ──────────────────────── - Alpine/ish-arm64-style rootfs - ├─ glibc compatibility layer (Ubuntu's, for the glibc tools below) - ├─ Swift aarch64 Linux toolchain ┐ - ├─ darwin Swift SDK ├─ in the bundled root, installed at - └─ xtool (aarch64) ┘ build time by install-toolchain.sh - → `xtool new` / `xtool dev build -s -i` -``` +## 5. What is backed up, and what is not -## 3b. What is backed up, and what is not +Everything XForge generates lives in `Documents`, which iOS backs up to iCloud and +— because the app declares `UIFileSharingEnabled` — shows in the Files app. Two +kinds of things live there and they want opposite treatment: -Everything XForge generates lives in `Documents` — which iOS backs up to iCloud and, -because the app declares `UIFileSharingEnabled`, shows in the Files app. Two kinds of -things live there and they want opposite treatment: +- **Regenerable**: downloads, the Darwin SDK, staged build artifacts, the log. + Marked `isExcludedFromBackup` at launch (`XForgeEnvironment.prepareStorage`). + Backing them up bloats every device backup with data the app can produce again, + which the storage guidelines forbid. +- **User data**: projects and exports. Both are backed up, deliberately. -- **Regenerable**: host-side downloads, staged build artifacts, the engine log. Marked - `isExcludedFromBackup` at launch (`XForgeEnvironment.prepareStorage`). Backing them up - bloats every device backup with data the app can produce again, which the storage - guidelines forbid. -- **The guest filesystem** (`Documents/embedded-linux`): excluded as well. It is the - bundled rootfs imported into fakefs, and with `XFORGE_PROVISION=all` that is several - gigabytes of Alpine, Swift, xtool and the Darwin SDK — the least suited thing to - upload to iCloud once per device. +**Projects are ordinary directories in the app container and are backed up.** That +is a reversal of the old position, and the reason it changed is worth recording: a +project used to live *inside* the guest filesystem, which had to be excluded as a +whole — several gigabytes of Linux, toolchain and SDK — and nothing could mark the +megabyte of source without also marking the gigabytes around it. The only way work +survived a device restore was to export it. Now that the toolchain runs in-process, +a project is a directory of files this process can open, so it is excluded from +nothing and restored like any other document. Export remains, and is still the way +work leaves the app, but it is a convenience rather than the only escape hatch. -The price of the second one is that **the user's projects live inside it**, in the same -opaque fakefs, and nothing in the app can mark one without the other. That is why the -export exists: "Export project" on a project's screen tars it *inside the guest* and -copies the archive to `Documents/exports`, which is deliberately **not** excluded — it is -user data, it is small, and it is the thing that should survive a device restore. The -`/host` share (also visible in the Files app) and the Terminal remain the escape hatch -for anything else. +The rule the code follows is: exclude what can be downloaded or rebuilt, back up +what the user wrote. `XForgeEnvironment` is the single place that decides which +directories these are, so the storage rules cannot drift from `prepareStorage`. -If projects ever need to be backed up as projects rather than as archives, they have to -live outside the fakefs. Nothing about a flag on the directory can do it. - -## 4. BuildExecutor abstraction +## 6. BuildExecutor, and what is left of the abstraction ```swift +@MainActor protocol BuildExecutor { - func createProject(_ template: ProjectTemplate) async throws + func bootstrap() async throws -> AsyncThrowingStream + func installSDK(from source: SDKSource) async throws + func createProject(named:organizationIdentifier:) async throws -> Project + func resolve(_ project: Project) async throws -> AsyncThrowingStream func build(_ project: Project, configuration: BuildConfiguration) async throws -> AsyncThrowingStream - func installSDK(_ source: SDKSource) async throws - func fetchToolchain() async throws } ``` -- `EmbeddedLinuxExecutor`: drives the embedded VM via the `LinuxVM` bridge (below). -- `RemoteExecutor` (future): same interface over SSH/WebSocket to a build server. - -## 4b. The Linux engine is **ish-arm64** (in-process, no subprocesses) - -**iOS cannot spawn subprocesses** (no `fork`/`exec`/`posix_spawn` in the app sandbox), -so the embedded Linux cannot run as a child process. It runs **in-process** as a -library. XForge uses [ish-arm64](https://github.com/OpenMinis/ish-arm64) for this: -it is a real Linux kernel + aarch64 emulator whose threaded-code interpreter -dispatches each guest instruction to a pre-compiled "gadget" function. It emits -**no machine code** and needs no executable memory, so it requires **no JIT -entitlement** and works in a sideloaded app. ish-arm64 is vendored as the -`Vendor/ish-arm64` git submodule and built for iOS by -`EmbeddedLinux/build-ish-core.sh`. - -That build requires **clang**: the aarch64 gadget sources alias registers with -`.req` (`_cpu .req x1`, `_pc .req x28`, …) and then use those names as operands, -which only clang's integrated assembler accepts — GNU `as` rejects every such -instruction. This is why the engine is built on a macOS runner rather than on -Linux, and why `build-ish-core.sh` probes for that capability up front instead -of failing with a wall of assembler errors. - -``` -BuildExecutor (EmbeddedLinuxExecutor) - │ drives - ▼ -LinuxVM (EmbeddedLinuxVM) ← command/file bridge, runs on MainActor - │ run / copyIn / copyOut - ▼ -LinuxEmulator (protocol) ← in-process execution engine - └─ ISHEmulator ← drives the embedded ish-arm64 core - │ one dedicated serial queue (the engine's `current` is thread-local) - ▼ -ISHBridge.c ← plain-C shim (bridging header → Swift) - │ - ▼ -libish + libish_emu + libfakefs + fakefs_import (built from Vendor/ish-arm64) -``` -- the engine's primitive is one-shot command capture - (`run_guest_command_capture_shell`), so `LinuxVM.run` executes a command and - forwards its merged stdout+stderr; `copyIn`/`copyOut` still move files via - base64 over the guest shell. -- **Boot** (`ISHBridge.c`) mirrors ish-arm64's own app: mount the imported - rootfs with `mount_root`, create init with `become_first_process`, then mount - `/proc`, `/sys`, `/dev/pts`. It does *not* run `/sbin/init` — XForge runs build - commands as fresh children of init, which is all the headless runner needs. -- The engine is built for the iOS **device** (arm64) only. Simulator builds (used - by unit tests) compile a stub in `ISHBridge.c` instead, so `make test` needs - neither the submodule nor the core libraries. - -## 5. Delivery / sideload pipeline - -- **CI (GitHub Actions, macOS runner)** builds the unsigned IPA (our proven pattern from - SideStore / tctiSH), commits it to a release. -- **Device install**: user installs XForge via SideStore/AltStore (already built by user). -- **Built apps**: XForge produces `.ipa`s that are handed to SideStore to install on the +`NativeBuildExecutor` is the only implementation. The protocol survives for two +reasons: it is the seam a future `RemoteExecutor` would implement, and it keeps the +`BuildManager` state machine honest about what a stage is allowed to assume. + +Every method streams `BuildEvent`s — progress lines for the console, structured +stage transitions for the step UI — so long work reports while it happens instead +of at the end. + +## 7. Honest performance note + +Compilation is native code now, not emulation: the old note about a JIT emulator +being slow no longer applies to the compiler itself. What is still true is that the +executor compiles translation units **one at a time**, so it does not use the +device's cores the way `swift build` or `xcodebuild` do, and that a phone compiles +more slowly than a Mac. How much of either matters for a real project is +**unverified** — it has not been measured on a device against a non-trivial +project, and it cannot even be approached until Swift compiles at all. XForge is a +*demonstrator and authoring tool* for the on-device path; a remote executor is the +sane answer for large projects, and the protocol above is where it would attach. + +## 8. Delivery / sideload pipeline + +- **CI (GitHub Actions, macOS runner)** builds the unsigned IPA, verifies it, and + publishes it as a release asset. +- **Device install**: the user installs XForge via SideStore/AltStore. +- **Built apps**: XForge produces `.ipa`s that are signed with XKit (with an + identity the user supplies) or handed to SideStore to sign and install on the same device. -## 6. Honest performance note - -Swift compilation under JIT emulation is slow. Tiny SwiftUI apps (hello-world / toy -packages) build in minutes. Real-world apps are impractical on-device; that's why the -`RemoteExecutor` exists. XForge is a *demonstrator + authoring tool* for the on-device -path, with a fast remote path available later. - -## 7. Repo layout +## 9. Repo layout ``` -.github/workflows/build-ipa.yml # CI: build the ish-arm64 core + unsigned XForge.ipa -.github/workflows/build-rootfs.yml # CI: build and publish the pinned Alpine rootfs -project.yml # XcodeGen definition -App/ # SwiftUI app sources (native shell) -App/EmbeddedVM/ # LinuxVM bridge, ISHEmulator, C bridge, rootfs unpack -Support/ # Info.plist, entitlements, Resources/ (bundled rootfs) -Vendor/ish-arm64/ # git submodule: the embedded Linux engine -Vendor/ish-arm64-build/ # core static libs (built, gitignored) -EmbeddedLinux/ # build-rootfs.sh, build-ish-core.sh, install-toolchain.sh, - # verify-rootfs.sh, xforge-login (installed into the guest) -Docs/ # this design doc + tutorials -Docs/ISH-ARM64-INTEGRATION.md # engine integration vs. OpenMinis's reference +.github/workflows/ build-ipa.yml (the unsigned IPA), native-toolchain.yml (the + iOS LLVM cross-build), ios-share.yml (a simulator build) +project.yml XcodeGen definition +App/ SwiftUI app sources +App/Build/ the pipeline: plan, invocation, executor, IPABuilder, manager +App/NativeToolchain/ the in-process compiler bridge + the Darwin SDK store +App/Services/ signing, project files, export, releases, log +App/Models/ pipeline, history, project store, XcodeProject +App/Views/ Projects, Build, Signing, Toolchain, Settings +AppTests/ unit tests (plan and argument construction) +Vendor/NativeToolchain/ the installed toolchain bundle (absent by default) +NativeToolchain/ install-bundle.sh, prepare-xcode.sh +Support/ Info.plist spec, entitlements, assets, the generated xcconfig +Tools/ gen-appicon.py +Docs/ this design doc + the pipeline, toolchain and Xcode-alternative notes ``` diff --git a/Docs/IPA-BUILD.md b/Docs/IPA-BUILD.md index c3634de..e80c2c2 100644 --- a/Docs/IPA-BUILD.md +++ b/Docs/IPA-BUILD.md @@ -1,15 +1,20 @@ # XForge IPA Build Tool — design -The on-device engine that turns a SwiftPM project into a signed, sideloadable `.ipa`. -This is the "build tool" layer between the GUI and the embedded Linux. +The on-device engine that turns an authored project into a signed, sideloadable +`.ipa`. This is the "build tool" layer between the GUI and the compiler libraries +linked into the app. ## 1. Goal From a project the user has authored, produce a valid `.ipa`: ``` -SwiftPM project ──► arm64-apple-ios .app ──► signed .ipa ──► SideStore/device - (editable) (compiled in guest) (packaged+sign on host) +project directory ──► NativeBuildPlan ──► arm64-apple-ios executable + (editable) (resolved, refused (clang / swift-frontend, + up front) linked with ld64.lld, + in this process) + ──► .app ──► unsigned .ipa ──► signed .ipa + (Payload/ + zip) (XKit, on the Signing screen) ``` Everything is driven from one orchestrator (`BuildManager`) that exposes a strict @@ -18,36 +23,91 @@ state machine to the UI, so the GUI always knows exactly which stage is running. ## 2. Pipeline stages (state machine) ``` -1. provision ensure embedded Linux is booted + Swift toolchain + xtool present -2. sdk ensure the `darwin` Swift SDK (arm64-apple-ios) is installed -3. configure write xtool.yml + Info.plist from the user's AppInfo/settings -4. resolve `swift package resolve` in the project (pull SPM deps) -5. compile `xtool dev build` → cross-compile to arm64-apple-ios .app -6. package host-side IPABuilder: assemble Payload, write Info.plist/entitlements, - codesign (real identity or ad-hoc), zip → .ipa -7. artifact stage the .ipa, hand it to the UI for export / install +1. provision report the linked toolchain (clang, ld64.lld, swift-frontend) and + require the Darwin SDK; fail with an explanation if either is absent +2. sdk install the Darwin SDK into the app container, if it is not there +3. configure check the project directory and record the app identity +4. resolve report "nothing to resolve", or refuse a project that declares + SwiftPM dependencies +5. compile plan → per-file compile → link → assemble .app → package unsigned .ipa +6. package accept the packaged artifact +7. artifact confirm the .ipa exists, is non-empty, and report its size ``` Each stage is idempotent (skippable if already done) and reports progress. Any stage -can fail; failures carry a human message + a machine-readable reason. +can fail; failures carry a human message, and the pipeline stops at the **first** +one, because everything after it fails as a consequence and would bury the cause. -## 3. Split of responsibilities +There is no "boot a guest" stage, and that is the whole point of the current design: +the compiler is a library call, so there is nothing to start before compiling. -| Concern | Where | Why | -|---|---|---| -| Provision, SDK install, resolve, compile | **Embedded Linux** (VM) via `BuildExecutor` | needs Swift + xtool + SDK | -| Config injection (xtool.yml, Info.plist) | host, before compile | user edits live in the GUI | -| Packaging (Payload, zip), final signing | **host** via `IPABuilder` | testable, works with any .app | -| Certificates / Apple ID | host via `SigningService` (XKit) | native on iOS | +### What each stage actually does -This split means `IPABuilder` is fully testable without the VM: give it any compiled -`.app` + `AppInfo` + a signer and it produces a valid `.ipa`. +| Stage | Implementation | Notes | +|---|---|---| +| provision | `NativeBuildExecutor.bootstrap` + `NativeToolchainCapabilities` | Reports one line per component. Fails when clang or LLD is not linked; *warns* when the Swift frontend is not. | +| sdk | `NativeSDK.install(fromRemote:)` | Skips itself when the bundle is already in the container. | +| configure | `BuildManager.configure` | Writes `/.xforge-build/build.json` (bundle ID, version, build number, configuration) so a failed build is reproducible, and throws `missingProjectDirectory` if the directory is gone. | +| resolve | `NativeBuildExecutor.resolve` + `NativeBuildPlanFactory.swiftPackageDependencies` | A textual scan of `Package.swift`; the answer it needs is only "are there any". | +| compile | `NativeBuildExecutor.build` | The real work: plan, compile, link, assemble, package. Also the stage that refuses Swift sources (`swiftFrontendMissing`). | +| package | `BuildManager.package` | The executor packages the IPA as the last step of compiling, so this stage accepts the result rather than producing it. | +| artifact | `BuildManager.stageArtifact` | A final stage that checks the file rather than reporting success unconditionally. | + +### The Swift frontend limitation, stated where it bites + +`NativeBuildExecutor.build` throws `NativeBuildError.swiftFrontendMissing(n)` — with +`n` the number of Swift files — when the plan has Swift sources and the frontend is +not linked. In the shipped toolchain bundle the frontend is never linked: the CI +workflow that builds the bundle builds Clang and Mach-O LLD only. So **a Swift +project cannot be compiled today**, and the failure happens before any file is +compiled, not after a partial build. + +The consequence for this pipeline is structural, not cosmetic: the stages that +exist are correct and exercised for C and Objective-C targets, and the Swift- +specific stages (module boundaries, a dependency graph, asset catalogs) are the +work that remains. Until then the pipeline can honestly describe itself as "a build +tool that compiles C", and the app says so rather than letting a user discover it +three stages in. -## 4. Signing strategy +## 3. Split of responsibilities -The current build exports an unsigned `.ipa`, which must be signed by Xcode, -SideStore/AltStore, or another signing service before installation. XKit-based -Apple ID authentication and signing are planned but are not wired into this build. +| Concern | Where | Why | +|---|---|---| +| Reading a project into a plan | `NativeBuildPlanFactory` (host) | Pure directory reading; no compiler needed, so it is fully testable | +| Argument construction | `NativeToolchainInvocation` (host) | Pure functions; the compiler only exists on a device, so this is the only half that can be unit-tested | +| Compiling and linking | `NativeToolchain*` (in-process libraries) | iOS cannot spawn a compiler | +| Bundle assembly, `Info.plist` merge | `NativeBuildExecutor` | Needs the plan and the compiled executable | +| Packaging (Payload, zip) | `IPABuilder` (host) | Testable, works with any `.app`, has no opinion about what built it | +| Signing | `AppBundleSigner` / `IPASigningJob` (XKit, in-process) | A separate step, on its own screen, so the unsigned IPA stays a first-class artifact | + +This split means `IPABuilder` is fully testable without a compiler: give it any +compiled `.app` + `AppInfo` and it produces a valid `.ipa`. + +## 4. Signing strategy: unsigned first, then signed + +The build produces an **unsigned** `.ipa`, staged at +`/staging/-.ipa`. Signing is `IPASigningJob`: + +1. Unzip the `.ipa` into a temporary directory and find the single `.app` under + `Payload/` (more or fewer than one is an error, not a guess). +2. Read the `.p12` through the Security framework (`PKCS12Identity.load`), with the + password held only for the duration of the attempt and cleared afterwards. The + private key is used in memory and written nowhere — which is why the old + `zsign` password-file workaround is gone with `zsign`. +3. Apply the identity overrides the user typed (bundle ID, display name, version) + to the bundle's `Info.plist`. Only those three: replacing the plist wholesale + would drop `CFBundleExecutable` and produce an app that cannot launch. +4. Write `embedded.mobileprovision` into the bundle **before** signing — the signer + reads it to decide which entitlements the signature may carry, and a signature + that does not match its profile is rejected at install time. +5. Sign with XKit's `Signer`, and re-zip the `Payload/` as + `/Signed-.ipa`. + +What this does **not** do is obtain a certificate. An Apple ID session needs +anisette + GrandSlam + 2FA (XKit's `SigningContext`/`DeveloperServices` path), which +is a separate, device-only piece of work; `XKitSigningService.signIn` throws rather +than reporting a signed-in account it does not have. The signer takes an existing +`.p12` and profile. ## 5. The pipeline's own types @@ -69,48 +129,66 @@ struct PipelineSnapshot: Equatable { } ``` -Each stage is a method on `BuildManager` run in order (`provision` → `ensureSDK` → -`configure` → `resolve` → `compile` → `package` → `stageArtifact`), and each one stops -the pipeline on failure. `BuildExecutor` (`App/Models/Project.swift`) is the seam to the -VM: `bootstrap()`, `createProject`, `installSDK`, `resolve` and `build` all return an -`AsyncThrowingStream`, which `BuildManager.consume` folds into the -snapshot. Packaging is a single host-side call: +Each stage is a method on `BuildManager` run in order, and each one stops the +pipeline on failure. `BuildExecutor` is the seam to the toolchain: `bootstrap()`, +`createProject`, `installSDK`, `resolve` and `build` all return an +`AsyncThrowingStream`, which `BuildManager.consume` folds into +the snapshot. Packaging inside the executor is a single host-side call: `IPABuilder.buildIPA(appBundle:appInfo:outputDir:)`. +The stage **title** for `package` in `BuildPipeline.swift` reads "Package & sign +.ipa", which is a leftover from when the stage did both; the stage now packages and +the Signing screen signs. + ## 6. Failure handling -- Stage-scoped: a failure marks that stage `.failed` and stops the pipeline (unless the - stage is retryable and the user retries). -- `BuildEvent` stream carries both progress lines (for the console) and structured - stage transitions (for the step UI). -- Staged artifacts are written under `Documents/staging/` and listed by the Artifacts - screen. +- Stage-scoped: a failure marks that stage `.failed` and stops the pipeline (unless + the stage is retryable and the user retries). +- The `BuildEvent` stream carries both progress lines (for the console) and + structured stage transitions (for the step UI). +- Compile and link failures carry the tool's diagnostics text; an empty diagnostics + buffer is reported as "reported a failure without diagnostics" rather than + silently producing a bare exit code. +- Staged artifacts are written under `Documents/staging/` and listed by the + Artifacts screen. Intermediate objects and the `.app` live under + `/.xforge-build`, which is discarded at the start of the next build. ## 7. Testability -- `IPABuilder` unit tests build a fake `.app` fixture in the test bundle and assert the - produced `.ipa` contains `Payload/.app/Info.plist` with the right values and is - a valid zip. -- `XcodeProjectTests` covers the `.xcodeproj` reader against an embedded project file. -- `ReleaseResolutionTests` drives `ToolchainManager` and `XForgeReleases` through a - stubbed `LinuxVM`/`ShellSession`, so SDK resolution is testable without the VM. +- `NativeBuildTests` covers the half of the build that can be tested without a + compiler: plan acceptance and refusals (including the SwiftPM-dependency refusal + and the textual dependency scan ignoring local targets), the Swift frontend and + linker argument lists, the `main.swift` case, and search-path filtering. +- `IPABuilderTests` builds a fake `.app` fixture in the test bundle and asserts the + produced `.ipa` contains `Payload/.app/Info.plist` with the right values and + is a valid zip. +- `XcodeProjectTests` covers the `.xcodeproj` reader against an embedded project + file. +- Nothing tests the compiler itself: it does not exist in a simulator build. That + is the boundary this design accepts, and the reason argument construction was + extracted into pure functions. ## 8. Files ``` -Docs/IPA-BUILD.md this design -App/Models/BuildPipeline.swift BuildStage, BuildStageState, PipelineSnapshot -App/Models/Project.swift BuildExecutor protocol, BuildEvent -App/Build/BuildManager.swift orchestrator / state machine -App/Build/IPABuilder.swift host-side packaging (Payload, Info.plist, zip) -App/Build/EmbeddedLinuxExecutor.swift the VM-backed BuildExecutor -App/Models/BuildHistoryStore.swift run history for the History screen -App/Services/SigningService.swift signing abstraction (integration pending) -AppTests/IPABuilderTests.swift packaging unit tests -App/Views/Build/* UI driven by BuildManager.snapshot +Docs/IPA-BUILD.md this design +App/Models/BuildPipeline.swift BuildStage, BuildStageState, PipelineSnapshot +App/Models/Project.swift BuildExecutor protocol, BuildEvent, SDKSource +App/Build/BuildManager.swift orchestrator / state machine +App/Build/NativeBuildPlan.swift the plan and the factory that refuses +App/Build/NativeToolchainInvocation.swift argument construction (pure) +App/Build/NativeBuildExecutor.swift the BuildExecutor: compile, link, assemble, package +App/Build/IPABuilder.swift host-side packaging (Payload, Info.plist, zip) +App/Build/NativeToolchainCapabilities.swift what this build can compile with +App/Services/AppBundleSigner.swift XKit signing, in process +App/Services/IPASigningJob.swift the signing workflow (unzip, overrides, re-zip) +App/Models/BuildHistoryStore.swift run history for the History screen +AppTests/NativeBuildTests.swift plan + argument unit tests +AppTests/IPABuilderTests.swift packaging unit tests +App/Views/Build/* UI driven by BuildManager.snapshot ``` -Docs that go with this one: `Docs/DESIGN.md` (the whole app), -`Docs/XCODE-ALTERNATIVE.md` (building existing Xcode projects on-device), -`Docs/NATIVE-TOOLCHAIN.md` (the in-process LLVM path) and -`Docs/ISH-ARM64-INTEGRATION.md` (the embedded Linux engine). +Docs that go with this one: `Docs/DESIGN.md` (the whole app) and +`Docs/XCODE-ALTERNATIVE.md` (building existing Xcode projects on-device). The +toolchain bundle that makes any of this run is described in +`Docs/NATIVE-TOOLCHAIN.md`. diff --git a/Docs/ISH-ARM64-INTEGRATION.md b/Docs/ISH-ARM64-INTEGRATION.md deleted file mode 100644 index 198ced2..0000000 --- a/Docs/ISH-ARM64-INTEGRATION.md +++ /dev/null @@ -1,204 +0,0 @@ -# Embedding ish-arm64: what the reference requires, and where XForge stands - -XForge's embedded Linux is **ish-arm64** (`Vendor/ish-arm64`, a git submodule of -[OpenMinis/ish-arm64](https://github.com/OpenMinis/ish-arm64)) running a **plain -Alpine aarch64 rootfs** that is shipped as an already-converted **fakefs ZIP**. - -The reference for that integration is OpenMinis's own app — the engine's author — -and its guide `deps/ISH_INTEGRATION.md` ("iSH-ARM64 iOS 集成指南"), plus the two -scripts that build the engine (`deps/build_ish.sh`) and the root -(`deps/prepare_alpine_rootfs.sh`). This document records what the guide requires, -what XForge does instead, and *why* where the two differ. It exists because the -differences are the interesting part: several of them are deliberate, and the ones -that are not were bugs. - -## The guide's checklist, item by item - -| The guide says | XForge | Where | -|---|---|---| -| Link `deps/libs/libish.a`, `libish_emu.a`, `libfakefs.a` | exactly those three | `project.yml` (`OTHER_LDFLAGS[sdk=iphoneos*]`) | -| Add system library `libsqlite3.tbd` | `-lsqlite3` | same | -| Header search path `deps/include` | `$(ISH_ROOT)` and `$(ISH_ROOT)/deps/libarchive/libarchive` | `project.yml` (`HEADER_SEARCH_PATHS[sdk=iphoneos*]`) | -| Other linker flags `-ObjC`, `-all_load` | **not used** — see below | — | -| Other C flags `-DISH_INTERNAL` | **not used** — see below | — | -| Bundle `alpine-rootfs.zip` | bundled, and CI asserts the app contains exactly one rootfs | `.github/workflows/build-ipa.yml`, "Verify the built app" | -| Bundle `libvdso.so.elf` (*optional* per the guide) | not bundled: it is compiled and linked **into `libish.a`** | `EmbeddedLinux/build-ish-core.sh` | -| Unzip the root to `Documents/alpine-rootfs`, mount `/data` | same layout, unpacked into a staging directory and moved into place | `App/EmbeddedVM/RootfsInstaller.swift` | -| `mount_root(&fakefs, /data)` then `become_first_process()` | same order, then device nodes, `/proc`, `/dev/pts` | `App/EmbeddedVM/ISHBridge.c` (`xf_ish_boot`) | -| `current->thread = pthread_self()` after `become_first_process()` | not needed: every guest task is started with `task_start()`, which sets `task->thread` via `pthread_create` (`kernel/task.c:243`) | `ISHBridge.c` (`xf_ish_start_init`, `xf_spawn_child`) | -| Register a custom tty driver: `DEFINE_TTY_DRIVER(..., TTY_CONSOLE_MAJOR, 8)` with `init`/`write`/`cleanup` | same driver on the same major, plus `/dev/tty1` and `set_console_device(4,1)` | `ISHBridge.c` (`xf_console_driver`) | -| Feed keystrokes with `tty_get()` + `tty_input()` + `tty_release()` | equivalent: the tty is recorded in the driver's `init` and typed into with `tty_input` (which reads no per-thread engine state) | `ISHBridge.c` (`xf_ish_console_write`, `xf_ish_console_*`) | -| `do_execve` + `task_start(current)` to run the console program | same, for pid 1 **and** for every command the app runs | `ISHBridge.c` | -| Build options (`build_ish.sh`): `--buildtype=release`, `-Db_ndebug=true`, `-Dlog=""`, `-Dlog_handler=nslog`, `-Dkernel=ish`, `-Dengine=asbestos`, `-Dguest_arch=arm64` | identical set | `EmbeddedLinux/build-ish-core.sh` | - -### Why no `libarchive` (or `libfakefsify`) - -The reference links libarchive and builds the fakefs tools for the app. XForge did -too, until the app's own header said otherwise: the bundled root is *already* a -fakefs ZIP, unpacked with ZIPFoundation into `Documents`, and `fakefs_import` is -never called — "there is no import step here: `fakefs_import` is a build-time tool -in this design, not a runtime one" (`App/EmbeddedVM/ISHBridge.h`). The `fakefsify` -the build machine needs is built by the engine's meson project on Linux, in -`EmbeddedLinux/build-rootfs.sh`. - -So `-larchive`, `-lfakefsify`, the libarchive header path, the -`deps/libarchive` submodule fetch and the libarchive iOS build are all gone from -the app's build. It was not just dead weight: libarchive has no iOS build system of -its own here, so `build-ish-core.sh` drove its Xcode project with a *nested* -`xcodebuild` — and nested inside Xcode's own build phase, that crashed the Xcode -build system (`error: unexpected service error: The Xcode build system has crashed`) -after 90/90 engine targets had already succeeded. Removing the dependency removed -the failure. - -### Why not `-ObjC -all_load` - -The guide asks for both, and the reason is a static-library one: a linker drops any -object file nothing references, and Objective-C categories (and code reachable only -through the ObjC runtime or a constructor) can be dropped with it. Neither applies -here: - -* XForge's engine libraries are **C** — there is no category to preserve — and the - bridge references every subsystem it needs directly, so a wrongly-dropped object - would be a *link error* rather than silent misbehaviour. -* The objects that *do* rely on a constructor (`fs/fake.c`'s `init_fake_fdops`, - `kernel/memory.c`'s `get_real_page_size`, `kernel/task.c`'s `create_attr`) are all - in objects that are already pulled in by symbols the app calls (`fakefs`, - `mount_root`, `task_start`). The engine's own authors avoid constructors precisely - where that is not true — see the comment in `kernel/native_offload_sym.c:111` - ("not via a constructor, which a static-library linker dead-strips when the TU is - otherwise unreferenced"). -* `-all_load` would then force the whole of `libish.a` and `libarchive.a` into the - app, for no reachability benefit. - -### Why not `-DISH_INTERNAL` - -The guide needs it because its `ISHKernel.m` includes `ish/fs/fake.h`, which refuses -to compile otherwise: - -```c -#ifndef ISH_INTERNAL -#error "for internal use only" -#endif -``` - -`fs/fake.h` is the *only* header with that gate. XForge's bridge needs the fakefs -instance, and that is declared in a header that is not gated -(`kernel/fs.h:184: extern const struct fs_ops fakefs;`), which the bridge already -includes. If it ever needs `fs/fake.h` (the bind-mount API there, for instance), -`-DISH_INTERNAL` has to be added to `OTHER_CFLAGS[sdk=iphoneos*]` in `project.yml`. - -## What the console is, and why XForge's is not the reference's - -The guide's terminal does this: - -```swift -create_stdio("/dev/tty1", TTY_CONSOLE_MAJOR, 1) // pid 1's stdio is the console -ISHKernel.execute(command: ["/bin/sh", "-l"]) // …then pid 1 *becomes* the shell -``` - -So in the reference the shell **is pid 1**, and the shell's controlling terminal is -the one pid 1 claimed when its stdio was wired. XForge instead boots a real init -(`/sbin/init`) which respawns `/sbin/xforge-login root` on `tty1`; that script reads -root's login shell out of `/etc/passwd` and execs it. That buys a guest that behaves -like a system — a real init for the guest's own scripts, `exit` giving a fresh -session, a motd — and it is what the Terminal tab and the whole rootfs design assume. - -It also had a consequence the reference never sees, because the reference never -creates a second session: **`create_stdio` opens the console without `O_NOCTTY`**, -so wiring pid 1's stdio made the console *pid 1's* controlling terminal and pinned -`tty->session` to it (`fs/tty.c:126`, `tty_open` hands a tty over only while -`tty->session == 0`). The login session init then started could never take it, and -the shell ran with no controlling terminal: bash printed - -``` --bash: cannot set terminal process group (-1): Not a tty --bash: no job control in this shell -``` - -and no `Ctrl-C`, `Ctrl-Z` or `SIGWINCH`-on-resize reached the shell. XForge's fix is -to open pid 1's stdio with `O_NOCTTY` (`xf_create_console_stdio` in -`App/EmbeddedVM/ISHBridge.c`), which is what Linux does too: init has no controlling -terminal, the console belongs to the login session that opens it. Same flags, same -`create_stdio` shape, one bit different — and the guest keeps its init. - -## The rootfs: what the reference configures, and what XForge adds - -`deps/prepare_alpine_rootfs.sh` downloads the plain Alpine minirootfs, converts it -with the engine's own `tools/fakefsify`, configures it in place (mount points, -`/etc/passwd`, `/etc/profile`, `/etc/motd`, `/etc/inittab`, `/etc/resolv.conf`, -`/etc/apk/repositories`) and packs it with `zip -r alpine-rootfs.zip`, excluding the -SQLite `-wal`/`-shm` sidecars. XForge's `EmbeddedLinux/build-rootfs.sh` does the same -thing and keeps the same ordering rule, which is the one that is easy to get wrong: - -> **`fakefsify` writes `meta.db` as an index of the tree as it stands.** Anything -> added to the root after the conversion exists in `data/` and is invisible to the -> guest. XForge installs the glibc layer and the guest's packages *before* the -> conversion, and asserts afterwards that they are indexed. - -On top of the reference's list, XForge's root also carries: - -* the **glibc compatibility layer** (`/opt/glibc`, wired through `/lib` and - `/usr/lib`) — xtool and the Swift toolchains are glibc binaries and Alpine is musl; -* the **console packages** (`bash coreutils less ncurses-terminfo`) and root's login - shell in `/etc/passwd`; -* `/sbin/xforge-login` and XForge's `/etc/inittab` (the stock one starts openrc, - which is not in this root, and six gettys on terminals that do not exist); -* `EmbeddedLinux/verify-rootfs.sh`, which runs the console program in a chroot of the - root — for a plain tree — and checks the fakefs ZIP through `meta.db`, which is the - database the engine actually resolves through. - -## Building it: the engine and the rootfs are prerequisites of the app - -The guide treats the engine as something you build first and then link. That is fine -for a human following instructions and wrong for a pipeline: any build that does not -know to run `deps/build_ish.sh` / `EmbeddedLinux/build-ish-core.sh` first fails -part-way through compiling the bridge, with - -``` -App/EmbeddedVM/ISHBridge.c:123:10: error: 'kernel/init.h' file not found -``` - -which reads like a missing header rather than a missing engine (and a fresh clone -has the submodule empty, so the header really is missing). So the app target now -carries a **first build phase**, `EmbeddedLinux/build-engine-for-xcode.sh`, which: - -* skips itself for simulator builds (which compile the stub branch in `ISHBridge.c`) - and for `XFORGE_SKIP_ENGINE=1`; -* fetches the engine's submodules if the checkout is empty; -* installs meson/ninja/llvm/lld/libarchive with Homebrew when they are missing - (Xcode build phases get a bare `PATH`, so Homebrew is put on it first); -* runs `build-ish-core.sh`, which is a no-op unless the engine revision, the script - or `project.yml` changed (`XFORGE_REBUILD_ENGINE=1` forces a rebuild). - -The result is that `xcodebuild`, Xcode itself, the IPA workflow and any other -pipeline all produce the same app from the same checkout. - -The **rootfs** is a prerequisite in the same sense, but it cannot be fetched from a -build phase: it is a *resource*, so it has to be on disk before the project is -generated (XcodeGen fixes the resource list when it walks the directory). Hence -`EmbeddedLinux/fetch-rootfs.sh`, which reads the pinned tag out of -`build-ipa.yml`, downloads the asset, verifies its published sha256 and checks it is -a fakefs ZIP — run by the IPA workflow before XcodeGen, by the generic iOS pipeline, -and by hand before opening the project in Xcode. A device build whose resource is -missing fails on purpose (`Check the bundled guest rootfs`), because an app without a -root is an app that cannot boot Linux, and finding that out on a phone is worse than -a red build. - -## Verification, and what is still only verified on a device - -What CI checks, on every IPA build (`.github/workflows/build-ipa.yml`): - -* the pinned rootfs release's sha256, its contents, and that the glibc layer and the - console paths are indexed in `meta.db`; -* that the engine libraries exist and that `libish_emu.a` carries the arm64 backend; -* that the app is arm64, contains exactly one rootfs (the fakefs ZIP), has no build - leftovers, and that its entitlements are the four the engine needs - (`extended-virtual-addressing` and `increased-memory-limit` among them); -* for the rootfs itself, that `/sbin/xforge-login` starts the shell `/etc/passwd` - names, as a login shell, in a chroot of the root. - -What that cannot check is the guest *running on iOS*: the engine executes in-process, -under the app's entitlements, with the host's real networking and the app's tty. That -is what the Terminal tab and the smoke harness (`Tools/engine-smoke/`) are for, and -it is why a change to the console path is worth trying on a device before believing -it. diff --git a/Docs/NATIVE-TOOLCHAIN.md b/Docs/NATIVE-TOOLCHAIN.md index fcbb2c0..492cd8b 100644 --- a/Docs/NATIVE-TOOLCHAIN.md +++ b/Docs/NATIVE-TOOLCHAIN.md @@ -1,95 +1,163 @@ -# Native iPhone toolchain proof of concept +# The native iPhone toolchain -This branch starts the migration away from the embedded Linux runtime. - -## Goal - -Run the compiler and Mach-O linker directly inside `XForge.app` on a stock iPhone: +XForge compiles in its own process: there is no Linux guest and no subprocess. The +compiler and the Mach-O linker are LLVM libraries **linked into the app binary**, +called through a small Objective-C++ bridge. ``` -source -> Clang/Swift frontend -> arm64 object -> LLD Mach-O -> .app -> sign -> IPA +source -> Clang / swift-frontend -> arm64 object -> ld64.lld -> .app -> .ipa + (in-process, no fork/exec/posix_spawn, no remote build host) ``` -No Linux rootfs, `fork`, `exec`, `posix_spawn`, or remote build host is required at runtime. - -## What is implemented - -- `NativeToolchainBridge.mm`: in-process Objective-C++ bridge for Clang codegen and - the LLD Darwin driver. -- `NativeToolchain.swift`: safe Swift wrapper plus an end-to-end C -> arm64 iOS - object smoke test. -- Stub behavior when the LLVM bundle is not linked, so normal XForge builds remain - buildable while the native toolchain artifact is being produced. -- `.github/workflows/native-toolchain.yml`: builds the iOS-hosted LLVM/Clang/LLD - libraries on a macOS GitHub runner and uploads them as - `XForgeNativeToolchain-arm64-ios.tar.gz`. -- `NativeToolchain/install-bundle.sh `: unpacks that artifact into - `Vendor/NativeToolchain` and runs `NativeToolchain/prepare-xcode.sh`, which writes - `Support/NativeToolchain.generated.xcconfig` — the file the app target consumes - (checked in with the backend disabled, so an ordinary clone builds without it). - `build-ipa.yml` runs `prepare-xcode.sh` too, so CI never depends on the bundle - being installed locally. - -The existing Alpine executor is intentionally retained as a fallback until the native -backend can compile Swift and package a complete app. - -## Runtime contract +## Goal -The native implementation is enabled only when the app target defines -`XFORGE_HAS_LLVM=1` and can see the LLVM/Clang/LLD headers. Otherwise the bridge -returns `Native LLVM backend not linked`. +Run the compiler and Mach-O linker directly inside `XForge.app` on a stock iPhone. +iOS cannot spawn a process, so "run clang" means "call clang's libraries"; the only +alternative would be a remote build host, which is a different product. -The first validation milestone is: +## What is implemented -``` -int xforge_native_smoke(void) { return 42; } - | - v -clang::CompilerInstance + EmitObjAction - | - v -arm64-apple-ios main.o +- `App/NativeToolchain/NativeToolchainBridge.mm` — the in-process bridge: Clang + codegen (`CompilerInvocation` + `EmitObjAction`), the `swift::performFrontend` + entry point, and the LLD Darwin driver. +- `App/NativeToolchain/NativeToolchain.swift` — the Swift wrapper, plus an + end-to-end C compile smoke test. +- `App/Build/NativeToolchainInvocation.swift` — the argument lists as pure + functions. The convention is not obvious and is worth stating: the frontend entry + point is handed everything *after* `-frontend`, so neither the program name nor + `-frontend` itself may appear in the array; passing it makes + `CompilerInvocation::parseArgs` reject the whole invocation. +- **Stub behaviour when the bundle is not linked**, so an ordinary clone (and every + simulator build) compiles and runs with no toolchain present. +- `.github/workflows/native-toolchain.yml` — builds the iOS-hosted LLVM/Clang/LLD + libraries and publishes the bundle. See below. +- `NativeToolchain/install-bundle.sh ` — installs that bundle. +- `NativeToolchain/prepare-xcode.sh` — writes the xcconfig the app target consumes. + `build-ipa.yml` runs it too, so CI never depends on a bundle being installed + locally; on a checkout with no bundle it writes the disabled configuration and CI + builds the stub. + +## What the bundle contains — and what it does not + +The workflow cross-builds, for iPhoneOS arm64: + +- **Clang**, as the library set the bridge calls (`clangCodeGen`, `clangFrontend`, + `clangFrontendTool`, `clangDriver`, `clangSerialization`, `clangSema`, `clangParse`, + `clangAST`, `clangLex`, `clangBasic`); +- **Mach-O LLD** (`lldMachO`, `lldCommon`). + +It does **not** contain the Swift frontend. `swift-frontend` is not part of +`llvm-project`: it has to be built from `swiftlang/swift` against +`swiftlang/llvm-project` **for iOS**, as a library set. That port is the long pole +and it is not done, so: + +- **Swift sources cannot be compiled today.** A plan that contains Swift files + fails at the compile stage with `swiftFrontendMissing`, naming the count of files. +- C, Objective-C and Objective-C++ sources compile and link today. +- The bridge does expose the entry point (`xf_native_swift_frontend`), gated behind + `XFORGE_HAS_SWIFT_FRONTEND`, so the port has somewhere to land. Until a frontend + library set exists, the gate is closed and the UI reports `swift-frontend: missing`. + +The two halves are deliberately independent: the Clang/LLD port can be validated +and used on its own while the considerably larger Swift compiler port is brought up. + +## How the bundle is built (CI) + +`.github/workflows/native-toolchain.yml`, job `llvm-ios`, on a `macos-15` runner: + +1. Build host TableGen tools (`llvm-tblgen`, `clang-tblgen`) natively — the iOS + cross-build needs a native TableGen to generate tables it can run. +2. Configure LLVM for iPhoneOS arm64 with `LLVM_ENABLE_PROJECTS="clang;lld"`, + `LLVM_TARGETS_TO_BUILD=AArch64`, deployment target 17.0, tools and examples off. +3. Build the Clang and LLD library targets listed above. +4. **Compile-check the bridge against the headers just built** — `clang++ -arch arm64 + -isysroot -DXFORGE_HAS_LLVM=1 -c App/NativeToolchain/NativeToolchainBridge.mm`. + This is the check that keeps a header change in LLVM from silently breaking the + bridge in a device build nobody can run yet. +5. Stage `include/` from `llvm-project` and the build directory, then **flatten every + static archive in the build into one** `dist/lib/libXForgeNativeToolchain.a` with + `libtool -static`. Flattening is deliberate: the app then links one archive + instead of depending on LLVM's internal archive ordering, which would otherwise + have to be reproduced by hand in the xcconfig. +6. **Link-check**: link a program that calls `xf_native_toolchain_available()` + against the flattened archive and the bridge object, and assert the result is a + real Mach-O. A library set that compiles but cannot link is the failure this step + exists to catch. +7. Write `manifest.txt` (`target=arm64-apple-ios`, `deployment_target=17.0`, + `llvm_commit=…`, `built_at=…`, `archive=…`) and tar the result as + `XForgeNativeToolchain-arm64-ios.tar.gz`, uploaded as the workflow artifact of the + same name. + +It is dispatched by hand (or by a pull request that touches `App/NativeToolchain/**`), +because it takes hours and its output changes only when the LLVM commit does. + +## How it is installed + +```bash +make toolchain ARCHIVE=XForgeNativeToolchain-arm64-ios.tar.gz +make gen ``` -The next milestone is linking that object with the Darwin LLD driver against a -minimal iPhoneOS SDK, followed by embedding the Swift frontend. - - -## Native Darwin SDK - -The app now has a host-side SDK store at: +`install-bundle.sh` unpacks the archive into `Vendor/NativeToolchain` and refuses an +archive that is missing `manifest.txt`, `include/` or `lib/` — a partial bundle would +otherwise surface as an inscrutable compile error much later. It then runs +`prepare-xcode.sh`, which writes `Support/NativeToolchain.generated.xcconfig`: ``` -Documents/native-sdk/darwin.artifactbundle +XFORGE_NATIVE_TOOLCHAIN_AVAILABLE = 1 +XFORGE_NATIVE_HEADER_SEARCH_PATHS = $(SRCROOT)/Vendor/NativeToolchain/include +XFORGE_NATIVE_LIBRARY_SEARCH_PATHS = $(SRCROOT)/Vendor/NativeToolchain/lib +XFORGE_NATIVE_CFLAGS = -DXFORGE_HAS_LLVM=1 +XFORGE_NATIVE_LDFLAGS = -lc++ -lz -liconv -lsqlite3 -framework Foundation ``` -It consumes xtool's existing `swift-sdk.json` metadata directly and resolves the -`arm64-apple-ios` SDK root, Swift resource directory and platform library search -paths without booting Alpine. - -The Native Toolchain settings screen can download the existing -`darwin.artifactbundle.zip` release directly into this store and run the C smoke test. - -## Swift frontend boundary +`project.yml` consumes those variables and nothing else, so the app target never +names a path inside the bundle directly. -The bridge now also exposes an optional Swift frontend: +When the bundle is **absent**, `prepare-xcode.sh` writes the same file with the +backend disabled. That file is **checked in, disabled**, so a plain clone builds: +the bridge compiles to a "not available" stub and the app says so. This is why +`make gen` alone is a complete build for someone who only wants to work on the UI. -``` -Swift source - | - v -xf_native_swift_frontend - | - v -swift::performFrontend(...) - | - v -arm64-apple-ios .o -``` - -This is compile-time gated behind `XFORGE_HAS_SWIFT_FRONTEND`. Until a Swift -frontend library set has been cross-built for iPhoneOS, the UI reports -`Swift frontend: Not linked` and the Swift smoke test remains disabled. +## The runtime contract -This separation is intentional: the LLVM/Clang/LLD port can be validated and used -independently while the significantly larger Swift compiler port is brought up. +The native implementation is enabled only when the app target defines +`XFORGE_HAS_LLVM=1` and can see the LLVM/Clang/LLD headers. Otherwise the bridge +answers `xf_native_toolchain_available() == 0` and every entry point fails +cleanly — no half-linked state, and no path where the app believes it has a +compiler it does not have. + +`NativeToolchainCapabilities.current` is the single source of that truth for the +UI. It reports clang, `ld64.lld`, `swift-frontend`, the SDK and the backend version +as separate lines, because "the toolchain is missing" is not one condition: + +- `canCompile` requires clang and LLD. It deliberately does **not** require the + Swift frontend: a C target links fine without it, and refusing to start would be + wrong about why the build fails. +- When clang or LLD is absent the app reports the missing toolchain and names the + way to fix it (`NativeToolchain/install-bundle.sh`, then rebuild), rather than + failing somewhere in the middle with a linker error. + +## The smoke test + +The Toolchain screen's **Verify** section compiles a C file +(`int xforge_native_smoke(void) { return 42; }`) through the linked clang, against +the installed SDK, in this process, and reports the object file it produced and its +size. The button needs both a linked compiler and an installed SDK, so it cannot +report success on a build that only has one of the two. + +This is what a real build does, at the smallest size that can fail: a compiler that +is present but mis-linked (wrong target triple, no SDK, missing resource directory) +fails here in a way the capability rows cannot detect. The screen's own footer +describes the check as compiling *and linking*; the button currently compiles only, +and the build-time link check lives in the toolchain workflow (step 6 above). + +## Known gap + +Until the Swift frontend libraries exist for iPhoneOS: + +- Swift projects cannot be built on device. This is not a fallback situation — there + is no guest to fall back to — it is the remaining port. +- The Swift capability row is reported honestly (`missing`), the Swift smoke test + remains unreachable, and a build that needs it fails up front with + `swiftFrontendMissing` and the file count. diff --git a/Docs/XCODE-ALTERNATIVE.md b/Docs/XCODE-ALTERNATIVE.md index c60d68b..c7f8c86 100644 --- a/Docs/XCODE-ALTERNATIVE.md +++ b/Docs/XCODE-ALTERNATIVE.md @@ -10,91 +10,123 @@ piece needs, and which parts cannot be done at all with the tools that exist. | Path | Status | |---|---| -| Authoring SwiftPM packages (project editor, manifest, files) | works | -| Building a **SwiftPM** package in the embedded Alpine guest via `xtool dev build` | works (slow: it is emulated) | +| Authoring a SwiftPM-shaped project (project editor, manifest, files) | works | +| Compiling and linking **C / Objective-C** sources in-process | works (`NativeBuildExecutor`) | +| Compiling **Swift** sources in-process | **does not work** — the frontend is not in the toolchain bundle | +| Resolving SwiftPM dependencies | **refused by name** — no package manager can run here | | Packaging an `.ipa` from a built `.app` | works (`IPABuilder`, host-side) | -| Signing (Apple ID / certificates / ad-hoc) | **stub** — `XKitSigningService` throws | +| Signing with an existing `.p12` + profile | works (`AppBundleSigner`, XKit, in-process) | +| Signing with an Apple ID (certificate issuance) | **not implemented** — `XKitSigningService.signIn` throws | | Installing to the device | **stub** — export to SideStore/AltStore is the working path | -| Building a **`.xcodeproj`** | not started; reading one now works (`XcodeProjectReader`) | -| Native (no-VM) compilation | POC branch: Clang + Mach-O LLD in-process, C only | +| Reading a `.xcodeproj` | works (`XcodeProjectReader`); nothing builds from it | +| Building a `.xcodeproj` | not started | +| Asset catalogs | **not compiled** — `.xcassets` are copied into the bundle with a warning | -Two things this table already implies: the guest can only build *SwiftPM* projects -(xtool is SwiftPM-only — its `XcodePacker` **generates** an Xcode project from a -SwiftPM package, the reverse of what we want), and nothing in the app has ever -looked at an `.xcodeproj`. +Three things that table already implies: + +- **The compiler runs in-process, not in a guest.** The embedded Linux is gone; the + Clang and LLD libraries are linked into the app. That removes the "emulated, so + slow" caveat, and it removes the escape hatch: there is no longer a guest that can + do the things the in-process path cannot (install packages, run shell, compile + Swift through a Linux toolchain). +- **Swift is the gate.** Everything interesting — which is to say almost every iOS + app — is Swift, and Swift does not compile on device yet. +- **The project model is read-only.** The reader parses an `.xcodeproj`; the import + flow still requires a `Package.swift` and rejects anything else, so there is no + path from a read `.xcodeproj` to a build. ## 2. What an Xcode project needs, and where each piece stands 1. **Project model** — targets, product types, sources, build settings. - **Readable on-device now** (`App/Models/XcodeProject.swift`). An `.xcodeproj` is a - property list; `tuist/XcodeProj` parses it in pure Swift on iOS 17. -2. **Build-setting *evaluation*** — the settings a build actually uses are computed, - not stored: `$(VAR)` expansion, `[sdk=…]`/`[config=…]` conditions, and the large - set of defaults Xcode supplies (`xcodebuild -showBuildSettings` output is not in - the pbxproj). The reader today reports values as written and merges the - inheritance layers in Xcode's order; full evaluation is its own piece of work and - is a prerequisite for the compiler command lines. -3. **A build driver** — dependency order between targets, Swift module boundaries - (`-emit-module` ordering, `.swiftmodule` search paths), bridging headers, + **Readable on-device now** (`App/Models/XcodeProject.swift`), with XForge's own + OpenStep parser (see the note in that file on why `tuist/XcodeProj` cannot be + used from this dependency graph). An `.xcodeproj` is a text property list. +2. **Build-setting *evaluation*** — **still open.** The settings a build actually + uses are computed, not stored: `$(VAR)` expansion, `[sdk=…]`/`[config=…]` + conditions, and the large set of defaults Xcode supplies. The reader today + merges the project's own inheritance layers in Xcode's order and reports values + as written; full evaluation is its own piece of work and is a prerequisite for + the compiler command lines. +3. **A build driver** — **still open, and now harder than it looked.** Dependency + order between targets, Swift module boundaries (`-emit-module` ordering, + `.swiftmodule` search paths), bridging headers, `SWIFT_OBJC_INTERFACE_HEADER_NAME`, entitlements, `Info.plist` variable - substitution, resource phases, and **script phases** (arbitrary shell, which the - guest can run — a useful hybrid). - Nothing here can be borrowed from SwiftPM: `swift-build`/SwiftPM spawn compilers - as processes, and iOS has no `fork`/`exec`. + substitution, resource phases. **Script phases are the part the platform takes + away**: they are arbitrary shell, and an iOS app cannot run shell, so a project + with a script phase cannot be built on device at all — not by XForge, and not by + any other on-device tool. Nothing here can be borrowed from SwiftPM either: + `swift-build` spawns compilers as processes, and iOS has no `fork`/`exec`. The + plan/executor split in `Docs/DESIGN.md` is the shape a driver would take, but + the driver itself (one target, one module) is what remains. 4. **Compilers and linkers.** - - C/Objective-C: `clang::CompilerInstance` driving `EmitObjAction` works in-process - (POC branch), given `-resource-dir` pointing at a staged `clang/lib/Headers`. - - Link: `ld64.lld` in-process (same branch). - - **Swift: the gate.** `swift-frontend` is not in `llvm-project`; it must be built - from `swiftlang/swift` against `swiftlang/llvm-project` **for iOS**, as a library - set, and it needs the same resource directory the guest's toolchain has. - Until that exists, on-device compilation is C/ObjC-only, and every Swift app - (i.e. almost all of them) still needs the emulated guest. + - C/Objective-C: clang in-process — **done**, not a POC any more. + - Link: `ld64.lld` in-process — **done**. + - **Swift: still the gate.** `swift-frontend` is not in `llvm-project`; it must + be built from `swiftlang/swift` against `swiftlang/llvm-project` **for iOS**, + as a library set, and it needs the same resource directory the SDK bundle + carries. The bridge already exposes the entry point behind + `XFORGE_HAS_SWIFT_FRONTEND`, and the CI workflow that produces the toolchain + artifact does not build it. Until it does, every Swift app is out of reach on + device — there is no guest to fall back to. - `dsymutil`/`strip` equivalents: LLVM ships `llvm-strip`, and debug maps can be skipped for a sideload build. -5. **Resources.** Copying files is trivial. **Compiling them is not:** - `actool` (asset catalogs → `Assets.car`) and `ibtool` (storyboards/XIB → `.nib`) - are Xcode's own macOS-only tools. They cannot run on iOS (wrong platform, and no - process spawning), and they cannot run in the Alpine guest either — which is why - **xtool has no asset-catalog support at all** (grep its sources: no `actool`, no - `xcassets`). There is no general open-source replacement to point at. - Practical consequence: an app whose UI is pure SwiftUI code and whose icon/drawables - are not in an asset catalog builds; one that needs `Assets.car` does not, on any - on-device toolchain. Shipping without an icon is an acceptable dev-build trade-off, - and a minimal `Assets.car` writer for the common cases (app icon + colours) is - conceivable but is its own project. -6. **Dependencies.** An Xcode project's SPM dependencies (`XCRemoteSwiftPackageReference`) - need a fetch (git over the network), a checkout, and then the same build treatment - as the project's own targets. CocoaPods projects are worse: their xcconfigs and - script phases assume a full Xcode/macOS toolchain. -7. **Signing and installation.** Signing is native-able today via XKit (in-process - zsign + Developer Services); installing to the device needs a usbmux transport — - the app sandbox cannot use the system `usbmuxd` socket, which is why SideStore - embeds a userspace one (Minimuxer) plus a pairing file. Exporting to SideStore - remains the pragmatic path. +5. **Resources.** Copying files is trivial and the executor does it. **Compiling + them is not:** `actool` (asset catalogs → `Assets.car`) and `ibtool` + (storyboards/XIB → `.nib`) are Xcode's own macOS-only tools. They cannot run on + iOS — wrong platform, and no process spawning — and no general open-source + replacement exists to point at. Practical consequence: an app whose UI is pure + SwiftUI code and whose icon and drawables are not in an asset catalog builds; + one that needs `Assets.car` does not, on any on-device toolchain. XForge copies + `.xcassets` into the bundle uncompiled and says so, so the failure is visible + rather than an app that ships without its icon and cannot say why. A minimal + `Assets.car` writer for the common cases (app icon + colours) is conceivable but + is its own project. +6. **Dependencies** — **still open, and unfetchable as things stand.** An Xcode + project's SPM dependencies (`XCRemoteSwiftPackageReference`) need a fetch (git + over the network), a checkout, and then the same build treatment as the + project's own targets. An iOS app cannot run `git`, so the fetch is not a matter + of writing a driver: it means reimplementing enough of git's wire protocol to + resolve and check out a package graph. For the same reason, XForge today refuses + a `Package.swift` that declares any dependency, by name. +7. **Signing and installation.** Signing **works** in-process via XKit with an + identity the user already has (`.p12` + profile); what is missing is *obtaining* + one — an Apple ID session needs anisette + GrandSlam + 2FA, which is a separate, + device-only piece. Installing to the device needs a usbmux transport: the app + sandbox cannot use the system `usbmuxd` socket, which is why SideStore embeds a + userspace one plus a pairing file. Exporting to SideStore remains the pragmatic + path. ## 3. Staged plan -- **M1 — Read the project.** `XcodeProjectReader` (done). Next: surface it in the - import flow so an `.xcodeproj` can be opened and its targets listed, instead of - being rejected as "not a SwiftPM package". -- **M2 — Build one app on-device.** A single app target, no asset catalog, SwiftUI in - one module: compile with the native path, link with `ld64.lld`, package with - `IPABuilder`, sign with XKit. This is the milestone that proves the whole concept - end to end, and it does **not** need the Swift frontend port if the target is - C/ObjC — but the interesting version of it does. +- **M1 — Read the project.** `XcodeProjectReader` (**done**). Next: surface it in + the import flow so an `.xcodeproj` can be opened and its targets listed, instead + of being rejected as "not a SwiftPM package". +- **M2 — Build one app on-device.** A single app target, no asset catalog, one + module: compile with the native path, link with `ld64.lld`, package with + `IPABuilder`, sign with XKit. The toolchain now does the C/ObjC half of this; + the interesting version needs Swift and therefore M4. This is the milestone that + proves the concept end to end. - **M3 — Build settings and the rest of the bundle.** Full `$(…)` evaluation, - `Info.plist` processing, resource copy phases, script phases delegated to the guest. -- **M4 — Swift.** Port `swift-frontend` for iOS (the long pole), then the build driver - for modules and dependencies. -- **M5 — SPM dependencies, then asset catalogs** (with the caveat in §2.5: this may - never cover the general case). + `Info.plist` processing, resource copy phases, and a decision about script phases + (which cannot be supported at all). +- **M4 — Swift.** Port `swift-frontend` for iOS (the long pole), then the build + driver for modules and dependencies. **Everything Swift waits on this.** +- **M5 — SPM dependencies, then asset catalogs** — with the caveats in §2.5 and + §2.6: the first needs a git implementation, and the second may never cover the + general case. ## 4. What this means for the goal -"An Xcode alternative on iOS" is achievable for a real class of projects — SwiftUI -apps that do not pull assets or storyboards out of a catalog — but the order matters: -the Swift frontend port is the gate on everything Swift, and asset catalogs and -interface files are walled off by tools that only exist on macOS. Until M4 lands, -the embedded guest (or a build server) remains the only way to compile Swift, and -that is a property of the platform, not of this app. +"An Xcode alternative on iOS" is achievable for a real class of projects — apps +whose sources are C/Objective-C, or SwiftUI apps once the frontend is ported, that +pull nothing from an asset catalog and run no script phase. The order matters: + +- the **Swift frontend port** is the gate on everything Swift, and not having it is + now a plain gap in the app rather than a reason to boot Linux instead; +- **asset catalogs and interface files** are walled off by tools that only exist on + macOS, and no on-device toolchain can change that; +- **dependency fetching** is a network-protocol problem, not a compiler problem; +- **signing** is done, and **install** is the part the sandbox will not allow. + +Until M4 lands, XForge is a working on-device C/Objective-C build tool, and a +Swift project cannot be built on device by anything at all. diff --git a/Makefile b/Makefile index f867346..d35ab27 100644 --- a/Makefile +++ b/Makefile @@ -18,7 +18,7 @@ toolchain: echo "usage: make toolchain ARCHIVE=XForgeNativeToolchain-arm64-ios.tar.gz"; \ echo ""; \ echo "Download the artifact from a successful 'Native iOS Toolchain' run:"; \ - echo " gh run download --repo R0GUEEE/XForge --name XForgeNativeToolchain-ios-arm64"; \ + echo " gh run download --repo R0GUEEE/XForge --name XForgeNativeToolchain-arm64-ios"; \ exit 64; \ fi @bash NativeToolchain/install-bundle.sh "$(ARCHIVE)" diff --git a/README.md b/README.md index 2933ca9..38a751a 100644 --- a/README.md +++ b/README.md @@ -1,77 +1,178 @@ # XForge -**Build iOS apps on your iPhone — powered by [xtool](https://github.com/xtool-org/xtool).** - -XForge embeds a real Linux userspace (running xtool + a Swift toolchain) inside an iOS -app. You author SwiftPM packages in the SwiftUI shell, and compile them into real, -signed iOS `.ipa` files entirely on-device. - -The Linux engine is **[ish-arm64](https://github.com/OpenMinis/ish-arm64)** — a real Linux -kernel + aarch64 emulator running **in-process** on iOS. Its aarch64 backend dispatches -guest instructions to pre-compiled "gadget" functions instead of emitting machine code, -so it needs no JIT entitlement and works in a sideloaded app. The Alpine aarch64 root filesystem ships -**inside the app**, so there is nothing to download after install. - -> **Status: active.** The SwiftUI shell, build pipeline, and CI are in place, and the -> embedded Linux is ish-arm64 with a bundled Alpine rootfs that already carries the -> build toolchain — xtool, the Swift toolchain and the `darwin` Swift SDK are in the -> root the app unpacks, so there is nothing to provision before the first build. The -> remaining work is wiring signing — see [Docs/DESIGN.md](Docs/DESIGN.md). - -## Why this works - -`xtool dev build` is SwiftPM cross-compilation to `arm64-apple-ios` using a Swift SDK -named `darwin`. All three heavyweight pieces are self-contained Linux artifacts: - -| Piece | Source | Notes | -|---|---|---| -| Linux engine | ish-arm64 (`Vendor/ish-arm64` submodule), built for iOS | runs in-process, no JIT entitlement | -| Alpine aarch64 rootfs | `alpine-rootfs.zip` (Alpine 3.21, engine `fakefs`) | **bundled in the app**, unpacked on first boot; pinned by release tag | -| Swift aarch64 Linux toolchain | swift.org, via `swiftly` | **in the bundled root**, installed at build time by `install-toolchain.sh` | -| `xtool` aarch64 binary | prebuilt `xtool-aarch64.AppImage` | **in the bundled root** | -| `darwin` Swift SDK (arm64-apple-ios) | XForge's `darwin-sdk-` release | **in the bundled root**; the Toolchain screen can also install one from your own `Xcode.xip` | - -The bundled root is **provisioned**: it is built by running the guest's own -installer (`EmbeddedLinux/install-toolchain.sh`) in a chroot of the root being -built, so the app unpacks a guest that already has xtool, the Swift toolchain and -the Darwin SDK — the whole chain is exercised on a build machine with a real CPU -and a fast network, once, instead of on a phone under emulation. Everything is -installed *before* the fakefs conversion, because the conversion indexes the tree -and anything added afterwards is on disk and invisible to the guest. The archive -is then converted to the engine's `fakefs` format, so the app only unzips. - -That makes the artifact ~1.6 GB, which is the price of arriving ready to build. -`EmbeddedLinux/build-rootfs.sh` takes `XFORGE_PROVISION=none` for a plain few-MB -Alpine root where the guest provisions itself on demand — the same script, the -same installer, run in the guest instead of at build time. The build also drops -what an iOS build never loads (the static Linux stdlib, lldb, the editor tooling, -and every download cache) and then *verifies the result compiles a Swift -program*, so a slimmed toolchain fails the build rather than a user's first build. +**Build iOS apps on your iPhone, with no Mac in the loop.** + +XForge is an on-device alternative to Xcode's build step. It is built on +[xtool](https://github.com/xtool-org/xtool)'s libraries — `XKit` for in-process +codesigning, and xtool's `darwin.artifactbundle` for the iPhoneOS SDK — and it +runs no helper process: the compiler and the Mach-O linker are linked into the app +binary and called in-process through a small C++ bridge. + +> **Status: active, and narrower than it looks.** A project authored in the SwiftUI +> shell is read into a build plan, compiled and linked in-process, packaged into an +> unsigned `.ipa`, and — with a certificate the user already has — signed with XKit. +> **C, Objective-C and Objective-C++ targets build today. Swift sources do not**, +> because the Swift frontend libraries are not part of the toolchain artifact yet; +> such a project stops at the compile stage with a message naming how many files it +> could not compile. See *What works today* and *What does not*. + +## How it is put together + +``` +project (a directory) → build plan → compile + link in-process → .app + → unsigned .ipa → XKit signing → signed .ipa +``` + +1. **Project.** A project is an ordinary directory in the app's container, + `/projects/`, holding a `Package.swift`, `Sources//`, + an optional `Support/Info.plist` and the `xtool.yml` metadata XForge and xtool + both read. `ProjectFiles` reads and writes it directly, keeping the + relative-path discipline the editor needs (a `..` is rejected rather than + normalised, because the paths come from editable text in the UI). A folder + picked in the Files app is copied in: an iOS process cannot run `git`, so a + clone is not something XForge can do. +2. **Build plan.** `NativeBuildPlanFactory` turns that directory into a fully + resolved `NativeBuildPlan` — module, sources, resources, frameworks, search + paths. It is deliberately conservative: it refuses, *by name*, the one thing an + in-process driver cannot resolve (SwiftPM dependencies, which need a package + manager that fetches and runs manifests) instead of failing halfway through a + build. +3. **Compile and link.** `NativeBuildExecutor` turns the plan into argument lists + (`NativeToolchainInvocation`, pure functions, so they can be unit-tested on a + simulator where no compiler exists) and calls the entry points behind the + bridge: clang through `CompilerInvocation` + `EmitObjAction`, `swift-frontend` + through `swift::performFrontend`, `ld64.lld` through `lld::lldMain`. The + blocking calls are pushed off the main actor so the Build screen keeps drawing. + The result is assembled into `.app` with an `Info.plist` and the + project's resources. +4. **Package.** `IPABuilder` puts the bundle under `Payload/` and zips it into an + unsigned `.ipa`, staged as `/staging/-.ipa`. +5. **Sign.** `IPASigningJob` + `AppBundleSigner` do the signing in this process + with XKit: the `.p12` is read through the Security framework, used in memory, + and never written anywhere. The result lands as `/Signed-.ipa`. + +Every screen reports from the same values, so the app cannot claim a capability it +does not have: the Build screen's stages, the Toolchain screen (capabilities, SDK, +smoke test) and the Settings screen (log, project files, diagnostics) all read the +in-process toolchain and the SDK in the container. + +## What works today + +- **Building a project on device.** C, Objective-C and Objective-C++ sources are + compiled by clang and linked by `ld64.lld` against the Darwin SDK, in-process, + into a `.app` and then an unsigned `.ipa`. +- **A project's own `Info.plist`.** A hand-written `Support/Info.plist` is merged + over the generated keys, so scene manifests and orientations survive a build. +- **Signing an app with an identity you already have** — a `.p12` and a + provisioning profile, with entitlements if you have them. +- **Exporting.** `ProjectExporter` zips a project into `/exports`, and + the file is visible in the Files app and shareable from there. Build output + (`.xforge-build`, `.build`) is left out of the archive. +- **Reading an `.xcodeproj`.** `App/Models/XcodeProject.swift` parses the project + model on device, with its own OpenStep parser. Nothing builds from it yet. +- **Building and testing XForge itself without any toolchain bundle.** A plain + clone compiles the bridge to a "not available" stub; `make test` runs on the + simulator. + +## What does not work (yet) + +- **Swift.** The Swift frontend libraries are not in the toolchain artifact: the + CI workflow that builds it builds Clang and LLD only. A plan with Swift sources + fails at the compile stage with `swiftFrontendMissing`, naming the number of + files, rather than pretending otherwise. This is the single largest gap; it is + the difference between "an Xcode alternative for C projects" and the thing the + app is for. +- **SwiftPM dependencies.** Refused by name at plan time. Fetching and resolving + packages means running a package manager, which cannot happen here — and a + dependency editor that could only produce unbuildable projects was removed + rather than kept as a trap. +- **Asset catalogs.** `.xcassets` are copied into the bundle uncompiled, with a + warning: `actool` is a macOS-only tool and there is no open-source replacement + to point at, so an app whose icon lives in a catalog ships without it. +- **Apple ID provisioning.** Obtaining a certificate without one already in hand + means anisette + GrandSlam + 2FA, which is a separate, device-only piece of + work. The signer takes an existing identity; `XKitSigningService` throws rather + than reporting a session it does not have. +- **Installing to the device.** Exporting to SideStore/AltStore remains the + working path; in-app install needs a usbmux transport the sandbox cannot reach. +- **A remote build server.** `RemoteExecutor` is still a plan, not code. + +## Build the app + +Requires macOS + Xcode + [XcodeGen](https://github.com/yonaskolb/XcodeGen). + +```bash +brew install xcodegen + +make gen && open XForge.xcodeproj +``` + +That is the whole of it for a plain clone: there is no submodule to initialise and +no second build system to drive. The app builds with the compiler bridge compiled +to a stub and says so on the Toolchain screen. + +To build with the compiler *linked in*, install the toolchain bundle first: + +```bash +make toolchain ARCHIVE=XForgeNativeToolchain-arm64-ios.tar.gz +make gen +``` + +`install-bundle.sh` unpacks the archive into `Vendor/NativeToolchain` and runs +`prepare-xcode.sh`, which writes `Support/NativeToolchain.generated.xcconfig` — +the file the app target consumes. That xcconfig is checked in *with the backend +disabled*, so a clone without the bundle is buildable; see +[Docs/NATIVE-TOOLCHAIN.md](Docs/NATIVE-TOOLCHAIN.md). + +The only build prerequisite beyond XcodeGen is the toolchain bundle, and only if +you want a working compiler. + +Or build the unsigned IPA for sideloading via GitHub Actions +(`.github/workflows/build-ipa.yml`) and install it with SideStore/AltStore. + +## On-device build pipeline + +1. **Check build toolchain** — `NativeToolchainCapabilities` reports clang, + `ld64.lld`, `swift-frontend` and the SDK; the stage fails with an explanation + when nothing is linked, rather than letting a build die in the middle. +2. **Install Darwin SDK** — the SDK is a folder in the app container. The stage + skips itself when it is already there, otherwise it downloads the newest + `darwin-sdk-*` release asset. +3. **Configure app** — records the app identity (bundle ID, version, build number, + configuration) into `/.xforge-build/build.json` so a failed build is + reproducible, and checks that the project directory exists. +4. **Resolve dependencies** — reports "nothing to resolve" for a project with no + dependencies, and refuses a project that declares any. +5. **Compile (arm64-apple-ios)** — the work described above; also assembles the + `.app` and packages the unsigned `.ipa`. +6. **Package & sign .ipa** — accepts the packaged artifact. (The signing itself is + on the Signing screen, not in this stage.) +7. **Stage artifact** — confirms the `.ipa` exists and is non-empty, and reports + its size. ## Repo layout ``` App/ SwiftUI app — project editing, build pipeline, signing -App/EmbeddedVM/ ish-arm64 bridge: ISHEmulator, C shim, rootfs unpack -App/Build/ BuildExecutor protocol + EmbeddedLinuxExecutor, IPABuilder +App/Build/ BuildExecutor protocol, the NativeBuild* pipeline, IPABuilder App/Models/ pipeline, history, project store, XcodeProject (reads .xcodeproj) -App/NativeToolchain/ in-process Clang/LLD bridge + host-side Darwin SDK store -App/Services/ auth/signing/device seams, downloads, toolchain components -App/Views/ screens: Projects, Build, Toolchain, Terminal, Settings -Vendor/ish-arm64/ git submodule: the embedded Linux engine -Vendor/NativeToolchain/ where an installed LLVM/Clang bundle goes (empty by default) -EmbeddedLinux/ build-rootfs.sh, build-ish-core.sh, install-toolchain.sh -NativeToolchain/ prepare-xcode.sh (wires a vendor bundle into the project) -Support/ entitlements, assets, Resources/ (bundled rootfs) -Tools/ gen-appicon.py, the engine smoke harness, rootfs test +App/NativeToolchain/ in-process Clang/LLD bridge + the Darwin SDK store +App/Services/ signing, project files, export, device seams, the log +App/Views/ screens: Projects, Build, Signing, Settings (Toolchain, …) +AppTests/ unit tests (plan and argument construction, which run without a compiler) +Vendor/NativeToolchain/ where an installed toolchain bundle goes (absent by default) +NativeToolchain/ install-bundle.sh, prepare-xcode.sh +Support/ entitlements, assets, the generated toolchain xcconfig +Tools/ gen-appicon.py (writes the app icon into the asset catalog) project.yml XcodeGen definition -.github/workflows/ build-ipa.yml (the IPA), build-rootfs.yml (the pinned root), - native-toolchain.yml (the iOS LLVM cross-build) +Makefile gen, build, test, ipa, toolchain +.github/workflows/ build-ipa.yml (the unsigned IPA), + native-toolchain.yml (the iOS LLVM cross-build), + ios-share.yml (a simulator build for MobAI) Docs/DESIGN.md full architecture write-up -Docs/ISH-ARM64-INTEGRATION.md the engine integration vs. its reference implementation Docs/IPA-BUILD.md the build pipeline and its stages +Docs/NATIVE-TOOLCHAIN.md the toolchain bundle, how it is built and linked Docs/XCODE-ALTERNATIVE.md what building existing Xcode projects on-device takes -Docs/NATIVE-TOOLCHAIN.md the in-process LLVM/Clang/LLD path CHANGELOG.md what changed, release by release ``` @@ -84,7 +185,7 @@ Everything about the app's identity lives in the project-level `settings` block |---|---| | Bundle identifier | `com.r0gueee.xforge` | | Display name | `XForge` | -| Apple team | set in `XFORGE_DEVELOPMENT_TEAM` (a wildcard `TEAMID.*` profile, so any bundle ID works) | +| Apple team | set in `XFORGE_DEVELOPMENT_TEAM` | | Marketing version | `XFORGE_MARKETING_VERSION` | | Build number | `XFORGE_BUILD_NUMBER` | | App icon | `Support/Assets.xcassets/AppIcon.appiconset` | @@ -111,73 +212,42 @@ the forge palette) and written into the asset catalog: make icon # rewrites Support/Assets.xcassets ``` -## Build the app - -Requires macOS + Xcode + [XcodeGen](https://github.com/yonaskolb/XcodeGen), meson and -ninja (for the ish-arm64 core). - -```bash -brew install xcodegen meson ninja llvm lld libarchive - -# One-time: engine sources, bundled rootfs, then the iOS engine libraries. -make bootstrap - -make gen && open XForge.xcodeproj -``` - -`make bootstrap` runs three steps: - -1. `git submodule update --init --depth 1 Vendor/ish-arm64` — the engine sources - (plus its `deps/libarchive` submodule, which the fakefs tools link against). -2. `EmbeddedLinux/build-rootfs.sh` — builds the Alpine aarch64 rootfs into - `Support/Resources/` so it is bundled into `XForge.app`. It shapes a plain - Alpine minirootfs, installs the glibc layer and the build toolchain into a - chroot of it (needs root: a chroot needs mounts), and converts the result to - the engine's fakefs format *on this machine*, so the app only has to unzip it. - `build-ipa.yml` downloads the published copy instead of rebuilding it — see - `EmbeddedLinux/build-rootfs.sh` and `build-rootfs.yml`. A local build without - root or without room for the toolchain can use `XFORGE_PROVISION=none`. -3. `EmbeddedLinux/build-ish-core.sh` — builds the engine's static libraries into - `Vendor/ish-arm64-build/lib` for the linker. This step needs macOS: the - aarch64 gadgets use `.req` register aliases that only clang's assembler - accepts. - -The engine is device-only; simulator builds (and `make test`) compile a stub instead -and need none of the above beyond a plain `make gen`. - -Or build the unsigned IPA for sideloading via GitHub Actions -(`.github/workflows/build-ipa.yml`) and install it with SideStore/AltStore. - -## On-device build pipeline - -1. **Embedded Linux** — ish-arm64 boots the bundled Alpine aarch64 rootfs, which is - already in the engine's `fakefs` format and is unpacked on first use. -2. **Toolchain** — nothing to do: the bundled root already carries the apk build - dependencies, the glibc compatibility layer, xtool, the Swift toolchain and the - Darwin SDK, all installed at build time. The Toolchain screen verifies them and - can install any of them into a guest that lacks one - (`sh /root/install-toolchain.sh all`, `sh /root/install-toolchain.sh sdk`), and - can replace the Darwin SDK with one built from your own `Xcode.xip`. -3. **Build** — `xtool dev build -s -i` runs in the guest; the `.ipa` is copied back out. -4. **Signing** — export the unsigned `.ipa` to SideStore/AltStore or another signing - service. Direct free-Apple-ID signing through XKit remains planned. +## Continuous integration + +- **`build-ipa.yml`** — runs the unit tests, archives the app unsigned + (`CODE_SIGNING_ALLOWED=NO`), verifies the built app and the final IPA, and + publishes the unsigned IPA for sideloading. It runs `prepare-xcode.sh` too, so CI + never depends on a toolchain bundle being installed in the checkout. +- **`native-toolchain.yml`** — cross-builds LLVM's Clang and Mach-O LLD for + iPhoneOS on a macOS runner, flattens the static dependency graph into one + archive, compile-checks and link-checks `NativeToolchainBridge.mm` against the + LLVM headers it just built, and uploads + `XForgeNativeToolchain-arm64-ios.tar.gz` as a workflow artifact. It is + dispatched by hand or by a pull request that touches `App/NativeToolchain/**`, + because it takes hours and its output changes only when the LLVM commit does. + See [Docs/NATIVE-TOOLCHAIN.md](Docs/NATIVE-TOOLCHAIN.md). +- **`ios-share.yml`** — builds the app for the simulator and keeps that simulator + usable from a machine that is not a Mac, so a build can be tried by hand. ## Roadmap -- [x] Embedded Linux engine: ish-arm64 built for iOS, running in-process -- [x] Alpine aarch64 rootfs bundled in the app and imported on first boot -- [x] Provisioned Alpine rootfs — bundled with the build dependencies, the glibc - layer, xtool, the Swift toolchain and the Darwin SDK, all installed at build - time; the Toolchain screen can still install or replace any of them in-guest -- [ ] XKit signing (free Apple ID) wired into the export flow +- [x] In-process compilation: clang + `ld64.lld` linked into the app, called + through a C++ bridge +- [x] Projects as ordinary directories in the app container +- [x] XKit signing wired into the export flow, for an identity the user has +- [x] Native toolchain bundle, built in CI and installed by `make toolchain` +- [ ] **The Swift frontend (`swift-frontend`) ported to iOS** — the gate on every + Swift project, and the reason the artifact carries Clang and LLD only +- [ ] Build-setting evaluation and a build driver for a dependency graph +- [ ] `actool`/`ibtool` replacements (asset catalogs, storyboards) +- [ ] Apple ID provisioning (certificate issuance, 2FA) in-app - [ ] Hand-off of built `.ipa` to SideStore/AltStore for install - [ ] `RemoteExecutor` (build server) for fast compilation of real apps ## Licence note -XForge links the engine's core. That core is GPLv2/GPLv3 depending on which guest -architectures and native programs are compiled in. XForge builds it with native -bash/zsh/dash/helix **disabled** and only the arm64 guest, which keeps the linked -subset to the engine's own GPLv2 kernel code. If you redistribute XForge you must comply -with those terms; see [the engine's README](https://github.com/OpenMinis/ish-arm64) for the -full breakdown. +XForge links no emulator and no kernel code; the app target is the SwiftUI sources +plus two SwiftPM dependencies, xtool's `XKit` and +[ZIPFoundation](https://github.com/weichsel/ZIPFoundation) (MIT). xtool's own terms +apply to `XKit` — see its repository. The GPL note that used to be here concerned a +bundled emulator that is no longer part of the app. diff --git a/Support/Resources/.gitkeep b/Support/Resources/.gitkeep deleted file mode 100644 index 88d9b83..0000000 --- a/Support/Resources/.gitkeep +++ /dev/null @@ -1 +0,0 @@ -# Keep the Resources directory in git; the rootfs archive is fetched at build time. diff --git a/Tools/test-verify-toolchain.sh b/Tools/test-verify-toolchain.sh deleted file mode 100644 index 580a683..0000000 --- a/Tools/test-verify-toolchain.sh +++ /dev/null @@ -1,199 +0,0 @@ -#!/usr/bin/env bash -# -# test-verify-toolchain.sh — check that verify-rootfs.sh holds a root to what its -# manifest claims about the build toolchain. -# -# This is the half of the rootfs verification that a *packed* root gets: a fakefs -# ZIP cannot be chrooted, so "the console starts" is checked on the tree during -# the build and "the toolchain is really in here" is checked structurally, on the -# artifact. The structural half is easy to get wrong in a way that passes -# everything — a check that looks for the wrong path, or that trusts the manifest -# instead of the root. -# -# So each case below builds a root that is complete EXCEPT for one thing, and -# asserts that verify-rootfs.sh fails, naming that thing. -# -# Usage: Tools/test-verify-toolchain.sh -# -set -euo pipefail - -HERE="$(cd "$(dirname "$0")" && pwd)" -# Overridable so this can be pointed at a copy of the checker; the default is the -# one that ships. -VERIFY="${XFORGE_VERIFY:-$HERE/../EmbeddedLinux/verify-rootfs.sh}" -[ -f "$VERIFY" ] || { echo "cannot find $VERIFY" >&2; exit 1; } -WORK="$(mktemp -d)" -trap 'rm -rf "$WORK"' EXIT - -pass=0 -fail=0 - -note() { printf ' %s\n' "$*"; } -ok() { pass=$((pass + 1)); } -bad() { fail=$((fail + 1)); printf 'FAIL: %s\n' "$*" >&2; } - -# A root with everything the console checks want, so the toolchain checks are the -# only thing under test. -make_root() { - local root="$1" - mkdir -p "$root/bin" "$root/sbin" "$root/etc/init.d" "$root/root" \ - "$root/usr/share/terminfo/x" "$root/usr/local/share/xforge" - printf '#!/bin/sh\n' > "$root/bin/busybox" - chmod +x "$root/bin/busybox" - printf '#!/bin/sh\n' > "$root/sbin/init" - chmod +x "$root/sbin/init" - ln -sf /bin/busybox "$root/bin/sh" - cp "$HERE/../EmbeddedLinux/xforge-login" "$root/sbin/xforge-login" - chmod +x "$root/sbin/xforge-login" - printf '::sysinit:/etc/init.d/rcS\ntty1::respawn:/sbin/xforge-login root\n' > "$root/etc/inittab" - printf '#!/bin/sh\nexit 0\n' > "$root/etc/init.d/rcS" - printf 'root:x:0:0:root:/root:/bin/sh\n' > "$root/etc/passwd" - printf 'root:x:0:0:root:/root:/bin/sh\n' > "$root/etc/shadow" - printf 'export TERM=xterm-256color\nexport PATH=/bin\n' > "$root/etc/profile" - mkdir -p "$root/etc/apk" - printf 'https://example.invalid/main\n' > "$root/etc/apk/repositories" - : > "$root/usr/share/terminfo/x/xterm-256color" - printf '#!/bin/sh\n' > "$root/root/install-toolchain.sh" - cat > "$root/usr/local/share/xforge/rootfs-manifest.txt" <<'EOF' -base: alpine-minirootfs-3.21.0-aarch64.tar.gz -rootfs: 3.21.0 -engine: ish-arm64 -format: fakefs-zip -shell: /bin/sh -console: /sbin/xforge-login root (tty1, respawned by init) -toolchain: not provisioned (the guest installs it on demand) -stamp: rootfs-v5 -EOF -} - -# Add a complete, working toolchain to a root. -add_toolchain() { - local root="$1" - mkdir -p "$root/usr/local/bin" "$root/opt/xtool/usr/bin" \ - "$root/usr/local/share/xforge" \ - "$root/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/bin" \ - "$root/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/lib/swift/linux/aarch64" \ - "$root/root/.swiftpm/swift-sdks/darwin.artifactbundle" - for tool in xtool swift swiftc swiftly; do - printf '#!/bin/sh\n' > "$root/usr/local/bin/$tool" - chmod +x "$root/usr/local/bin/$tool" - done - printf 'XFORGE_GLIBC_LD=/opt/glibc/usr/lib/aarch64-linux-gnu/ld-linux-aarch64.so.1\n' \ - > "$root/usr/local/share/xforge/glibc.env" - printf '#!/bin/sh\n' > "$root/opt/xtool/usr/bin/xtool" - chmod +x "$root/opt/xtool/usr/bin/xtool" - : > "$root/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/bin/swift-frontend" - : > "$root/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/lib/swift/linux/aarch64/Swift.swiftmodule" - : > "$root/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/lib/swift/linux/libswiftCore.so" - printf '{"name": "darwin"}\n' > "$root/root/.swiftpm/swift-sdks/darwin.artifactbundle/info.json" - cat > "$root/usr/local/share/xforge/darwin-sdk.txt" <<'EOF' -tag: darwin-sdk-7 -asset: darwin.artifactbundle.zip -sha256: 0000000000000000000000000000000000000000000000000000000000000000 -path: /root/.swiftpm/swift-sdks/darwin.artifactbundle -EOF - cat > "$root/usr/local/share/xforge/rootfs-manifest.txt" <<'EOF' -base: alpine-minirootfs-3.21.0-aarch64.tar.gz -rootfs: 3.21.0 -engine: ish-arm64 -format: fakefs-zip -shell: /bin/sh -console: /sbin/xforge-login root (tty1, respawned by init) -toolchain: xtool, swiftly, swift -darwin-sdk: darwin-sdk-7 (darwin.artifactbundle.zip) -sdk-sha256: 0000000000000000000000000000000000000000000000000000000000000000 -darwin-sdk-path: /root/.swiftpm/swift-sdks/darwin.artifactbundle -stamp: rootfs-v5 -EOF -} - -# run_verify — with the console check skipped: this is about the files, and -# running the console needs the *root's own* shell, which a synthetic root has not -# got. -run_verify() { - XFORGE_VERIFY_SKIP_CONSOLE=1 bash "$VERIFY" "$1" > "$WORK/out.txt" 2>&1 -} - -check_passes() { - local name="$1" root="$2" - if run_verify "$root"; then ok; note "$name: passes"; else - bad "$name should have passed"; sed 's/^/ /' "$WORK/out.txt" | tail -5 - fi -} - -check_fails() { - local name="$1" root="$2" expect="$3" - if run_verify "$root"; then - bad "$name should have failed" - elif grep -q "$expect" "$WORK/out.txt"; then - ok; note "$name: fails, naming '$expect'" - else - bad "$name failed without naming '$expect'"; sed 's/^/ /' "$WORK/out.txt" | tail -5 - fi -} - -printf '\n==> A root with no toolchain claim\n' -make_root "$WORK/plain" -check_passes "plain root" "$WORK/plain" - -printf '\n==> A root that claims a toolchain and has one\n' -make_root "$WORK/full" -add_toolchain "$WORK/full" -check_passes "provisioned root" "$WORK/full" - -printf '\n==> A root that claims a toolchain and does not have one\n' -make_root "$WORK/liar" -sed -i 's|^toolchain:.*|toolchain: xtool, swiftly, swift|' \ - "$WORK/liar/usr/local/share/xforge/rootfs-manifest.txt" -check_fails "manifest without tools" "$WORK/liar" 'usr/local/bin/xtool is missing' - -printf '\n==> The Swift toolchain, one piece at a time\n' -for piece in \ - 'usr/local/bin/xtool' \ - 'usr/local/bin/swift' \ - 'usr/local/bin/swiftc' \ - 'usr/local/bin/swiftly' \ - 'opt/xtool/usr/bin/xtool' \ - 'usr/local/share/xforge/glibc.env' ; do - rm -rf "$WORK/missing" - cp -a "$WORK/full" "$WORK/missing" - rm -rf "$WORK/missing/$piece" - check_fails "no $piece" "$WORK/missing" '/'"$piece"' is missing' -done - -rm -rf "$WORK/nofrontend" -cp -a "$WORK/full" "$WORK/nofrontend" -rm -f "$WORK/nofrontend/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/bin/swift-frontend" -check_fails "no swift-frontend" "$WORK/nofrontend" 'no Swift toolchain under' - -rm -rf "$WORK/nostdlib" -cp -a "$WORK/full" "$WORK/nostdlib" -rm -f "$WORK/nostdlib/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/lib/swift/linux/aarch64/Swift.swiftmodule" -check_fails "no stdlib interface" "$WORK/nostdlib" 'no Linux stdlib interface' - -rm -rf "$WORK/nostdlibso" -cp -a "$WORK/full" "$WORK/nostdlibso" -rm -f "$WORK/nostdlibso/root/.local/share/swiftly/toolchains/6.2-RELEASE/usr/lib/swift/linux/libswiftCore.so" -check_fails "no stdlib library" "$WORK/nostdlibso" 'no Linux stdlib under' - -rm -rf "$WORK/nosdkpath" -cp -a "$WORK/full" "$WORK/nosdkpath" -sed -i '/^path:/d' "$WORK/nosdkpath/usr/local/share/xforge/darwin-sdk.txt" -check_fails "SDK record without a path" "$WORK/nosdkpath" 'does not name where the SDK went' - -rm -rf "$WORK/nosdk" -cp -a "$WORK/full" "$WORK/nosdk" -rm -rf "$WORK/nosdk/root/.swiftpm/swift-sdks/darwin.artifactbundle" -check_fails "SDK missing" "$WORK/nosdk" 'which is not in this root' - -printf '\n==> A root with xtool but no Darwin SDK is fine (XFORGE_PROVISION_SDK=0)\n' -rm -rf "$WORK/nosdkclaim" -cp -a "$WORK/full" "$WORK/nosdkclaim" -rm -rf "$WORK/nosdkclaim/usr/local/share/xforge/darwin-sdk.txt" \ - "$WORK/nosdkclaim/root/.swiftpm" -sed -i -e '/^darwin-sdk:/d' -e '/^sdk-sha256:/d' -e '/^darwin-sdk-path:/d' \ - "$WORK/nosdkclaim/usr/local/share/xforge/rootfs-manifest.txt" -check_passes "toolchain without the SDK" "$WORK/nosdkclaim" - -printf '\n%s checks passed, %s failed\n' "$pass" "$fail" -[ "$fail" -eq 0 ] diff --git a/project.yml b/project.yml index 31f0127..c728e1b 100644 --- a/project.yml +++ b/project.yml @@ -51,9 +51,6 @@ targets: sources: - path: App - path: Support - - path: Support/Resources - buildPhase: resources - optional: true dependencies: - package: xtool product: XKit @@ -75,7 +72,7 @@ targets: ASSETCATALOG_COMPILER_GENERATE_ASSET_SYMBOLS: YES # --- signing ------------------------------------------------------ # Automatic signing with the team above. CI builds pass - # CODE_SIGNING_ALLOWED=NO (see .github/workflows/unsigned-ipa.yml), so + # CODE_SIGNING_ALLOWED=NO (see .github/workflows/build-ipa.yml), so # an unsigned IPA can be produced without any certificate. DEVELOPMENT_TEAM: $(XFORGE_DEVELOPMENT_TEAM) CODE_SIGN_STYLE: Automatic From e7ab12a31003f4871936ccb6839bfcacfeda0e8c Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 06:21:14 -0400 Subject: [PATCH 08/54] cleanup: one toolchain screen, no failing make target, no guest-era comments MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The review of the rewritten documentation turned up four leftovers: - NativeToolchainView and ToolchainView showed the same three things — compiler status, SDK install/remove, smoke tests — so the older one is gone. The surviving screen reports capabilities, installs or imports the SDK, and runs a compile-and-link smoke test (plus the Swift one when the frontend is linked). - `make bootstrap` depended on ARCHIVE and therefore failed on a plain clone; the target is gone rather than left as a trap. `make gen` is what a clone needs. - Project.rootPath and NativeSDK still described themselves in terms of the guest (a guest path, SwiftPM's SDK store inside a toolchain installation) that no longer exists. - the docs' own follow-ups (smoke test that claimed to link, a stage titled 'Package & sign', a stale workflow name, the artifact-name mismatch in the Makefile hint, the orphaned verify-rootfs harness, the empty Support/Resources resource directory) landed with the documentation commit. --- App/Models/Project.swift | 7 +- App/NativeToolchain/NativeSDK.swift | 7 +- App/Views/Settings/SettingsView.swift | 3 - App/Views/Toolchain/NativeToolchainView.swift | 150 ------------------ Makefile | 5 +- 5 files changed, 9 insertions(+), 163 deletions(-) delete mode 100644 App/Views/Toolchain/NativeToolchainView.swift diff --git a/App/Models/Project.swift b/App/Models/Project.swift index c2df697..d15819d 100644 --- a/App/Models/Project.swift +++ b/App/Models/Project.swift @@ -20,9 +20,10 @@ struct Project: Identifiable, Hashable, Codable { /// The project directory in XForge's own container. /// - /// Derived from `name` rather than stored. `rootPath` is the guest path this - /// model was designed around, and two stored locations for one project is how - /// a build ends up reading a directory nobody created. + /// Derived from `name` rather than stored: `rootPath` is the string this model + /// carries around for display and for the safety check, and two stored + /// locations for one project is how a build ends up reading a directory nobody + /// created. var rootURL: URL { XForgeEnvironment.projectsDirectory.appendingPathComponent(name, isDirectory: true) } diff --git a/App/NativeToolchain/NativeSDK.swift b/App/NativeToolchain/NativeSDK.swift index 56c389b..54d1076 100644 --- a/App/NativeToolchain/NativeSDK.swift +++ b/App/NativeToolchain/NativeSDK.swift @@ -53,9 +53,10 @@ enum NativeSDKError: LocalizedError { } } -/// Reads the same swift-sdk.json layout that xtool's Darwin SDK builder emits, -/// but resolves it directly from the iOS app sandbox rather than through SwiftPM -/// in the Alpine guest. +/// Reads the same `swift-sdk.json` layout that xtool's Darwin SDK builder emits, +/// but resolves it directly from the app's container: SwiftPM's SDK store is a +/// directory inside a toolchain installation, and there is no installation here — +/// the bundle is a folder the app downloaded and unpacked itself. enum NativeSDK { static var installedBundle: URL { XForgeEnvironment.nativeSDKDirectory diff --git a/App/Views/Settings/SettingsView.swift b/App/Views/Settings/SettingsView.swift index b9997c9..a7a3971 100644 --- a/App/Views/Settings/SettingsView.swift +++ b/App/Views/Settings/SettingsView.swift @@ -15,9 +15,6 @@ struct SettingsView: View { NavigationLink { ToolchainView() } label: { Label("Toolchain & SDK", systemImage: "wrench.and.screwdriver") } - NavigationLink { NativeToolchainView() } label: { - Label("Native Compiler", systemImage: "cpu") - } NavigationLink { SandboxBrowserView(root: XForgeEnvironment.documentDirectory) } label: { diff --git a/App/Views/Toolchain/NativeToolchainView.swift b/App/Views/Toolchain/NativeToolchainView.swift deleted file mode 100644 index 41ad56e..0000000 --- a/App/Views/Toolchain/NativeToolchainView.swift +++ /dev/null @@ -1,150 +0,0 @@ -import SwiftUI - -@MainActor -struct NativeToolchainView: View { - @State private var status = "" - @State private var busy = false - @State private var sdkSummary = "Not installed" - - var body: some View { - List { - Section("Compiler") { - LabeledContent( - "Backend", - value: NativeToolchain.isAvailable ? "Native LLVM" : "Not linked" - ) - LabeledContent("Version", value: NativeToolchain.version) - LabeledContent( - "Swift frontend", - value: NativeToolchain.isSwiftAvailable ? "Linked" : "Not linked" - ) - - if NativeToolchain.isAvailable { - Label("Clang and LLD run in-process", systemImage: "checkmark.circle.fill") - .foregroundStyle(.green) - } else { - Text("Install the native toolchain bundle before generating the Xcode project.") - .font(.caption) - .foregroundStyle(.secondary) - } - } - - Section("Darwin SDK") { - LabeledContent("Status", value: sdkSummary) - - Button { - installSDK() - } label: { - Label("Download Native Darwin SDK", systemImage: "arrow.down.circle") - } - .disabled(busy) - - if NativeSDK.isInstalled { - Button(role: .destructive) { - removeSDK() - } label: { - Label("Remove Native SDK", systemImage: "trash") - } - .disabled(busy) - } - } - - Section("Validation") { - Button { - runSmokeTest() - } label: { - Label("Compile Native C Smoke Test", systemImage: "hammer") - } - .disabled(busy || !NativeToolchain.isAvailable || !NativeSDK.isInstalled) - - Button { - runSwiftSmokeTest() - } label: { - Label("Compile Native Swift Smoke Test", systemImage: "swift") - } - .disabled(busy || !NativeToolchain.isSwiftAvailable || !NativeSDK.isInstalled) - - if !status.isEmpty { - Text(status) - .font(.caption.monospaced()) - .textSelection(.enabled) - } - } - } - .navigationTitle("Native Toolchain") - .task { refreshSDKStatus() } - } - - private func refreshSDKStatus() { - guard NativeSDK.isInstalled else { - sdkSummary = "Not installed" - return - } - do { - let layout = try NativeSDK.layout() - sdkSummary = layout.sdkRoot.lastPathComponent - } catch { - sdkSummary = "Invalid" - status = error.localizedDescription - } - } - - private func installSDK() { - busy = true - status = "Resolving Darwin SDK release…" - Task { - defer { busy = false } - do { - try await NativeSDK.installLatestPrebuilt() - refreshSDKStatus() - status = "Native Darwin SDK installed." - } catch { - status = error.localizedDescription - } - } - } - - private func removeSDK() { - do { - try NativeSDK.remove() - refreshSDKStatus() - status = "Native Darwin SDK removed." - } catch { - status = error.localizedDescription - } - } - - private func runSmokeTest() { - busy = true - status = "Compiling C in-process…" - Task { - defer { busy = false } - do { - let layout = try NativeSDK.layout() - let object = try NativeToolchain.smokeCompile(sdk: layout.sdkRoot) - let attributes = try FileManager.default.attributesOfItem(atPath: object.path) - let size = (attributes[.size] as? NSNumber)?.int64Value ?? 0 - status = "C success: \(object.lastPathComponent) (\(size) bytes)" - } catch { - status = error.localizedDescription - } - } - } - - private func runSwiftSmokeTest() { - busy = true - status = "Compiling Swift in-process…" - Task { - defer { busy = false } - do { - let layout = try NativeSDK.layout() - let object = try NativeToolchain.smokeCompileSwift(sdk: layout) - let attributes = try FileManager.default.attributesOfItem(atPath: object.path) - let size = (attributes[.size] as? NSNumber)?.int64Value ?? 0 - status = "Swift success: \(object.lastPathComponent) (\(size) bytes)" - } catch { - status = error.localizedDescription - } - } - } -} diff --git a/Makefile b/Makefile index d35ab27..be2c4a2 100644 --- a/Makefile +++ b/Makefile @@ -1,11 +1,8 @@ -.PHONY: bootstrap toolchain icon gen build test ipa clean +.PHONY: toolchain icon gen build test ipa clean XCODE := xcodebuild SCHEME := XForge -## Everything needed to build locally, in order. -bootstrap: toolchain gen - ## Install the native toolchain bundle into Vendor/NativeToolchain. ## ## The bundle is built by the `Native iOS Toolchain` workflow (Clang + Mach-O LLD From 864d72c34b90fac895d16818890dbbc2ffba8824 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 07:25:55 -0400 Subject: [PATCH 09/54] ci: build Swift's frontend libraries for iPhoneOS MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The toolchain bundle is Clang + Mach-O LLD, which is why a Swift project cannot be compiled on device. This adds the missing half: an opt-in stage that builds `swiftFrontendTool` — the static library holding swift::performFrontend — for arm64-apple-ios, merges it into the same archive, and stages Swift's headers so the bridge's Swift code path compiles. The pieces that are not obvious: - **Swift needs its own LLVM.** The frontend is developed against swiftlang/llvm-project (the swiftasync calling convention, the coroutine passes, the C++ API it is compiled against), so the Swift path checks out that fork at the paired swift/release/6.2 branch. Upstream is kept for the LLVM-only path. - **`-DSWIFT_INSTALL_COMPONENTS=` is the fix for the configure failure** that stops a Swift-for-iOS configure dead: Swift's CMake writes that variable as a *cache* entry, so an empty value on the command line survives its default and no component is installed, which means `swift_install_in_component` returns early and its `install(TARGETS ...)` never runs. CMake refuses that rule for an executable with no BUNDLE destination on an iOS target, and Swift's host tools have none. - **The version has to match the SDK's runtime.** The darwin SDK the app compiles against was built from an Xcode, and Xcode 26.6 ships Swift 6.2.4, so the frontend is pinned to swift-6.2.4-RELEASE; a mismatch means the frontend cannot read the standard library's modules. It is an input, not a constant. - **`configure_only`** exists because the configure is where this fails, and waiting hours to learn that is a waste: it stops after configuring (and after the header compile-check, which proves the bridge finds Swift's headers) so the expensive part is only paid once the shape is right. - The iOS builds are cached under a key that deliberately ignores this file — editing the workflow is what iterating on it looks like, and it must not rebuild a compiler that did not change. `cache_salt` forces one. --- .github/workflows/native-toolchain.yml | 307 +++++++++++++++++++++++-- 1 file changed, 289 insertions(+), 18 deletions(-) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 900c15d..3cd8457 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -1,7 +1,54 @@ name: Native iOS Toolchain +# Builds the compiler that XForge links into the app: Clang + Mach-O LLD for +# iPhoneOS, and — when `with_swift` is set — Swift's frontend libraries. +# +# Two things about the Swift half are worth knowing before editing it: +# +# * Swift's frontend is *not* built against upstream LLVM. Swift depends on the +# LLVM fork it is developed with (the `swiftasync` calling convention, the +# coroutine passes, and the C++ API surface it compiles against), so the Swift +# path checks out `swiftlang/llvm-project` at the matching `swift/release/X.Y` +# branch. The upstream checkout is kept for the LLVM-only path. +# * `-DSWIFT_INSTALL_COMPONENTS=` is load-bearing. Swift's CMake writes that as a +# *cache* variable, so an empty value on the command line survives its default +# and no component is ever installed — which means `swift_install_in_component` +# returns early and its `install(TARGETS ...)` never runs. That rule is what +# aborts the configure when Swift is cross-configured for iOS: CMake refuses +# `install(TARGETS)` for an executable without a BUNDLE destination on an iOS +# target, and Swift's host tools have no such destination. +# +# The Swift version must match the Swift runtime inside the darwin SDK the app +# compiles against (that SDK is built from an Xcode; Xcode 26.6 ships Swift +# 6.2.4), otherwise the frontend cannot read the SDK's standard-library modules. + on: workflow_dispatch: + inputs: + with_swift: + description: Build the Swift frontend libraries as well + type: boolean + default: false + swift_ref: + description: swiftlang/swift tag or branch + type: string + default: swift-6.2.4-RELEASE + swift_llvm_ref: + description: swiftlang/llvm-project branch paired with swift_ref + type: string + default: swift/release/6.2 + configure_only: + description: Stop after configuring the Swift build (cheap iteration) + type: boolean + default: false + deployment_target: + description: Minimum iOS version + type: string + default: "17.0" + cache_salt: + description: Bump to discard the cached iOS builds + type: string + default: "1" pull_request: paths: - "App/NativeToolchain/**" @@ -16,24 +63,98 @@ on: jobs: llvm-ios: runs-on: macos-15 - timeout-minutes: 180 + timeout-minutes: 350 + env: + IOS_DEPLOYMENT_TARGET: ${{ inputs.deployment_target || '17.0' }} + SWIFT_REF: ${{ inputs.swift_ref || 'swift-6.2.4-RELEASE' }} + SWIFT_LLVM_REF: ${{ inputs.swift_llvm_ref || 'swift/release/6.2' }} steps: - uses: actions/checkout@v4 - name: Install build tools run: brew install cmake ninja + # The LLVM the compiler is built from: upstream for the LLVM-only bundle, + # Swift's own fork when the frontend is part of it. - name: Checkout LLVM uses: actions/checkout@v4 with: - repository: llvm/llvm-project + repository: ${{ inputs.with_swift && 'swiftlang/llvm-project' || 'llvm/llvm-project' }} + ref: ${{ inputs.with_swift && env.SWIFT_LLVM_REF || '' }} path: llvm-project fetch-depth: 1 + - name: Checkout Swift + if: ${{ inputs.with_swift }} + uses: actions/checkout@v4 + with: + repository: swiftlang/swift + ref: ${{ env.SWIFT_REF }} + path: swift + fetch-depth: 1 + + # Swift's configure step executes `cmark --version` and generates + # documentation with it, so a native cmark has to exist before the + # cross-configure runs. + - name: Checkout cmark + if: ${{ inputs.with_swift }} + uses: actions/checkout@v4 + with: + repository: swiftlang/swift-cmark + ref: release/6.2 + path: swift-cmark + fetch-depth: 1 + + # Only needed to *find* the sources at configure time; the macro and ASTGen + # libraries are switched off, because a macro plugin is a separate process + # and iOS has no way to run one. + - name: Checkout swift-syntax + if: ${{ inputs.with_swift }} + uses: actions/checkout@v4 + with: + repository: swiftlang/swift-syntax + ref: release/6.2 + path: swift-syntax + fetch-depth: 1 + + - name: Checkout swift-experimental-string-processing + if: ${{ inputs.with_swift }} + uses: actions/checkout@v4 + with: + repository: swiftlang/swift-experimental-string-processing + ref: release/6.2 + path: swift-experimental-string-processing + fetch-depth: 1 + + - name: Identify the iOS compiler toolchain + id: toolchain-versions + run: | + { + echo "value=$(xcodebuild -version | tr '\n' ' ')-$(xcrun --sdk iphoneos --show-sdk-version)" + } >> "$GITHUB_OUTPUT" + + # The expensive half is LLVM's cross-build and (when enabled) Swift's. The + # key deliberately does not include the workflow file: editing this file — + # which is what iterating on it looks like — must not force a rebuild of a + # compiler that did not change. `cache_salt` is the way to ask for one. + - name: Cache the iOS builds + uses: actions/cache@v4 + with: + path: | + llvm-host-build + llvm-build + cmark-host-build + swift-build + key: ${{ runner.os }}-${{ runner.arch }}-toolchain-${{ steps.toolchain-versions.outputs.value }}-${{ env.SWIFT_REF }}-${{ inputs.with_swift && env.SWIFT_LLVM_REF || 'upstream' }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '1' }} + - name: Build host TableGen tools shell: bash run: | set -euxo pipefail + if [ -x llvm-host-build/bin/llvm-tblgen ] && [ -x llvm-host-build/bin/clang-tblgen ]; then + echo "host TableGen tools already cached" + exit 0 + fi cmake -S llvm-project/llvm -B llvm-host-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DLLVM_ENABLE_PROJECTS="clang" \ @@ -48,6 +169,22 @@ jobs: -DLLVM_ENABLE_CURL=OFF ninja -C llvm-host-build llvm-tblgen clang-tblgen + - name: Build native cmark + if: ${{ inputs.with_swift }} + shell: bash + run: | + set -euxo pipefail + if [ -x cmark-host-build/src/cmark-gfm ]; then + echo "native cmark already cached" + exit 0 + fi + cmake -S swift-cmark -B cmark-host-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMARK_TESTS=OFF + cmake --build cmark-host-build --target cmark-gfm + mkdir -p cmark-host-build/src + ln -sf cmark-host-build/src/cmark-gfm cmark-host-build/src/cmark + - name: Configure LLVM for iPhoneOS arm64 shell: bash run: | @@ -59,11 +196,11 @@ jobs: -DCMAKE_MACOSX_BUNDLE=OFF \ -DCMAKE_OSX_SYSROOT="$SDK" \ -DCMAKE_OSX_ARCHITECTURES=arm64 \ - -DCMAKE_OSX_DEPLOYMENT_TARGET=17.0 \ - -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios17.0 \ - -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios17.0 \ - -DLLVM_HOST_TRIPLE=arm64-apple-ios17.0 \ - -DLLVM_DEFAULT_TARGET_TRIPLE=arm64-apple-ios17.0 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ + -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_HOST_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_DEFAULT_TARGET_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ -DLLVM_NATIVE_TOOL_DIR="$PWD/llvm-host-build/bin" \ -DLLVM_ENABLE_PROJECTS="clang;lld" \ -DLLVM_TARGETS_TO_BUILD=AArch64 \ @@ -85,6 +222,7 @@ jobs: -DLLD_BUILD_TOOLS=OFF - name: Build Clang and Mach-O LLD libraries + if: ${{ !inputs.configure_only }} run: | set -euxo pipefail ninja -C llvm-build \ @@ -101,27 +239,138 @@ jobs: lldMachO \ lldCommon + # The step this workflow exists for. Everything above is shared with the + # LLVM-only bundle; everything below only happens when Swift is asked for. + - name: Configure the Swift frontend for iPhoneOS + if: ${{ inputs.with_swift }} + shell: bash + run: | + set -euxo pipefail + SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + HOST_SWIFT_BIN="$(dirname "$(xcrun -f swiftc)")" + + cmake -S swift -B swift-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_SYSTEM_NAME=iOS \ + -DCMAKE_MACOSX_BUNDLE=OFF \ + -DCMAKE_OSX_SYSROOT="$SDK" \ + -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ + -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DCMAKE_Swift_COMPILER="$(xcrun -f swiftc)" \ + -DCMAKE_Swift_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_DIR="$PWD/llvm-build/lib/cmake/llvm" \ + -DClang_DIR="$PWD/llvm-build/lib/cmake/clang" \ + -DLLVM_HOST_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_DEFAULT_TARGET_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DSWIFT_NATIVE_LLVM_TOOLS_PATH="$PWD/llvm-host-build/bin" \ + -DSWIFT_NATIVE_CLANG_TOOLS_PATH="$PWD/llvm-host-build/bin" \ + -DSWIFT_NATIVE_SWIFT_TOOLS_PATH="$HOST_SWIFT_BIN" \ + -DSWIFT_PATH_TO_CMARK_BUILD="$PWD/cmark-host-build" \ + -DSWIFT_PATH_TO_CMARK_SOURCE="$PWD/swift-cmark" \ + -DSWIFT_PATH_TO_STRING_PROCESSING_SOURCE="$PWD/swift-experimental-string-processing" \ + -DSWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE="$PWD/swift-syntax" \ + -DSWIFT_HOST_VARIANT=iphoneos \ + -DSWIFT_HOST_VARIANT_SDK=IOS \ + -DSWIFT_HOST_VARIANT_ARCH=arm64 \ + -DSWIFT_DARWIN_DEPLOYMENT_VERSION_IOS="$IOS_DEPLOYMENT_TARGET" \ + -DSWIFT_BUILD_DYNAMIC_STDLIB=OFF \ + -DSWIFT_BUILD_STATIC_STDLIB=OFF \ + -DSWIFT_BUILD_DYNAMIC_SDK_OVERLAY=OFF \ + -DSWIFT_BUILD_STATIC_SDK_OVERLAY=OFF \ + -DSWIFT_BUILD_STDLIB_EXTRA_TOOLCHAIN_CONTENT=OFF \ + -DSWIFT_BUILD_SOURCEKIT=OFF \ + -DSWIFT_BUILD_SWIFT_SYNTAX=OFF \ + -DSWIFT_BUILD_REMOTE_MIRROR=OFF \ + -DSWIFT_BUILD_PERF_TESTSUITE=OFF \ + -DSWIFT_INCLUDE_TESTS=OFF \ + -DSWIFT_INCLUDE_DOCS=OFF \ + -DSWIFT_INSTALL_COMPONENTS= + + # Proof that the configure produced what the app will link, before the + # hours of building. + test -f swift-build/lib/swift/host/libswiftFrontendTool.a \ + || test -f swift-build/lib/libswiftFrontendTool.a \ + || echo "::warning::swiftFrontendTool archive not found where expected; listing swift-build/lib" + ls -la swift-build/lib | head -40 || true + + - name: Build the Swift frontend libraries + if: ${{ inputs.with_swift && !inputs.configure_only }} + shell: bash + run: | + set -euxo pipefail + # The static library holding `swift::performFrontend`. Its transitive + # dependencies (swiftAST, swiftSema, the SIL and IRGen libraries, the + # Clang importer and Swift's own Clang/LLVM) come along with it; the + # swift/driver executable libraries are deliberately not built, because + # nothing on the device runs a driver. + ninja -C swift-build swiftFrontendTool + + - name: Verify the Swift frontend + if: ${{ inputs.with_swift && !inputs.configure_only }} + shell: bash + run: | + set -euxo pipefail + find swift-build/lib -maxdepth 3 -type f -name 'libswift*.a' | sort > /tmp/swift-archives.txt + test -s /tmp/swift-archives.txt || { echo "::error::no Swift libraries were produced"; exit 1; } + echo "Swift archives: $(wc -l < /tmp/swift-archives.txt)" + + # The symbol the app's bridge calls. Without it the bundle would look + # complete and only fail when the app links, on a device. + found="" + while IFS= read -r library; do + archs="$(lipo -archs "$library" 2>/dev/null || true)" + case " $archs " in + *" arm64 "*) ;; + *) echo "::error::$library is not arm64 ($archs)"; exit 1 ;; + esac + if [ -z "$found" ] && nm -gU "$library" 2>/dev/null | grep -q 'performFrontend'; then + found="$library" + fi + done < /tmp/swift-archives.txt + test -n "$found" || { echo "::error::no Swift library exports performFrontend"; exit 1; } + echo "performFrontend is in $found" + - name: Compile-check XForge bridge against LLVM headers shell: bash run: | set -euxo pipefail SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + EXTRA_DEFINES=() + EXTRA_INCLUDES=() + if [ "${{ inputs.with_swift }}" = "true" ]; then + # Exercises the Swift half of the bridge: with the define and the + # headers present, NativeToolchainBridge.mm compiles the code path + # that calls swift::performFrontend instead of the stub. + EXTRA_DEFINES+=(-DXFORGE_HAS_SWIFT_FRONTEND=1) + EXTRA_INCLUDES+=(-I swift/include) + if [ -d swift-build/include ]; then + EXTRA_INCLUDES+=(-I swift-build/include) + fi + if [ -d swift-build/tools/swift/include ]; then + EXTRA_INCLUDES+=(-I swift-build/tools/swift/include) + fi + fi xcrun --sdk iphoneos clang++ \ -std=c++20 \ -arch arm64 \ - -miphoneos-version-min=17.0 \ + -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET \ -isysroot "$SDK" \ -DXFORGE_HAS_LLVM=1 \ + "${EXTRA_DEFINES[@]}" \ -I llvm-project/llvm/include \ -I llvm-build/include \ -I llvm-project/clang/include \ -I llvm-build/tools/clang/include \ -I llvm-project/lld/include \ + "${EXTRA_INCLUDES[@]}" \ -I App/NativeToolchain \ -c App/NativeToolchain/NativeToolchainBridge.mm \ -o /tmp/NativeToolchainBridge.o - name: Stage native toolchain SDK + if: ${{ !inputs.configure_only }} shell: bash run: | set -euxo pipefail @@ -134,15 +383,29 @@ jobs: fi cp -R llvm-project/lld/include/lld dist/include/ - # Flatten the static dependency graph into one archive. XForge then - # links a single library instead of depending on LLVM's internal - # archive ordering. - find llvm-build/lib -maxdepth 1 -type f -name '*.a' -print | sort > /tmp/llvm-archives.txt - test -s /tmp/llvm-archives.txt + if [ "${{ inputs.with_swift }}" = "true" ]; then + cp -R swift/include/swift dist/include/ + if [ -d swift-build/include/swift ]; then + cp -R swift-build/include/swift dist/include/ + fi + if [ -d swift-build/tools/swift/include/swift ]; then + cp -R swift-build/tools/swift/include/swift dist/include/ + fi + fi + + # Flatten the static dependency graph — LLVM's, Clang's, LLD's and (when + # built) Swift's — into one archive. XForge then links a single library + # instead of depending on archive ordering, and a Swift frontend that + # links an LLVM built in the same configuration cannot drift from it. + find llvm-build/lib -maxdepth 1 -type f -name '*.a' -print | sort > /tmp/archives.txt + if [ "${{ inputs.with_swift }}" = "true" ]; then + find swift-build/lib -maxdepth 3 -type f -name '*.a' -print | sort >> /tmp/archives.txt + fi + test -s /tmp/archives.txt ARCHIVES=() while IFS= read -r archive; do ARCHIVES+=("$archive") - done < /tmp/llvm-archives.txt + done < /tmp/archives.txt /usr/bin/libtool -static -o dist/lib/libXForgeNativeToolchain.a "${ARCHIVES[@]}" cat > /tmp/native_link_smoke.mm <<'EOF' @@ -156,7 +419,7 @@ jobs: xcrun --sdk iphoneos clang++ \ -std=c++20 \ -arch arm64 \ - -miphoneos-version-min=17.0 \ + -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET \ -isysroot "$SDK" \ -I App/NativeToolchain \ -c /tmp/native_link_smoke.mm \ @@ -164,14 +427,19 @@ jobs: xcrun --sdk iphoneos clang++ \ -arch arm64 \ - -miphoneos-version-min=17.0 \ + -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET \ -isysroot "$SDK" \ /tmp/native_link_smoke.o \ /tmp/NativeToolchainBridge.o \ dist/lib/libXForgeNativeToolchain.a \ -lc++ -lz -liconv -lsqlite3 \ -framework Foundation \ - -o /tmp/XForgeNativeToolchainSmoke + -o /tmp/XForgeNativeToolchainSmoke \ + 2> /tmp/native-link-errors.txt || { + echo "::error::linking the toolchain into an app-shaped target failed" + tail -60 /tmp/native-link-errors.txt + exit 1 + } file /tmp/XForgeNativeToolchainSmoke file /tmp/NativeToolchainBridge.o @@ -179,14 +447,17 @@ jobs: cat > dist/manifest.txt < Date: Thu, 24 Sep 2026 07:45:26 -0400 Subject: [PATCH 10/54] toolchain: compile the Swift frontend in when the bundle carries it The bundle's manifest says whether it includes Swift's frontend libraries, and the bridge needs the matching define: without -DXFORGE_HAS_SWIFT_FRONTEND the Swift entry point compiles to a stub and the app reports 'swift-frontend: missing' for a bundle that has it. Two fixes found while testing the three paths (no bundle / Clang and LLD only / with the frontend) against a fixture bundle: the archive list came from process substitution, which needs /dev/fd, and the revision line came from a grep pipeline that leaves a first read of the manifest awkward on a bare shell. Both are plain redirections now, and the fixture test asserts the resulting XFORGE_NATIVE_CFLAGS for each case. --- NativeToolchain/prepare-xcode.sh | 44 +++++++++++++++++++++++++++----- 1 file changed, 37 insertions(+), 7 deletions(-) diff --git a/NativeToolchain/prepare-xcode.sh b/NativeToolchain/prepare-xcode.sh index 004d8c1..d3470d3 100644 --- a/NativeToolchain/prepare-xcode.sh +++ b/NativeToolchain/prepare-xcode.sh @@ -1,13 +1,23 @@ #!/usr/bin/env bash set -euo pipefail +# Turns Vendor/NativeToolchain — the bundle the `Native iOS Toolchain` workflow +# uploads — into the xcconfig the app target is built with. Nothing here runs on +# iOS: this is a build-time step, and what it really decides is whether the C +# bridge compiles the compiler in or the "not available" stub. +# +# A missing bundle is not an error. A clone without one still builds, and the +# Toolchain screen says the compiler is not linked; that is a better default than +# refusing to build. + ROOT="$(cd "$(dirname "$0")/.." && pwd)" BUNDLE="$ROOT/Vendor/NativeToolchain" OUT="$ROOT/Support/NativeToolchain.generated.xcconfig" +MANIFEST="$BUNDLE/manifest.txt" mkdir -p "$(dirname "$OUT")" -if [ ! -f "$BUNDLE/manifest.txt" ] || [ ! -d "$BUNDLE/include" ] || [ ! -d "$BUNDLE/lib" ]; then +if [ ! -f "$MANIFEST" ] || [ ! -d "$BUNDLE/include" ] || [ ! -d "$BUNDLE/lib" ]; then cat >"$OUT" <"$ARCHIVE_LIST" +while IFS= read -r lib; do + [ -n "$lib" ] || continue ARCHIVES+=("\$(SRCROOT)/\${lib#"$ROOT/"}") -done < <(find "$BUNDLE/lib" -maxdepth 1 -type f -name '*.a' -print0 | sort -z) +done <"$ARCHIVE_LIST" -if [ "\${#ARCHIVES[@]}" -eq 0 ]; then +if [ "${#ARCHIVES[@]}" -eq 0 ]; then echo "error: Native toolchain bundle has no static libraries in $BUNDLE/lib" >&2 exit 1 fi +# The bundle declares what it contains, and the bridge needs the matching defines: +# XFORGE_HAS_LLVM gates the Clang/LLD entry points, XFORGE_HAS_SWIFT_FRONTEND the +# one that calls swift::performFrontend. Without the second, a bundle that carries +# the Swift libraries still reports "swift-frontend: missing". +CFLAGS="-DXFORGE_HAS_LLVM=1" +if grep -q '^swift_frontend=1' "$MANIFEST" 2>/dev/null; then + CFLAGS="$CFLAGS -DXFORGE_HAS_SWIFT_FRONTEND=1" + SWIFT_REF="$(sed -n 's/^swift_ref=//p' "$MANIFEST" 2>/dev/null)" + echo "Native toolchain: Swift frontend included (${SWIFT_REF:-unknown revision})" +else + echo "Native toolchain: Clang and LLD only (no Swift frontend in this bundle)" +fi + { echo "XFORGE_NATIVE_TOOLCHAIN_AVAILABLE = 1" echo "XFORGE_NATIVE_HEADER_SEARCH_PATHS = \$(SRCROOT)/Vendor/NativeToolchain/include" echo "XFORGE_NATIVE_LIBRARY_SEARCH_PATHS = \$(SRCROOT)/Vendor/NativeToolchain/lib" - echo "XFORGE_NATIVE_CFLAGS = -DXFORGE_HAS_LLVM=1" + echo "XFORGE_NATIVE_CFLAGS = $CFLAGS" printf "XFORGE_NATIVE_LDFLAGS =" - for archive in "\${ARCHIVES[@]}"; do + for archive in "${ARCHIVES[@]}"; do printf " %s" "$archive" done echo " -lc++ -lz -liconv -lsqlite3 -framework Foundation" } >"$OUT" -echo "Native toolchain: enabled (\${#ARCHIVES[@]} archives)" +echo "Native toolchain: enabled (${#ARCHIVES[@]} archives)" From e26d96d7bf09bf6da31ddbf664facea63a04fb5f Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 07:46:43 -0400 Subject: [PATCH 11/54] ci: satisfy Swift's host-tablegen requirement, and cache past a failure The first Swift configure stopped at SwiftSharedCMakeConfig.cmake:62: Attempting to cross-compile swift standalone but no native LLVM build found. Please cross-compile LLVM as well. That check only runs when `LLVM_TABLEGEN` is unset; setting it (and CLANG_TABLEGEN) to the host tools this workflow already builds is what it is asking for. actions/cache also only writes on success, so a run that failed in the Swift configure stored nothing and would have rebuilt LLVM from scratch on every iteration. Restore and save are separate steps now, and the save runs with `if: always()` before the step most likely to fail. --- .github/workflows/native-toolchain.yml | 19 +++++++++++++++++-- 1 file changed, 17 insertions(+), 2 deletions(-) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 3cd8457..42b3e30 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -137,8 +137,9 @@ jobs: # key deliberately does not include the workflow file: editing this file — # which is what iterating on it looks like — must not force a rebuild of a # compiler that did not change. `cache_salt` is the way to ask for one. - - name: Cache the iOS builds - uses: actions/cache@v4 + - name: Restore the iOS builds + id: toolchain-cache + uses: actions/cache/restore@v4 with: path: | llvm-host-build @@ -241,6 +242,18 @@ jobs: # The step this workflow exists for. Everything above is shared with the # LLVM-only bundle; everything below only happens when Swift is asked for. + # Saved here, before the step most likely to fail, and with `if: always()`: + # a failed Swift configure must not cost another LLVM cross-build. + - name: Save the iOS builds + if: ${{ always() && steps.toolchain-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + key: ${{ steps.toolchain-cache.outputs.cache-primary-key }} + path: | + llvm-host-build + llvm-build + cmark-host-build + - name: Configure the Swift frontend for iPhoneOS if: ${{ inputs.with_swift }} shell: bash @@ -264,6 +277,8 @@ jobs: -DClang_DIR="$PWD/llvm-build/lib/cmake/clang" \ -DLLVM_HOST_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ -DLLVM_DEFAULT_TARGET_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_TABLEGEN="$PWD/llvm-host-build/bin/llvm-tblgen" \ + -DCLANG_TABLEGEN="$PWD/llvm-host-build/bin/clang-tblgen" \ -DSWIFT_NATIVE_LLVM_TOOLS_PATH="$PWD/llvm-host-build/bin" \ -DSWIFT_NATIVE_CLANG_TOOLS_PATH="$PWD/llvm-host-build/bin" \ -DSWIFT_NATIVE_SWIFT_TOOLS_PATH="$HOST_SWIFT_BIN" \ From 9d5bb7ada08b1b22457a36a7e2e504c6be9ec33f Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 07:59:26 -0400 Subject: [PATCH 12/54] ci: configure Swift as a Darwin target, not an iOS one MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The configure got past the tablegen requirement and stopped on two Linux-isms: cmake/modules/Libdispatch.cmake:28: libdispatch requires a newer clang compiler lib/Basic/CMakeLists.txt:10: Could NOT find UUID (missing: UUID_LIBRARY) Both branches are selected by `CMAKE_SYSTEM_NAME STREQUAL "Darwin"`, which CMAKE_SYSTEM_NAME=iOS fails. Swift's own build-script-impl configures iOS the same way this now does — sysroot, architectures, deployment target, compiler target triple and SWIFT_HOST_VARIANT_SDK, with no system name — so the target is unchanged and only the platform *branch* is corrected. One consequence of that: the build is no longer "cross-compiling" as far as CMake is concerned, so Swift's CMake takes the host branch and looks for its native tools in LLVM_TOOLS_BINARY_DIR — the iOS LLVM build's bin, which LLVM_BUILD_TOOLS=OFF leaves empty. The host tools are mirrored there now. --- .github/workflows/native-toolchain.yml | 23 ++++++++++++++++++++++- 1 file changed, 22 insertions(+), 1 deletion(-) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 42b3e30..59a772e 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -170,6 +170,15 @@ jobs: -DLLVM_ENABLE_CURL=OFF ninja -C llvm-host-build llvm-tblgen clang-tblgen + # Swift's CMake resolves its native tools through LLVM_TOOLS_BINARY_DIR, + # which for the iOS build is a directory with no tools in it + # (LLVM_BUILD_TOOLS=OFF). Mirroring the host tools there means every + # path it might take finds them. + mkdir -p llvm-build/bin + for tool in llvm-host-build/bin/*; do + ln -sf "$PWD/$tool" "llvm-build/bin/$(basename "$tool")" + done + - name: Build native cmark if: ${{ inputs.with_swift }} shell: bash @@ -262,9 +271,21 @@ jobs: SDK="$(xcrun --sdk iphoneos --show-sdk-path)" HOST_SWIFT_BIN="$(dirname "$(xcrun -f swiftc)")" + # CMAKE_SYSTEM_NAME=Darwin, not iOS, and that is deliberate. Swift's own + # iOS builds are configured exactly this way (build-script-impl passes + # the sysroot and SWIFT_HOST_VARIANT_SDK and never sets the system + # name), because its CMake branches on `CMAKE_SYSTEM_NAME STREQUAL + # "Darwin"` for Apple: + # * lib/Basic/CMakeLists.txt does `find_package(UUID REQUIRED)` on + # anything else — a Linux dependency for Swift-on-Linux builds; + # * SwiftSharedCMakeConfig's Libdispatch.cmake errors out with + # "libdispatch requires a newer clang compiler" unless it is Darwin + # (on Darwin, dispatch comes from the SDK). + # The target is still iOS: the sysroot, the architectures, the + # deployment target and the compiler target triple below say so. cmake -S swift -B swift-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_SYSTEM_NAME=iOS \ + -DCMAKE_SYSTEM_NAME=Darwin \ -DCMAKE_MACOSX_BUNDLE=OFF \ -DCMAKE_OSX_SYSROOT="$SDK" \ -DCMAKE_OSX_ARCHITECTURES=arm64 \ From 84dd1a37d05f89dea37f453cbf3f9375d34c6e1e Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 08:09:08 -0400 Subject: [PATCH 13/54] ci: keep the clang target defined for Swift's generation stage MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit With the platform branch corrected, the Swift configure now completes ("Configuring done") and fails while generating: CMake Error at lib/Tooling/libSwiftScan/CMakeLists.txt:19 (add_dependencies): The dependency target "clang" of target "libSwiftScan" does not exist. That comes from CLANG_BUILD_TOOLS=OFF: a unified build normally defines the `clang` executable target, and Swift's CMake depends on it existing even when nobody builds it. Turning it on defines the target and nothing builds it — the Swift build still asks ninja for one static library. --- .github/workflows/native-toolchain.yml | 10 +++++++++- 1 file changed, 9 insertions(+), 1 deletion(-) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 59a772e..e095013 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -199,6 +199,14 @@ jobs: shell: bash run: | set -euxo pipefail + # CLANG_BUILD_TOOLS is ON on purpose, and nothing builds that target. + # Swift's CMake assumes the unified build it is configured against has a + # `clang` executable target — lib/Tooling/libSwiftScan/CMakeLists.txt + # does `add_dependencies(libSwiftScan clang …)` — and generation fails + # outright when the target does not exist ("The dependency target + # \"clang\" of target \"libSwiftScan\" does not exist"), even though + # neither target is ever built here. Defining it is cheaper than + # disabling Swift's subdirectories one generation error at a time. SDK="$(xcrun --sdk iphoneos --show-sdk-path)" cmake -S llvm-project/llvm -B llvm-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ @@ -227,7 +235,7 @@ jobs: -DLLVM_ENABLE_CURL=OFF \ -DLLVM_ENABLE_RTTI=ON \ -DCLANG_INCLUDE_TESTS=OFF \ - -DCLANG_BUILD_TOOLS=OFF \ + -DCLANG_BUILD_TOOLS=ON \ -DLLD_INCLUDE_TESTS=OFF \ -DLLD_BUILD_TOOLS=OFF From cf352a2c01af6a0d91d358de932c94c7fad342f6 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 08:20:20 -0400 Subject: [PATCH 14/54] ci: the Swift configure passes; gate the header check on a real build MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit configure_only reached the end of the Swift configure for iPhoneOS: -- Configuring done (9.1s) and then failed in the bridge compile-check on 'clang/Basic/DiagnosticGroups.inc' file not found — a header the Clang build generates, which configure_only does not run. The check belongs to a real build, so it is skipped with the rest of the build steps, and the configure-only path reports what it actually produced (the tablegen'd SwiftOptions archive, and the listing of swift-build/lib). --- .github/workflows/native-toolchain.yml | 4 ++++ 1 file changed, 4 insertions(+) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index e095013..3b1f4e6 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -376,7 +376,11 @@ jobs: test -n "$found" || { echo "::error::no Swift library exports performFrontend"; exit 1; } echo "performFrontend is in $found" + # Skipped in configure_only mode for a real reason: the headers this + # compiles against are *generated* by the Clang build (DiagnosticGroups.inc + # and friends), which configure_only deliberately does not run. - name: Compile-check XForge bridge against LLVM headers + if: ${{ !inputs.configure_only }} shell: bash run: | set -euxo pipefail From 5aaee0fe55dcf36f7c277fe5011f958752a5bf27 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 10:26:38 -0400 Subject: [PATCH 15/54] ci: cache the Swift build, and report the symbol instead of gating on it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Swift frontend built (937/937 edges, lib/libswiftFrontendTool.a), and then my own verification failed it: 29 archives, all arm64, and 'nm -gU … | grep performFrontend' found nothing. The symbol is defined in lib/FrontendTool/FrontendTool.cpp, so it is in that archive — the check was wrong, not the build, and it stopped the job five seconds after an hour of work. - the symbol search now uses the host llvm-nm when it exists, prints what it found, and *warns* rather than failing. The gate is the link below it, which compiles the bridge with XFORGE_HAS_SWIFT_FRONTEND and fails on a genuinely missing symbol — a functional check instead of a fragile one; - the architecture check stays an error, because a mixed-architecture bundle is worse than one that fails here; - the cache is split. The old single save ran before swift-build existed, so the 52-minute Swift build was being thrown away on every iteration; LLVM's half and Swift's are separate entries now, each saved right after it is produced. --- .github/workflows/native-toolchain.yml | 62 ++++++++++++++++++-------- 1 file changed, 44 insertions(+), 18 deletions(-) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 3b1f4e6..3807c36 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -137,16 +137,25 @@ jobs: # key deliberately does not include the workflow file: editing this file — # which is what iterating on it looks like — must not force a rebuild of a # compiler that did not change. `cache_salt` is the way to ask for one. - - name: Restore the iOS builds - id: toolchain-cache + - name: Restore the LLVM builds + id: llvm-cache uses: actions/cache/restore@v4 with: path: | llvm-host-build llvm-build cmark-host-build - swift-build - key: ${{ runner.os }}-${{ runner.arch }}-toolchain-${{ steps.toolchain-versions.outputs.value }}-${{ env.SWIFT_REF }}-${{ inputs.with_swift && env.SWIFT_LLVM_REF || 'upstream' }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '1' }} + key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ steps.toolchain-versions.outputs.value }}-${{ inputs.with_swift && env.SWIFT_LLVM_REF || 'upstream' }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '1' }} + + # Swift's own build is the longer half and is cached separately, because it + # is saved *after* the LLVM half has been built and Swift's has too. + - name: Restore the Swift build + if: ${{ inputs.with_swift }} + id: swift-cache + uses: actions/cache/restore@v4 + with: + path: swift-build + key: ${{ runner.os }}-${{ runner.arch }}-swift-${{ env.SWIFT_REF }}-${{ env.SWIFT_LLVM_REF }}-${{ steps.toolchain-versions.outputs.value }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '1' }} - name: Build host TableGen tools shell: bash @@ -261,11 +270,11 @@ jobs: # LLVM-only bundle; everything below only happens when Swift is asked for. # Saved here, before the step most likely to fail, and with `if: always()`: # a failed Swift configure must not cost another LLVM cross-build. - - name: Save the iOS builds - if: ${{ always() && steps.toolchain-cache.outputs.cache-hit != 'true' }} + - name: Save the LLVM builds + if: ${{ always() && steps.llvm-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@v4 with: - key: ${{ steps.toolchain-cache.outputs.cache-primary-key }} + key: ${{ steps.llvm-cache.outputs.cache-primary-key }} path: | llvm-host-build llvm-build @@ -351,6 +360,13 @@ jobs: # nothing on the device runs a driver. ninja -C swift-build swiftFrontendTool + - name: Save the Swift build + if: ${{ always() && inputs.with_swift && !inputs.configure_only && steps.swift-cache.outputs.cache-hit != 'true' }} + uses: actions/cache/save@v4 + with: + key: ${{ steps.swift-cache.outputs.cache-primary-key }} + path: swift-build + - name: Verify the Swift frontend if: ${{ inputs.with_swift && !inputs.configure_only }} shell: bash @@ -360,25 +376,35 @@ jobs: test -s /tmp/swift-archives.txt || { echo "::error::no Swift libraries were produced"; exit 1; } echo "Swift archives: $(wc -l < /tmp/swift-archives.txt)" - # The symbol the app's bridge calls. Without it the bundle would look - # complete and only fail when the app links, on a device. - found="" + # Every archive must be arm64: a bundle that mixes architectures links + # and then misbehaves, which is worse than one that fails here. while IFS= read -r library; do archs="$(lipo -archs "$library" 2>/dev/null || true)" case " $archs " in *" arm64 "*) ;; *) echo "::error::$library is not arm64 ($archs)"; exit 1 ;; esac - if [ -z "$found" ] && nm -gU "$library" 2>/dev/null | grep -q 'performFrontend'; then - found="$library" - fi done < /tmp/swift-archives.txt - test -n "$found" || { echo "::error::no Swift library exports performFrontend"; exit 1; } - echo "performFrontend is in $found" + echo "every Swift archive is arm64" + + # `performFrontend` lives in libswiftFrontendTool.a (FrontendTool.cpp). + # The search is reported, not enforced: the gate that matters is the + # link below, which fails if the symbol is missing for real. An earlier + # version of this step failed the job because `nm` did not print the + # symbol for a library that had just been built. + FRONTEND_ARCHIVE="$PWD/swift-build/lib/libswiftFrontendTool.a" + test -f "$FRONTEND_ARCHIVE" || { echo "::error::libswiftFrontendTool.a was not produced"; exit 1; } + NM="" + if [ -x llvm-host-build/bin/llvm-nm ]; then NM="$PWD/llvm-host-build/bin/llvm-nm"; else NM="nm"; fi + echo "--- $NM -gU $FRONTEND_ARCHIVE | grep -c performFrontend" + SYMBOLS="$("$NM" -gU "$FRONTEND_ARCHIVE" 2>&1 || true)" + if printf '%s\n' "$SYMBOLS" | grep -q performFrontend; then + echo "performFrontend is in libswiftFrontendTool.a" + else + echo "::warning::no performFrontend symbol in libswiftFrontendTool.a; the link below decides" + printf '%s\n' "$SYMBOLS" | head -20 + fi - # Skipped in configure_only mode for a real reason: the headers this - # compiles against are *generated* by the Clang build (DiagnosticGroups.inc - # and friends), which configure_only deliberately does not run. - name: Compile-check XForge bridge against LLVM headers if: ${{ !inputs.configure_only }} shell: bash From 58d4499c0e58a9d694b6dd47af2565b38653fbeb Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 12:17:49 -0400 Subject: [PATCH 16/54] native: compile the bridge against either clang generation MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Swift frontend builds now (937/937 edges, cached), and the job failed in the bridge compile-check with two independent problems: 1. Apple's SDK predefines IBAction and IBOutlet as macros, and clang's generated AttrList.inc expands ATTR(IBAction) into `class IBAction##Attr;` — the macro is pasted into the token paste and the header cannot compile. Both are undefined before any clang header is included, which is what clang's own headers do on the revisions where this bites. 2. Swift's LLVM fork is the *older* clang API generation: DiagnosticOptions is still reference-counted, TextDiagnosticPrinter takes a DiagnosticOptions*, and CompilerInstance has no constructor taking an invocation. Upstream LLVM is the newer one: a plain value taken by reference, invocation in the constructor. The bundle can be built from either — this workflow builds the LLVM-only bundle from upstream and the Swift-carrying one from the fork — so both have to compile. The split is a template parameter rather than an `if constexpr` in place: outside a template the discarded branch is still type-checked, which would defeat the point. The marker for the generation is the base class (is_base_of RefCountedBase), not a member — Retain() is protected, so a member-based trait returns false for both and silently picks the wrong branch. --- App/NativeToolchain/NativeToolchainBridge.mm | 134 +++++++++++++++---- 1 file changed, 106 insertions(+), 28 deletions(-) diff --git a/App/NativeToolchain/NativeToolchainBridge.mm b/App/NativeToolchain/NativeToolchainBridge.mm index 4d3441e..d19971d 100644 --- a/App/NativeToolchain/NativeToolchainBridge.mm +++ b/App/NativeToolchain/NativeToolchainBridge.mm @@ -18,15 +18,34 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity __has_include() && \ __has_include() +// Apple's SDK predefines IBAction and IBOutlet as macros (`#define IBOutlet +// __attribute__((iboutlet))`). Clang's generated AttrList.inc expands +// `INHERITABLE_ATTR(IBAction)` into `ATTR(IBAction)` → `class IBAction##Attr;`, so +// the predefined macro is pasted into the token paste and the header does not +// compile: "pasting formed ')Attr', an invalid preprocessing token". Clang's own +// headers undefine them for this reason on some revisions and not others, so it is +// done here, before any clang header is included. +#ifdef IBAction +#undef IBAction +#endif +#ifdef IBOutlet +#undef IBOutlet +#endif + #include +#include #include #include #include #include #include #include +#include #include +#include +#include + #if defined(XFORGE_HAS_SWIFT_FRONTEND) && __has_include() #include #define XFORGE_SWIFT_FRONTEND_READY 1 @@ -96,6 +115,86 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity #endif } +namespace { + +/// Whether this clang's `DiagnosticOptions` is still reference-counted. +/// +/// Swift's LLVM fork (`swift/release/6.2`) is the older generation: options are +/// refcounted, `TextDiagnosticPrinter` takes a `DiagnosticOptions *`, and +/// `CompilerInstance` has no constructor taking an invocation. Upstream LLVM is the +/// newer one: a plain value taken by reference, and the invocation goes in through +/// the constructor. The bundle can be built from either, so both have to compile. +// The marker is the base class, not a member: `Retain()` lives in +// `RefCountedBase` and is protected, so a trait that calls it would fail access +// checking and report "false" for both generations. Deriving from +// `RefCountedBase` is what actually distinguishes them. +constexpr bool xf_clang_uses_refcounted_diagnostics() { + return std::is_base_of_v, + clang::DiagnosticOptions>; +} + +/// Run one cc1-style invocation through a `CompilerInstance`. +template +bool xf_clang_execute(std::shared_ptr invocation, + std::unique_ptr printer) { + if constexpr (RefCountedOptions) { + // Older generation: the instance starts empty and is handed its + // invocation afterwards. + clang::CompilerInstance compiler; + compiler.setInvocation(std::move(invocation)); + compiler.createDiagnostics(printer.release(), true); + if (!compiler.hasDiagnostics()) return false; + clang::EmitObjAction action; + return compiler.ExecuteAction(action); + } else { + clang::CompilerInstance compiler(std::move(invocation)); + compiler.createDiagnostics(printer.release(), true); + if (!compiler.hasDiagnostics()) return false; + clang::EmitObjAction action; + return compiler.ExecuteAction(action); + } +} + +template +int xf_clang_compile_impl(const std::vector &args, + llvm::raw_string_ostream &diagOS, + std::string &diagText, + char *diagnostics, + size_t diagnostics_capacity) { + auto diagIDs = llvm::IntrusiveRefCntPtr(new clang::DiagnosticIDs()); + auto invocation = std::make_shared(); + + if constexpr (RefCountedOptions) { + auto diagOpts = llvm::makeIntrusiveRefCnt(); + auto printer = std::make_unique(diagOS, diagOpts.get()); + clang::DiagnosticsEngine diags(diagIDs, diagOpts, printer.get(), false); + if (!clang::CompilerInvocation::CreateFromArgs(*invocation, args, diags)) { + diagOS.flush(); + xf_copy_diag(diagText, diagnostics, diagnostics_capacity); + return 65; + } + const bool ok = xf_clang_execute(invocation, std::move(printer)); + diagOS.flush(); + xf_copy_diag(diagText, diagnostics, diagnostics_capacity); + return ok ? 0 : 1; + } else { + clang::DiagnosticOptions diagOpts; + auto printer = std::make_unique(diagOS, diagOpts); + clang::DiagnosticsEngine diags(diagIDs, diagOpts, printer.get(), false); + if (!clang::CompilerInvocation::CreateFromArgs(*invocation, args, diags)) { + diagOS.flush(); + xf_copy_diag(diagText, diagnostics, diagnostics_capacity); + return 65; + } + const bool ok = xf_clang_execute(invocation, std::move(printer)); + diagOS.flush(); + xf_copy_diag(diagText, diagnostics, diagnostics_capacity); + return ok ? 0 : 1; + } +} + +} // namespace + extern "C" int xf_native_clang_compile(const char *source_path, const char *object_path, const char *sdk_path, @@ -111,14 +210,6 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity std::string diagText; llvm::raw_string_ostream diagOS(diagText); - // DiagnosticOptions is a plain value and is taken by reference by both the - // printer and the engine; it is no longer refcounted (LLVM 19+), so it must - // not be wrapped in an IntrusiveRefCntPtr. - clang::DiagnosticOptions diagOpts; - auto diagPrinter = std::make_unique(diagOS, diagOpts); - auto diagIDs = llvm::IntrusiveRefCntPtr(new clang::DiagnosticIDs()); - clang::DiagnosticsEngine diags(diagIDs, diagOpts, diagPrinter.get(), false); - std::vector owned = { "-triple", target_triple, "-emit-obj", @@ -144,27 +235,14 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity args.reserve(owned.size()); for (const auto &arg : owned) args.push_back(arg.c_str()); - auto invocation = std::make_shared(); - if (!clang::CompilerInvocation::CreateFromArgs(*invocation, args, diags)) { - diagOS.flush(); - xf_copy_diag(diagText, diagnostics, diagnostics_capacity); - return 65; + // The two clang generations differ in exactly two places, and both are wired + // to one template parameter so that only the matching half is ever + // instantiated: `if constexpr` outside a template still type-checks the + // branch it discards, so the split has to be a template to be meaningful. + if (xf_clang_uses_refcounted_diagnostics()) { + return xf_clang_compile_impl(args, diagOS, diagText, diagnostics, diagnostics_capacity); } - - // CompilerInstance takes its invocation through the constructor now; - // setInvocation() no longer exists. - clang::CompilerInstance compiler(invocation); - compiler.createDiagnostics(diagPrinter.release(), true); - if (!compiler.hasDiagnostics()) { - xf_copy_diag("native clang: failed to create diagnostics engine", diagnostics, diagnostics_capacity); - return 66; - } - - clang::EmitObjAction action; - const bool ok = compiler.ExecuteAction(action); - diagOS.flush(); - xf_copy_diag(diagText, diagnostics, diagnostics_capacity); - return ok ? 0 : 1; + return xf_clang_compile_impl(args, diagOS, diagText, diagnostics, diagnostics_capacity); } extern "C" int xf_native_lld_link(int argc, From 053702f6cd38ca30082621c34de68aa5629512a6 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 12:29:56 -0400 Subject: [PATCH 17/54] ci: make restored build trees newer than the sources MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Both caches were restored and ninja rebuilt anyway: a restored output carries its original timestamp, the checkout gives the sources 'now', and ninja trusts timestamps — so an hour of LLVM and another of Swift were repeated for nothing. The restored trees are touched after extraction, which is safe precisely because any change to a source ref changes the cache key. --- .github/workflows/native-toolchain.yml | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 3807c36..74c31b2 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -157,6 +157,22 @@ jobs: path: swift-build key: ${{ runner.os }}-${{ runner.arch }}-swift-${{ env.SWIFT_REF }}-${{ env.SWIFT_LLVM_REF }}-${{ steps.toolchain-versions.outputs.value }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '1' }} + # A restored build tree looks *older* than the sources that were just + # checked out, and ninja compares timestamps: it would rebuild everything the + # cache was meant to save. Touching the restored outputs puts them ahead of + # the sources again, which is sound because a change to any source ref + # changes the cache key and therefore misses the cache entirely. + - name: Make the restored builds newer than the sources + if: ${{ steps.llvm-cache.outputs.cache-hit == 'true' || steps.swift-cache.outputs.cache-hit == 'true' }} + shell: bash + run: | + set -euxo pipefail + for tree in llvm-host-build llvm-build cmark-host-build swift-build; do + if [ -d "$tree" ]; then + find "$tree" -exec touch -c {} + 2>/dev/null || true + fi + done + - name: Build host TableGen tools shell: bash run: | From 80eac5682607fd705516fe932aaa86f27a5a33ba Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 14:00:29 -0400 Subject: [PATCH 18/54] native: instantiate one clang generation, not both MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The previous attempt still failed, and the log says why in one line: the build was in 'xf_clang_compile_impl' — the *newer* API — against the fork's headers. Two mistakes, both mine: - the dispatch was 'if (xf_clang_uses_refcounted_diagnostics())', a *runtime* condition, so both specialisations were instantiated and the one that does not belong to this clang had to compile anyway. The whole entry point is now a template and the choice is its template argument, a constant expression, so exactly one instantiation exists; - the trait was is_base_of on RefCountedBase. The printer's constructor is the honest test — the older generation takes a DiagnosticOptions*, the newer one a reference, and a pointer never converts to a reference. The lesson generalises: 'if constexpr' outside a template discards nothing, so a generation split has to be a template parameter from the start. --- App/NativeToolchain/NativeToolchainBridge.mm | 116 ++++++++++--------- 1 file changed, 64 insertions(+), 52 deletions(-) diff --git a/App/NativeToolchain/NativeToolchainBridge.mm b/App/NativeToolchain/NativeToolchainBridge.mm index d19971d..5b572ef 100644 --- a/App/NativeToolchain/NativeToolchainBridge.mm +++ b/App/NativeToolchain/NativeToolchainBridge.mm @@ -124,13 +124,16 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity /// `CompilerInstance` has no constructor taking an invocation. Upstream LLVM is the /// newer one: a plain value taken by reference, and the invocation goes in through /// the constructor. The bundle can be built from either, so both have to compile. -// The marker is the base class, not a member: `Retain()` lives in -// `RefCountedBase` and is protected, so a trait that calls it would fail access -// checking and report "false" for both generations. Deriving from -// `RefCountedBase` is what actually distinguishes them. +// The marker is the constructor the *printer* takes, because that is the thing +// being branched on: the older generation takes `DiagnosticOptions *`, the newer +// one takes a reference, and a pointer never converts to a reference, so this is +// exactly the generation test. (A trait on a member would not work: `Retain()` +// lives in `RefCountedBase` and is protected, so probing it fails access checking +// and answers "new" for both generations.) constexpr bool xf_clang_uses_refcounted_diagnostics() { - return std::is_base_of_v, - clang::DiagnosticOptions>; + return std::is_constructible_v; } /// Run one cc1-style invocation through a `CompilerInstance`. @@ -155,16 +158,59 @@ bool xf_clang_execute(std::shared_ptr invocation, } } +/// The whole C entry point, as a template. +/// +/// It has to be a template, and it has to be instantiated exactly once, for the +/// `if constexpr` below to mean anything: a discarded branch is only exempt from +/// instantiation inside a templated entity, and a *runtime* condition — +/// `if (xf_clang_uses_refcounted_diagnostics())` — instantiates both halves and +/// then fails on whichever one does not belong to this clang. template -int xf_clang_compile_impl(const std::vector &args, - llvm::raw_string_ostream &diagOS, - std::string &diagText, - char *diagnostics, - size_t diagnostics_capacity) { +int xf_clang_compile_entry(const char *source_path, + const char *object_path, + const char *sdk_path, + const char *target_triple, + const char *language, + char *diagnostics, + size_t diagnostics_capacity) { + if (!source_path || !object_path || !sdk_path || !target_triple) { + xf_copy_diag("native clang: missing required argument", diagnostics, diagnostics_capacity); + return 64; + } + + std::string diagText; + llvm::raw_string_ostream diagOS(diagText); + + std::vector owned = { + "-triple", target_triple, + "-emit-obj", + "-o", object_path, + "-isysroot", sdk_path, + "-fblocks", + "-fobjc-arc" + }; + + const std::string lang = language ? language : "c"; + if (lang == "objective-c" || lang == "objc") { + owned.insert(owned.end(), {"-x", "objective-c"}); + } else if (lang == "objective-c++" || lang == "objc++") { + owned.insert(owned.end(), {"-x", "objective-c++"}); + } else if (lang == "c++" || lang == "cpp") { + owned.insert(owned.end(), {"-x", "c++"}); + } else { + owned.insert(owned.end(), {"-x", "c"}); + } + owned.push_back(source_path); + + std::vector args; + args.reserve(owned.size()); + for (const auto &arg : owned) args.push_back(arg.c_str()); + auto diagIDs = llvm::IntrusiveRefCntPtr(new clang::DiagnosticIDs()); auto invocation = std::make_shared(); if constexpr (RefCountedOptions) { + // Older generation: reference-counted options, a pointer-taking printer. auto diagOpts = llvm::makeIntrusiveRefCnt(); auto printer = std::make_unique(diagOS, diagOpts.get()); clang::DiagnosticsEngine diags(diagIDs, diagOpts, printer.get(), false); @@ -178,6 +224,7 @@ int xf_clang_compile_impl(const std::vector &args, xf_copy_diag(diagText, diagnostics, diagnostics_capacity); return ok ? 0 : 1; } else { + // Newer generation: a plain value taken by reference. clang::DiagnosticOptions diagOpts; auto printer = std::make_unique(diagOS, diagOpts); clang::DiagnosticsEngine diags(diagIDs, diagOpts, printer.get(), false); @@ -202,47 +249,12 @@ int xf_clang_compile_impl(const std::vector &args, const char *language, char *diagnostics, size_t diagnostics_capacity) { - if (!source_path || !object_path || !sdk_path || !target_triple) { - xf_copy_diag("native clang: missing required argument", diagnostics, diagnostics_capacity); - return 64; - } - - std::string diagText; - llvm::raw_string_ostream diagOS(diagText); - - std::vector owned = { - "-triple", target_triple, - "-emit-obj", - "-o", object_path, - "-isysroot", sdk_path, - "-fblocks", - "-fobjc-arc" - }; - - const std::string lang = language ? language : "c"; - if (lang == "objective-c" || lang == "objc") { - owned.insert(owned.end(), {"-x", "objective-c"}); - } else if (lang == "objective-c++" || lang == "objc++") { - owned.insert(owned.end(), {"-x", "objective-c++"}); - } else if (lang == "c++" || lang == "cpp") { - owned.insert(owned.end(), {"-x", "c++"}); - } else { - owned.insert(owned.end(), {"-x", "c"}); - } - owned.push_back(source_path); - - std::vector args; - args.reserve(owned.size()); - for (const auto &arg : owned) args.push_back(arg.c_str()); - - // The two clang generations differ in exactly two places, and both are wired - // to one template parameter so that only the matching half is ever - // instantiated: `if constexpr` outside a template still type-checks the - // branch it discards, so the split has to be a template to be meaningful. - if (xf_clang_uses_refcounted_diagnostics()) { - return xf_clang_compile_impl(args, diagOS, diagText, diagnostics, diagnostics_capacity); - } - return xf_clang_compile_impl(args, diagOS, diagText, diagnostics, diagnostics_capacity); + // A constant expression as the template argument: exactly one instantiation + // exists in a build, so the branch for the other generation is never + // type-checked. + return xf_clang_compile_entry( + source_path, object_path, sdk_path, target_triple, language, + diagnostics, diagnostics_capacity); } extern "C" int xf_native_lld_link(int argc, From 3797ab3d36db4962c765cd0c79a3030d23f8820b Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 14:17:50 -0400 Subject: [PATCH 19/54] ci: touch the restored trees after the configure, not before MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The early touch ran before CMake's configure, and configure rewrites generated headers (llvm/Config/…). A header newer than the outputs invalidates nearly every edge that includes it, so ninja rebuilt the world anyway. The touches now sit immediately before the two build steps, where nothing regenerates inputs in between. --- .github/workflows/native-toolchain.yml | 39 +++++++++++++++----------- 1 file changed, 23 insertions(+), 16 deletions(-) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 74c31b2..9f7ff8e 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -157,22 +157,6 @@ jobs: path: swift-build key: ${{ runner.os }}-${{ runner.arch }}-swift-${{ env.SWIFT_REF }}-${{ env.SWIFT_LLVM_REF }}-${{ steps.toolchain-versions.outputs.value }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '1' }} - # A restored build tree looks *older* than the sources that were just - # checked out, and ninja compares timestamps: it would rebuild everything the - # cache was meant to save. Touching the restored outputs puts them ahead of - # the sources again, which is sound because a change to any source ref - # changes the cache key and therefore misses the cache entirely. - - name: Make the restored builds newer than the sources - if: ${{ steps.llvm-cache.outputs.cache-hit == 'true' || steps.swift-cache.outputs.cache-hit == 'true' }} - shell: bash - run: | - set -euxo pipefail - for tree in llvm-host-build llvm-build cmark-host-build swift-build; do - if [ -d "$tree" ]; then - find "$tree" -exec touch -c {} + 2>/dev/null || true - fi - done - - name: Build host TableGen tools shell: bash run: | @@ -264,6 +248,22 @@ jobs: -DLLD_INCLUDE_TESTS=OFF \ -DLLD_BUILD_TOOLS=OFF + # A restored build tree looks *older* than the sources that were just checked + # out, and ninja compares timestamps — so it would rebuild everything the + # cache exists to save. Touching the outputs puts them ahead again. It has to + # happen *after* CMake's configure, not before: configure rewrites generated + # headers (llvm/Config/…), and a header newer than the outputs invalidates + # every edge that includes it, which is nearly all of them. Sound either way, + # because a change to any source ref changes the cache key. + - name: Make the restored LLVM build newer than its inputs + if: ${{ steps.llvm-cache.outputs.cache-hit == 'true' }} + shell: bash + run: | + set -euxo pipefail + for tree in llvm-host-build llvm-build cmark-host-build; do + [ -d "$tree" ] && find "$tree" -exec touch -c {} + 2>/dev/null || true + done + - name: Build Clang and Mach-O LLD libraries if: ${{ !inputs.configure_only }} run: | @@ -364,6 +364,13 @@ jobs: || echo "::warning::swiftFrontendTool archive not found where expected; listing swift-build/lib" ls -la swift-build/lib | head -40 || true + - name: Make the restored Swift build newer than its inputs + if: ${{ steps.swift-cache.outputs.cache-hit == 'true' }} + shell: bash + run: | + set -euxo pipefail + [ -d swift-build ] && find swift-build -exec touch -c {} + 2>/dev/null || true + - name: Build the Swift frontend libraries if: ${{ inputs.with_swift && !inputs.configure_only }} shell: bash From 268c175548f12962f82a96df87f1037b801a4ea2 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 14:29:04 -0400 Subject: [PATCH 20/54] ci: a switch that builds the app with the compiler linked in MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every run so far has built the app *without* a toolchain: the bundle is a separate workflow's artifact, so prepare-xcode.sh writes the disabled xcconfig and the bridge compiles to the 'not available' stub. Nothing in CI has ever made the app link LLVM, Clang, LLD or Swift. `with_toolchain: true` downloads the artifact, installs it where the xcconfig points, prints the manifest and the generated xcconfig, and builds on from there — which is also the only way to find out what linking the merged archive costs. --- .github/workflows/build-ipa.yml | 22 ++++++++++++++++++++++ 1 file changed, 22 insertions(+) diff --git a/.github/workflows/build-ipa.yml b/.github/workflows/build-ipa.yml index 26cbed2..ad24c16 100644 --- a/.github/workflows/build-ipa.yml +++ b/.github/workflows/build-ipa.yml @@ -23,6 +23,10 @@ on: description: Run simulator unit tests before archiving type: boolean default: false + with_toolchain: + description: Download the native toolchain artifact and link it into the app + type: boolean + default: false permissions: contents: write @@ -67,6 +71,24 @@ jobs: echo "value=$(xcodebuild -version | tr '\n' ' ')-$(clang --version | head -n 1)" } >> "$GITHUB_OUTPUT" + # The build that proves the compiler really is in the app: the bundle is + # installed where the generated xcconfig points, so the bridge compiles with + # XFORGE_HAS_LLVM / XFORGE_HAS_SWIFT_FRONTEND and the app *links* the merged + # archive — the step that no other workflow exercises. + - name: Install the native toolchain bundle + if: ${{ inputs.with_toolchain }} + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euo pipefail + gh run download --repo "$GITHUB_REPOSITORY" \ + --name XForgeNativeToolchain-arm64-ios \ + --dir .toolchain-artifact + bash NativeToolchain/install-bundle.sh \ + .toolchain-artifact/XForgeNativeToolchain-arm64-ios.tar.gz + cat Vendor/NativeToolchain/manifest.txt + cat Support/NativeToolchain.generated.xcconfig + - name: Prepare the native toolchain configuration run: | set -euo pipefail From 5d0c7292360a9f1994b8aa0d5d9a903269bc12f0 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 16:16:38 -0400 Subject: [PATCH 21/54] native: one clang generation, and the bridge written for it MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The template dispatch still failed, and the log explains why: the errors were in the *newer* API branch against the fork's headers even though the trait should have selected the other one. The reason is the rule I misread: a discarded `if constexpr` branch inside a template is exempt from instantiation only for constructs that depend on the template parameter. Mine did not — `DiagnosticOptions diagOpts;` is a concrete type — so both halves were checked and one of them could never compile. There is no way to hold two API generations in one function that does not depend on them. So there is one generation now, and it is the fork's (reference-counted `DiagnosticOptions`, pointer-taking printer, `setInvocation`), which is also the one Swift itself is built against. The workflow checks out `swiftlang/llvm-project` for *every* bundle instead of upstream for the LLVM-only one, which is what makes a single bridge possible — and it pins a revision that was previously an unpinned checkout of llvm-project main. This deletes the matching machinery rather than adding to it: the trait, the two templates and the dispatcher are gone, and the compile path is a straight line. --- .github/workflows/native-toolchain.yml | 30 ++-- App/NativeToolchain/NativeToolchainBridge.mm | 137 +++++-------------- 2 files changed, 55 insertions(+), 112 deletions(-) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 9f7ff8e..ff3a020 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -5,11 +5,16 @@ name: Native iOS Toolchain # # Two things about the Swift half are worth knowing before editing it: # -# * Swift's frontend is *not* built against upstream LLVM. Swift depends on the -# LLVM fork it is developed with (the `swiftasync` calling convention, the -# coroutine passes, and the C++ API surface it compiles against), so the Swift -# path checks out `swiftlang/llvm-project` at the matching `swift/release/X.Y` -# branch. The upstream checkout is kept for the LLVM-only path. +# * Every bundle is built from Swift's LLVM fork (`swiftlang/llvm-project` at the +# branch paired with the Swift version), never from upstream. Swift is developed +# against that fork (the `swiftasync` calling convention, the coroutine passes, +# the C++ API it compiles against), and — the part that decided it — upstream's +# clang is a different *API generation*: value-typed `DiagnosticOptions` and +# reference-taking constructors, against the fork's reference-counted options and +# pointer-taking ones. The app's C bridge is compiled against whichever bundle is +# installed, so it can only be written for one generation. Pinning the fork for +# both bundles is also what removes the old "unpinned checkout of llvm-project +# main" this workflow used to have. # * `-DSWIFT_INSTALL_COMPONENTS=` is load-bearing. Swift's CMake writes that as a # *cache* variable, so an empty value on the command line survives its default # and no component is ever installed — which means `swift_install_in_component` @@ -74,13 +79,18 @@ jobs: - name: Install build tools run: brew install cmake ninja - # The LLVM the compiler is built from: upstream for the LLVM-only bundle, - # Swift's own fork when the frontend is part of it. + # One LLVM for every bundle this workflow builds: Swift's fork, at the branch + # paired with the Swift version. Upstream's clang is a *different API + # generation* from the fork's (reference-counted options and pointer-taking + # printers versus value options and references), and the C bridge that the app + # compiles against the bundle can only be written for one of them — so the + # bundle must always be the same generation. Pinning it also removes the + # "unpinned checkout of llvm-project main" this workflow used to have. - name: Checkout LLVM uses: actions/checkout@v4 with: - repository: ${{ inputs.with_swift && 'swiftlang/llvm-project' || 'llvm/llvm-project' }} - ref: ${{ inputs.with_swift && env.SWIFT_LLVM_REF || '' }} + repository: swiftlang/llvm-project + ref: ${{ env.SWIFT_LLVM_REF }} path: llvm-project fetch-depth: 1 @@ -145,7 +155,7 @@ jobs: llvm-host-build llvm-build cmark-host-build - key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ steps.toolchain-versions.outputs.value }}-${{ inputs.with_swift && env.SWIFT_LLVM_REF || 'upstream' }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '1' }} + key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ steps.toolchain-versions.outputs.value }}-${{ env.SWIFT_LLVM_REF }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '1' }} # Swift's own build is the longer half and is cached separately, because it # is saved *after* the LLVM half has been built and Swift's has too. diff --git a/App/NativeToolchain/NativeToolchainBridge.mm b/App/NativeToolchain/NativeToolchainBridge.mm index 5b572ef..d4c93d9 100644 --- a/App/NativeToolchain/NativeToolchainBridge.mm +++ b/App/NativeToolchain/NativeToolchainBridge.mm @@ -44,7 +44,6 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity #include #include -#include #if defined(XFORGE_HAS_SWIFT_FRONTEND) && __has_include() #include @@ -115,64 +114,13 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity #endif } -namespace { - -/// Whether this clang's `DiagnosticOptions` is still reference-counted. -/// -/// Swift's LLVM fork (`swift/release/6.2`) is the older generation: options are -/// refcounted, `TextDiagnosticPrinter` takes a `DiagnosticOptions *`, and -/// `CompilerInstance` has no constructor taking an invocation. Upstream LLVM is the -/// newer one: a plain value taken by reference, and the invocation goes in through -/// the constructor. The bundle can be built from either, so both have to compile. -// The marker is the constructor the *printer* takes, because that is the thing -// being branched on: the older generation takes `DiagnosticOptions *`, the newer -// one takes a reference, and a pointer never converts to a reference, so this is -// exactly the generation test. (A trait on a member would not work: `Retain()` -// lives in `RefCountedBase` and is protected, so probing it fails access checking -// and answers "new" for both generations.) -constexpr bool xf_clang_uses_refcounted_diagnostics() { - return std::is_constructible_v; -} - -/// Run one cc1-style invocation through a `CompilerInstance`. -template -bool xf_clang_execute(std::shared_ptr invocation, - std::unique_ptr printer) { - if constexpr (RefCountedOptions) { - // Older generation: the instance starts empty and is handed its - // invocation afterwards. - clang::CompilerInstance compiler; - compiler.setInvocation(std::move(invocation)); - compiler.createDiagnostics(printer.release(), true); - if (!compiler.hasDiagnostics()) return false; - clang::EmitObjAction action; - return compiler.ExecuteAction(action); - } else { - clang::CompilerInstance compiler(std::move(invocation)); - compiler.createDiagnostics(printer.release(), true); - if (!compiler.hasDiagnostics()) return false; - clang::EmitObjAction action; - return compiler.ExecuteAction(action); - } -} - -/// The whole C entry point, as a template. -/// -/// It has to be a template, and it has to be instantiated exactly once, for the -/// `if constexpr` below to mean anything: a discarded branch is only exempt from -/// instantiation inside a templated entity, and a *runtime* condition — -/// `if (xf_clang_uses_refcounted_diagnostics())` — instantiates both halves and -/// then fails on whichever one does not belong to this clang. -template -int xf_clang_compile_entry(const char *source_path, - const char *object_path, - const char *sdk_path, - const char *target_triple, - const char *language, - char *diagnostics, - size_t diagnostics_capacity) { +extern "C" int xf_native_clang_compile(const char *source_path, + const char *object_path, + const char *sdk_path, + const char *target_triple, + const char *language, + char *diagnostics, + size_t diagnostics_capacity) { if (!source_path || !object_path || !sdk_path || !target_triple) { xf_copy_diag("native clang: missing required argument", diagnostics, diagnostics_capacity); return 64; @@ -181,6 +129,20 @@ int xf_clang_compile_entry(const char *source_path, std::string diagText; llvm::raw_string_ostream diagOS(diagText); + // One clang generation, and it is the older one on purpose. Every bundle + // XForge builds comes from Swift's LLVM fork (`swiftlang/llvm-project`), which + // is where this API lives: `DiagnosticOptions` is reference-counted, + // `TextDiagnosticPrinter` takes a `DiagnosticOptions *`, and a + // `CompilerInstance` is handed its invocation rather than constructed with it. + // Supporting upstream's newer API as well was tried and abandoned: the two + // differ in three places, and a shared function cannot hold both, because a + // discarded `if constexpr` branch is still type-checked when its code does not + // depend on the template parameter. + auto diagOpts = llvm::makeIntrusiveRefCnt(); + auto diagPrinter = std::make_unique(diagOS, diagOpts.get()); + auto diagIDs = llvm::IntrusiveRefCntPtr(new clang::DiagnosticIDs()); + clang::DiagnosticsEngine diags(diagIDs, diagOpts, diagPrinter.get(), false); + std::vector owned = { "-triple", target_triple, "-emit-obj", @@ -206,55 +168,26 @@ int xf_clang_compile_entry(const char *source_path, args.reserve(owned.size()); for (const auto &arg : owned) args.push_back(arg.c_str()); - auto diagIDs = llvm::IntrusiveRefCntPtr(new clang::DiagnosticIDs()); auto invocation = std::make_shared(); - - if constexpr (RefCountedOptions) { - // Older generation: reference-counted options, a pointer-taking printer. - auto diagOpts = llvm::makeIntrusiveRefCnt(); - auto printer = std::make_unique(diagOS, diagOpts.get()); - clang::DiagnosticsEngine diags(diagIDs, diagOpts, printer.get(), false); - if (!clang::CompilerInvocation::CreateFromArgs(*invocation, args, diags)) { - diagOS.flush(); - xf_copy_diag(diagText, diagnostics, diagnostics_capacity); - return 65; - } - const bool ok = xf_clang_execute(invocation, std::move(printer)); + if (!clang::CompilerInvocation::CreateFromArgs(*invocation, args, diags)) { diagOS.flush(); xf_copy_diag(diagText, diagnostics, diagnostics_capacity); - return ok ? 0 : 1; - } else { - // Newer generation: a plain value taken by reference. - clang::DiagnosticOptions diagOpts; - auto printer = std::make_unique(diagOS, diagOpts); - clang::DiagnosticsEngine diags(diagIDs, diagOpts, printer.get(), false); - if (!clang::CompilerInvocation::CreateFromArgs(*invocation, args, diags)) { - diagOS.flush(); - xf_copy_diag(diagText, diagnostics, diagnostics_capacity); - return 65; - } - const bool ok = xf_clang_execute(invocation, std::move(printer)); - diagOS.flush(); - xf_copy_diag(diagText, diagnostics, diagnostics_capacity); - return ok ? 0 : 1; + return 65; } -} -} // namespace + clang::CompilerInstance compiler; + compiler.setInvocation(invocation); + compiler.createDiagnostics(diagPrinter.release(), true); + if (!compiler.hasDiagnostics()) { + xf_copy_diag("native clang: failed to create diagnostics engine", diagnostics, diagnostics_capacity); + return 66; + } -extern "C" int xf_native_clang_compile(const char *source_path, - const char *object_path, - const char *sdk_path, - const char *target_triple, - const char *language, - char *diagnostics, - size_t diagnostics_capacity) { - // A constant expression as the template argument: exactly one instantiation - // exists in a build, so the branch for the other generation is never - // type-checked. - return xf_clang_compile_entry( - source_path, object_path, sdk_path, target_triple, language, - diagnostics, diagnostics_capacity); + clang::EmitObjAction action; + const bool ok = compiler.ExecuteAction(action); + diagOS.flush(); + xf_copy_diag(diagText, diagnostics, diagnostics_capacity); + return ok ? 0 : 1; } extern "C" int xf_native_lld_link(int argc, From 8cf187be22e20b54792f737e65ec073f0a8ee8b3 Mon Sep 17 00:00:00 2001 From: XForge Date: Thu, 24 Sep 2026 14:41:19 -0400 Subject: [PATCH 22/54] Stage generated headers into their own root MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The staging step copies the build tree's Swift headers on top of the source tree's, and those two disagree about one path: the build generates a *file* at `swift/bridging` (the C++ interop header, included as ) while `swift/include/swift/` has a *directory* of the same name, holding ASTGen.h and MacroEvaluation.h. On a case-insensitive filesystem that is a single path, so the copy dies with cp: ...: Is a directory and stages nothing. It has not bitten yet only because step 20 is skipped while the Swift frontend check above it fails; the moment that step passes, this one fails instead. Same fix as the one the unified toolchain workflow needed. Sources go to `dist/include`, the build tree to `dist/include-generated`, and consumers search the generated root first — the order a normal Swift/LLVM cross-build uses. `cp -R -L` is applied for a second reason: the build tree mirrors headers as symlinks into this checkout, and a bundle shipping those links is dead on any other machine. Guarded with a required-roots check and a symlink check so a partial or non-self-contained bundle cannot be published. --- .github/workflows/native-toolchain.yml | 51 +++++++++++++++++++------- Docs/NATIVE-TOOLCHAIN.md | 19 +++++++--- NativeToolchain/install-bundle.sh | 2 +- NativeToolchain/prepare-xcode.sh | 7 +++- 4 files changed, 58 insertions(+), 21 deletions(-) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index ff3a020..1b4745e 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -481,23 +481,47 @@ jobs: shell: bash run: | set -euxo pipefail - mkdir -p dist/include dist/lib - cp -R llvm-project/llvm/include/llvm dist/include/ - cp -R llvm-build/include/llvm dist/include/ - cp -R llvm-project/clang/include/clang dist/include/ + # Two header roots, searched in this order, and they must not be merged: + # the build tree generates a *file* at `swift/bridging` (the C++ interop + # header, included as ) while the sources have a + # *directory* of the same name (`include/swift/Bridging/`, holding + # ASTGen.h and MacroEvaluation.h). On a case-insensitive filesystem that + # is a single path, so copying the build tree over the source tree fails + # with "cp: .../swift/bridging: Is a directory" and stages nothing. + # + # `-L` is load-bearing too: the build tree mirrors headers as symlinks + # into this checkout, and a bundle that ships those links is dead + # anywhere but this runner. + mkdir -p dist/include-generated dist/include dist/lib + stage_headers() { cp -R -L "$1" "$2"; } + + stage_headers llvm-project/llvm/include/llvm dist/include/ + stage_headers llvm-project/clang/include/clang dist/include/ + stage_headers llvm-project/lld/include/lld dist/include/ + + stage_headers llvm-build/include/llvm dist/include-generated/ if [ -d llvm-build/tools/clang/include/clang ]; then - cp -R llvm-build/tools/clang/include/clang dist/include/ + stage_headers llvm-build/tools/clang/include/clang dist/include-generated/ fi - cp -R llvm-project/lld/include/lld dist/include/ if [ "${{ inputs.with_swift }}" = "true" ]; then - cp -R swift/include/swift dist/include/ - if [ -d swift-build/include/swift ]; then - cp -R swift-build/include/swift dist/include/ - fi - if [ -d swift-build/tools/swift/include/swift ]; then - cp -R swift-build/tools/swift/include/swift dist/include/ - fi + stage_headers swift/include/swift dist/include/ + for swift_headers in swift-build/include/swift swift-build/tools/swift/include/swift; do + if [ -d "$swift_headers" ]; then + stage_headers "$swift_headers" dist/include-generated/ + fi + done + fi + + for required in dist/include/llvm dist/include/clang dist/include/lld; do + [ -d "$required" ] || { echo "::error::missing staged header root $required"; exit 1; } + done + + # A symlink here would point into the runner's checkout. + if [ -n "$(find dist/include dist/include-generated -type l -print -quit)" ]; then + echo "::error::staged headers still contain symlinks:" + find dist/include dist/include-generated -type l -print + exit 1 fi # Flatten the static dependency graph — LLVM's, Clang's, LLD's and (when @@ -560,6 +584,7 @@ jobs: swift_ref=${{ inputs.with_swift && env.SWIFT_REF || 'none' }} built_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) archive=lib/libXForgeNativeToolchain.a + header_roots=include-generated include EOF tar -C dist -czf XForgeNativeToolchain-arm64-ios.tar.gz . diff --git a/Docs/NATIVE-TOOLCHAIN.md b/Docs/NATIVE-TOOLCHAIN.md index 492cd8b..634c03f 100644 --- a/Docs/NATIVE-TOOLCHAIN.md +++ b/Docs/NATIVE-TOOLCHAIN.md @@ -74,7 +74,9 @@ and used on its own while the considerably larger Swift compiler port is brought -isysroot -DXFORGE_HAS_LLVM=1 -c App/NativeToolchain/NativeToolchainBridge.mm`. This is the check that keeps a header change in LLVM from silently breaking the bridge in a device build nobody can run yet. -5. Stage `include/` from `llvm-project` and the build directory, then **flatten every +5. Stage the headers into two roots — sources into `include/`, the build tree into + `include-generated/`, because they disagree about `swift/bridging` — then + **flatten every static archive in the build into one** `dist/lib/libXForgeNativeToolchain.a` with `libtool -static`. Flattening is deliberate: the app then links one archive instead of depending on LLVM's internal archive ordering, which would otherwise @@ -99,18 +101,25 @@ make gen ``` `install-bundle.sh` unpacks the archive into `Vendor/NativeToolchain` and refuses an -archive that is missing `manifest.txt`, `include/` or `lib/` — a partial bundle would -otherwise surface as an inscrutable compile error much later. It then runs -`prepare-xcode.sh`, which writes `Support/NativeToolchain.generated.xcconfig`: +archive that is missing `manifest.txt`, `include/`, `include-generated/` or `lib/` — a +partial bundle would otherwise surface as an inscrutable compile error much later. It +then runs `prepare-xcode.sh`, which writes `Support/NativeToolchain.generated.xcconfig`: ``` XFORGE_NATIVE_TOOLCHAIN_AVAILABLE = 1 -XFORGE_NATIVE_HEADER_SEARCH_PATHS = $(SRCROOT)/Vendor/NativeToolchain/include +XFORGE_NATIVE_HEADER_SEARCH_PATHS = $(SRCROOT)/Vendor/NativeToolchain/include-generated $(SRCROOT)/Vendor/NativeToolchain/include XFORGE_NATIVE_LIBRARY_SEARCH_PATHS = $(SRCROOT)/Vendor/NativeToolchain/lib XFORGE_NATIVE_CFLAGS = -DXFORGE_HAS_LLVM=1 XFORGE_NATIVE_LDFLAGS = -lc++ -lz -liconv -lsqlite3 -framework Foundation ``` +The two header roots are searched in that order and are deliberately not merged. The +build tree *generates a file* at `swift/bridging` (the C++ interop header, included as +``) while Swift's sources have a *directory* of the same name +(`include/swift/Bridging/`). On a case-insensitive filesystem that is one path, so +staging both into a single root fails — `cp: .../swift/bridging: Is a directory` — and +the ordered pair is what a normal Swift/LLVM cross-build uses as well. + `project.yml` consumes those variables and nothing else, so the app target never names a path inside the bundle directly. diff --git a/NativeToolchain/install-bundle.sh b/NativeToolchain/install-bundle.sh index 3f32fb1..f05e2dc 100644 --- a/NativeToolchain/install-bundle.sh +++ b/NativeToolchain/install-bundle.sh @@ -16,7 +16,7 @@ rm -rf "$DEST" mkdir -p "$DEST" tar -xzf "$ARCHIVE" -C "$DEST" -for required in manifest.txt include lib; do +for required in manifest.txt include include-generated lib; do [ -e "$DEST/$required" ] || { echo "error: native toolchain bundle is missing $required" >&2 rm -rf "$DEST" diff --git a/NativeToolchain/prepare-xcode.sh b/NativeToolchain/prepare-xcode.sh index d3470d3..dc296a6 100644 --- a/NativeToolchain/prepare-xcode.sh +++ b/NativeToolchain/prepare-xcode.sh @@ -17,7 +17,7 @@ MANIFEST="$BUNDLE/manifest.txt" mkdir -p "$(dirname "$OUT")" -if [ ! -f "$MANIFEST" ] || [ ! -d "$BUNDLE/include" ] || [ ! -d "$BUNDLE/lib" ]; then +if [ ! -f "$MANIFEST" ] || [ ! -d "$BUNDLE/include" ] || [ ! -d "$BUNDLE/include-generated" ] || [ ! -d "$BUNDLE/lib" ]; then cat >"$OUT" <> "$GITHUB_OUTPUT" + + # The same keys the toolchain workflow writes, so this reuses what it built. + - name: Restore the LLVM builds + id: llvm-cache + uses: actions/cache/restore@v4 + with: + path: | + llvm-host-build + llvm-build + cmark-host-build + key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ steps.toolchain-versions.outputs.value }}-${{ env.SWIFT_LLVM_REF }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '2' }} + + - name: Restore the Swift build + id: swift-cache + uses: actions/cache/restore@v4 + with: + path: swift-build + key: ${{ runner.os }}-${{ runner.arch }}-swift-${{ env.SWIFT_REF }}-${{ env.SWIFT_LLVM_REF }}-${{ steps.toolchain-versions.outputs.value }}-${{ env.IOS_DEPLOYMENT_TARGET }}-1 + + - name: Report what was available + run: | + set -euxo pipefail + echo "llvm cache hit: ${{ steps.llvm-cache.outputs.cache-hit }}" + echo "swift cache hit: ${{ steps.swift-cache.outputs.cache-hit }}" + ls llvm-build/include 2>/dev/null | head -5 || echo "no llvm-build/include" + ls llvm-build/tools/clang/include/clang/Basic/DiagnosticGroups.inc 2>/dev/null \ + || echo "no generated clang headers (expected on a cold cache)" + ls swift-build/include 2>/dev/null | head -5 || echo "no swift-build/include" + + # Configure both trees so the generated headers match the sources, then make + # everything newer than the checkouts — the same reason the toolchain workflow + # does it: restored outputs look older than a fresh checkout. + - name: Configure LLVM for iPhoneOS arm64 + shell: bash + run: | + set -euxo pipefail + SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + cmake -S llvm-project/llvm -B llvm-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_SYSTEM_NAME=iOS \ + -DCMAKE_MACOSX_BUNDLE=OFF \ + -DCMAKE_OSX_SYSROOT="$SDK" \ + -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ + -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_HOST_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_DEFAULT_TARGET_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_HOST_BUILD_DIR="$PWD/llvm-host-build" \ + -DLLVM_ENABLE_PROJECTS="clang;lld" \ + -DLLVM_TARGETS_TO_BUILD=AArch64 \ + -DLLVM_INCLUDE_TESTS=OFF \ + -DLLVM_INCLUDE_EXAMPLES=OFF \ + -DLLVM_INCLUDE_BENCHMARKS=OFF \ + -DLLVM_INCLUDE_TOOLS=ON \ + -DLLVM_BUILD_TOOLS=OFF \ + -DLLVM_ENABLE_TERMINFO=OFF \ + -DLLVM_ENABLE_ZLIB=OFF \ + -DLLVM_ENABLE_ZSTD=OFF \ + -DLLVM_ENABLE_LIBXML2=OFF \ + -DLLVM_ENABLE_CURL=OFF \ + -DLLVM_ENABLE_RTTI=ON \ + -DCLANG_INCLUDE_TESTS=OFF \ + -DCLANG_BUILD_TOOLS=ON + + - name: Configure the Swift frontend for iPhoneOS + if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} + shell: bash + run: | + set -euxo pipefail + SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + cmake -S swift -B swift-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMAKE_SYSTEM_NAME=Darwin \ + -DCMAKE_OSX_SYSROOT="$SDK" \ + -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ + -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DCMAKE_Swift_COMPILER="$(xcrun -f swiftc)" \ + -DCMAKE_Swift_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DLLVM_DIR="$PWD/llvm-build/lib/cmake/llvm" \ + -DClang_DIR="$PWD/llvm-build/lib/cmake/clang" \ + -DLLVM_TABLEGEN="$PWD/llvm-host-build/bin/llvm-tblgen" \ + -DCLANG_TABLEGEN="$PWD/llvm-host-build/bin/clang-tblgen" \ + -DSWIFT_NATIVE_LLVM_TOOLS_PATH="$PWD/llvm-host-build/bin" \ + -DSWIFT_NATIVE_CLANG_TOOLS_PATH="$PWD/llvm-host-build/bin" \ + -DSWIFT_NATIVE_SWIFT_TOOLS_PATH="$(dirname "$(xcrun -f swiftc)")" \ + -DSWIFT_PATH_TO_CMARK_BUILD="$PWD/cmark-host-build" \ + -DSWIFT_PATH_TO_CMARK_SOURCE="$PWD/swift-cmark" \ + -DSWIFT_HOST_VARIANT=iphoneos \ + -DSWIFT_HOST_VARIANT_SDK=IOS \ + -DSWIFT_HOST_VARIANT_ARCH=arm64 \ + -DSWIFT_DARWIN_DEPLOYMENT_VERSION_IOS="$IOS_DEPLOYMENT_TARGET" \ + -DSWIFT_INCLUDE_TOOLS=ON \ + -DSWIFT_BUILD_DYNAMIC_STDLIB=OFF \ + -DSWIFT_BUILD_STATIC_STDLIB=OFF \ + -DSWIFT_BUILD_SOURCEKIT=OFF \ + -DSWIFT_BUILD_SWIFT_SYNTAX=OFF \ + -DSWIFT_INCLUDE_TESTS=OFF \ + -DSWIFT_INSTALL_COMPONENTS= + + - name: Make the trees newer than the sources + shell: bash + run: | + set -euxo pipefail + for tree in llvm-host-build llvm-build cmark-host-build swift-build; do + [ -d "$tree" ] && find "$tree" -exec touch -c {} + 2>/dev/null || true + done + + # The check itself, identical to the toolchain workflow's, so the two cannot + # drift: the bridge compiled for iOS with the headers of the exact LLVM and + # Swift the bundle will carry. + - name: Compile-check XForge bridge against LLVM headers + shell: bash + run: | + set -euxo pipefail + SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + EXTRA_DEFINES=() + EXTRA_INCLUDES=() + if [ "${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }}" = "true" ]; then + EXTRA_DEFINES+=(-DXFORGE_HAS_SWIFT_FRONTEND=1) + EXTRA_INCLUDES+=(-I swift/include) + [ -d swift-build/include ] && EXTRA_INCLUDES+=(-I swift-build/include) + [ -d swift-build/tools/swift/include ] && EXTRA_INCLUDES+=(-I swift-build/tools/swift/include) + fi + xcrun --sdk iphoneos clang++ \ + -std=c++20 \ + -arch arm64 \ + -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET \ + -isysroot "$SDK" \ + -DXFORGE_HAS_LLVM=1 \ + "${EXTRA_DEFINES[@]}" \ + -I llvm-project/llvm/include \ + -I llvm-build/include \ + -I llvm-project/clang/include \ + -I llvm-build/tools/clang/include \ + -I llvm-project/lld/include \ + "${EXTRA_INCLUDES[@]}" \ + -I App/NativeToolchain \ + -c App/NativeToolchain/NativeToolchainBridge.mm \ + -o /tmp/NativeToolchainBridge.o + file /tmp/NativeToolchainBridge.o From f3f6e5080a536a1b34d0aaa8aff804c2164e67ac Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 17:32:20 -0400 Subject: [PATCH 25/54] native: name the PCH container operations, as the shipped signature does R0GUEEE hit the same three signatures from the other branch (4fd80cc) and wrote the constructor argument out explicitly rather than relying on the default. Taking that form, plus the __has_include guard on PCHContainerOperations.h it needs. Their commit and 5d0c729 are the same fix reached independently; the difference is that this branch also pins every bundle to that LLVM fork, without which the LLVM-only bundle is a different API generation from the bridge that compiles against it. --- App/NativeToolchain/NativeToolchainBridge.mm | 12 +++++++++++- 1 file changed, 11 insertions(+), 1 deletion(-) diff --git a/App/NativeToolchain/NativeToolchainBridge.mm b/App/NativeToolchain/NativeToolchainBridge.mm index d4c93d9..d9d5488 100644 --- a/App/NativeToolchain/NativeToolchainBridge.mm +++ b/App/NativeToolchain/NativeToolchainBridge.mm @@ -16,6 +16,7 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity __has_include() && \ __has_include() && \ __has_include() && \ + __has_include() && \ __has_include() // Apple's SDK predefines IBAction and IBOutlet as macros (`#define IBOutlet @@ -38,6 +39,7 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity #include #include #include +#include #include #include #include @@ -175,7 +177,15 @@ static void xf_copy_diag(const std::string &value, char *buffer, size_t capacity return 65; } - clang::CompilerInstance compiler; + // The three signatures that differ between clang revisions, as this one + // declares them (swiftlang/llvm-project swift/release/6.2): + // DiagnosticsEngine(IntrusiveRefCntPtr, + // IntrusiveRefCntPtr, DiagnosticConsumer *, + // bool) ← options are refcounted here + // TextDiagnosticPrinter(raw_ostream &, DiagnosticOptions *) ← pointer to them + // CompilerInstance(shared_ptr, ModuleCache *) + // and the invocation goes in through setInvocation(). + clang::CompilerInstance compiler(std::make_shared()); compiler.setInvocation(invocation); compiler.createDiagnostics(diagPrinter.release(), true); if (!compiler.hasDiagnostics()) { From 31e5b939ae4087ff764d094102034b7a71765e20 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 17:36:25 -0400 Subject: [PATCH 26/54] ci: give the fast loop the sources its Swift configure needs MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit It reached the Swift configure in two minutes and failed there — my own workflow's fault, not the bridge's: Swift's configure executes a native cmark and resolves its sibling sources by path, and this workflow checked out neither. Both are mirrored from the toolchain workflow now, and the configure args match it argument for argument, so a failure here means an API problem rather than a missing argument. --- .github/workflows/bridge-api-check.yml | 43 ++++++++++++++++++++++++++ 1 file changed, 43 insertions(+) diff --git a/.github/workflows/bridge-api-check.yml b/.github/workflows/bridge-api-check.yml index c872b94..5c40908 100644 --- a/.github/workflows/bridge-api-check.yml +++ b/.github/workflows/bridge-api-check.yml @@ -71,6 +71,33 @@ jobs: path: swift fetch-depth: 1 + - name: Checkout cmark + if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} + uses: actions/checkout@v4 + with: + repository: swiftlang/swift-cmark + ref: release/6.2 + path: swift-cmark + fetch-depth: 1 + + - name: Checkout swift-syntax + if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} + uses: actions/checkout@v4 + with: + repository: swiftlang/swift-syntax + ref: release/6.2 + path: swift-syntax + fetch-depth: 1 + + - name: Checkout swift-experimental-string-processing + if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} + uses: actions/checkout@v4 + with: + repository: swiftlang/swift-experimental-string-processing + ref: release/6.2 + path: swift-experimental-string-processing + fetch-depth: 1 + - name: Identify the iOS compiler toolchain id: toolchain-versions run: | @@ -142,6 +169,20 @@ jobs: -DCLANG_INCLUDE_TESTS=OFF \ -DCLANG_BUILD_TOOLS=ON + - name: Build native cmark + if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} + shell: bash + run: | + set -euxo pipefail + if [ -x cmark-host-build/src/cmark-gfm ]; then + echo "native cmark already cached" + exit 0 + fi + cmake -S swift-cmark -B cmark-host-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release -DCMARK_TESTS=OFF + cmake --build cmark-host-build --target cmark-gfm + ln -sf "$PWD/cmark-host-build/src/cmark-gfm" cmark-host-build/src/cmark + - name: Configure the Swift frontend for iPhoneOS if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} shell: bash @@ -167,6 +208,8 @@ jobs: -DSWIFT_NATIVE_SWIFT_TOOLS_PATH="$(dirname "$(xcrun -f swiftc)")" \ -DSWIFT_PATH_TO_CMARK_BUILD="$PWD/cmark-host-build" \ -DSWIFT_PATH_TO_CMARK_SOURCE="$PWD/swift-cmark" \ + -DSWIFT_PATH_TO_STRING_PROCESSING_SOURCE="$PWD/swift-experimental-string-processing" \ + -DSWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE="$PWD/swift-syntax" \ -DSWIFT_HOST_VARIANT=iphoneos \ -DSWIFT_HOST_VARIANT_SDK=IOS \ -DSWIFT_HOST_VARIANT_ARCH=arm64 \ From 63606e00ac511fe15b50ceb022db83d64daff585 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 18:07:01 -0400 Subject: [PATCH 27/54] toolchain: cmark for iOS, and a fast loop that links MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The bridge compiled (step 23 passed for the first time) and staging failed at the link smoke with: "_cmark_iter_free", referenced from: parseDocumentImpl(swift::markup::MarkupContext&, …) "_cmark_node_get_type", … ld: symbol(s) not found for architecture arm64 Swift's Markup — the doc-comment parser — calls into cmark, and this workflow only ever built cmark as a *host* tool for Swift's configure step. Nothing built it for iPhoneOS, so nothing provided those symbols; the same is true of cmark-gfm-extensions, which is a separate archive. Both are built for the device now and merged into the bundle. The second half of the commit is the reason the first half was found late: the fast loop only compiled the bridge, so anything wrong with staging or linking was discovered by the 90-minute toolchain run — which is exactly what happened, twice. It now stages and links too, with the same commands, and prints the undefined symbols on failure (the raw log buries them under forty lines of libtool notes). --- .github/workflows/bridge-api-check.yml | 100 ++++++++++++++++++++++++- .github/workflows/native-toolchain.yml | 32 +++++++- 2 files changed, 130 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bridge-api-check.yml b/.github/workflows/bridge-api-check.yml index 5c40908..f09d1b6 100644 --- a/.github/workflows/bridge-api-check.yml +++ b/.github/workflows/bridge-api-check.yml @@ -114,6 +114,7 @@ jobs: llvm-host-build llvm-build cmark-host-build + cmark-ios-build key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ steps.toolchain-versions.outputs.value }}-${{ env.SWIFT_LLVM_REF }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '2' }} - name: Restore the Swift build @@ -183,6 +184,31 @@ jobs: cmake --build cmark-host-build --target cmark-gfm ln -sf "$PWD/cmark-host-build/src/cmark-gfm" cmark-host-build/src/cmark + - name: Build cmark for iPhoneOS + if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} + shell: bash + run: | + set -euxo pipefail + if [ -f cmark-ios-build/src/libcmark-gfm.a ]; then + echo "iOS cmark already cached" + exit 0 + fi + SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + cmake -S swift-cmark -B cmark-ios-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMARK_TESTS=OFF \ + -DCMARK_SHARED=OFF \ + -DCMAKE_SYSTEM_NAME=iOS \ + -DCMAKE_OSX_SYSROOT="$SDK" \ + -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ + -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET + # cmark-gfm-extensions is a separate archive and swift's Markup references + # symbols from both. + ninja -C cmark-ios-build cmark-gfm cmark-gfm-extensions + find cmark-ios-build -name 'libcmark-gfm*.a' + - name: Configure the Swift frontend for iPhoneOS if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} shell: bash @@ -226,7 +252,7 @@ jobs: shell: bash run: | set -euxo pipefail - for tree in llvm-host-build llvm-build cmark-host-build swift-build; do + for tree in llvm-host-build llvm-build cmark-host-build cmark-ios-build swift-build; do [ -d "$tree" ] && find "$tree" -exec touch -c {} + 2>/dev/null || true done @@ -263,3 +289,75 @@ jobs: -c App/NativeToolchain/NativeToolchainBridge.mm \ -o /tmp/NativeToolchainBridge.o file /tmp/NativeToolchainBridge.o + + # The staging and link half, deliberately the same commands as the toolchain + # workflow's: header roots (sources and generated kept apart — `swift/bridging` + # is a generated file and `swift/Bridging/` a source directory), one merged + # archive, and a link into an app-shaped target. Without this the fast loop + # would answer the compile question and leave the link question to a 90-minute + # run, which is how the cmark symbols went unnoticed. + - name: Stage the toolchain + shell: bash + run: | + set -euxo pipefail + mkdir -p dist/include dist/include-generated dist/lib + stage_headers() { cp -R -L "$1" "$2"; } + + stage_headers llvm-project/llvm/include/llvm dist/include/ + stage_headers llvm-project/clang/include/clang dist/include/ + stage_headers llvm-project/lld/include/lld dist/include/ + + [ -d llvm-build/include/llvm ] && stage_headers llvm-build/include/llvm dist/include-generated/ + [ -d llvm-build/tools/clang/include/clang ] && stage_headers llvm-build/tools/clang/include/clang dist/include-generated/ + + if [ "${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }}" = "true" ]; then + stage_headers swift/include/swift dist/include/ + for swift_headers in swift-build/include/swift swift-build/tools/swift/include/swift; do + [ -d "$swift_headers" ] && stage_headers "$swift_headers" dist/include-generated/ + done + fi + + find llvm-build/lib -maxdepth 1 -type f -name '*.a' -print | sort > /tmp/archives.txt + if [ -d cmark-ios-build ]; then + find cmark-ios-build -type f -name 'libcmark-gfm*.a' -print | sort >> /tmp/archives.txt + fi + if [ "${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }}" = "true" ]; then + find swift-build/lib -maxdepth 3 -type f -name '*.a' -print | sort >> /tmp/archives.txt + fi + test -s /tmp/archives.txt + ARCHIVES=() + while IFS= read -r archive; do ARCHIVES+=("$archive"); done < /tmp/archives.txt + echo "merging ${#ARCHIVES[@]} archives" + /usr/bin/libtool -static -o dist/lib/libXForgeNativeToolchain.a "${ARCHIVES[@]}" + ls -la dist/lib/libXForgeNativeToolchain.a + + - name: Link an app-shaped target + shell: bash + run: | + set -euxo pipefail + SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + cat > /tmp/native_link_smoke.mm <<'EOF' + #include "NativeToolchainBridge.h" + int main(void) { + return xf_native_toolchain_available() ? 0 : 1; + } + EOF + xcrun --sdk iphoneos clang++ -std=c++20 -arch arm64 \ + -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET -isysroot "$SDK" \ + -I App/NativeToolchain -c /tmp/native_link_smoke.mm -o /tmp/native_link_smoke.o + set +e + xcrun --sdk iphoneos clang++ -arch arm64 \ + -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET -isysroot "$SDK" \ + /tmp/native_link_smoke.o /tmp/NativeToolchainBridge.o \ + dist/lib/libXForgeNativeToolchain.a \ + -lc++ -lz -liconv -lsqlite3 -framework Foundation \ + -o /tmp/XForgeNativeToolchainSmoke 2> /tmp/native-link-errors.txt + STATUS=$? + set -e + if [ "$STATUS" != "0" ]; then + echo "::error::linking the toolchain into an app-shaped target failed" + grep -E '^( "_|ld: )' /tmp/native-link-errors.txt | sort -u | head -40 + tail -40 /tmp/native-link-errors.txt + exit 1 + fi + file /tmp/XForgeNativeToolchainSmoke diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index b77c93d..bf209af 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -155,6 +155,7 @@ jobs: llvm-host-build llvm-build cmark-host-build + cmark-ios-build key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ steps.toolchain-versions.outputs.value }}-${{ env.SWIFT_LLVM_REF }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '1' }} # Swift's own build is the longer half and is cached separately, because it @@ -218,6 +219,31 @@ jobs: mkdir -p cmark-host-build/src ln -sf cmark-host-build/src/cmark-gfm cmark-host-build/src/cmark + - name: Build cmark for iPhoneOS + if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} + shell: bash + run: | + set -euxo pipefail + if [ -f cmark-ios-build/src/libcmark-gfm.a ]; then + echo "iOS cmark already cached" + exit 0 + fi + SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + cmake -S swift-cmark -B cmark-ios-build -G Ninja \ + -DCMAKE_BUILD_TYPE=Release \ + -DCMARK_TESTS=OFF \ + -DCMARK_SHARED=OFF \ + -DCMAKE_SYSTEM_NAME=iOS \ + -DCMAKE_OSX_SYSROOT="$SDK" \ + -DCMAKE_OSX_ARCHITECTURES=arm64 \ + -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ + -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ + -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET + # cmark-gfm-extensions is a separate archive and swift's Markup references + # symbols from both. + ninja -C cmark-ios-build cmark-gfm cmark-gfm-extensions + find cmark-ios-build -name 'libcmark-gfm*.a' + - name: Configure LLVM for iPhoneOS arm64 shell: bash run: | @@ -274,7 +300,7 @@ jobs: shell: bash run: | set -euxo pipefail - for tree in llvm-host-build llvm-build cmark-host-build; do + for tree in llvm-host-build llvm-build cmark-host-build cmark-ios-build; do [ -d "$tree" ] && find "$tree" -exec touch -c {} + 2>/dev/null || true done @@ -309,6 +335,7 @@ jobs: llvm-host-build llvm-build cmark-host-build + cmark-ios-build - name: Configure the Swift frontend for iPhoneOS if: ${{ inputs.with_swift }} @@ -533,6 +560,9 @@ jobs: # instead of depending on archive ordering, and a Swift frontend that # links an LLVM built in the same configuration cannot drift from it. find llvm-build/lib -maxdepth 1 -type f -name '*.a' -print | sort > /tmp/archives.txt + if [ -d cmark-ios-build ]; then + find cmark-ios-build -type f -name 'libcmark-gfm*.a' -print | sort >> /tmp/archives.txt + fi if [ "${{ inputs.with_swift }}" = "true" ]; then find swift-build/lib -maxdepth 3 -type f -name '*.a' -print | sort >> /tmp/archives.txt fi From d942590e49e5f4694f67e11b5cffc28d1886c1d4 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 18:24:17 -0400 Subject: [PATCH 28/54] toolchain: build the iOS cmark as a Darwin target MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Same trap as the Swift configure, one layer down: CMake Error at src/CMakeLists.txt:64 (install): install TARGETS given no BUNDLE DESTINATION for MACOSX_BUNDLE executable target "cmark-gfm" On an iOS platform CMake refuses install(TARGETS) for an executable without a bundle destination, and cmark installs its command-line tool. Configured as Darwin with the iPhoneOS sysroot and an arm64-apple-ios compiler target — the shape Swift's own iOS builds use — the rule is valid, the objects are still arm64 iOS, and nothing is installed. --- .github/workflows/bridge-api-check.yml | 7 ++++++- .github/workflows/native-toolchain.yml | 7 ++++++- 2 files changed, 12 insertions(+), 2 deletions(-) diff --git a/.github/workflows/bridge-api-check.yml b/.github/workflows/bridge-api-check.yml index f09d1b6..5f8547b 100644 --- a/.github/workflows/bridge-api-check.yml +++ b/.github/workflows/bridge-api-check.yml @@ -194,11 +194,16 @@ jobs: exit 0 fi SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + # CMAKE_SYSTEM_NAME=Darwin, not iOS, for the same reason the Swift + # configure uses it: on iOS CMake refuses \`install(TARGETS ...)\` for an + # executable without a BUNDLE destination, and cmark installs its + # command-line tool. The target is still iOS — sysroot, architecture and + # compiler target below say so — and nothing is installed here anyway. cmake -S swift-cmark -B cmark-ios-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DCMARK_TESTS=OFF \ -DCMARK_SHARED=OFF \ - -DCMAKE_SYSTEM_NAME=iOS \ + -DCMAKE_SYSTEM_NAME=Darwin \ -DCMAKE_OSX_SYSROOT="$SDK" \ -DCMAKE_OSX_ARCHITECTURES=arm64 \ -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index bf209af..c731010 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -229,11 +229,16 @@ jobs: exit 0 fi SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + # CMAKE_SYSTEM_NAME=Darwin, not iOS, for the same reason the Swift + # configure uses it: on iOS CMake refuses \`install(TARGETS ...)\` for an + # executable without a BUNDLE destination, and cmark installs its + # command-line tool. The target is still iOS — sysroot, architecture and + # compiler target below say so — and nothing is installed here anyway. cmake -S swift-cmark -B cmark-ios-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ -DCMARK_TESTS=OFF \ -DCMARK_SHARED=OFF \ - -DCMAKE_SYSTEM_NAME=iOS \ + -DCMAKE_SYSTEM_NAME=Darwin \ -DCMAKE_OSX_SYSROOT="$SDK" \ -DCMAKE_OSX_ARCHITECTURES=arm64 \ -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ From 41dc82fce7c749761cc9968d9859e61adf5f8adc Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 18:41:52 -0400 Subject: [PATCH 29/54] toolchain: name cmark's library targets correctly MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The Darwin configure worked and ninja rejected the target: cmark's archives are 'libcmark-gfm' and 'libcmark-gfm-extensions' (the bare names are the command-line tool, which only the host build needs). Also drops two cmark options this version does not define — static archives are its default, and the unused-variable warnings were the only effect. --- .github/workflows/bridge-api-check.yml | 8 +++++--- .github/workflows/native-toolchain.yml | 8 +++++--- 2 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.github/workflows/bridge-api-check.yml b/.github/workflows/bridge-api-check.yml index 5f8547b..6eb903c 100644 --- a/.github/workflows/bridge-api-check.yml +++ b/.github/workflows/bridge-api-check.yml @@ -199,10 +199,10 @@ jobs: # executable without a BUNDLE destination, and cmark installs its # command-line tool. The target is still iOS — sysroot, architecture and # compiler target below say so — and nothing is installed here anyway. + # Static archives are cmark's default; naming CMARK_TESTS/CMARK_SHARED here + # only produces "manually-specified variables were not used" warnings. cmake -S swift-cmark -B cmark-ios-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ - -DCMARK_TESTS=OFF \ - -DCMARK_SHARED=OFF \ -DCMAKE_SYSTEM_NAME=Darwin \ -DCMAKE_OSX_SYSROOT="$SDK" \ -DCMAKE_OSX_ARCHITECTURES=arm64 \ @@ -211,7 +211,9 @@ jobs: -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET # cmark-gfm-extensions is a separate archive and swift's Markup references # symbols from both. - ninja -C cmark-ios-build cmark-gfm cmark-gfm-extensions + # The *library* targets are libcmark-gfm and libcmark-gfm-extensions; the + # bare names are the command-line tool, which only the host build needs. + ninja -C cmark-ios-build libcmark-gfm libcmark-gfm-extensions find cmark-ios-build -name 'libcmark-gfm*.a' - name: Configure the Swift frontend for iPhoneOS diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index c731010..2958e0d 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -234,10 +234,10 @@ jobs: # executable without a BUNDLE destination, and cmark installs its # command-line tool. The target is still iOS — sysroot, architecture and # compiler target below say so — and nothing is installed here anyway. + # Static archives are cmark's default; naming CMARK_TESTS/CMARK_SHARED here + # only produces "manually-specified variables were not used" warnings. cmake -S swift-cmark -B cmark-ios-build -G Ninja \ -DCMAKE_BUILD_TYPE=Release \ - -DCMARK_TESTS=OFF \ - -DCMARK_SHARED=OFF \ -DCMAKE_SYSTEM_NAME=Darwin \ -DCMAKE_OSX_SYSROOT="$SDK" \ -DCMAKE_OSX_ARCHITECTURES=arm64 \ @@ -246,7 +246,9 @@ jobs: -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET # cmark-gfm-extensions is a separate archive and swift's Markup references # symbols from both. - ninja -C cmark-ios-build cmark-gfm cmark-gfm-extensions + # The *library* targets are libcmark-gfm and libcmark-gfm-extensions; the + # bare names are the command-line tool, which only the host build needs. + ninja -C cmark-ios-build libcmark-gfm libcmark-gfm-extensions find cmark-ios-build -name 'libcmark-gfm*.a' - name: Configure LLVM for iPhoneOS arm64 From a1868ab4033fd38519ced4b433f71dcf056fc03e Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 18:54:36 -0400 Subject: [PATCH 30/54] toolchain: gate the iOS cmark build like the Swift stages MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit On a pull_request run `inputs.with_swift` is empty, so the cmark *checkout* is skipped while my new cmark-for-iOS step ran anyway — 'The source directory swift-cmark does not exist'. It now carries the same condition as the Swift checkouts and stages, which is the condition that actually decides whether cmark is needed: Swift's Markup is the only consumer. --- .github/workflows/native-toolchain.yml | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 2958e0d..0c6d720 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -219,8 +219,11 @@ jobs: mkdir -p cmark-host-build/src ln -sf cmark-host-build/src/cmark-gfm cmark-host-build/src/cmark + # Gated exactly like the Swift checkouts and the Swift stages: it is only + # needed when the frontend is, and on a PR/push run the cmark sources are not + # checked out at all — which is how this step failed its first two runs. - name: Build cmark for iPhoneOS - if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} + if: ${{ inputs.with_swift }} shell: bash run: | set -euxo pipefail From 5a0d26228b0b5f43b607acd1885b68088627f223 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 19:28:34 -0400 Subject: [PATCH 31/54] ci: iterate on an artifact, not on a cache MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Caches were the wrong substrate for the fast loop and I should have seen it sooner: the toolchain workflow's saves take six seconds for multi-gigabyte trees, which is not a save, and the fast loop reported 'llvm cache hit:' empty — so it failed a step the toolchain run passes with no generated headers at all. The toolchain workflow now packages the static archives, cmark and the generated headers into an artifact (with `always()`, so a failure after the builds still publishes them — the case the fast loop exists for), and the fast loop downloads it. Object files and CMake metadata stay out: most of the bytes, none of the use. --- .github/workflows/bridge-api-check.yml | 18 +++++++++++ .github/workflows/native-toolchain.yml | 43 ++++++++++++++++++++++++++ 2 files changed, 61 insertions(+) diff --git a/.github/workflows/bridge-api-check.yml b/.github/workflows/bridge-api-check.yml index 6eb903c..c2d95e7 100644 --- a/.github/workflows/bridge-api-check.yml +++ b/.github/workflows/bridge-api-check.yml @@ -105,6 +105,24 @@ jobs: echo "value=$(xcodebuild -version | tr '\n' ' ')-$(xcrun --sdk iphoneos --show-sdk-version)" } >> "$GITHUB_OUTPUT" + # The build trees, from the last toolchain run that produced them. This is what + # makes the staging and link half checkable in minutes: the archives and + # generated headers are the expensive part, and they arrive as an artifact + # rather than as a cache entry that may or may not be there. + - name: Download the build trees + env: + GH_TOKEN: ${{ github.token }} + run: | + set -euxo pipefail + mkdir -p build-trees + if gh run download --repo "$GITHUB_REPOSITORY" --name XForgeBuildTrees --dir build-trees; then + tar -xf build-trees/build-trees.tar + echo "build trees restored:" + ls -d llvm-build/lib swift-build/lib cmark-ios-build 2>/dev/null || true + else + echo "::warning::no XForgeBuildTrees artifact yet; falling back to the caches, then to configure only" + fi + # The same keys the toolchain workflow writes, so this reuses what it built. - name: Restore the LLVM builds id: llvm-cache diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 0c6d720..9e836f3 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -517,6 +517,49 @@ jobs: -c App/NativeToolchain/NativeToolchainBridge.mm \ -o /tmp/NativeToolchainBridge.o + # The iteration substrate. Caches were the wrong tool for this: the entries + # either never stored (a 6-second "save" of a multi-gigabyte tree) or were + # evicted, so a workflow that restored them found nothing and failed steps this + # one passes. Artifacts are not evicted, and this is only what a check needs — + # the static archives and the generated headers — not the object files and + # CMake metadata that make up most of the trees. + # + # Uploaded with `always()` so a failure *after* the builds still publishes them: + # that is exactly the case the fast loop exists to iterate on. + - name: Package the build trees + if: ${{ always() && !inputs.configure_only }} + shell: bash + run: | + set -euxo pipefail + mkdir -p build-trees + ITEMS=() + for item in \ + llvm-build/lib \ + swift-build/lib \ + cmark-ios-build \ + llvm-build/include \ + llvm-build/tools/clang/include \ + swift-build/include \ + swift-build/tools/swift/include; do + [ -d "$item" ] && ITEMS+=("$item") + done + if [ "${#ITEMS[@]}" -eq 0 ]; then + echo "nothing to package" + exit 0 + fi + tar -cf build-trees/build-trees.tar "${ITEMS[@]}" + du -sh build-trees/build-trees.tar + + - name: Upload the build trees + if: ${{ always() && !inputs.configure_only }} + uses: actions/upload-artifact@v4 + with: + name: XForgeBuildTrees + path: build-trees/build-trees.tar + if-no-files-found: warn + retention-days: 7 + compression-level: 0 + - name: Stage native toolchain SDK if: ${{ !inputs.configure_only }} shell: bash From 7496e7a34563b3e5c113fb779d9f1bb6d0110ae9 Mon Sep 17 00:00:00 2001 From: R0GUEEE Date: Thu, 24 Sep 2026 20:42:31 -0400 Subject: [PATCH 32/54] toolchain: merge archives without dropping members MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Staging failed on undefined JIT symbols that were referenced from an object that *is* in the archive — the signature of a merge that lost objects. It had: libtool: warning duplicate member name 'X86.cpp.o' from 'libclangCodeGen.a' and … `libtool -static` de-duplicates members by name, across clang's libraries and Swift's alike, and a dropped member is a silently missing object. The archive is now merged with GNU MRI (`llvm-ar -M` + `addlib`, which copies every member, duplicates included), and the step asserts that the merged member count is at least the sum of the inputs' — so this class of bug fails where it happens instead of surfacing as a missing-library link error two steps later. The fast loop keeps staging archives unmerged and force-loading them in its link smoke, which is deliberately order-independent: it is a check, not a product. --- .github/workflows/bridge-api-check.yml | 15 +++++--- .github/workflows/native-toolchain.yml | 51 ++++++++++++++++++++------ 2 files changed, 49 insertions(+), 17 deletions(-) diff --git a/.github/workflows/bridge-api-check.yml b/.github/workflows/bridge-api-check.yml index c2d95e7..bc60563 100644 --- a/.github/workflows/bridge-api-check.yml +++ b/.github/workflows/bridge-api-check.yml @@ -350,11 +350,14 @@ jobs: find swift-build/lib -maxdepth 3 -type f -name '*.a' -print | sort >> /tmp/archives.txt fi test -s /tmp/archives.txt - ARCHIVES=() - while IFS= read -r archive; do ARCHIVES+=("$archive"); done < /tmp/archives.txt - echo "merging ${#ARCHIVES[@]} archives" - /usr/bin/libtool -static -o dist/lib/libXForgeNativeToolchain.a "${ARCHIVES[@]}" - ls -la dist/lib/libXForgeNativeToolchain.a + # Staged unmerged: `libtool -static` de-duplicates members by name + # ("duplicate member name 'X86.cpp.o' from …"), and a dropped member is a + # silently missing object — which is precisely how the JIT symbols went + # undefined while their object was in the archive. + while IFS= read -r archive; do cp "$archive" dist/lib/; done < /tmp/archives.txt + cp /tmp/archives.txt dist/lib/XForgeToolchain.libraries.txt + echo "staged $(wc -l < /tmp/archives.txt) archives, unmerged:" + ls dist/lib | head -10 - name: Link an app-shaped target shell: bash @@ -374,7 +377,7 @@ jobs: xcrun --sdk iphoneos clang++ -arch arm64 \ -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET -isysroot "$SDK" \ /tmp/native_link_smoke.o /tmp/NativeToolchainBridge.o \ - dist/lib/libXForgeNativeToolchain.a \ + $(for library in dist/lib/*.a; do printf ' -Wl,-force_load,%s' "$library"; done) \ -lc++ -lz -liconv -lsqlite3 -framework Foundation \ -o /tmp/XForgeNativeToolchainSmoke 2> /tmp/native-link-errors.txt STATUS=$? diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 9e836f3..c1ef9e6 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -192,7 +192,7 @@ jobs: -DLLVM_ENABLE_ZSTD=OFF \ -DLLVM_ENABLE_LIBXML2=OFF \ -DLLVM_ENABLE_CURL=OFF - ninja -C llvm-host-build llvm-tblgen clang-tblgen + ninja -C llvm-host-build llvm-tblgen clang-tblgen llvm-ar llvm-nm # Swift's CMake resolves its native tools through LLVM_TOOLS_BINARY_DIR, # which for the iOS build is a directory with no tools in it @@ -608,10 +608,17 @@ jobs: exit 1 fi - # Flatten the static dependency graph — LLVM's, Clang's, LLD's and (when - # built) Swift's — into one archive. XForge then links a single library - # instead of depending on archive ordering, and a Swift frontend that - # links an LLVM built in the same configuration cannot drift from it. + # Archives are staged *unmerged*, deliberately. + # + # This used to flatten every library into one archive with + # `libtool -static`, which was wrong: libtool de-duplicates members by name + # ("duplicate member name 'X86.cpp.o' from libclangCodeGen.a and …"), and a + # dropped member is a silently missing object. That is what produced + # undefined JIT symbols referenced from an object that was demonstrably in + # the archive. Swift's libraries collide the same way. + # + # The app already links a list — `prepare-xcode.sh` globs `lib/*.a` into + # XFORGE_NATIVE_LDFLAGS — so the merge only ever saved a line of shell. find llvm-build/lib -maxdepth 1 -type f -name '*.a' -print | sort > /tmp/archives.txt if [ -d cmark-ios-build ]; then find cmark-ios-build -type f -name 'libcmark-gfm*.a' -print | sort >> /tmp/archives.txt @@ -621,10 +628,32 @@ jobs: fi test -s /tmp/archives.txt ARCHIVES=() - while IFS= read -r archive; do - ARCHIVES+=("$archive") - done < /tmp/archives.txt - /usr/bin/libtool -static -o dist/lib/libXForgeNativeToolchain.a "${ARCHIVES[@]}" + while IFS= read -r archive; do ARCHIVES+=("$archive"); done < /tmp/archives.txt + cp /tmp/archives.txt dist/lib/XForgeToolchain.libraries.txt + + # One archive, merged with GNU MRI rather than `libtool -static`. + # libtool de-duplicates members by name — the log warns "duplicate member + # name 'X86.cpp.o' from libclangCodeGen.a and …" for clang's libraries and + # the same happens across Swift's — and a dropped member is a silently + # missing object, which is how the JIT symbols came out undefined while + # their object was demonstrably in the archive. `addlib` copies every + # member, duplicates included. + AR="$PWD/llvm-host-build/bin/llvm-ar" + MRI="$(mktemp)" + echo "create dist/lib/libXForgeNativeToolchain.a" > "$MRI" + for archive in "${ARCHIVES[@]}"; do echo "addlib $archive" >> "$MRI"; done + echo "save" >> "$MRI" + echo "end" >> "$MRI" + "$AR" -M < "$MRI" + + # The guard: a merge that loses members is an error here, not a link + # failure two steps later that looks like a missing library. + MERGED=$("$AR" t dist/lib/libXForgeNativeToolchain.a | wc -l | tr -d ' ') + EXPECTED=$(for archive in "${ARCHIVES[@]}"; do "$AR" t "$archive"; done | wc -l | tr -d ' ') + echo "merged members: $MERGED, sources: $EXPECTED in ${#ARCHIVES[@]} archives" + [ "$MERGED" -ge "$EXPECTED" ] \ + || { echo "::error::the merge dropped members ($MERGED < $EXPECTED)"; exit 1; } + ls -la dist/lib/libXForgeNativeToolchain.a cat > /tmp/native_link_smoke.mm <<'EOF' #include "NativeToolchainBridge.h" @@ -649,7 +678,7 @@ jobs: -isysroot "$SDK" \ /tmp/native_link_smoke.o \ /tmp/NativeToolchainBridge.o \ - dist/lib/libXForgeNativeToolchain.a \ + $(for library in dist/lib/*.a; do printf ' -Wl,-force_load,%s' "$library"; done) \ -lc++ -lz -liconv -lsqlite3 \ -framework Foundation \ -o /tmp/XForgeNativeToolchainSmoke \ @@ -670,7 +699,7 @@ jobs: swift_frontend=${{ inputs.with_swift && '1' || '0' }} swift_ref=${{ inputs.with_swift && env.SWIFT_REF || 'none' }} built_at=$(date -u +%Y-%m-%dT%H:%M:%SZ) - archive=lib/libXForgeNativeToolchain.a + archives=lib/XForgeToolchain.libraries.txt header_roots=include-generated include EOF tar -C dist -czf XForgeNativeToolchain-arm64-ios.tar.gz . From 7459c492bd00224e69230e78ffddf2b6e0ffb1ae Mon Sep 17 00:00:00 2001 From: Felix Date: Thu, 24 Sep 2026 22:42:58 -0400 Subject: [PATCH 33/54] ci(toolchain): make the caches actually work, and close the link MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Every run rebuilt LLVM (37 min) and Swift (34 min) from scratch even when both caches reported a hit. The restored trees were stamped with `find | touch` after configure, and that stamps files in batches milliseconds apart, so a generator input could end up newer than build.ninja: ninja re-ran CMake, CMake rewrote llvm/Config/*.h, and all 2601 edges were dirty again. The runs show it — restore "hit" at 00:45:08, `[1/2601]` at 00:45:25. * age the source checkouts to a fixed past date instead, so every file the cache restores is newer than every source — true for the object edges and for the generator edge alike; * skip the configure when the restored tree is already configured and built (checked before use, never assumed); * save only trees the build actually finished. The old `always()` save stored whatever existed when the build failed, and since a cache key can never be rewritten, every later run "hit" that partial tree and rebuilt anyway; * new keys (cache_salt 3, Swift -v2) to get past the stored ones. Add the two libraries only Swift references, which the force-loaded smoke link fails on: LLVMOrcJIT (swift's JIT in SwiftMaterializationUnit.cpp) and clangDependencyScanning (ClangImporter's module dependency scanning). Print every undefined symbol on that failure instead of the last 60 lines. --- .github/workflows/native-toolchain.yml | 161 +++++++++++++++++++------ 1 file changed, 126 insertions(+), 35 deletions(-) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index c1ef9e6..1df0d6c 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -51,9 +51,9 @@ on: type: string default: "17.0" cache_salt: - description: Bump to discard the cached iOS builds + description: Bump to discard the cached iOS builds (do this when the configure recipe below changes) type: string - default: "1" + default: "3" pull_request: paths: - "App/NativeToolchain/**" @@ -143,10 +143,41 @@ jobs: echo "value=$(xcodebuild -version | tr '\n' ' ')-$(xcrun --sdk iphoneos --show-sdk-version)" } >> "$GITHUB_OUTPUT" + # A restored build tree carries the timestamps of the run that saved it, and + # those are *older* than the sources this run has just checked out. Ninja + # compares timestamps, so the tree it just restored would be rebuilt in full + # — the one thing the cache exists to prevent. The fix is to age the sources + # instead of refreshing the tree: "every file in the tree is newer than every + # source" then holds for whatever the cache restores, and it holds for ninja's + # generator check as well, because build.ninja is an *output* of that edge and + # any source newer than it sends ninja into a re-configure. + # + # Stamping the tree with `find … -exec touch` (what this workflow used to do, + # after configure) is unsound: the files are stamped in batches milliseconds + # apart, so whether a generator *input* ends up after build.ninja is decided + # by directory order. When it does, ninja re-runs CMake, CMake rewrites + # llvm/Config/*.h, and every object that includes them is dirty again — a + # full 2601-edge rebuild that the restore paid seconds to avoid. + - name: Age the source checkouts + shell: bash + run: | + set -euxo pipefail + for dir in llvm-project swift swift-cmark swift-syntax \ + swift-experimental-string-processing; do + [ -d "$dir" ] && find "$dir" -exec touch -h -t 202001010000 {} + 2>/dev/null || true + done + # The expensive half is LLVM's cross-build and (when enabled) Swift's. The # key deliberately does not include the workflow file: editing this file — # which is what iterating on it looks like — must not force a rebuild of a # compiler that did not change. `cache_salt` is the way to ask for one. + # + # Bump it whenever the *recipe* changes too. A cache entry is immutable: once + # a partial tree has been stored under a key, every later run gets a "hit" + # for it, the save below is skipped (the key already exists, so the upload + # would be refused anyway) and the key can never be repaired. That is the + # trap the old `always()`-gated save walked into — it stored whatever existed + # even when the build it was saving had just failed. - name: Restore the LLVM builds id: llvm-cache uses: actions/cache/restore@v4 @@ -156,21 +187,23 @@ jobs: llvm-build cmark-host-build cmark-ios-build - key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ steps.toolchain-versions.outputs.value }}-${{ env.SWIFT_LLVM_REF }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '1' }} + key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ steps.toolchain-versions.outputs.value }}-${{ env.SWIFT_LLVM_REF }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '3' }} # Swift's own build is the longer half and is cached separately, because it # is saved *after* the LLVM half has been built and Swift's has too. Its key # deliberately ignores `cache_salt`: that salt exists to discard an LLVM tree # when the LLVM source changes, and the Swift tree is a function of the Swift # refs and the SDK alone — discarding it too would throw away the 45-minute - # half for a change to the other one. + # half for a change to the other one. The `-vN` suffix is this half's own + # salt: bump it when the Swift configure or build recipe changes, because a + # stored tree can never be replaced under the key it was stored with. - name: Restore the Swift build if: ${{ inputs.with_swift }} id: swift-cache uses: actions/cache/restore@v4 with: path: swift-build - key: ${{ runner.os }}-${{ runner.arch }}-swift-${{ env.SWIFT_REF }}-${{ env.SWIFT_LLVM_REF }}-${{ steps.toolchain-versions.outputs.value }}-${{ env.IOS_DEPLOYMENT_TARGET }}-1 + key: ${{ runner.os }}-${{ runner.arch }}-swift-${{ env.SWIFT_REF }}-${{ env.SWIFT_LLVM_REF }}-${{ steps.toolchain-versions.outputs.value }}-${{ env.IOS_DEPLOYMENT_TARGET }}-v2 - name: Build host TableGen tools shell: bash @@ -254,7 +287,34 @@ jobs: ninja -C cmark-ios-build libcmark-gfm libcmark-gfm-extensions find cmark-ios-build -name 'libcmark-gfm*.a' + # Skipping the configure below is only sound if the restored tree really is + # the configured-and-built one it was saved from. Checking is cheap, it has to + # happen *before* the step it gates (a step's output does not exist yet when + # the condition above it is evaluated), and the alternative is a confusing + # failure — no build.ninja — where the old workflow had a slow-but-working path. + - name: Verify the restored LLVM build tree + if: ${{ steps.llvm-cache.outputs.cache-hit == 'true' }} + id: llvm-tree + shell: bash + run: | + set -euxo pipefail + MISSING="" + for file in llvm-build/build.ninja llvm-build/CMakeCache.txt \ + llvm-build/include/llvm/Config/llvm-config.h \ + llvm-host-build/bin/llvm-tblgen llvm-host-build/bin/clang-tblgen; do + [ -e "$file" ] || MISSING="$MISSING $file" + done + OBJECTS="$(find llvm-build -name '*.o' -print -quit)" + if [ -n "$MISSING" ] || [ -z "$OBJECTS" ]; then + echo "::warning::restored LLVM tree is not usable (missing:$MISSING, objects:$([ -n "$OBJECTS" ] && echo present || echo none)); configuring from scratch" + echo "reusable=false" >> "$GITHUB_OUTPUT" + else + echo "restored LLVM tree is configured and built; the configure below is skipped" + echo "reusable=true" >> "$GITHUB_OUTPUT" + fi + - name: Configure LLVM for iPhoneOS arm64 + if: ${{ steps.llvm-cache.outputs.cache-hit != 'true' || steps.llvm-tree.outputs.reusable != 'true' }} shell: bash run: | set -euxo pipefail @@ -298,23 +358,25 @@ jobs: -DLLD_INCLUDE_TESTS=OFF \ -DLLD_BUILD_TOOLS=OFF - # A restored build tree looks *older* than the sources that were just checked - # out, and ninja compares timestamps — so it would rebuild everything the - # cache exists to save. Touching the outputs puts them ahead again. It has to - # happen *after* CMake's configure, not before: configure rewrites generated - # headers (llvm/Config/…), and a header newer than the outputs invalidates - # every edge that includes it, which is nearly all of them. Sound either way, - # because a change to any source ref changes the cache key. - - name: Make the restored LLVM build newer than its inputs - if: ${{ steps.llvm-cache.outputs.cache-hit == 'true' }} - shell: bash - run: | - set -euxo pipefail - for tree in llvm-host-build llvm-build cmark-host-build cmark-ios-build; do - [ -d "$tree" ] && find "$tree" -exec touch -c {} + 2>/dev/null || true - done - + # Cheap by construction when the tree above was restored: the sources are all + # older than the objects, so ninja has nothing to do. It is also where the + # cache is proven — a restore that does not come back built shows up here as + # a 2601-edge rebuild in the log, not as a mystery. + # + # The target list is not "everything the app might use" but the closure the + # bundle has to satisfy, because the smoke link force-loads every archive: + # each member's undefined symbols must resolve inside the bundle. Two of these + # are invisible to clang's own libraries and only Swift references them, which + # is why they were missing: + # * LLVMOrcJIT — swift's JIT (SwiftMaterializationUnit.cpp) and LLJIT; + # * clangDependencyScanning — ClangImporter's explicit-module dependency + # scanning (ModuleDepCollector/ScanAndUpdateArgs). + # Their transitive dependencies (JITLink, ExecutionEngine, RuntimeDyld, + # OrcShared/OrcTargetProcess, clangTooling, clangCAS, …) come with them. + # Adding a target here changes the delivered bundle, so bump `cache_salt` + # when you do: the cache key cannot see this list. - name: Build Clang and Mach-O LLD libraries + id: llvm-build if: ${{ !inputs.configure_only }} run: | set -euxo pipefail @@ -329,15 +391,21 @@ jobs: clangAST \ clangLex \ clangBasic \ + clangDependencyScanning \ lldMachO \ - lldCommon + lldCommon \ + LLVMOrcJIT # The step this workflow exists for. Everything above is shared with the # LLVM-only bundle; everything below only happens when Swift is asked for. - # Saved here, before the step most likely to fail, and with `if: always()`: - # a failed Swift configure must not cost another LLVM cross-build. + # Saved here, before the step most likely to fail, so that a failed Swift + # configure does not cost another LLVM cross-build. + # + # Gated on the build having *succeeded*, not on `always()`: saving a tree + # ninja never finished stores a partial one under a key that can never be + # rewritten, and every later run then "hits" it and rebuilds anyway. - name: Save the LLVM builds - if: ${{ always() && steps.llvm-cache.outputs.cache-hit != 'true' }} + if: ${{ steps.llvm-build.outcome == 'success' && steps.llvm-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@v4 with: key: ${{ steps.llvm-cache.outputs.cache-primary-key }} @@ -347,8 +415,31 @@ jobs: cmark-host-build cmark-ios-build + # Same contract as the LLVM tree check above, and the same reason: the + # configure is skipped when the tree came back built, so a tree that came back + # *unbuilt* would fail on the missing archive instead of just rebuilding. + - name: Verify the restored Swift build tree + if: ${{ inputs.with_swift && steps.swift-cache.outputs.cache-hit == 'true' }} + id: swift-tree + shell: bash + run: | + set -euxo pipefail + MISSING="" + for file in swift-build/build.ninja swift-build/CMakeCache.txt \ + swift-build/lib/libswiftFrontendTool.a; do + [ -e "$file" ] || MISSING="$MISSING $file" + done + OBJECTS="$(find swift-build -name '*.o' -print -quit)" + if [ -n "$MISSING" ] || [ -z "$OBJECTS" ]; then + echo "::warning::restored Swift tree is not usable (missing:$MISSING, objects:$([ -n "$OBJECTS" ] && echo present || echo none)); configuring from scratch" + echo "reusable=false" >> "$GITHUB_OUTPUT" + else + echo "restored Swift tree is configured and built; the configure below is skipped" + echo "reusable=true" >> "$GITHUB_OUTPUT" + fi + - name: Configure the Swift frontend for iPhoneOS - if: ${{ inputs.with_swift }} + if: ${{ inputs.with_swift && (steps.swift-cache.outputs.cache-hit != 'true' || steps.swift-tree.outputs.reusable != 'true') }} shell: bash run: | set -euxo pipefail @@ -415,14 +506,8 @@ jobs: || echo "::warning::swiftFrontendTool archive not found where expected; listing swift-build/lib" ls -la swift-build/lib | head -40 || true - - name: Make the restored Swift build newer than its inputs - if: ${{ steps.swift-cache.outputs.cache-hit == 'true' }} - shell: bash - run: | - set -euxo pipefail - [ -d swift-build ] && find swift-build -exec touch -c {} + 2>/dev/null || true - - name: Build the Swift frontend libraries + id: swift-build if: ${{ inputs.with_swift && !inputs.configure_only }} shell: bash run: | @@ -435,7 +520,7 @@ jobs: ninja -C swift-build swiftFrontendTool - name: Save the Swift build - if: ${{ always() && inputs.with_swift && !inputs.configure_only && steps.swift-cache.outputs.cache-hit != 'true' }} + if: ${{ inputs.with_swift && !inputs.configure_only && steps.swift-build.outcome == 'success' && steps.swift-cache.outputs.cache-hit != 'true' }} uses: actions/cache/save@v4 with: key: ${{ steps.swift-cache.outputs.cache-primary-key }} @@ -684,7 +769,13 @@ jobs: -o /tmp/XForgeNativeToolchainSmoke \ 2> /tmp/native-link-errors.txt || { echo "::error::linking the toolchain into an app-shaped target failed" - tail -60 /tmp/native-link-errors.txt + # ld prints one ` "", referenced from:` line per undefined + # symbol. `tail -60` showed the last of them and hid the rest, which + # is how a whole missing library stayed a mystery for a run: print + # every one, deduplicated, so the next iteration is not another guess. + grep 'referenced from:' /tmp/native-link-errors.txt \ + | sed 's/, referenced from:$//' | sort -u || true + tail -20 /tmp/native-link-errors.txt exit 1 } From ee9333d589fd2a429be291825d583d94574aef67 Mon Sep 17 00:00:00 2001 From: Felix Date: Thu, 24 Sep 2026 22:44:17 -0400 Subject: [PATCH 34/54] ci(toolchain): the host-tools check has to cover all four tools MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `llvm-ar` and `llvm-nm` come out of the same step as the TableGen pair, and the packaging step merges the archives with the restored `llvm-ar`. Checking only the TableGen tools let a restored tree missing it pass, and the run died three seconds into staging with "llvm-host-build/bin/llvm-ar: No such file or directory" — after an hour of building. Check all four, and refresh the llvm-build/bin mirror on the cached path too (previously the early exit skipped it). --- .github/workflows/native-toolchain.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 1df0d6c..deb62ec 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -209,8 +209,19 @@ jobs: shell: bash run: | set -euxo pipefail - if [ -x llvm-host-build/bin/llvm-tblgen ] && [ -x llvm-host-build/bin/clang-tblgen ]; then - echo "host TableGen tools already cached" + # All four, not just the TableGen pair: `llvm-ar` is what the packaging + # step merges the archives with, and a restored tree that has the two + # TableGen tools but not it used to pass this check and fail three seconds + # into staging ("llvm-host-build/bin/llvm-ar: No such file or directory"). + if [ -x llvm-host-build/bin/llvm-tblgen ] \ + && [ -x llvm-host-build/bin/clang-tblgen ] \ + && [ -x llvm-host-build/bin/llvm-ar ] \ + && [ -x llvm-host-build/bin/llvm-nm ]; then + echo "host tools already built" + mkdir -p llvm-build/bin + for tool in llvm-host-build/bin/*; do + ln -sf "$PWD/$tool" "llvm-build/bin/$(basename "$tool")" + done exit 0 fi cmake -S llvm-project/llvm -B llvm-host-build -G Ninja \ From ce467aa706f7133567adaee738c2d942f46eea5b Mon Sep 17 00:00:00 2001 From: Felix Date: Thu, 24 Sep 2026 23:11:53 -0400 Subject: [PATCH 35/54] toolchain: publish the bundle as a release, install it by tag The whole point of the toolchain build is that it only has to happen once per recipe. A workflow artifact cannot carry that: it expires, and it cannot be fetched by name, so "install the toolchain" meant "find the right run first". Publish instead, tagged by what is inside: `toolchain---ios-sdk`. Two builds of the same inputs land on the same tag, and a bundle that could not be substituted for another (no Swift frontend) can never be handed over as "the newest one" -- only hand-dispatched runs publish, so the LLVM-only half cannot take that slot. - native-toolchain.yml records the tag, the Xcode version and the SDK version in manifest.txt, and uploads the tarball to the release (clobbering a previous upload of the same tag: rebuilding identical inputs is normal). - install-bundle.sh gained `--release [tag]` (and `--release artifact` for a bundle built but not published) so a clone that never ran the workflow can still install the compiler: `make toolchain-release`. - build-ipa.yml installs from the newest toolchain-* release, takes an optional `toolchain_tag`, and warns when the bundle's Swift revision or SDK disagrees with the Xcode it is being linked into -- the frontend reads the deployment SDK's standard library modules, so that mismatch is the failure mode nobody can debug on a device. Docs updated where they claimed the bundle was a workflow artifact, and where they described the old `libtool -static` flatten (the merge is `llvm-ar -M` with an MRI script, because libtool drops members with duplicate names). --- .github/workflows/build-ipa.yml | 81 +++++++++++- .github/workflows/native-toolchain.yml | 99 +++++++++++++++ CHANGELOG.md | 27 ++-- Docs/DESIGN.md | 14 ++- Docs/NATIVE-TOOLCHAIN.md | 166 +++++++++++++++++-------- Makefile | 23 ++-- NativeToolchain/install-bundle.sh | 124 +++++++++++++++++- README.md | 28 +++-- 8 files changed, 468 insertions(+), 94 deletions(-) diff --git a/.github/workflows/build-ipa.yml b/.github/workflows/build-ipa.yml index ad24c16..98bec6d 100644 --- a/.github/workflows/build-ipa.yml +++ b/.github/workflows/build-ipa.yml @@ -24,9 +24,13 @@ on: type: boolean default: false with_toolchain: - description: Download the native toolchain artifact and link it into the app + description: Install the published native toolchain bundle and link it into the app type: boolean default: false + toolchain_tag: + description: Toolchain release tag to install (default - the newest toolchain-* release, or "artifact" for the newest workflow artifact) + type: string + default: "" permissions: contents: write @@ -75,20 +79,85 @@ jobs: # installed where the generated xcconfig points, so the bridge compiles with # XFORGE_HAS_LLVM / XFORGE_HAS_SWIFT_FRONTEND and the app *links* the merged # archive — the step that no other workflow exercises. + # + # From the published release, not a workflow artifact: the toolchain build + # takes ~90 minutes and its output only changes when an LLVM or Swift revision + # does, so making the app build depend on a toolchain run would put that cost + # on every app build for no reason. The tag names its contents, so "the newest + # `toolchain-*` release" is a well-defined thing to ask for. - name: Install the native toolchain bundle if: ${{ inputs.with_toolchain }} env: GH_TOKEN: ${{ github.token }} + REQUESTED_TAG: ${{ inputs.toolchain_tag }} run: | set -euo pipefail - gh run download --repo "$GITHUB_REPOSITORY" \ - --name XForgeNativeToolchain-arm64-ios \ - --dir .toolchain-artifact - bash NativeToolchain/install-bundle.sh \ - .toolchain-artifact/XForgeNativeToolchain-arm64-ios.tar.gz + ARCHIVE_NAME="XForgeNativeToolchain-arm64-ios.tar.gz" + tag="$REQUESTED_TAG" + + if [ "$tag" = "artifact" ]; then + # Escape hatch for testing a bundle that was just built and has not been + # published: the newest artifact of that name. + echo "installing the newest workflow artifact" + gh run download --repo "$GITHUB_REPOSITORY" \ + --name XForgeNativeToolchain-arm64-ios --dir .toolchain-artifact + else + if [ -z "$tag" ]; then + # `gh release list` is newest-first, so the first toolchain-* tag is + # the newest published bundle. + tag="$(gh release list --repo "$GITHUB_REPOSITORY" --limit 100 \ + --json tagName --jq '[.[] | select(.tagName | startswith("toolchain-"))][0].tagName')" + fi + if [ -z "$tag" ] || [ "$tag" = "null" ]; then + echo "::error::no published toolchain release found." + echo "Run the 'Native iOS Toolchain' workflow by hand (with_swift=true) and let it publish, or pass toolchain_tag." + exit 1 + fi + echo "installing the toolchain bundle from release $tag" + gh release download "$tag" --repo "$GITHUB_REPOSITORY" \ + --pattern "XForgeNativeToolchain-*.tar.gz" --dir .toolchain-artifact + fi + + bash NativeToolchain/install-bundle.sh ".toolchain-artifact/$ARCHIVE_NAME" cat Vendor/NativeToolchain/manifest.txt cat Support/NativeToolchain.generated.xcconfig + # The frontend can only read the deployment SDK's standard-library modules when + # it is the same Swift version as the app it is linked into. The bundle records + # what it was built from; this says so loudly instead of shipping an app whose + # Swift support fails on a device nobody can debug here. A mismatch is a + # warning, not a failure: the exact version string of an Apple toolchain is not + # something to bet the build on. + - name: Check the toolchain bundle against this Xcode + if: ${{ inputs.with_toolchain }} + run: | + set -euo pipefail + MANIFEST="Vendor/NativeToolchain/manifest.txt" + get() { sed -n "s/^$1=//p" "$MANIFEST"; } + + echo "bundle: $(get bundle_tag)" + echo " built with Xcode $(get xcode), iPhoneOS SDK $(get ios_sdk)" + echo " LLVM commit $(get llvm_commit)" + + runner_sdk="$(xcrun --sdk iphoneos --show-sdk-version)" + if [ "$(get ios_sdk)" != "$runner_sdk" ]; then + echo "::warning::the bundle was built against iPhoneOS SDK $(get ios_sdk) but this runner has $runner_sdk" + fi + + if ! grep -q '^swift_frontend=1' "$MANIFEST"; then + echo "this bundle has no Swift frontend; Swift targets will report it missing" + exit 0 + fi + + built_version="$(get swift_ref | sed -n 's/^swift-\([0-9][0-9.]*\).*/\1/p')" + app_version="$(xcrun swift --version 2>/dev/null | head -n 1)" + echo " Swift frontend built from $(get swift_ref)" + echo " this runner's Swift: $app_version" + case "$app_version" in + *"$built_version"*) echo "Swift versions agree" ;; + *) echo "::warning::the bundle's Swift frontend is $built_version, this Xcode's Swift is '$app_version' — the frontend may not read this SDK's standard library modules" ;; + esac + - name: Prepare the native toolchain configuration run: | set -euo pipefail diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index deb62ec..138c78a 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -54,6 +54,10 @@ on: description: Bump to discard the cached iOS builds (do this when the configure recipe below changes) type: string default: "3" + publish: + description: Publish the finished bundle as a release asset + type: boolean + default: true pull_request: paths: - "App/NativeToolchain/**" @@ -69,6 +73,11 @@ jobs: llvm-ios: runs-on: macos-15 timeout-minutes: 350 + # `contents: write` is for the publish step at the bottom: the bundle is worth + # more as a release asset than as a workflow artifact, because artifacts expire + # (7-90 days, at GitHub's discretion) and cannot be fetched by tag. + permissions: + contents: write env: IOS_DEPLOYMENT_TARGET: ${{ inputs.deployment_target || '17.0' }} SWIFT_REF: ${{ inputs.swift_ref || 'swift-6.2.4-RELEASE' }} @@ -143,6 +152,30 @@ jobs: echo "value=$(xcodebuild -version | tr '\n' ' ')-$(xcrun --sdk iphoneos --show-sdk-version)" } >> "$GITHUB_OUTPUT" + # The bundle's *name*, decided from what goes into it rather than from when it + # was built: the LLVM commit, the Swift revision (or the fact that it is not + # included) and the deployment target decide which archives exist, and the + # SDK decides what they were compiled against. Two builds of the same inputs + # therefore land on the same tag, and a build that could not be substituted + # for another can never shadow it — which is what makes + # `build-ipa.yml`'s "the newest `toolchain-*` release" a safe rule. + - name: Name the bundle + id: bundle + shell: bash + run: | + set -euxo pipefail + LLVM_SHA="$(git -C llvm-project rev-parse --short=12 HEAD)" + if [ "${{ inputs.with_swift }}" = "true" ]; then + # `/` is legal in a tag and awkward in a URL and a shell word. + SWIFT_MARK="swift-$(printf '%s' "$SWIFT_REF" | tr '/' '-')" + else + SWIFT_MARK="noswift" + fi + SDK_VERSION="$(xcrun --sdk iphoneos --show-sdk-version)" + TAG="toolchain-${LLVM_SHA}-${SWIFT_MARK}-ios${IOS_DEPLOYMENT_TARGET}-sdk${SDK_VERSION}" + echo "tag=$TAG" >> "$GITHUB_OUTPUT" + echo "bundle tag: $TAG" + # A restored build tree carries the timestamps of the run that saved it, and # those are *older* than the sources this run has just checked out. Ninja # compares timestamps, so the tree it just restored would be rebuilt in full @@ -657,6 +690,7 @@ jobs: compression-level: 0 - name: Stage native toolchain SDK + id: stage if: ${{ !inputs.configure_only }} shell: bash run: | @@ -794,12 +828,21 @@ jobs: file /tmp/NativeToolchainBridge.o cp /tmp/NativeToolchainBridge.o dist/ + # The manifest is what the consumers read, so everything they have to + # agree with is in it: `bundle_tag` is the release this can be fetched + # from, `xcode`/`ios_sdk` are the SDK the archives were compiled against + # (a different SDK is a different bundle), and `swift_ref` is the Swift + # version the frontend speaks — the app's own Swift must match it, because + # the frontend reads the SDK's standard-library modules. cat > dist/manifest.txt </dev/null 2>&1; then + echo "release $TAG already exists; replacing its asset" + else + gh release create "$TAG" --repo "$GITHUB_REPOSITORY" \ + --title "Native toolchain $TAG" \ + --notes-file dist/manifest.txt + fi + gh release upload "$TAG" "$ARCHIVE" --repo "$GITHUB_REPOSITORY" --clobber + + # The point of publishing is that someone else can consume it, so assert + # what the consumer will see rather than what we just sent. + gh release view "$TAG" --repo "$GITHUB_REPOSITORY" \ + --json tagName,assets \ + --jq '.assets[] | select(.name == "XForgeNativeToolchain-arm64-ios.tar.gz") | "\(.name): \(.size) bytes"' + { + echo '### Published toolchain bundle' + echo + echo "- tag: \`$TAG\`" + echo "- asset: \`$ARCHIVE\` ($((bytes / 1048576)) MB)" + echo "- install in an app build: \`with_toolchain: true\`" + } >> "$GITHUB_STEP_SUMMARY" diff --git a/CHANGELOG.md b/CHANGELOG.md index 80252bc..37939fd 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,17 @@ they are ordinary directories in the app's container. - **`App/Views/Toolchain/ToolchainView.swift`** — the toolchain and SDK screen, now native: it reports which libraries are linked in, installs or removes the Darwin SDK in the app container, and runs a compile smoke test. +- **The toolchain bundle is published, not just uploaded.** `native-toolchain.yml` + attaches the finished bundle to a release tagged + `toolchain---ios-sdk` — the tag names what is + inside, because the tag is what a consumer asks for and a bundle that could not be + substituted for another must never be handed over as "the newest one". A dispatched + run is the only thing that publishes; a pull request or push run builds the + LLVM-only half and stays out of the way. `NativeToolchain/install-bundle.sh --release + [tag]` (and `make toolchain-release`) installs one without ever having run the + workflow, and `build-ipa.yml` takes `with_toolchain: true` plus an optional + `toolchain_tag` to link it into the app. Workflow artifacts expire and cannot be + fetched by tag; a release can. ### Changed - **Build path.** `NativeBuildExecutor` compiles each translation unit through the @@ -77,13 +88,15 @@ they are ordinary directories in the app's container. directory in the container rather than a file inside an opaque fakefs. ### Known gap -- **The Swift frontend is not in the toolchain artifact.** - `.github/workflows/native-toolchain.yml` builds Clang and Mach-O LLD only, so a - project whose sources are Swift still cannot be compiled. The build now fails at - the Swift compile step with `swiftFrontendMissing`, naming the count of files, - instead of silently needing a guest. C and Objective-C targets compile and link - today. The guest is not coming back as a fallback: the honest replacement for it - is a toolchain artifact that carries the frontend. +- **Swift needs a bundle that carries the frontend.** `.github/workflows/native-toolchain.yml` + builds the Swift frontend libraries when it is dispatched with `with_swift=true`, + and that bundle says `swift_frontend=1` in its manifest; the default dispatch and + every pull-request run build the Clang/LLD half only. Against a bundle without the + frontend a project whose sources are Swift still fails at the Swift compile step + with `swiftFrontendMissing`, naming the count of files, instead of silently needing + a guest. C and Objective-C targets compile and link either way. The guest is not + coming back as a fallback: the honest replacement for it is a bundle that carries + the frontend, which is now a thing you can build and install. - **Asset catalogs are copied uncompiled**, with a warning: `actool` is a macOS-only tool with no open-source replacement. diff --git a/Docs/DESIGN.md b/Docs/DESIGN.md index 33e0ca8..e74b9e5 100644 --- a/Docs/DESIGN.md +++ b/Docs/DESIGN.md @@ -24,8 +24,9 @@ call in XForge's own process: paths. Nothing compiles until this value exists, and nothing in the executor re-decides any of it. 2. **Compile.** Swift sources would go through `swift::performFrontend` — but the - frontend libraries are not in the toolchain artifact yet, so a plan with Swift - files is refused at this point (§2, third bullet). C, Objective-C and + frontend libraries are only in a bundle built with `with_swift`, so a plan with + Swift files is refused at this point when the installed bundle has none (§2, + third bullet). C, Objective-C and Objective-C++ sources compile through clang's `CompilerInstance` with `EmitObjAction`. Both take an argument list built by `NativeToolchainInvocation`, against the iPhoneOS SDK in the Darwin SDK bundle. @@ -72,10 +73,11 @@ what is needed, and the executor states plainly where it can only copy: ## 2. Where the toolchain comes from The Clang/LLD libraries are **cross-built for iPhoneOS in CI** on a macOS runner -(`.github/workflows/native-toolchain.yml`), flattened into a single static archive -and published as a workflow artifact. `NativeToolchain/install-bundle.sh` unpacks -that artifact into `Vendor/NativeToolchain` and `NativeToolchain/prepare-xcode.sh` -writes the xcconfig the app target consumes. +(`.github/workflows/native-toolchain.yml`), merged into a single static archive and +published as a release asset (tagged `toolchain---ios…`). +`NativeToolchain/install-bundle.sh --release` fetches it, or unpacks a local archive, +into `Vendor/NativeToolchain`, and `NativeToolchain/prepare-xcode.sh` writes the +xcconfig the app target consumes. The consequences of linking the compiler in rather than shipping it as data: diff --git a/Docs/NATIVE-TOOLCHAIN.md b/Docs/NATIVE-TOOLCHAIN.md index 634c03f..30cf3f5 100644 --- a/Docs/NATIVE-TOOLCHAIN.md +++ b/Docs/NATIVE-TOOLCHAIN.md @@ -31,76 +31,136 @@ alternative would be a remote build host, which is a different product. simulator build) compiles and runs with no toolchain present. - `.github/workflows/native-toolchain.yml` — builds the iOS-hosted LLVM/Clang/LLD libraries and publishes the bundle. See below. -- `NativeToolchain/install-bundle.sh ` — installs that bundle. +- `NativeToolchain/install-bundle.sh | --release [tag]` — installs that + bundle, from a local file or from its release. - `NativeToolchain/prepare-xcode.sh` — writes the xcconfig the app target consumes. `build-ipa.yml` runs it too, so CI never depends on a bundle being installed locally; on a checkout with no bundle it writes the disabled configuration and CI builds the stub. -## What the bundle contains — and what it does not +## What a bundle contains — and which half it has The workflow cross-builds, for iPhoneOS arm64: - **Clang**, as the library set the bridge calls (`clangCodeGen`, `clangFrontend`, `clangFrontendTool`, `clangDriver`, `clangSerialization`, `clangSema`, `clangParse`, - `clangAST`, `clangLex`, `clangBasic`); -- **Mach-O LLD** (`lldMachO`, `lldCommon`). + `clangAST`, `clangLex`, `clangBasic`), plus `clangDependencyScanning`; +- **Mach-O LLD** (`lldMachO`, `lldCommon`); +- **`LLVMOrcJIT`**, which Swift's in-process JIT needs; +- and, in a `with_swift` build, **Swift's frontend libraries** (`swiftFrontendTool` + and the swiftAST / swiftSema / IRGen / ClangImporter set it pulls in), built from + `swiftlang/swift` against `swiftlang/llvm-project` — for iOS, as a library set. + +The target list is not "everything the app might use" but the closure the bundle has +to satisfy, because the link check force-loads every archive: each member's undefined +symbols must resolve *inside* the bundle. `clangDependencyScanning` (used by +`ClangImporter` to scan a target's module dependencies) and `LLVMOrcJIT` (used by +`SwiftMaterializationUnit.cpp`) are the two that no clang library pulls in by itself, +and leaving them out produced a bundle that compiled and then failed to link — with +the missing symbols visible only as a truncated tail. The link check now prints all +of them. + +`manifest.txt` is how a consumer knows what it has: -It does **not** contain the Swift frontend. `swift-frontend` is not part of -`llvm-project`: it has to be built from `swiftlang/swift` against -`swiftlang/llvm-project` **for iOS**, as a library set. That port is the long pole -and it is not done, so: +``` +bundle_tag=toolchain---ios-sdk +target=arm64-apple-ios +deployment_target=17.0 +llvm_commit=… +swift_frontend=0|1 +swift_ref=swift-6.2.4-RELEASE|none +xcode=16.4 +ios_sdk=18.5 +built_at=… +archives=lib/XForgeToolchain.libraries.txt +header_roots=include-generated include +``` -- **Swift sources cannot be compiled today.** A plan that contains Swift files - fails at the compile stage with `swiftFrontendMissing`, naming the count of files. -- C, Objective-C and Objective-C++ sources compile and link today. -- The bridge does expose the entry point (`xf_native_swift_frontend`), gated behind - `XFORGE_HAS_SWIFT_FRONTEND`, so the port has somewhere to land. Until a frontend - library set exists, the gate is closed and the UI reports `swift-frontend: missing`. +`prepare-xcode.sh` reads `swift_frontend` and defines `XFORGE_HAS_SWIFT_FRONTEND` +accordingly, so a bundle without the frontend still builds the whole app: the bridge +compiles to the stub for that entry point and the UI reports `swift-frontend: missing` +rather than claiming a compiler it does not have. -The two halves are deliberately independent: the Clang/LLD port can be validated -and used on its own while the considerably larger Swift compiler port is brought up. +The two halves are deliberately independent. The Clang/LLD half is a complete +deliverable for C, Objective-C and Objective-C++ targets and can be validated on its +own while the considerably larger Swift port is brought up; the tag says which one a +bundle is (`…-swift--…` versus `…-noswift-…`), so they never have to be guessed +apart. ## How the bundle is built (CI) `.github/workflows/native-toolchain.yml`, job `llvm-ios`, on a `macos-15` runner: -1. Build host TableGen tools (`llvm-tblgen`, `clang-tblgen`) natively — the iOS - cross-build needs a native TableGen to generate tables it can run. -2. Configure LLVM for iPhoneOS arm64 with `LLVM_ENABLE_PROJECTS="clang;lld"`, +1. Check out Swift's LLVM fork (`swiftlang/llvm-project`, the branch paired with the + Swift version) and, for a `with_swift` build, `swiftlang/swift`, `swift-cmark`, + `swift-syntax` and `swift-experimental-string-processing`. +2. Restore the build trees from the Actions cache. The key names the LLVM revision, + the deployment target and the runner's Xcode/SDK, so a hit means "the compiler for + exactly these inputs" — and the sources are then aged to a fixed past date, so the + restored tree is newer than every source. Without that ageing ninja treats the + whole tree as stale and rebuilds it: the restore paid seconds to avoid a + 2601-edge rebuild and got it anyway. +3. Build the host tools natively (`llvm-tblgen`, `clang-tblgen`, `llvm-ar`, + `llvm-nm`) and, with Swift, a native `cmark` plus the iOS cmark archives. +4. Configure LLVM for iPhoneOS arm64 with `LLVM_ENABLE_PROJECTS="clang;lld"`, `LLVM_TARGETS_TO_BUILD=AArch64`, deployment target 17.0, tools and examples off. -3. Build the Clang and LLD library targets listed above. -4. **Compile-check the bridge against the headers just built** — `clang++ -arch arm64 + Skipped when the restored tree came back already configured and built. +5. Build the library targets listed above (~37 minutes cold), then, with Swift, + the frontend libraries (~34 minutes cold). +6. **Compile-check the bridge against the headers just built** — `clang++ -arch arm64 -isysroot -DXFORGE_HAS_LLVM=1 -c App/NativeToolchain/NativeToolchainBridge.mm`. This is the check that keeps a header change in LLVM from silently breaking the bridge in a device build nobody can run yet. -5. Stage the headers into two roots — sources into `include/`, the build tree into - `include-generated/`, because they disagree about `swift/bridging` — then - **flatten every - static archive in the build into one** `dist/lib/libXForgeNativeToolchain.a` with - `libtool -static`. Flattening is deliberate: the app then links one archive - instead of depending on LLVM's internal archive ordering, which would otherwise - have to be reproduced by hand in the xcconfig. -6. **Link-check**: link a program that calls `xf_native_toolchain_available()` - against the flattened archive and the bridge object, and assert the result is a - real Mach-O. A library set that compiles but cannot link is the failure this step - exists to catch. -7. Write `manifest.txt` (`target=arm64-apple-ios`, `deployment_target=17.0`, - `llvm_commit=…`, `built_at=…`, `archive=…`) and tar the result as - `XForgeNativeToolchain-arm64-ios.tar.gz`, uploaded as the workflow artifact of the - same name. - -It is dispatched by hand (or by a pull request that touches `App/NativeToolchain/**`), -because it takes hours and its output changes only when the LLVM commit does. +7. Stage the headers into two roots — sources into `include/`, the build tree into + `include-generated/`, because they disagree about `swift/bridging` — and merge + every static archive in the build into one `dist/lib/libXForgeNativeToolchain.a` + with `llvm-ar -M` and an MRI `addlib` script. `addlib` copies *every* member: + `libtool -static` de-duplicates them by name (`duplicate member name 'X86.cpp.o' + from libclangCodeGen.a and …`), and a dropped member is a silently missing object — + which is how the JIT symbols once came out undefined while their object was + demonstrably in the archive. The merge fails if it produced fewer members than it + consumed. +8. **Link-check**: link a program that calls `xf_native_toolchain_available()` against + the merged archive and the bridge object, with every archive force-loaded, and + assert the result is a real Mach-O. Force-loading makes this the strictest check + available: it fails unless every member resolves inside the bundle. On failure it + prints every undefined symbol, not a tail of them. +9. Write `manifest.txt` and tar the result as `XForgeNativeToolchain-arm64-ios.tar.gz` + — uploaded as the workflow artifact of the same name, and, on a hand-dispatched + run that succeeded, **published as a release asset** tagged + `toolchain---ios-sdk`. + +The cache is why none of this needs doing twice. A run whose inputs are unchanged +restores both trees and goes straight to the staging and link checks. Everything +unusual about the workflow — the key salts, the save gating (a tree is stored only if +the build that produced it finished), the "skip the configure when the tree is already +built" rule, the aged sources — exists for that one property, and each was added after +a run that spent an hour rebuilding what the cache already had. + +Publishing is what takes the cost off everyone else. `build-ipa.yml` installs the +bundle from the newest `toolchain-*` release, so building the app never waits for a +toolchain run and a toolchain run never has to be in the same repository state as the +app it serves. Only dispatched runs publish: a pull request or push run builds the +LLVM-only half, and publishing that as the newest bundle is exactly what a consumer +asking for "the newest one" must not get. ## How it is installed ```bash -make toolchain ARCHIVE=XForgeNativeToolchain-arm64-ios.tar.gz +make toolchain-release # the newest published bundle make gen ``` -`install-bundle.sh` unpacks the archive into `Vendor/NativeToolchain` and refuses an +A specific release, a local tarball, or a bundle that was built but not published: + +```bash +make toolchain-release TAG=toolchain-<…> +make toolchain ARCHIVE=XForgeNativeToolchain-arm64-ios.tar.gz +NativeToolchain/install-bundle.sh --release artifact +``` + +`install-bundle.sh` fetches the asset (the GitHub CLI if it is installed, `curl` +otherwise) or unpacks the local archive into `Vendor/NativeToolchain`, and refuses an archive that is missing `manifest.txt`, `include/`, `include-generated/` or `lib/` — a partial bundle would otherwise surface as an inscrutable compile error much later. It then runs `prepare-xcode.sh`, which writes `Support/NativeToolchain.generated.xcconfig`: @@ -161,12 +221,18 @@ fails here in a way the capability rows cannot detect. The screen's own footer describes the check as compiling *and linking*; the button currently compiles only, and the build-time link check lives in the toolchain workflow (step 6 above). -## Known gap - -Until the Swift frontend libraries exist for iPhoneOS: - -- Swift projects cannot be built on device. This is not a fallback situation — there - is no guest to fall back to — it is the remaining port. -- The Swift capability row is reported honestly (`missing`), the Swift smoke test - remains unreachable, and a build that needs it fails up front with - `swiftFrontendMissing` and the file count. +## Which half you have + +The frontend is in the bundle or it is not, and `manifest.txt` says which — so the +honest statement is per bundle rather than per project: + +- A **`noswift`** bundle has no frontend. Swift projects cannot be built from it and + fail up front with `swiftFrontendMissing` and the file count; that is not a fallback + situation — there is no guest to fall back to — it is the other half of the port, + while C, Objective-C and Objective-C++ work in full. +- A **`swift-*`** bundle carries the frontend, so the Swift capability row reports it + like any other part and the Swift smoke test is reachable. Its Swift revision has to + match the app's: the frontend reads the deployment SDK's standard-library modules, + which is why the workflow pins the LLVM/Swift refs to the pair the Xcode in use + ships, and why `build-ipa.yml` warns when the installed bundle and the runner's + Xcode disagree. diff --git a/Makefile b/Makefile index be2c4a2..43058e5 100644 --- a/Makefile +++ b/Makefile @@ -1,25 +1,30 @@ -.PHONY: toolchain icon gen build test ipa clean +.PHONY: toolchain toolchain-release icon gen build test ipa clean XCODE := xcodebuild SCHEME := XForge ## Install the native toolchain bundle into Vendor/NativeToolchain. ## -## The bundle is built by the `Native iOS Toolchain` workflow (Clang + Mach-O LLD -## for iPhoneOS, cross-built on a Mac) and attached to the run as an artifact. -## Without it the app still builds and runs: the C bridge compiles to a -## "not available" stub and the Toolchain screen says so, which is what a plain -## clone should do. +## The bundle is built by the `Native iOS Toolchain` workflow and published as a +## release asset whose tag names its contents, so the newest one is a well-defined +## thing to ask for. Without a bundle the app still builds and runs: the C bridge +## compiles to a "not available" stub and the Toolchain screen says so. +## +## make toolchain-release # newest published bundle +## make toolchain-release TAG=... # a specific one +## make toolchain ARCHIVE=... # a local tarball toolchain: @if [ -z "$(ARCHIVE)" ]; then \ echo "usage: make toolchain ARCHIVE=XForgeNativeToolchain-arm64-ios.tar.gz"; \ - echo ""; \ - echo "Download the artifact from a successful 'Native iOS Toolchain' run:"; \ - echo " gh run download --repo R0GUEEE/XForge --name XForgeNativeToolchain-arm64-ios"; \ + echo " make toolchain-release [TAG=...]"; \ exit 64; \ fi @bash NativeToolchain/install-bundle.sh "$(ARCHIVE)" +## Install the newest published toolchain bundle (see `toolchain` above) +toolchain-release: + @bash NativeToolchain/install-bundle.sh --release $(TAG) + ## Regenerate the app icon + accent colour asset catalog (needs Pillow) icon: python3 Tools/gen-appicon.py diff --git a/NativeToolchain/install-bundle.sh b/NativeToolchain/install-bundle.sh index f05e2dc..842e904 100644 --- a/NativeToolchain/install-bundle.sh +++ b/NativeToolchain/install-bundle.sh @@ -1,15 +1,129 @@ #!/usr/bin/env bash set -euo pipefail -if [ "$#" -ne 1 ]; then - echo "usage: $0 XForgeNativeToolchain-arm64-ios.tar.gz" >&2 - exit 64 -fi +# usage: install-bundle.sh +# install-bundle.sh --release [tag] +# install-bundle.sh --release artifact # newest workflow artifact +# +# The bundle is built by the `Native iOS Toolchain` workflow, which publishes it as +# a release asset whose tag names its contents (`toolchain--swift--ios…`). +# `--release` fetches that asset, so a machine that has never run the workflow — or +# came to it after the workflow artifact expired — can still install the compiler. +# With no tag, the newest published `toolchain-*` release is installed. + +REPO="${XFORGE_REPO:-R0GUEEE/XForge}" +ASSET="XForgeNativeToolchain-arm64-ios.tar.gz" +ARTIFACT_NAME="XForgeNativeToolchain-arm64-ios" ROOT="$(cd "$(dirname "$0")/.." && pwd)" -ARCHIVE="$1" DEST="$ROOT/Vendor/NativeToolchain" +TMP="" +# `if` rather than `[ … ] && …`: a false test would leave the trap returning +# non-zero, which is not something to have running at exit. +cleanup() { if [ -n "$TMP" ]; then rm -rf "$TMP"; fi; } +trap cleanup EXIT + +usage() { + cat >&2 < + $0 --release [tag] + $0 --release artifact + + --release [tag] fetch a published toolchain release asset (default: newest) + artifact the newest workflow artifact instead (needs the GitHub CLI) +EOF +} + +have_gh() { command -v gh >/dev/null 2>&1; } + +# The newest `toolchain-*` release. `gh release list` is newest-first; without the +# CLI this asks the API directly, which is rate-limited for anonymous callers but +# fine for the one-off case this is for. +api() { + # A token is only used when one happens to be in the environment: it is what makes + # this work against a private fork, and it raises the API's rate limit otherwise. + local token="${GH_TOKEN:-${GITHUB_TOKEN:-}}" + if [ -n "$token" ]; then + curl -fsSL -H "Authorization: Bearer $token" "$1" + else + curl -fsSL "$1" + fi +} + +newest_tag() { + if have_gh; then + gh release list --repo "$REPO" --limit 100 --json tagName \ + --jq '[.[] | select(.tagName | startswith("toolchain-"))][0].tagName' + return + fi + command -v python3 >/dev/null 2>&1 || { + echo "error: need either the GitHub CLI or python3 to find the newest release" >&2 + exit 69 + } + api "https://api.github.com/repos/$REPO/releases?per_page=100" \ + | python3 -c 'import json,sys +try: + releases = json.load(sys.stdin) +except ValueError: + releases = [] +if not isinstance(releases, list): + releases = [] +print(next((r["tag_name"] for r in releases + if isinstance(r, dict) and r.get("tag_name", "").startswith("toolchain-")), ""))' +} + +case "${1:-}" in + "") + usage + exit 64 + ;; + -h|--help) + usage + exit 0 + ;; + --release) + TAG="${2:-}" + TMP="$(mktemp -d)" + + if [ "$TAG" = "artifact" ]; then + have_gh || { echo "error: --release artifact needs the GitHub CLI" >&2; exit 69; } + gh run download --repo "$REPO" --name "$ARTIFACT_NAME" --dir "$TMP" + else + if [ -z "$TAG" ]; then + TAG="$(newest_tag)" + fi + [ -n "$TAG" ] && [ "$TAG" != "null" ] || { + echo "error: no published toolchain release in $REPO" >&2 + echo " run the 'Native iOS Toolchain' workflow by hand, with_swift=true" >&2 + exit 66 + } + echo "installing toolchain bundle from release $TAG" + if have_gh; then + gh release download "$TAG" --repo "$REPO" \ + --pattern "XForgeNativeToolchain-*.tar.gz" --dir "$TMP" || { + echo "error: no $ASSET asset in release $TAG of $REPO" >&2 + exit 66 + } + else + curl -fSL --retry 3 -o "$TMP/$ASSET" \ + "https://github.com/$REPO/releases/download/$TAG/$ASSET" || { + echo "error: no $ASSET asset in release $TAG of $REPO" >&2 + exit 66 + } + fi + fi + ARCHIVE="$TMP/$ASSET" + ;; + -*) + usage + exit 64 + ;; + *) + ARCHIVE="$1" + ;; +esac + [ -f "$ARCHIVE" ] || { echo "error: archive not found: $ARCHIVE" >&2; exit 66; } rm -rf "$DEST" diff --git a/README.md b/README.md index 38a751a..e81d6c5 100644 --- a/README.md +++ b/README.md @@ -114,14 +114,16 @@ to a stub and says so on the Toolchain screen. To build with the compiler *linked in*, install the toolchain bundle first: ```bash -make toolchain ARCHIVE=XForgeNativeToolchain-arm64-ios.tar.gz +make toolchain-release # the newest published bundle make gen ``` -`install-bundle.sh` unpacks the archive into `Vendor/NativeToolchain` and runs -`prepare-xcode.sh`, which writes `Support/NativeToolchain.generated.xcconfig` — -the file the app target consumes. That xcconfig is checked in *with the backend -disabled*, so a clone without the bundle is buildable; see +`install-bundle.sh --release` fetches the bundle from its release (a specific tag, +or a local tarball, also work — see `make toolchain`), unpacks it into +`Vendor/NativeToolchain` and runs `prepare-xcode.sh`, which writes +`Support/NativeToolchain.generated.xcconfig` — the file the app target consumes. +That xcconfig is checked in *with the backend disabled*, so a clone without the +bundle is buildable; see [Docs/NATIVE-TOOLCHAIN.md](Docs/NATIVE-TOOLCHAIN.md). The only build prerequisite beyond XcodeGen is the toolchain bundle, and only if @@ -219,12 +221,16 @@ make icon # rewrites Support/Assets.xcassets publishes the unsigned IPA for sideloading. It runs `prepare-xcode.sh` too, so CI never depends on a toolchain bundle being installed in the checkout. - **`native-toolchain.yml`** — cross-builds LLVM's Clang and Mach-O LLD for - iPhoneOS on a macOS runner, flattens the static dependency graph into one - archive, compile-checks and link-checks `NativeToolchainBridge.mm` against the - LLVM headers it just built, and uploads - `XForgeNativeToolchain-arm64-ios.tar.gz` as a workflow artifact. It is - dispatched by hand or by a pull request that touches `App/NativeToolchain/**`, - because it takes hours and its output changes only when the LLVM commit does. + iPhoneOS (and, with `with_swift=true`, Swift's frontend libraries) on a macOS + runner, merges the static archives into one, compile-checks and link-checks + `NativeToolchainBridge.mm` against the LLVM headers it just built, and publishes + `XForgeNativeToolchain-arm64-ios.tar.gz` as a release asset tagged + `toolchain---ios-sdk` (and as a workflow + artifact). It is dispatched by hand or by a pull request that touches + `App/NativeToolchain/**`; only a dispatched run publishes, because a run that + never built the Swift half must not become "the newest bundle". Its build trees + are cached, so a run whose inputs are unchanged finishes in minutes rather than + the ~80 minutes a cold build takes. See [Docs/NATIVE-TOOLCHAIN.md](Docs/NATIVE-TOOLCHAIN.md). - **`ios-share.yml`** — builds the app for the simulator and keeps that simulator usable from a machine that is not a Mac, so a build can be tried by hand. From 93425c8b32422dbe2bb0a324c95c2d7150f75eeb Mon Sep 17 00:00:00 2001 From: Felix Date: Thu, 24 Sep 2026 23:13:40 -0400 Subject: [PATCH 36/54] gitignore: an installed toolchain bundle is not source Installing the bundle is now an ordinary local action (`make toolchain-release`), and it unpacks hundreds of megabytes of archives into the checkout, where every file showed up as untracked. They are reproducible from the release the manifest names, so they are ignored; Vendor/NativeToolchain/.gitkeep stays tracked so a clone still has the directory to install into. --- .gitignore | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/.gitignore b/.gitignore index bb9d76c..76f5d84 100644 --- a/.gitignore +++ b/.gitignore @@ -12,6 +12,17 @@ XForge.xcodeproj/ # Where build-ipa.yml clones package checkouts (`clonedSourcePackagesDirPath`). .spm/ +# The native toolchain bundle. Installing it (`make toolchain-release`) unpacks +# hundreds of megabytes of LLVM/Swift archives into the checkout; they are +# reproducible from the release the manifest names, and committing them is never +# what anyone means to do. The directory and the gcc-style placeholder stay, so a +# clone has somewhere to install into. +Vendor/NativeToolchain/* +!Vendor/NativeToolchain/.gitkeep + +# Where build-ipa.yml unpacks a fetched toolchain bundle. +.toolchain-artifact/ + # macOS .DS_Store From e6173ec0f71aaaadfbb23d6934bfb2749711d117 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 00:58:46 -0400 Subject: [PATCH 37/54] toolchain: stage the whole header tree, and check the bundle, not the sources The first app build that linked the published bundle failed on `Vendor/NativeToolchain/include/llvm/Support/DataTypes.h:19: 'llvm-c/DataTypes.h' file not found`. `llvm/include/llvm-c/` was never staged: the staging copied `llvm/`, `clang/` and `lld/`, which reads like the whole tree and is not. The reason it went unnoticed is the shape of the check: the bridge compile-check compiles against the *source* tree, where `llvm-c/` is present. Nothing compiled the bridge against the staged bundle until a host app did, 20 minutes into an app build. - Stage every top-level directory under each project's `include/`, sources and generated both, instead of naming the ones that looked right. `llvm-c` is required from now on, so leaving it behind fails the packaging step. - The staging step now compiles NativeToolchainBridge.mm against `dist/include` and `dist/include-generated` and links the smoke test with *that* object, so a header the bundle is missing fails here in seconds. - build-ipa's bundle check no longer cries wolf about the runner's Swift version: that Swift only bootstrapped the frontend's build. What has to match is the Swift inside the Darwin SDK the app installs at runtime, which this step cannot see, so it prints both and says which one governs. --- .github/workflows/build-ipa.yml | 26 +++++------ .github/workflows/native-toolchain.yml | 61 +++++++++++++++++++++----- 2 files changed, 63 insertions(+), 24 deletions(-) diff --git a/.github/workflows/build-ipa.yml b/.github/workflows/build-ipa.yml index 98bec6d..0acadb8 100644 --- a/.github/workflows/build-ipa.yml +++ b/.github/workflows/build-ipa.yml @@ -122,12 +122,11 @@ jobs: cat Vendor/NativeToolchain/manifest.txt cat Support/NativeToolchain.generated.xcconfig - # The frontend can only read the deployment SDK's standard-library modules when - # it is the same Swift version as the app it is linked into. The bundle records - # what it was built from; this says so loudly instead of shipping an app whose - # Swift support fails on a device nobody can debug here. A mismatch is a - # warning, not a failure: the exact version string of an Apple toolchain is not - # something to bet the build on. + # The manifest is what the app bundle's compiler is made of, so say what it is + # before building with it: the SDK the archives and the bridge were compiled + # against, and the Swift revision the frontend speaks. Both have to line up + # with what the app will actually compile against on a device — the Darwin SDK + # it installs at runtime — and neither is visible anywhere else in this log. - name: Check the toolchain bundle against this Xcode if: ${{ inputs.with_toolchain }} run: | @@ -149,14 +148,15 @@ jobs: exit 0 fi - built_version="$(get swift_ref | sed -n 's/^swift-\([0-9][0-9.]*\).*/\1/p')" - app_version="$(xcrun swift --version 2>/dev/null | head -n 1)" + # The frontend's own version matters, and the runner's is not it: this + # Swift is what *bootstrapped* the frontend's build, while the frontend + # itself comes from the ref below and has to match the Swift version + # inside the Darwin SDK the app installs on device. Printing both is the + # point — a mismatch between them is expected and is not a problem; a + # mismatch between the bundle and the Darwin SDK is, and this step cannot + # see that one. echo " Swift frontend built from $(get swift_ref)" - echo " this runner's Swift: $app_version" - case "$app_version" in - *"$built_version"*) echo "Swift versions agree" ;; - *) echo "::warning::the bundle's Swift frontend is $built_version, this Xcode's Swift is '$app_version' — the frontend may not read this SDK's standard library modules" ;; - esac + echo " this runner's Swift (used to bootstrap it): $(xcrun swift --version 2>/dev/null | head -n 1)" - name: Prepare the native toolchain configuration run: | diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 138c78a..deb90d8 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -709,14 +709,30 @@ jobs: mkdir -p dist/include-generated dist/include dist/lib stage_headers() { cp -R -L "$1" "$2"; } - stage_headers llvm-project/llvm/include/llvm dist/include/ - stage_headers llvm-project/clang/include/clang dist/include/ - stage_headers llvm-project/lld/include/lld dist/include/ + # Every top-level header root the sources ship, not just the one named + # after the project. `llvm/include/llvm-c/DataTypes.h` is included from + # `llvm/include/llvm/Support/DataTypes.h`, and staging only `llvm/` left it + # out of the bundle: the app build failed on it while the bridge + # compile-check passed, because that check compiles against the source + # tree rather than against what is shipped. Same for `clang-c`, and for + # anything a future revision grows. + for source_root in llvm-project/llvm/include \ + llvm-project/clang/include \ + llvm-project/lld/include; do + for dir in "$source_root"/*/; do + stage_headers "${dir%/}" dist/include/ + done + done - stage_headers llvm-build/include/llvm dist/include-generated/ - if [ -d llvm-build/tools/clang/include/clang ]; then - stage_headers llvm-build/tools/clang/include/clang dist/include-generated/ - fi + # The generated headers, likewise per top-level directory: the build tree + # writes `llvm/Config/…` and `clang/…`, and `llvm-c` if a revision ever + # generates one. + for generated_root in llvm-build/include llvm-build/tools/clang/include; do + [ -d "$generated_root" ] || continue + for dir in "$generated_root"/*/; do + stage_headers "${dir%/}" dist/include-generated/ + done + done if [ "${{ inputs.with_swift }}" = "true" ]; then stage_headers swift/include/swift dist/include/ @@ -727,7 +743,7 @@ jobs: done fi - for required in dist/include/llvm dist/include/clang dist/include/lld; do + for required in dist/include/llvm dist/include/llvm-c dist/include/clang dist/include/lld; do [ -d "$required" ] || { echo "::error::missing staged header root $required"; exit 1; } done @@ -793,6 +809,29 @@ jobs: EOF SDK="$(xcrun --sdk iphoneos --show-sdk-path)" + + # Compile the bridge against the *staged* roots and link the result. This + # is the check that the shipped bundle is self-contained — the earlier + # compile-check compiles against the source tree, which says nothing about + # what was staged. A header missing here is a header missing from every + # app that links this bundle, and finding that in a 20-minute app build + # instead of here is how `llvm-c/` stayed missing. + BRIDGE_DEFINES=(-DXFORGE_HAS_LLVM=1) + if [ "${{ inputs.with_swift }}" = "true" ]; then + BRIDGE_DEFINES+=(-DXFORGE_HAS_SWIFT_FRONTEND=1) + fi + xcrun --sdk iphoneos clang++ \ + -std=c++20 \ + -arch arm64 \ + -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET \ + -isysroot "$SDK" \ + "${BRIDGE_DEFINES[@]}" \ + -I dist/include \ + -I dist/include-generated \ + -I App/NativeToolchain \ + -c App/NativeToolchain/NativeToolchainBridge.mm \ + -o /tmp/StageBundleBridge.o + xcrun --sdk iphoneos clang++ \ -std=c++20 \ -arch arm64 \ @@ -807,7 +846,7 @@ jobs: -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET \ -isysroot "$SDK" \ /tmp/native_link_smoke.o \ - /tmp/NativeToolchainBridge.o \ + /tmp/StageBundleBridge.o \ $(for library in dist/lib/*.a; do printf ' -Wl,-force_load,%s' "$library"; done) \ -lc++ -lz -liconv -lsqlite3 \ -framework Foundation \ @@ -825,8 +864,8 @@ jobs: } file /tmp/XForgeNativeToolchainSmoke - file /tmp/NativeToolchainBridge.o - cp /tmp/NativeToolchainBridge.o dist/ + file /tmp/StageBundleBridge.o + cp /tmp/StageBundleBridge.o dist/ # The manifest is what the consumers read, so everything they have to # agree with is in it: `bundle_tag` is the release this can be fetched From 59642ca4627ea86225381bb965183a08f4943c46 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 01:44:30 -0400 Subject: [PATCH 38/54] bundle: the xcconfig pointed the linker at an empty path MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `prepare-xcode.sh` builds XFORGE_NATIVE_LDFLAGS from the archives in the bundle, escaping `$(SRCROOT)` so Xcode keeps it. It escaped the parameter expansion too: ARCHIVES+=("\$(SRCROOT)/\${lib#"$ROOT/"}") so the *literal* text `${lib#/Users/runner/work/XForge/XForge/}` went into the xcconfig. Xcode reads that as a variable reference, finds nothing, and hands the linker `$(SRCROOT)/` — the archive path with the archive missing: ld: file cannot be mmap()ed, errno=22 path=/Users/runner/work/XForge/XForge/ Every build with the compiler linked in failed there, which is why no app build had ever linked the bundle. `${lib#…}` is now expanded here, and the script fails outright if the file it is about to write still contains a `${…}` or an empty archive path, so this cannot reach a linker again. Also `continue-on-error` on both cache saves: an entry whose key already exists cannot be reserved, and that is not a reason to fail a build that succeeded. Verified against a fixture bundle for all three paths (no bundle, bundle with no archives, bundle with one archive) and against an injected `${…}`. --- .github/workflows/native-toolchain.yml | 5 ++++ NativeToolchain/prepare-xcode.sh | 32 +++++++++++++++++++++++++- 2 files changed, 36 insertions(+), 1 deletion(-) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index deb90d8..343b024 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -450,6 +450,10 @@ jobs: # rewritten, and every later run then "hits" it and rebuilds anyway. - name: Save the LLVM builds if: ${{ steps.llvm-build.outcome == 'success' && steps.llvm-cache.outputs.cache-hit != 'true' }} + # A refused save must not fail a build that succeeded: an entry whose key + # already exists cannot be reserved, and the other workflow that stored it + # stored the same tree. The cache is an optimisation; the build is the work. + continue-on-error: true uses: actions/cache/save@v4 with: key: ${{ steps.llvm-cache.outputs.cache-primary-key }} @@ -565,6 +569,7 @@ jobs: - name: Save the Swift build if: ${{ inputs.with_swift && !inputs.configure_only && steps.swift-build.outcome == 'success' && steps.swift-cache.outputs.cache-hit != 'true' }} + continue-on-error: true uses: actions/cache/save@v4 with: key: ${{ steps.swift-cache.outputs.cache-primary-key }} diff --git a/NativeToolchain/prepare-xcode.sh b/NativeToolchain/prepare-xcode.sh index dc296a6..f235f97 100644 --- a/NativeToolchain/prepare-xcode.sh +++ b/NativeToolchain/prepare-xcode.sh @@ -36,9 +36,16 @@ ARCHIVES=() ARCHIVE_LIST="$(mktemp)" trap 'rm -f "$ARCHIVE_LIST"' EXIT find "$BUNDLE/lib" -maxdepth 1 -type f -name '*.a' >"$ARCHIVE_LIST" +# Only `$(SRCROOT)` is escaped: that one is meant for Xcode to expand. The +# parameter expansion is not — it has to happen here, in this shell. Escaping it +# too (`\${lib#…}`) put the literal text into the xcconfig, where Xcode read it as +# a variable named `lib#/Users/… `, resolved it to nothing, and handed the linker +# `$(SRCROOT)/` as a file name: `ld: file cannot be mmap()ed, errno=22 +# path=/Users/runner/work/XForge/XForge/`. Every build with the compiler linked in +# failed like that, and the guard at the end of this script now catches it here. while IFS= read -r lib; do [ -n "$lib" ] || continue - ARCHIVES+=("\$(SRCROOT)/\${lib#"$ROOT/"}") + ARCHIVES+=("\$(SRCROOT)/${lib#"$ROOT/"}") done <"$ARCHIVE_LIST" if [ "${#ARCHIVES[@]}" -eq 0 ]; then @@ -74,4 +81,27 @@ fi echo " -lc++ -lz -liconv -lsqlite3 -framework Foundation" } >"$OUT" +# A `$(…)` is for Xcode; a `${…}` is this script failing to expand something, and +# Xcode will resolve it to the empty string. That is worth failing for: it took a +# full archive of a 20-minute app build to find the last one. Written as `case` +# rather than `grep -q`: no pipeline, no dependence on which grep is installed. +while IFS= read -r line; do + case "$line" in + *'${'*) + echo "error: $OUT contains an unexpanded shell expansion:" >&2 + echo " $line" >&2 + exit 65 + ;; + esac +done <"$OUT" + +for archive in "${ARCHIVES[@]}"; do + case "$archive" in + *'$(SRCROOT)/'|*'$(SRCROOT)/ ') + echo "error: an archive path came out empty: $archive" >&2 + exit 65 + ;; + esac +done + echo "Native toolchain: enabled (${#ARCHIVES[@]} archives)" From 400d858598338538f095112d64556cedc6c22a07 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 02:00:16 -0400 Subject: [PATCH 39/54] ci(toolchain): the LLVM-only compile-check died before compiling anything MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit `EXTRA_DEFINES[@]: unbound variable`. `shell: bash` on a macOS runner is macOS's bash 3.2, where `set -u` plus an *empty* `"${ARRAY[@]}"` is an error — and those arrays are empty exactly when the Swift half was not built, which is every pull request and push run. So the half of CI that runs unattended was red for a reason that had nothing to do with the bridge, while dispatched `with_swift=true` runs passed. The flags are accumulated in a string now. Also stop publishing a `noswift` bundle. It is a complete deliverable for C and Objective-C targets, but `build-ipa.yml` resolves "the newest toolchain-* release", and a newer bundle with no frontend in it is the wrong thing to hand over. It stays available as a workflow artifact. --- .github/workflows/native-toolchain.yml | 28 +++++++++++++++++--------- Docs/NATIVE-TOOLCHAIN.md | 10 ++++++--- 2 files changed, 26 insertions(+), 12 deletions(-) diff --git a/.github/workflows/native-toolchain.yml b/.github/workflows/native-toolchain.yml index 343b024..9fd897f 100644 --- a/.github/workflows/native-toolchain.yml +++ b/.github/workflows/native-toolchain.yml @@ -619,19 +619,24 @@ jobs: run: | set -euxo pipefail SDK="$(xcrun --sdk iphoneos --show-sdk-path)" - EXTRA_DEFINES=() - EXTRA_INCLUDES=() + # Accumulated in a *string*, not an array. `shell: bash` on a macOS runner + # is macOS's bash 3.2, where `set -u` plus an empty `"${ARRAY[@]}"` is an + # unbound-variable error — and the arrays here are empty on every run that + # does not build the Swift half, which is every pull request and push run. + # The step died on `EXTRA_DEFINES[@]: unbound variable` before compiling + # anything, so the half of CI that runs unattended was red for a reason + # that had nothing to do with the bridge. + EXTRA_FLAGS="" if [ "${{ inputs.with_swift }}" = "true" ]; then # Exercises the Swift half of the bridge: with the define and the # headers present, NativeToolchainBridge.mm compiles the code path # that calls swift::performFrontend instead of the stub. - EXTRA_DEFINES+=(-DXFORGE_HAS_SWIFT_FRONTEND=1) - EXTRA_INCLUDES+=(-I swift/include) + EXTRA_FLAGS="$EXTRA_FLAGS -DXFORGE_HAS_SWIFT_FRONTEND=1 -I swift/include" if [ -d swift-build/include ]; then - EXTRA_INCLUDES+=(-I swift-build/include) + EXTRA_FLAGS="$EXTRA_FLAGS -I swift-build/include" fi if [ -d swift-build/tools/swift/include ]; then - EXTRA_INCLUDES+=(-I swift-build/tools/swift/include) + EXTRA_FLAGS="$EXTRA_FLAGS -I swift-build/tools/swift/include" fi fi xcrun --sdk iphoneos clang++ \ @@ -640,13 +645,12 @@ jobs: -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET \ -isysroot "$SDK" \ -DXFORGE_HAS_LLVM=1 \ - "${EXTRA_DEFINES[@]}" \ + $EXTRA_FLAGS \ -I llvm-project/llvm/include \ -I llvm-build/include \ -I llvm-project/clang/include \ -I llvm-build/tools/clang/include \ -I llvm-project/lld/include \ - "${EXTRA_INCLUDES[@]}" \ -I App/NativeToolchain \ -c App/NativeToolchain/NativeToolchainBridge.mm \ -o /tmp/NativeToolchainBridge.o @@ -912,8 +916,14 @@ jobs: # consumer asking for "the newest one" a bundle with no Swift frontend in it. # The tag already tells the two apart; this keeps them from racing for the # same "newest" slot in the first place. + # + # And only when the Swift half was built, for the same reason one level down: + # a `noswift` bundle is a complete deliverable for C and Objective-C targets, + # but a consumer that asks for "the newest toolchain release" means the full + # one. Clang-only bundles stay reachable as workflow artifacts + # (`toolchain_tag: artifact` in build-ipa.yml, `--release artifact` locally). - name: Publish the toolchain bundle - if: ${{ !inputs.configure_only && github.event_name == 'workflow_dispatch' && inputs.publish && steps.stage.outcome == 'success' }} + if: ${{ !inputs.configure_only && inputs.with_swift && github.event_name == 'workflow_dispatch' && inputs.publish && steps.stage.outcome == 'success' }} env: GH_TOKEN: ${{ github.token }} shell: bash diff --git a/Docs/NATIVE-TOOLCHAIN.md b/Docs/NATIVE-TOOLCHAIN.md index 30cf3f5..e006602 100644 --- a/Docs/NATIVE-TOOLCHAIN.md +++ b/Docs/NATIVE-TOOLCHAIN.md @@ -126,9 +126,13 @@ apart. available: it fails unless every member resolves inside the bundle. On failure it prints every undefined symbol, not a tail of them. 9. Write `manifest.txt` and tar the result as `XForgeNativeToolchain-arm64-ios.tar.gz` - — uploaded as the workflow artifact of the same name, and, on a hand-dispatched - run that succeeded, **published as a release asset** tagged - `toolchain---ios-sdk`. + — uploaded as the workflow artifact of the same name, and, when a hand-dispatched + run that built the Swift half succeeded, **published as a release asset** tagged + `toolchain--swift--ios-sdk`. A `noswift` bundle is a + complete deliverable for C and Objective-C targets but is not published: a + consumer asking for "the newest toolchain release" means the full one, and the + tag would otherwise be the only thing standing between them and a bundle with no + frontend in it. The cache is why none of this needs doing twice. A run whose inputs are unchanged restores both trees and goes straight to the staging and link checks. Everything From faf9971c38267560039b9004e96348a815c138c0 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 02:51:21 -0400 Subject: [PATCH 40/54] App: import an Xcode.xip as the Darwin SDK MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The SDK the compiler reads has to come from somewhere, and until now the only somewhere was a bundle we publish or one you built on a Mac with `xtool sdk build`. That is the wrong shape for a phone: the app is the tool. `XipArchive` reads Apple's .xip — xar header and table of contents, the pbzx stream in its Content member, the odc cpio archive that decompresses to — in a single streaming pass, one block at a time. Apple's own `Compression` decodes the LZMA2 blocks, the same decoder `unxip` uses on Apple platforms, so nothing is vendored. `DarwinSDKBuilder` turns the entries it wants into the bundle xtool's builder would have produced: xtool's path list (minus prebuilt-modules, with hard links kept as hard links and symlinks as symlinks) and its swift-sdk.json, restricted to iPhoneOS.platform because XForge only ever compiles for the device. `NativeSDK` installs an imported bundle by moving it, and deletes the copy the document picker made, which for a xip is most of the free space on the device. The Toolchain screen takes .xip, .zip and folders, checks free space before it starts, streams progress through an AsyncStream into a @MainActor ObservableObject (the shape the rest of the app uses for state a long job updates), and cleans up afterwards. The tests build a xip layer by layer — odc cpio in pbzx blocks in a xar — with the TOC checksum that a sloppier offset reader would take for the data's, and check the LZMA decode against a blob produced by liblzma, so the decoder is validated against a different implementation than its own encoder. None of it needed a real 11 GB xip. Verified locally: both cpio layouts against real `cpio -H odc` and `-H newc` output, and the whole pipeline (xar offset base, block framing, path filter, bundle layout, swift-sdk.json) against a mirror of the same fixture — which is how the bundle path that dropped `Developer/` was caught before CI could. --- App/Models/SystemInfo.swift | 4 +- App/NativeToolchain/DarwinSDKBuilder.swift | 337 +++++++++++++ App/NativeToolchain/NativeSDK.swift | 70 +++ App/NativeToolchain/XipArchive.swift | 554 +++++++++++++++++++++ App/Views/Toolchain/ToolchainView.swift | 107 +++- AppTests/XipArchiveTests.swift | 402 +++++++++++++++ CHANGELOG.md | 11 + Docs/DESIGN.md | 47 +- Docs/NATIVE-TOOLCHAIN.md | 33 ++ 9 files changed, 1544 insertions(+), 21 deletions(-) create mode 100644 App/NativeToolchain/DarwinSDKBuilder.swift create mode 100644 App/NativeToolchain/XipArchive.swift create mode 100644 AppTests/XipArchiveTests.swift diff --git a/App/Models/SystemInfo.swift b/App/Models/SystemInfo.swift index aa9b686..1707d00 100644 --- a/App/Models/SystemInfo.swift +++ b/App/Models/SystemInfo.swift @@ -21,8 +21,8 @@ enum SystemInfo { /// Free and total space. /// - /// Free is the *important usage* figure — the same one the app decides with - /// (`XForgeEnvironment.availableBytes`, which guards the Xcode.xip import) — so + /// Free is the *important usage* figure — the same one an import decides with + /// (`DarwinSDKBuilder.availableBytes`, which guards the `Xcode.xip` build) — so /// this screen cannot report space that an import then refuses. The two come /// from different APIs and disagree: `systemFreeSize` does not count the space /// iOS would free up for you. diff --git a/App/NativeToolchain/DarwinSDKBuilder.swift b/App/NativeToolchain/DarwinSDKBuilder.swift new file mode 100644 index 0000000..e1f81a4 --- /dev/null +++ b/App/NativeToolchain/DarwinSDKBuilder.swift @@ -0,0 +1,337 @@ +import Foundation + +/// Builds a `darwin.artifactbundle` — the Darwin SDK the native compiler reads — +/// out of an `Xcode.xip`, on the device. +/// +/// This is what `xtool sdk build ` does on a Mac, and it has to exist +/// here because that is exactly what a phone cannot do: xtool is a macOS program +/// that shells out to `xar`, and there is no guest to run it in any more. The +/// layout it produces is reproduced rather than invented, so a bundle this writes +/// is the same shape as the one the app downloads: +/// +/// ``` +/// darwin.artifactbundle/ +/// swift-sdk.json the metadata NativeSDK reads +/// darwin-sdk-version.txt +/// Developer/ +/// Platforms/iPhoneOS.platform/ +/// Info.plist +/// Developer/SDKs/iPhoneOS27.0.sdk/… +/// Developer/usr/lib/… (the tbd stubs' link paths) +/// Developer/Library/Frameworks/… (XCTest and friends, linked from the SDK) +/// Toolchains/XcodeDefault.xctoolchain/usr/lib/ +/// swift/… the Swift modules and resource dir +/// swift_static/iphoneos/… the static runtime a sideloaded app links +/// clang/… the clang resource dir +/// ``` +/// +/// Which paths are kept is not a guess: `SDKBuilder.wanted` in xtool enumerates +/// them, and this follows it. One deliberate difference — xtool builds a bundle for +/// three platforms (device, simulator, macOS) because a Mac builds for all of them; +/// XForge only ever compiles for the device, so only `iPhoneOS.platform` is kept and +/// `swift-sdk.json` declares only `arm64-apple-ios`. That is roughly a third of the +/// extraction work and a third of the disk for exactly the same capability. +enum DarwinSDKBuilder { + /// What the caller gets back. `bundle` is inside the staging directory; the + /// caller installs it (`NativeSDK.install(preparedBundle:)`). + struct Result { + var bundle: URL + var sdkRoot: String + var files: Int + var bytes: Int64 + var skipped: Int + } + + struct Progress: Sendable { + /// 0…1 over the whole job (extraction dominates). + var fraction: Double + var message: String + } + + enum Error: LocalizedError { + case noXcodeInside + case noiPhoneOSSDK + case notEnoughSpace(needed: Int64, available: Int64) + case noSwiftStaticRuntime(String) + + var errorDescription: String? { + switch self { + case .noXcodeInside: + return "The .xip does not contain an Xcode.app." + case .noiPhoneOSSDK: + return "Xcode was found, but it has no iPhoneOS SDK — the xip is probably " + + "a Command Line Tools or a simulator-only archive." + case .notEnoughSpace(let needed, let available): + return "Not enough space: building the SDK needs about \(Self.size(needed)) " + + "free, and there is \(Self.size(available)). Delete some projects or " + + "downloads and try again." + case .noSwiftStaticRuntime(let path): + return "The xip has no Swift static runtime at \(path), so a sideloaded app " + + "could not link the Swift runtime into itself." + } + } + + private static func size(_ bytes: Int64) -> String { + ByteCountFormatter.string(fromByteCount: bytes, countStyle: .file) + } + } + + /// Roughly what the iPhoneOS-only bundle occupies. Used for the space guard + /// before the expensive part starts, not as a hard limit. + static let estimatedBytes: Int64 = 1_500_000_000 + + // MARK: - Entry point + + /// Extract an `Xcode.xip` into `destination/`. + /// + /// - Parameters: + /// - xip: the archive, wherever the document picker put it. + /// - destination: the directory the bundle is written into (its parent, not + /// the bundle itself) — `NativeSDK`'s staging directory, in practice. + /// - progress: called often, from the calling thread. + static func build(fromXip xip: URL, + into destination: URL, + progress: (@Sendable (Progress) -> Void)? = nil) throws -> Result { + let available = availableBytes(at: destination) + if available > 0, available < estimatedBytes { + throw Error.notEnoughSpace(needed: estimatedBytes, available: available) + } + + let staging = destination.appendingPathComponent( + "darwin.artifactbundle.building-\(UUID().uuidString)", isDirectory: true) + do { + let content = try XipArchive.content(of: xip) + let writer = Writer(staging: staging) + + // Extraction is essentially all of the wall clock, so the fraction of the + // compressed member that has been consumed is the honest fraction of the + // whole job; the metadata write at the end gets the last few percent. + try XipArchive.forEachEntry(in: xip, content: content, progress: { fraction in + progress?(Progress(fraction: fraction * 0.95, + message: "Extracting Xcode — \(Int(fraction * 100))%")) + }) { entry, payload in + try writer.accept(entry, payload) + } + + let result = try writer.finish(progress: progress) + + try FileManager.default.createDirectory(at: destination, withIntermediateDirectories: true) + return result + } catch { + try? FileManager.default.removeItem(at: staging) + throw error + } + } + + // MARK: - Which paths become the bundle + + /// The subset of `Xcode.app/Contents/Developer` a Darwin SDK needs. + /// + /// Mirrors `SDKBuilder.SDKEntry.wanted` in xtool, restricted to the device + /// platform, plus its one exclusion (`swift/prebuilt-modules`, which is + /// per-configuration build output measured in gigabytes). + /// + /// Written as explicit list comparisons rather than the nested matcher xtool + /// uses: there are five leaves, and a reader can check this against xtool's table + /// line by line. + static func isWanted(_ path: String) -> Bool { + var components = path.split(separator: "/").map(String.init) + if components.first == "." { components.removeFirst() } + // The archive's own root is `Xcode.app/…`; stripping it lets the same filter + // read both that and a bare `Contents/Developer/…`. + if components.first?.hasSuffix(".app") == true { components.removeFirst() } + + guard components.starts(with: ["Contents", "Developer"]) else { return false } + let rest = Array(components.dropFirst(2)) + + // The toolchain: the Swift modules, the static runtime, the clang resource + // directory. These are the parts a compiler needs that the SDK does not carry. + if rest.starts(with: ["Toolchains", "XcodeDefault.xctoolchain", "usr", "lib"]) { + let tail = Array(rest.dropFirst(4)) + guard let name = tail.first, + ["swift", "swift_static", "clang"].contains(name) else { return false } + if name == "swift", tail.dropFirst().first == "prebuilt-modules" { return false } + return true + } + + guard rest.starts(with: ["Platforms", "iPhoneOS.platform"]) else { return false } + let tail = Array(rest.dropFirst(2)) + + // `Info.plist` describes the platform, and the SDK's own tooling reads it. + if tail == ["Info.plist"] { return true } + + guard tail.first == "Developer" else { return false } + let developer = Array(tail.dropFirst()) + if developer.first == "SDKs" || developer.first == "usr" { return true } + // XCTest and Testing live in the platform's Library rather than in the SDK, + // and the SDK links to them from there (see `swift-sdk.json`'s search paths). + return developer.starts(with: ["Library", "Frameworks"]) + || developer.starts(with: ["Library", "PrivateFrameworks"]) + } + + // MARK: - Writing the bundle + + /// Turns entries into the bundle tree, one at a time. + /// + /// Stateful because the archive is: a hard link is only recognisable once its + /// first occurrence has been seen, and a directory entry is the only thing that + /// says a directory exists at all. + private final class Writer { + private let staging: URL + private var hardLinks: [UInt64: URL] = [:] + private var files = 0 + private var bytes: Int64 = 0 + private var skipped = 0 + private var sdkRoot: String? + private var sdkVersion: String? + private var sawSwiftStaticRuntime = false + + init(staging: URL) { + self.staging = staging + } + + func accept(_ entry: XipArchive.Entry, _ payload: XipArchive.Payload) throws { + guard Self.isWanted(entry.name) else { + skipped += 1 + return + } + // `Xcode.app/` prefixes everything; the bundle re-roots at `Developer/`, + // which is what `swift-sdk.json`'s paths are relative to. + let relative = Self.bundlePath(for: entry.name) + let target = staging.appendingPathComponent(relative) + + if entry.isDirectory { + try FileManager.default.createDirectory(at: target, withIntermediateDirectories: true) + if relative.contains("Developer/SDKs/"), relative.hasSuffix(".sdk") { + // `iPhoneOS.sdk` is a symlink to `iPhoneOS27.0.sdk` in recent + // Xcodes; the versioned name is the one worth declaring, so it + // wins whenever it turns up. + let name = target.lastPathComponent + if sdkRoot == nil || name != "iPhoneOS.sdk" { + sdkRoot = relative + sdkVersion = Self.platformVersion(fromSDKName: name) + } + } + } else if entry.isSymlink { + let link = String(decoding: try payload.readAll(), as: UTF8.self) + try FileManager.default.createDirectory( + at: target.deletingLastPathComponent(), withIntermediateDirectories: true) + try? FileManager.default.removeItem(at: target) + try FileManager.default.createSymbolicLink(atPath: target.path, + withDestinationPath: link) + files += 1 + } else if entry.isRegular { + try FileManager.default.createDirectory( + at: target.deletingLastPathComponent(), withIntermediateDirectories: true) + + // A second occurrence of the same inode is a hard link, and Xcode's + // trees are full of them: writing the bytes again would multiply the + // bundle by the link count. When the first occurrence was filtered out + // there is nothing to link to, and the data is written instead. + if entry.linkCount > 1 { + let key = UInt64(entry.device) << 32 | UInt64(entry.inode) + if let original = hardLinks[key] { + try? FileManager.default.removeItem(at: target) + try FileManager.default.linkItem(at: original, to: target) + files += 1 + return + } + hardLinks[key] = target + } + + if relative.contains("swift_static/iphoneos/") { + sawSwiftStaticRuntime = true + } + try payload.write(to: target) + files += 1 + bytes += entry.size + } else { + // Device nodes, fifos and sockets: nothing an SDK is built from. + skipped += 1 + } + } + + /// Write the metadata, having seen the whole tree. + func finish(progress: (@Sendable (Progress) -> Void)?) throws -> Result { + guard let sdkRoot else { throw Error.noiPhoneOSSDK } + guard sawSwiftStaticRuntime else { + throw Error.noSwiftStaticRuntime("Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static/iphoneos") + } + progress?(Progress(fraction: 0.97, message: "Writing swift-sdk.json…")) + + let json = SDKDefinition( + schemaVersion: "4.0", + // One triple, unlike xtool's five: XForge compiles for the device. + targetTriples: [ + "arm64-apple-ios": SDKDefinition.Triple( + sdkRootPath: sdkRoot, + includeSearchPaths: ["Developer/Platforms/iPhoneOS.platform/Developer/usr/lib"], + librarySearchPaths: ["Developer/Platforms/iPhoneOS.platform/Developer/usr/lib"], + swiftResourcesPath: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift", + swiftStaticResourcesPath: "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static" + ), + ] + ) + let encoder = JSONEncoder() + encoder.outputFormatting = [.prettyPrinted, .sortedKeys] + try encoder.encode(json).write(to: staging.appendingPathComponent("swift-sdk.json")) + + // xtool's marker, with the platform version rather than an SDK-build + // counter: this bundle was not built by xtool, and claiming otherwise + // would make `xtool sdk` try to update it. + try Data("\(sdkVersion ?? "unknown")\n".utf8) + .write(to: staging.appendingPathComponent("darwin-sdk-version.txt")) + + progress?(Progress(fraction: 1, message: "Done")) + return Result(bundle: staging, sdkRoot: sdkRoot, files: files, + bytes: bytes, skipped: skipped) + } + + /// `Xcode.app/Contents/Developer/…` → `Developer/…`. + /// + /// Only `Contents` is dropped: `Developer` is kept because it *is* the + /// bundle's root — `swift-sdk.json` names `Developer/Platforms/…`, and + /// dropping it too would put the SDK where the metadata does not look for it. + static func bundlePath(for entry: String) -> String { + var components = entry.split(separator: "/").map(String.init) + if components.first == "." { components.removeFirst() } + if components.first?.hasSuffix(".app") == true { components.removeFirst() } + if components.first == "Contents" { components.removeFirst() } + return components.joined(separator: "/") + } + + /// `iPhoneOS27.0.sdk` → `27.0`; a bare `iPhoneOS.sdk` has no version to give. + static func platformVersion(fromSDKName name: String) -> String? { + let trimmed = name.hasSuffix(".sdk") ? String(name.dropLast(4)) : name + let digits = trimmed.drop { !$0.isNumber } + return digits.isEmpty ? nil : String(digits) + } + } + + // MARK: - Metadata + + private struct SDKDefinition: Encodable { + struct Triple: Encodable { + var sdkRootPath: String + var includeSearchPaths: [String] + var librarySearchPaths: [String] + var swiftResourcesPath: String + var swiftStaticResourcesPath: String + // xtool also writes `toolsetPaths`, pointing at a `toolset.json` it + // generates for macro plugins. XForge never runs a plugin (a macro is a + // separate process, and iOS has none), so listing a file that is not + // there would be a worse lie than omitting the key. + } + + var schemaVersion: String + var targetTriples: [String: Triple] + } + + /// Free space on the volume the bundle will be written to. + static func availableBytes(at url: URL) -> Int64 { + let probe = url.deletingLastPathComponent().path + let values = try? URL(fileURLWithPath: probe) + .resourceValues(forKeys: [.volumeAvailableCapacityForImportantUsageKey]) + return Int64(values?.volumeAvailableCapacityForImportantUsage ?? 0) + } +} diff --git a/App/NativeToolchain/NativeSDK.swift b/App/NativeToolchain/NativeSDK.swift index 54d1076..75b182f 100644 --- a/App/NativeToolchain/NativeSDK.swift +++ b/App/NativeToolchain/NativeSDK.swift @@ -186,4 +186,74 @@ enum NativeSDK { guard FileManager.default.fileExists(atPath: installedBundle.path) else { return } try FileManager.default.removeItem(at: installedBundle) } + + // MARK: - Importing + + /// Install a bundle the app built itself. + /// + /// A *move* where `install(from:)` copies: this bundle is already inside the + /// app's container, so copying a gigabyte only to delete the original costs a + /// gigabyte of writes and a second gigabyte of free space for nothing. + static func install(preparedBundle: URL) throws { + let fm = FileManager.default + let destination = installedBundle + try fm.createDirectory(at: XForgeEnvironment.nativeSDKDirectory, + withIntermediateDirectories: true) + _ = try layout(at: preparedBundle) // validate before replacing what is there + if fm.fileExists(atPath: destination.path) { + try fm.removeItem(at: destination) + } + try fm.moveItem(at: preparedBundle, to: destination) + } + + /// Install from whatever the document picker handed over. + /// + /// Three shapes arrive this way, and they are told apart by what they are rather + /// than by what they are called: + /// + /// - a `darwin.artifactbundle` **folder** — already built, perhaps by + /// `xtool sdk build` on a Mac; + /// - a **zip** of one, which is how the hosted bundle is published; + /// - an Apple **`Xcode.xip`**, which `DarwinSDKBuilder` turns into one here. + /// That is the case that needs a Mac otherwise, and the reason the picker no + /// longer filters for folders only. + static func installImported(from source: URL, + progress: (@Sendable (DarwinSDKBuilder.Progress) -> Void)? = nil) throws { + let accessed = source.startAccessingSecurityScopedResource() + defer { if accessed { source.stopAccessingSecurityScopedResource() } } + + var isDirectory: ObjCBool = false + let exists = FileManager.default.fileExists(atPath: source.path, isDirectory: &isDirectory) + let kind = source.pathExtension.lowercased() + + if exists, isDirectory.boolValue { + try install(from: source) + } else if kind == "xip" { + let built = try DarwinSDKBuilder.build(fromXip: source, + into: XForgeEnvironment.nativeSDKDirectory, + progress: progress) + try install(preparedBundle: built.bundle) + } else if kind == "zip" { + try installDownloadedArchive(at: source) + } else { + throw NativeSDKError.missingBundle + } + } + + /// Delete the copy the document picker made of an imported archive. + /// + /// `fileImporter` hands over a copy inside the app's own container, and for an + /// `Xcode.xip` that copy is ~10 GB: leaving it behind would fill the device on + /// the first import. Only a file that really is one of ours is removed — the + /// picker's import directories are under `tmp/`, and a URL the picker returned + /// in place (or a folder the user owns) is left alone. + static func discardImportCopy(at source: URL) { + let fm = FileManager.default + let path = source.standardizedFileURL.path + let temporary = fm.temporaryDirectory.standardizedFileURL.path + let inbox = XForgeEnvironment.documentDirectory + .appendingPathComponent("Inbox", isDirectory: true).standardizedFileURL.path + guard path.hasPrefix(temporary + "/") || path.hasPrefix(inbox + "/") else { return } + try? fm.removeItem(at: source) + } } diff --git a/App/NativeToolchain/XipArchive.swift b/App/NativeToolchain/XipArchive.swift new file mode 100644 index 0000000..2bf99ba --- /dev/null +++ b/App/NativeToolchain/XipArchive.swift @@ -0,0 +1,554 @@ +import Compression +import Foundation + +/// Reads an Apple `.xip` — the container Xcode is distributed in — far enough to +/// pull out the paths a Darwin SDK bundle needs. +/// +/// The format is not documented, but it is stable and public in the sense that +/// tools read it (`xar`, `pbzx`, `unxip`): +/// +/// **xip** is a xar archive: a 28-byte header (`xar!`, header size, version, the +/// compressed and uncompressed size of the table of contents, a checksum +/// algorithm), a zlib-compressed XML TOC listing the members, then the members' +/// data. An Xcode xip has exactly one member worth reading: `Content`, which +/// holds `Xcode.app`. +/// +/// **Content** is a pbzx stream: the bytes `pbzx`, an 8-byte big-endian chunk +/// size, then blocks of (decompressed size, compressed size, bytes) — repeated +/// while a block's decompressed size equals the chunk size. A block is either +/// stored or LZMA-compressed, which the two sizes reveal. +/// +/// **decompressed** it is an `odc` cpio archive (magic `070707`, six-digit octal +/// fields, no alignment padding) of the `Xcode.app` tree, ending at a +/// `TRAILER!!!` entry. `newc` (`070701`, eight-digit hex, four-byte padding) is +/// accepted too: it is what other tools produce, and the cost of accepting both +/// is a branch. +/// +/// Two properties shape the API: +/// +/// - **It streams.** A full Xcode is ~15 GB and the part that belongs in a build +/// SDK is a small fraction of it, so nothing is ever materialised whole: the +/// walk hands each entry to a closure and only what that closure writes reaches +/// the disk. Peak memory is one decompressed chunk (16 MB for the xips Apple +/// ships) plus the caller's own buffers. +/// - **It is synchronous.** The work is blocking file I/O over a local file, so +/// there is no async surface to get wrong; a caller that does not want to block +/// runs it inside a detached task. +/// +/// LZMA and zlib are Apple's own (`Compression`), the same decoders `unxip` uses on +/// Apple platforms, so this needs no vendored library. +enum XipArchive { + /// A byte range inside the xip, as the archive's TOC records it. + struct Member: Sendable { + var offset: Int64 + var length: Int64 + } + + /// One cpio entry. `mode` decides what it is: `0o040000` directory, + /// `0o120000` symbolic link, `0o100000` regular file. + struct Entry: Sendable { + var name: String + var mode: UInt32 + var size: Int64 + var inode: UInt32 + var device: UInt32 + var linkCount: UInt32 + + var isDirectory: Bool { mode & 0o170000 == 0o040000 } + var isSymlink: Bool { mode & 0o170000 == 0o120000 } + var isRegular: Bool { mode & 0o170000 == 0o100000 } + } + + enum Error: LocalizedError { + case notAXip + case unsupportedArchive(String) + case noContentMember + case unsupportedPayload(String) + case truncated(String) + case decompressionFailed(Int64) + + var errorDescription: String? { + switch self { + case .notAXip: + return "That file is not an Apple .xip: it does not start with 'xar!'." + case .unsupportedArchive(let detail): + return "Unsupported .xip structure: \(detail)." + case .noContentMember: + return "The .xip has no 'Content' member, so there is no Xcode inside it." + case .unsupportedPayload(let detail): + return "Unrecognised .xip payload: \(detail)." + case .truncated(let what): + return "The .xip ended in the middle of \(what). It is probably incomplete." + case .decompressionFailed(let offset): + return "Could not decompress the .xip payload at byte \(offset). The file may be corrupt." + } + } + } + + // MARK: - The xar header and TOC + + /// The `Content` member's byte range, read from the xar table of contents. + static func content(of url: URL) throws -> Member { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + + /// `read(upToCount:)` is allowed to return short, and for a table of contents + /// or a header that would surface as "the .xip is truncated" on a file that + /// is perfectly fine. + func read(_ count: Int) throws -> Data { + var data = Data() + while data.count < count { + guard let piece = try handle.read(upToCount: count - data.count), + !piece.isEmpty else { + throw Error.truncated("the archive header") + } + data += piece + } + return data + } + + guard try read(4).elementsEqual(Array("xar!".utf8)) else { throw Error.notAXip } + + let headerSize = Int(try read(2).bigEndian(as: UInt16.self)) + let version = try read(2).bigEndian(as: UInt16.self) + guard version == 1 else { throw Error.unsupportedArchive("xar version \(version)") } + let tocCompressedSize = Int(try read(8).bigEndian(as: UInt64.self)) + let tocUncompressedSize = Int(try read(8).bigEndian(as: UInt64.self)) + _ = try read(4) // checksum algorithm + + // The header is padded to `headerSize`; skip whatever is left of it. + let consumed = 28 + if headerSize > consumed { + _ = try read(headerSize - consumed) + } + + let compressedTOC = try read(tocCompressedSize) + let toc = try inflate(compressedTOC, to: tocUncompressedSize, what: "the table of contents") + + // A member's `` is relative to the *heap*, which starts immediately + // after the compressed TOC — not to the start of the file. Reading it as an + // absolute offset lands somewhere inside the header and fails on the `pbzx` + // magic, which is at least a loud way to be wrong. + let member = try contentMember(inTOC: toc) + return Member(offset: Int64(headerSize + tocCompressedSize) + member.offset, + length: member.length) + } + + /// The `Content` member of a xar TOC. + /// + /// Parsed with `XMLParser` rather than a string scan: the TOC is the one part of + /// the format whose shape is not fixed (a xip can carry signatures and + /// metadata), and fishing offsets out of XML by hand is how a format reader + /// starts to fail on the next release. + private static func contentMember(inTOC toc: Data) throws -> Member { + let parser = TOCParser() + let xml = XMLParser(data: toc) + xml.delegate = parser + guard xml.parse() else { + throw Error.unsupportedArchive(xml.parserError?.localizedDescription ?? "unreadable TOC") + } + guard let content = parser.members["Content"] else { throw Error.noContentMember } + return content + } + + /// Collects `………`. + /// + /// The offset and length are only read *inside* ``: a member's entry can + /// carry a `` with an offset of its own, and taking the last + /// offset seen would point the payload at the checksum instead of the data. + private final class TOCParser: NSObject, XMLParserDelegate { + private(set) var members: [String: Member] = [:] + private var name: String? + private var offset: Int64? + private var length: Int64? + private var inData = false + private var text = "" + + func parser(_ parser: XMLParser, didStartElement element: String, + namespaceURI: String?, qualifiedName: String?, + attributes: [String: String] = [:]) { + text = "" + switch element { + case "data": inData = true + case "name": name = nil + case "offset" where inData: offset = nil + case "length" where inData: length = nil + default: break + } + } + + func parser(_ parser: XMLParser, foundCharacters string: String) { text += string } + + func parser(_ parser: XMLParser, didEndElement element: String, + namespaceURI: String?, qualifiedName: String?) { + let value = text.trimmingCharacters(in: .whitespacesAndNewlines) + switch element { + case "name": name = value + case "offset" where inData: offset = Int64(value) + case "length" where inData: length = Int64(value) + case "data": inData = false + case "file": + // A `` closes each member, and its `` comes before it, so + // by now the name/offset/length that belong to it are the last ones + // seen. + if let name, let offset, let length { + members[name] = Member(offset: offset, length: length) + } + name = nil; offset = nil; length = nil; inData = false + default: + break + } + text = "" + } + } + + // MARK: - The pbzx stream + + /// Walk every cpio entry in `Content`. + /// + /// `body` is called once per entry with the entry's header and a payload reader + /// for its data — and it must either read that data or leave it: the walker + /// consumes whatever is left when `body` returns, because cpio is sequential and + /// a skipped entry cannot be seeked past. `progress` is handed the fraction of + /// the *compressed* member consumed, which is monotonic and good enough for a + /// progress bar. + /// + /// Entries are visited in archive order, which is also the order they have to be + /// written in: a directory always precedes the files inside it. + static func forEachEntry( + in url: URL, + content: Member, + progress: (@Sendable (Double) -> Void)? = nil, + _ body: (Entry, Payload) throws -> Void + ) throws { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + try handle.seek(toOffset: UInt64(content.offset)) + let end = content.offset + content.length + + let stream = try BlockStream(handle: handle, end: end, contentLength: content.length, + progress: progress) + try readCPIO(from: stream, body) + } + + /// Decompressed bytes of the payload, one pbzx block at a time. + /// + /// A cursor over a sequence of decompressed blocks rather than one buffer: the + /// payload is ~15 GB decompressed, and a cpio entry — let alone a whole block — + /// does not respect block boundaries, so reads have to be able to span them. + private final class BlockStream { + private let handle: FileHandle + private let end: Int64 + private let contentLength: Int64 + private let progress: (@Sendable (Double) -> Void)? + private var buffer: [UInt8] = [] + private var position = 0 + private var chunkSize: Int64 = 0 + private var finished = false + private var consumed: Int64 = 0 + + init(handle: FileHandle, end: Int64, contentLength: Int64, + progress: (@Sendable (Double) -> Void)?) throws { + self.handle = handle + self.end = end + self.contentLength = contentLength + self.progress = progress + + guard try readExact(4).elementsEqual(Array("pbzx".utf8)) else { + throw Error.unsupportedPayload("the payload does not start with 'pbzx'") + } + chunkSize = Int64(try readExact(8).bigEndian(as: UInt64.self)) + guard chunkSize > 0, chunkSize <= 1 << 30 else { + throw Error.unsupportedPayload("implausible chunk size \(chunkSize)") + } + } + + /// The next decompressed block, or `nil` at the end of the payload. + private func nextBlock() throws -> [UInt8]? { + if finished { return nil } + if handle.offsetInFile >= end { return nil } + let header = try readUpTo(16) + guard header.count == 16 else { return nil } + let decompressedSize = Int64(header[0..<8].bigEndian(as: UInt64.self)) + let compressedSize = Int64(header[8..<16].bigEndian(as: UInt64.self)) + guard decompressedSize > 0, compressedSize > 0 else { return nil } + + // Never read past the member: the archive's own signature and metadata + // follow it, and swallowing those as payload would be reported as a + // corrupt xip rather than as the end of the stream. + let available = end - handle.offsetInFile + let block = try readExact(Int(min(compressedSize, available))) + // The block is stored when it did not shrink; otherwise it is LZMA. + if decompressedSize < chunkSize { + finished = true + } + if Int64(block.count) == decompressedSize { + return [UInt8](block) + } + return try XipArchive.inflateLZMA(block, to: Int(decompressedSize)) + } + + private func loadNext() throws -> Bool { + guard let block = try nextBlock(), !block.isEmpty else { return false } + buffer = block + position = 0 + consumed = handle.offsetInFile + if let progress, contentLength > 0 { + progress(min(1, Double(consumed) / Double(contentLength))) + } + return true + } + + /// Read up to `count` bytes, across block boundaries. Empty at the end. + func read(upTo count: Int) throws -> [UInt8] { + var result: [UInt8] = [] + result.reserveCapacity(count) + while result.count < count { + if position >= buffer.count { + buffer = [] + position = 0 + guard try loadNext() else { break } + } + let take = min(count - result.count, buffer.count - position) + result.append(contentsOf: buffer[position..<(position + take)]) + position += take + } + return result + } + + /// Discard `count` bytes. Same failure mode as `read`, minus the copying. + func skip(_ count: Int64) throws { + var remaining = count + while remaining > 0 { + if position >= buffer.count { + buffer = [] + position = 0 + guard try loadNext() else { + throw Error.truncated("a cpio entry") + } + } + let take = min(Int64(buffer.count - position), remaining) + position += Int(take) + remaining -= take + } + } + + private func readExact(_ count: Int) throws -> Data { + var data = Data() + while data.count < count { + guard let piece = try handle.read(upToCount: count - data.count), + !piece.isEmpty else { + throw Error.truncated("the payload") + } + data += piece + } + consumed = handle.offsetInFile + return data + } + + private func readUpTo(_ count: Int) throws -> Data { + let data = try handle.read(upToCount: count) ?? Data() + consumed = handle.offsetInFile + return data + } + } + + /// A cursor over one entry's data, valid only for the duration of the callback. + final class Payload { + private let stream: BlockStream + private var remaining: Int64 + + init(stream: BlockStream, size: Int64) { + self.stream = stream + self.remaining = size + } + + var bytesRemaining: Int64 { remaining } + + /// Up to `count` bytes. Shorter than requested only at the end of the entry. + func read(upTo count: Int) throws -> [UInt8] { + guard remaining > 0 else { return [] } + let chunk = try stream.read(upTo: Int(min(Int64(count), remaining))) + remaining -= Int64(chunk.count) + return chunk + } + + /// Everything left of this entry. + func readAll() throws -> [UInt8] { + var result: [UInt8] = [] + while remaining > 0 { + result.append(contentsOf: try read(upTo: 1 << 20)) + } + return result + } + + /// Write the rest of this entry to `url` in `bufferSize` pieces. + func write(to url: URL, bufferSize: Int = 1 << 20) throws { + try? FileManager.default.removeItem(at: url) + FileManager.default.createFile(atPath: url.path, contents: nil) + let output = try FileHandle(forWritingTo: url) + defer { try? output.close() } + while remaining > 0 { + try Task.checkCancellation() + let chunk = try read(upTo: min(bufferSize, Int(remaining))) + guard !chunk.isEmpty else { break } + try output.write(contentsOf: chunk) + } + } + + /// Skip to the end of this entry. + func discard() throws { + guard remaining > 0 else { return } + try stream.skip(remaining) + remaining = 0 + } + } + + // MARK: - cpio + + /// The numeric part of a cpio entry header. + /// + /// Two dialects, and they differ in every field: `odc` (`070707`, which Apple's + /// xips use) is six-digit octal and has no alignment padding, `newc` (`070701`) + /// is eight-digit hex and pads the name to four bytes. Only the fields a Darwin + /// SDK needs are kept — ownership, times and device numbers are dropped rather + /// than carried around unread. + private struct RawHeader { + var mode: UInt32 + var inode: UInt32 + var device: UInt32 + var linkCount: UInt32 + var size: Int64 + var nameLength: Int + var padded: Bool + } + + private static func readCPIO(from stream: BlockStream, + _ body: (Entry, Payload) throws -> Void) throws { + while true { + try Task.checkCancellation() + let magic = try stream.read(upTo: 6) + guard magic.count == 6 else { throw Error.truncated("a cpio entry") } + + let header: RawHeader + switch String(decoding: magic, as: UTF8.self) { + case "070707": header = try odcHeader(from: stream) + case "070701", "070702": header = try newcHeader(from: stream) + case let other: throw Error.unsupportedPayload("cpio magic '\(other)'") + } + + let raw = try stream.read(upTo: header.nameLength) + guard raw.count == header.nameLength, raw.last == 0 else { + throw Error.unsupportedPayload("unnamed cpio entry") + } + let name = String(decoding: raw.dropLast(), as: UTF8.self) + if header.padded { + // `newc` aligns the whole header-plus-name to four bytes; the header + // is 110 bytes, so only the name can push it out of alignment. + let padding = (4 - ((110 + header.nameLength) % 4)) % 4 + if padding > 0 { try stream.skip(Int64(padding)) } + } + + // The end marker: there is nothing after it, and its size is zero. + if name == "TRAILER!!!" { return } + + let payload = Payload(stream: stream, size: header.size) + let entry = Entry(name: name, mode: header.mode, size: header.size, + inode: header.inode, device: header.device, + linkCount: header.linkCount) + try body(entry, payload) + // Whatever the caller did not read still has to come off the stream: + // cpio is sequential, so an entry cannot be seeked past. + try payload.discard() + } + } + + /// `odc`: magic consumed, then six-digit octal fields. + private static func odcHeader(from stream: BlockStream) throws -> RawHeader { + func octal(_ digits: Int, as type: UInt32.Type = UInt32.self) throws -> UInt32 { + let bytes = try stream.read(upTo: digits) + guard bytes.count == digits, + let value = UInt32(String(decoding: bytes, as: UTF8.self), radix: 8) else { + throw Error.unsupportedPayload("unreadable cpio field") + } + return value + } + let device = try octal(6) + let inode = try octal(6) + let mode = try octal(6) + _ = try octal(6) // uid + _ = try octal(6) // gid + let linkCount = try octal(6) + _ = try octal(6) // rdev + _ = try stream.read(upTo: 11) // mtime + let nameLength = try octal(6) + let size = try octal(11) + + return RawHeader(mode: mode, inode: inode, device: device, linkCount: linkCount, + size: Int64(size), nameLength: Int(nameLength), padded: false) + } + + /// `newc`: magic consumed, then eight-digit hex fields. + private static func newcHeader(from stream: BlockStream) throws -> RawHeader { + func hex(_ digits: Int) throws -> UInt32 { + let bytes = try stream.read(upTo: digits) + guard bytes.count == digits, + let value = UInt32(String(decoding: bytes, as: UTF8.self), radix: 16) else { + throw Error.unsupportedPayload("unreadable cpio field") + } + return value + } + let inode = try hex(8) + let mode = try hex(8) + _ = try hex(8) // uid + _ = try hex(8) // gid + let linkCount = try hex(8) + _ = try hex(8) // mtime + let size = try hex(8) + let deviceMajor = try hex(8) + let deviceMinor = try hex(8) + _ = try hex(8) // rdev major + _ = try hex(8) // rdev minor + let nameLength = try hex(8) + _ = try hex(8) // check + + return RawHeader(mode: mode, inode: inode, + device: (deviceMajor << 16) | deviceMinor, + linkCount: linkCount, size: Int64(size), + nameLength: Int(nameLength), padded: true) + } + + // MARK: - Decompression + + /// zlib inflate into a buffer of the declared size. + static func inflate(_ data: Data, to size: Int, what: String) throws -> Data { + guard size > 0 else { throw Error.unsupportedPayload("\(what) declares no size") } + var output = [UInt8](repeating: 0, count: size) + let written = compression_decode_buffer(&output, size, [UInt8](data), data.count, + nil, COMPRESSION_ZLIB) + guard written == size else { throw Error.unsupportedArchive("could not inflate \(what)") } + return Data(output) + } + + /// Raw LZMA2 as pbzx stores it, decoded by Apple's own implementation. + static func inflateLZMA(_ data: Data, to size: Int) throws -> [UInt8] { + var output = [UInt8](repeating: 0, count: size) + let written = compression_decode_buffer(&output, size, [UInt8](data), data.count, + nil, COMPRESSION_LZMA) + guard written == size else { throw Error.decompressionFailed(Int64(size)) } + return output + } +} + +private extension Data { + /// Big-endian integer from the leading bytes of this data. + func bigEndian(as type: T.Type) -> T { + var value: T = 0 + withUnsafeMutableBytes(of: &value) { destination in + for (index, byte) in prefix(MemoryLayout.size).enumerated() { + destination[index] = byte + } + } + return value.bigEndian + } +} diff --git a/App/Views/Toolchain/ToolchainView.swift b/App/Views/Toolchain/ToolchainView.swift index a7ac5d2..5c64053 100644 --- a/App/Views/Toolchain/ToolchainView.swift +++ b/App/Views/Toolchain/ToolchainView.swift @@ -23,6 +23,19 @@ struct ToolchainView: View { @State private var message: String? @State private var smokeResult: String? @State private var confirmingDownload = false + @StateObject private var importState = SDKImportState() + + /// What the picker offers: a built bundle (a folder, or a zip of one) or Apple's + /// `Xcode.xip`, which the app turns into one. `.xip` has no system type, so it is + /// declared here by extension — without it the file is greyed out in Files and + /// the one import this screen exists for cannot be started. + private static var importableTypes: [UTType] { + var types: [UTType] = [.folder, .zip] + if let xip = UTType(filenameExtension: "xip", conformingTo: .data) { + types.append(xip) + } + return types + } private var capabilities: NativeToolchainCapabilities { .current } @@ -57,7 +70,10 @@ struct ToolchainView: View { } footer: { Text("The SDK is xtool's darwin.artifactbundle: the iPhoneOS headers, the " + "tbd stubs and the static Swift runtime, read straight out of the " - + "app's container. About 460 MB.") + + "app's container. About 460 MB downloaded, or built here from an " + + "Xcode.xip — which the picker copies into the container first, so " + + "an 11 GB xip needs roughly 12 GB free, plus about 1.5 GB for the " + + "bundle. Extraction takes a few minutes.") } Section { @@ -71,10 +87,19 @@ struct ToolchainView: View { Button { importing = true } label: { - Label("Install from a bundle in Files…", systemImage: "doc.badge.plus") + Label("Install from an Xcode.xip or a bundle…", systemImage: "doc.badge.plus") } .disabled(working) + if importState.isRunning { + VStack(alignment: .leading, spacing: 4) { + ProgressView(value: importState.fraction) + Text(importState.status) + .font(.caption) + .foregroundStyle(.secondary) + } + } + if sdkPath != nil { Button(role: .destructive) { removeSDK() @@ -120,11 +145,11 @@ struct ToolchainView: View { .task { refresh() } .fileImporter( isPresented: $importing, - allowedContentTypes: [.folder], + allowedContentTypes: Self.importableTypes, allowsMultipleSelection: false ) { result in guard case .success(let urls) = result, let url = urls.first else { return } - install(from: url) + Task { await install(from: url) } } .confirmationDialog( "Download the Darwin SDK?", @@ -176,17 +201,55 @@ struct ToolchainView: View { } } - private func install(from url: URL) { + /// Install whatever was picked, off the main actor, with progress. + /// + /// The work is minutes of blocking decompression, so it runs in a detached task. + /// Its progress comes back through a stream rather than a closure that writes to + /// this view's state: the closure handed to the detached task is `@Sendable`, and + /// capturing the view in one is the sort of thing that compiles until it does not + /// — the receiver is a `@MainActor` type, which is `Sendable`, so it can be + /// captured freely. Same shape as the build pipeline's stage streams. + private func install(from url: URL) async { working = true message = nil - defer { working = false } + sdkError = nil + + // The object, not the property wrapper: this is what the consumer task + // captures, so that no closure here has to capture the view. + let state = importState + state.begin("Reading \(url.lastPathComponent)…") + + let (updates, continuation) = AsyncStream.makeStream() + let consumer = Task { @MainActor in + for await update in updates { + state.update(fraction: update.fraction, status: update.message) + } + state.finish() + } + do { - try NativeSDK.install(from: url) + try await Task.detached(priority: .userInitiated) { + defer { continuation.finish() } + try NativeSDK.installImported(from: url) { continuation.yield($0) } + }.value refresh() message = "Darwin SDK installed from \(url.lastPathComponent)." + } catch is CancellationError { + message = "Import cancelled." } catch { sdkError = error.localizedDescription } + + continuation.finish() + await consumer.value + state.finish() + working = false + + // The picker's copy of the archive is the app's to delete, and for an + // `Xcode.xip` it is most of the free space on the device. + await Task.detached(priority: .utility) { + NativeSDK.discardImportCopy(at: url) + }.value } private func removeSDK() { @@ -251,3 +314,33 @@ struct ToolchainView: View { } } } + +/// Progress of a Darwin SDK import, as the screen renders it. +/// +/// A reference type and a plain `ObservableObject` — the shape the rest of the app +/// uses for state a long job updates — rather than two `@State` fields, because the +/// consumer of the progress stream has to hold *something* across an actor boundary: +/// this object is `@MainActor`, and therefore `Sendable`, while the view is not. +/// Holding the view in a task's closure is the kind of capture that compiles until +/// it does not. +@MainActor +final class SDKImportState: ObservableObject { + @Published private(set) var fraction: Double = 0 + @Published private(set) var status = "" + @Published private(set) var isRunning = false + + func begin(_ status: String) { + fraction = 0 + self.status = status + isRunning = true + } + + func update(fraction: Double, status: String) { + self.fraction = fraction + self.status = status + } + + func finish() { + isRunning = false + } +} diff --git a/AppTests/XipArchiveTests.swift b/AppTests/XipArchiveTests.swift new file mode 100644 index 0000000..5077118 --- /dev/null +++ b/AppTests/XipArchiveTests.swift @@ -0,0 +1,402 @@ +import Compression +import XCTest +@testable import XForge + +/// The `.xip` reader and the Darwin SDK builder. +/// +/// There is no real Xcode xip to test against — they are 11 GB and need an Apple ID — +/// so these tests build one, layer by layer, the way the format notes in `XipArchive` +/// describe it: `odc` cpio inside pbzx blocks inside a xar. Two things that would +/// otherwise go untested are covered anyway: the pbzx framing is exercised across +/// block boundaries and with both a stored and a compressed block, and the LZMA +/// decode is checked against a fixture produced by **liblzma** (Python's `lzma` +/// module), so the decoder is validated against a different implementation than its +/// own encoder. +final class XipArchiveTests: XCTestCase { + + // MARK: - Fixture scaffolding + + private struct Spec { + var path: String + var mode: UInt32 + var data: Data = Data() + var inode: UInt32 + var links: UInt32 = 1 + } + + private static let directory: UInt32 = 0o040755 + private static let regular: UInt32 = 0o100644 + private static let symlink: UInt32 = 0o120777 + + /// The paths of the fake Xcode tree, rooted at `Xcode.app` as the archive's are. + private static let developer = "Xcode.app/Contents/Developer" + private static let platform = "\(developer)/Platforms/iPhoneOS.platform" + private static let toolchain = "\(developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib" + + private static func fakeXcode() -> [Spec] { + var inode: UInt32 = 0 + func next() -> UInt32 { inode += 1; return inode } + + let directories = [ + "Xcode.app", "Xcode.app/Contents", developer, "\(developer)/Platforms", + platform, "\(platform)/Developer", + "\(platform)/Developer/SDKs", + "\(platform)/Developer/SDKs/iPhoneOS27.0.sdk", + "\(platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr", + "\(platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr/include", + "\(developer)/Toolchains", "\(developer)/Toolchains/XcodeDefault.xctoolchain", + "\(developer)/Toolchains/XcodeDefault.xctoolchain/usr", + "\(toolchain)", "\(toolchain)/swift_static", "\(toolchain)/swift_static/iphoneos", + "\(toolchain)/swift", "\(toolchain)/swift/prebuilt-modules", + "Xcode.app/Contents/Resources", + "\(developer)/Platforms/MacOSX.platform", + ] + var specs = directories.map { Spec(path: $0, mode: directory, inode: next()) } + + // Kept: the platform manifest, a header from the SDK, the Swift static runtime. + specs.append(Spec(path: "\(platform)/Info.plist", mode: regular, + data: Data("".utf8), inode: next())) + specs.append(Spec(path: "\(platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr/include/stdio.h", + mode: regular, data: Data("#pragma once\n".utf8), inode: next())) + let runtimeInode = next() + specs.append(Spec(path: "\(toolchain)/swift_static/iphoneos/libswiftCore.a", + mode: regular, data: Data(repeating: 0xAB, count: 5_000), + inode: runtimeInode, links: 2)) + + // Dropped: Xcode's own build output, another platform, anything under + // Contents that is not the Developer directory. + specs.append(Spec(path: "\(toolchain)/swift/prebuilt-modules/Foundation.swiftmodule", + mode: regular, data: Data(repeating: 0xFF, count: 64), inode: next())) + specs.append(Spec(path: "\(developer)/Platforms/MacOSX.platform/Info.plist", + mode: regular, data: Data("".utf8), inode: next())) + specs.append(Spec(path: "Xcode.app/Contents/Resources/Dropped.bin", + mode: regular, data: Data(repeating: 0x11, count: 32), inode: next())) + specs.append(Spec(path: "Xcode.app/Contents/version.plist", mode: regular, + data: Data("".utf8), inode: next())) + + // The second link to the runtime: same inode, and both are inside the filter, + // so the bundle should carry one file and one hard link, not two copies. + specs.append(Spec(path: "\(toolchain)/swift/libswiftCore.a", mode: regular, + data: Data(), inode: runtimeInode, links: 2)) + + // `iPhoneOS.sdk` is a symlink to the versioned directory in recent Xcodes. + specs.append(Spec(path: "\(platform)/Developer/SDKs/iPhoneOS.sdk", mode: symlink, + data: Data("iPhoneOS27.0.sdk".utf8), inode: next())) + return specs + } + + private static func cpio(_ specs: [Spec]) -> Data { + var out = Data() + for spec in specs { out += odcEntry(spec) } + out += odcEntry(Spec(path: "TRAILER!!!", mode: 0, inode: 0)) + return out + } + + /// `odc`: magic, six-digit octal fields, an 11-digit size, then the name and data. + private static func odcEntry(_ spec: Spec) -> Data { + func octal(_ value: UInt32, _ width: Int) -> Data { + Data(String(format: "%0\(width)o", value).utf8) + } + var out = Data("070707".utf8) + out += octal(0, 6) // device + out += octal(spec.inode, 6) // inode + out += octal(spec.mode, 6) // mode + out += octal(0, 6) // uid + out += octal(0, 6) // gid + out += octal(spec.links, 6) // link count + out += octal(0, 6) // rdev + out += octal(0, 11) // mtime + out += octal(UInt32(spec.path.utf8.count + 1), 6) // name length + out += octal(UInt32(spec.data.count), 11) // file size + out += Data(spec.path.utf8) + out += Data([0]) + out += spec.data + return out + } + + /// pbzx-wrap a payload. The first block is LZMA-compressed when the platform can + /// encode it (Apple's LZMA encoder is decode-mostly), the rest are stored — the + /// same mix a real xip has. + private static func pbzx(_ payload: Data, chunkSize: Int = 2_048) -> Data { + var out = Data("pbzx".utf8) + out += bigEndian(UInt64(chunkSize), 8) + var index = payload.startIndex + var block = 0 + while index < payload.endIndex { + let end = payload.index(index, offsetBy: chunkSize, limitedBy: payload.endIndex) + ?? payload.endIndex + let uncompressed = Data(payload[index..` is heap-relative. + /// + /// The `` is not decoration: it carries an `` of its + /// own, and a reader that takes the last offset it sees points the payload at the + /// checksum. A real xip's TOC has one, so this fixture does too. + private static func xar(content: Data) -> Data { + let toc = """ + + 0Contentfile\ + 0\(content.count)\(content.count)\ + \(content.count)\ + 20 + """ + let tocData = Data(toc.utf8) + let compressed = zlibEncode(tocData) ?? tocData + var out = Data("xar!".utf8) + out += bigEndian(UInt16(28), 2) + out += bigEndian(UInt16(1), 2) + out += bigEndian(UInt64(compressed.count), 8) + out += bigEndian(UInt64(tocData.count), 8) + out += bigEndian(UInt32(0), 4) + out += compressed + out += content + return out + } + + private static func bigEndian(_ value: T, _ width: Int) -> Data { + var big = value.bigEndian + return withUnsafeBytes(of: &big) { Data($0.suffix(width)) } + } + + private static func zlibEncode(_ data: Data) -> Data? { + var buffer = [UInt8](repeating: 0, count: data.count + 4_096) + let written = compression_encode_buffer(&buffer, buffer.count, [UInt8](data), data.count, + nil, COMPRESSION_ZLIB) + return written > 0 ? Data(buffer[0.. Data? { + var buffer = [UInt8](repeating: 0, count: data.count * 2 + 4_096) + let written = compression_encode_buffer(&buffer, buffer.count, [UInt8](data), data.count, + nil, COMPRESSION_LZMA) + return written > 0 ? Data(buffer[0.. URL { + let directory = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("xip-tests-\(UUID().uuidString)", isDirectory: true) + try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) + addTeardownBlock { try? FileManager.default.removeItem(at: directory) } + + let cpio = payload ?? Self.cpio(Self.fakeXcode()) + let url = directory.appendingPathComponent("Xcode_27.xip") + try Self.xar(content: Self.pbzx(cpio)).write(to: url) + return url + } + + // MARK: - The xar layer + + /// The `Content` member's byte range. The TOC records a *heap-relative* offset, so + /// a reader that treats it as absolute lands inside the header and finds no pbzx + /// magic — which is what this pins down. + func testContentMemberIsLocatedAfterTheTableOfContents() throws { + let url = try makeXip() + let content = try XipArchive.content(of: url) + + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + try handle.seek(toOffset: UInt64(content.offset)) + let magic = try handle.read(upToCount: 4) + XCTAssertEqual(magic.map { [UInt8]($0) }, Array("pbzx".utf8), + "the Content member should start at the pbzx stream") + XCTAssertGreaterThan(content.length, 0) + } + + func testRejectsAFileThatIsNotAXip() throws { + let url = URL(fileURLWithPath: NSTemporaryDirectory()) + .appendingPathComponent("not-a-xip-\(UUID().uuidString).xip") + try Data("this is just a file".utf8).write(to: url) + addTeardownBlock { try? FileManager.default.removeItem(at: url) } + + XCTAssertThrowsError(try XipArchive.content(of: url)) { error in + guard let xipError = error as? XipArchive.Error, case .notAXip = xipError else { + return XCTFail("expected notAXip, got \(error)") + } + } + } + + // MARK: - The pbzx and cpio layers + + /// Every entry, in order, with the shapes intact — and across block boundaries: + /// the fixture's chunk size is 2 KB and the fake tree is larger than that. + func testWalkYieldsEveryEntry() throws { + let url = try makeXip() + let content = try XipArchive.content(of: url) + + var entries: [XipArchive.Entry] = [] + var payloads: [String: Int] = [:] + try XipArchive.forEachEntry(in: url, content: content) { entry, payload in + entries.append(entry) + payloads[entry.name] = try payload.readAll().count + } + + let names = entries.map(\.name) + XCTAssertTrue(names.contains("\(Self.platform)/Info.plist")) + XCTAssertFalse(names.contains("TRAILER!!!"), "the trailer ends the walk") + XCTAssertEqual(entries.filter(\.isDirectory).count, + Self.fakeXcode().filter { $0.mode == Self.directory }.count) + + // A file's data has to survive whole even when it straddles blocks. + XCTAssertEqual(payloads["\(Self.toolchain)/swift_static/iphoneos/libswiftCore.a"], 5_000) + XCTAssertEqual(payloads["\(Self.platform)/Info.plist"], Data("".utf8).count) + + // A symlink's "data" is its target, which is how the bundle recreates it. + let link = entries.first { $0.name.hasSuffix("/SDKs/iPhoneOS.sdk") } + XCTAssertEqual(link?.isSymlink, true) + XCTAssertEqual(payloads["\(Self.platform)/Developer/SDKs/iPhoneOS.sdk"], 16) + } + + /// The one part that has to agree with another implementation of the format. + /// + /// The blob is 420 bytes of text compressed to 85 bytes of **raw LZMA2** by + /// liblzma, which is what a pbzx block holds. Apple's decoder has to reproduce it + /// byte for byte; an encoder round-trip through this process would have proved + /// nothing, since two matching bugs look exactly like correctness. + func testDecodesRawLZMA2ProducedByLiblzma() throws { + let base64 = """ + 4AGjAE1dACwaSgQTYMiZxu0IMKQ6+2aC3gaIG0RBZoQd0tB/LAu8gQdovsXeV7hHLo7E0RzTAygy\ + 1lQcyW1wiJyvt23W96lpWfeimqcQPsLwdAAAAA== + """ + let compressed = try XCTUnwrap(Data(base64Encoded: base64)) + let expected = Data(String(repeating: "XipArchive: a pbzx block is raw LZMA2, " + + "decoded with COMPRESSION_LZMA. ", count: 6).utf8) + XCTAssertEqual(expected.count, 420, "the fixture and this payload must stay in step") + + let decoded = try XipArchive.inflateLZMA(compressed, to: expected.count) + XCTAssertEqual(Data(decoded), expected) + } + + // MARK: - What becomes the bundle + + /// The filter, checked against the list xtool keeps (`SDKBuilder.wanted`), with + /// XForge's one deliberate narrowing: the device platform only. + func testWantedPaths() { + let kept = [ + "\(developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/iphoneos/Swift.swiftmodule", + "\(toolchain)/swift_static/iphoneos/libswiftCore.a", + "\(toolchain)/clang/include/stdarg.h", + "\(platform)/Info.plist", + "\(platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr/include/stdio.h", + "\(platform)/Developer/usr/lib/libSystem.tbd", + "\(platform)/Developer/Library/Frameworks/XCTest.framework/XCTest", + ] + for path in kept { + XCTAssertTrue(DarwinSDKBuilder.isWanted(path), "should be kept: \(path)") + } + + let dropped = [ + "\(toolchain)/swift/prebuilt-modules/Foundation.swiftmodule", + "\(developer)/Platforms/MacOSX.platform/Info.plist", + "\(developer)/Applications/Whatever.app/Whatever", + "Contents/Resources/English.lproj/InfoPlist.strings", + "Xcode.app/Contents/version.plist", + "\(toolchain)/swift_static/iphoneos", + "\(developer)", + ] + for path in dropped { + XCTAssertFalse(DarwinSDKBuilder.isWanted(path), "should be dropped: \(path)") + } + } + + // MARK: - End to end + + /// An `.xip` in, a `darwin.artifactbundle` out — one the app's own reader accepts. + func testBuildsAnInstallableBundleFromAXip() throws { + let url = try makeXip() + let destination = url.deletingLastPathComponent() + .appendingPathComponent("out", isDirectory: true) + + let result = try DarwinSDKBuilder.build(fromXip: url, into: destination) + XCTAssertGreaterThan(result.files, 5) + XCTAssertGreaterThan(result.skipped, 0, "the unwanted paths should be counted as skipped") + XCTAssertEqual(result.sdkRoot, + "Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS27.0.sdk") + + let bundle = result.bundle + let fileManager = FileManager.default + func exists(_ relative: String) -> Bool { + fileManager.fileExists(atPath: bundle.appendingPathComponent(relative).path) + } + + XCTAssertTrue(exists("Developer/Platforms/iPhoneOS.platform/Info.plist")) + XCTAssertTrue(exists("Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS27.0.sdk/usr/include/stdio.h")) + XCTAssertTrue(exists("Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static/iphoneos/libswiftCore.a")) + + // The tree is re-rooted at Developer/, not copied as Xcode.app. + XCTAssertFalse(exists("Xcode.app")) + + XCTAssertFalse(exists("Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/prebuilt-modules/Foundation.swiftmodule")) + XCTAssertFalse(exists("Developer/Platforms/MacOSX.platform/Info.plist")) + XCTAssertFalse(exists("Contents/Resources/Dropped.bin")) + + // The symlink is a symlink, pointing at the versioned SDK directory. + let sdkLink = bundle.appendingPathComponent( + "Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS.sdk") + XCTAssertEqual(try fileManager.destinationOfSymbolicLink(atPath: sdkLink.path), + "iPhoneOS27.0.sdk") + + // The two links to one inode stay one file plus a link. + func inode(_ relative: String) throws -> NSNumber? { + let attributes = try fileManager.attributesOfItem( + atPath: bundle.appendingPathComponent(relative).path) + return attributes[.systemFileNumber] as? NSNumber + } + let runtime = try inode("Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static/iphoneos/libswiftCore.a") + let other = try inode("Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/libswiftCore.a") + XCTAssertEqual(runtime, other, "the archive's hard link should stay a hard link") + + // The metadata, as `NativeSDK` reads it. + let json = try XCTUnwrap( + try JSONSerialization.jsonObject( + with: Data(contentsOf: bundle.appendingPathComponent("swift-sdk.json"))) + as? [String: Any]) + XCTAssertEqual(json["schemaVersion"] as? String, "4.0") + let triples = try XCTUnwrap(json["targetTriples"] as? [String: Any]) + let triple = try XCTUnwrap(triples["arm64-apple-ios"] as? [String: Any]) + XCTAssertEqual(triple["sdkRootPath"] as? String, result.sdkRoot) + XCTAssertEqual(triple["swiftStaticResourcesPath"] as? String, + "Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static") + XCTAssertEqual(try String(contentsOf: bundle.appendingPathComponent("darwin-sdk-version.txt")) + .trimmingCharacters(in: .whitespacesAndNewlines), "27.0") + + // And the app's own reader resolves it — the assertion that matters, because a + // bundle nothing can read is not an SDK. + let layout = try NativeSDK.layout(at: bundle) + XCTAssertEqual(layout.sdkRoot.lastPathComponent, "iPhoneOS27.0.sdk") + XCTAssertEqual(layout.swiftStaticResources?.lastPathComponent, "swift_static") + XCTAssertTrue(layout.swiftRuntimeLibraryPaths.contains { + $0.path.hasSuffix("/swift_static/iphoneos") + }) + } + + /// A xip without an iPhoneOS SDK is refused with a message that says so, rather + /// than installing a bundle that fails later. + func testRejectsAXipWithNoIPhoneOSSDK() throws { + let specs = [ + Spec(path: "Xcode.app/Contents/Developer/Platforms/MacOSX.platform/Info.plist", + mode: Self.regular, data: Data("".utf8), inode: 1), + ] + let url = try makeXip(payload: Self.cpio(specs)) + let destination = url.deletingLastPathComponent() + .appendingPathComponent("out", isDirectory: true) + + XCTAssertThrowsError(try DarwinSDKBuilder.build(fromXip: url, into: destination)) { error in + guard case DarwinSDKBuilder.Error.noiPhoneOSSDK = error else { + return XCTFail("expected noiPhoneOSSDK, got \(error)") + } + } + XCTAssertFalse(FileManager.default.fileExists( + atPath: destination.appendingPathComponent("darwin.artifactbundle").path)) + } +} diff --git a/CHANGELOG.md b/CHANGELOG.md index 37939fd..34c82ff 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -48,6 +48,17 @@ they are ordinary directories in the app's container. - **`App/Views/Toolchain/ToolchainView.swift`** — the toolchain and SDK screen, now native: it reports which libraries are linked in, installs or removes the Darwin SDK in the app container, and runs a compile smoke test. +- **Importing an `Xcode.xip` on the device.** The Toolchain screen accepts Apple's + xip and builds the Darwin SDK out of it here — the job `xtool sdk build` does on a + Mac, and the reason a phone with no Mac could not use a newer SDK than the one we + publish. A xip is a xar archive holding a pbzx stream of LZMA2 blocks that + decompresses to a cpio archive of `Xcode.app`; `XipArchive` reads it in one + streaming pass (Apple's `Compression` decodes the LZMA2, so nothing is vendored) + and `DarwinSDKBuilder` keeps xtool's own list of paths, restricted to + `iPhoneOS.platform` because XForge only compiles for the device. The picker also + takes a zip or a folder, for a bundle built elsewhere. Free space is checked + before extraction starts, progress is reported while it runs, and the picker's own + copy of the archive — gigabytes, for a xip — is deleted afterwards. - **The toolchain bundle is published, not just uploaded.** `native-toolchain.yml` attaches the finished bundle to a release tagged `toolchain---ios-sdk` — the tag names what is diff --git a/Docs/DESIGN.md b/Docs/DESIGN.md index e74b9e5..06f8dc4 100644 --- a/Docs/DESIGN.md +++ b/Docs/DESIGN.md @@ -96,18 +96,41 @@ The consequences of linking the compiler in rather than shipping it as data: ## 3. Where the `darwin` SDK comes from -`xtool sdk build ` produces the `darwin` SDK from a real Xcode — -impossible on a phone. So the `darwin.artifactbundle` is **built once in CI** and -published as a release asset under XForge's own `darwin-sdk-` release series, -and the app resolves the newest matching release at build time. - -The bundle is installed into the app container at `/native-sdk/darwin.artifactbundle` -and read directly: `NativeSDK` parses the same `swift-sdk.json` layout xtool's -builder emits, resolving the SDK root, the Swift resource directory and the static -runtime search paths. It is about 460 MB. The Toolchain screen can also install one -from a folder picked in Files, which is how a user with their own `Xcode.xip` — or -a newer artifact — replaces it; the install is staged and validated before it -replaces what is there, so a bad bundle cannot leave the app without an SDK. +`xtool sdk build ` produces the `darwin` SDK from a real Xcode. That is a +macOS program shelling out to `xar`, so the `.xip` path used to stop at "impossible +on a phone" — and the default is still the cheap one: a `darwin.artifactbundle` is +**built once in CI** and published as a release asset under XForge's own +`darwin-sdk-` release series, and the app resolves the newest matching release at +build time. About 460 MB. + +The bundle is installed into the app container at +`/native-sdk/darwin.artifactbundle` and read directly: `NativeSDK` parses +the same `swift-sdk.json` layout xtool's builder emits, resolving the SDK root, the +Swift resource directory and the static runtime search paths. + +Three things can be installed from the Toolchain screen, told apart by what they are +rather than by what they are called: that hosted bundle (a zip), a +`darwin.artifactbundle` **folder** (perhaps built by `xtool sdk build` on a Mac), and +an Apple **`Xcode.xip`**, which the app now builds into a bundle itself +(`DarwinSDKBuilder` + `XipArchive`). The third is the one that needs no Mac: + +- a `.xip` is a xar archive whose `Content` member is a pbzx stream of LZMA2 blocks, + which decompresses to an `odc` cpio archive of `Xcode.app`. `XipArchive` reads it + in one pass, streaming, and hands only the wanted entries to disk; +- which entries those are is xtool's own list (`SDKBuilder.wanted`), not a guess: + `Contents/Developer/Platforms/iPhoneOS.platform/{Info.plist,Developer/{SDKs,usr/lib,Library/Frameworks,Library/PrivateFrameworks}}` + and `Contents/Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/{swift,swift_static,clang}`, + minus `swift/prebuilt-modules` (per-configuration build output measured in GB); +- one deliberate narrowing: xtool builds a bundle for three platforms because a Mac + builds for all of them, and XForge only ever compiles for the device, so only + `iPhoneOS.platform` is extracted and `swift-sdk.json` declares only + `arm64-apple-ios`. A third of the work and a third of the disk for the same + capability. + +Extraction is minutes of CPU and ~1.5 GB of disk on top of whatever the document +picker copied, and both are checked before it starts rather than discovered halfway +through. The install is staged and validated (`NativeSDK.layout`) before it replaces +what is there, so a bad bundle cannot leave the app without an SDK. > Note: Apple also publishes official iOS Swift SDKs on swift.org, but they use the > triple `aarch64-apple-ios` under a different bundle name. xtool hardcodes diff --git a/Docs/NATIVE-TOOLCHAIN.md b/Docs/NATIVE-TOOLCHAIN.md index e006602..7739940 100644 --- a/Docs/NATIVE-TOOLCHAIN.md +++ b/Docs/NATIVE-TOOLCHAIN.md @@ -33,6 +33,14 @@ alternative would be a remote build host, which is a different product. libraries and publishes the bundle. See below. - `NativeToolchain/install-bundle.sh | --release [tag]` — installs that bundle, from a local file or from its release. +- `App/NativeToolchain/XipArchive.swift` — reads Apple's `.xip` container far enough + to take an SDK out of it. xar header and table of contents, the pbzx stream inside + the `Content` member, and the `odc` cpio archive that decompresses to — streamed, + one 16 MB block at a time, with Apple's own `Compression` decoding the LZMA2 blocks + so the app carries no decompressor of its own. +- `App/NativeToolchain/DarwinSDKBuilder.swift` — turns an `Xcode.xip` into a + `darwin.artifactbundle` on the device: xtool's path list, its bundle layout, and + the `swift-sdk.json` `NativeSDK` reads. - `NativeToolchain/prepare-xcode.sh` — writes the xcconfig the app target consumes. `build-ipa.yml` runs it too, so CI never depends on a bundle being installed locally; on a checkout with no bundle it writes the disabled configuration and CI @@ -192,6 +200,31 @@ backend disabled. That file is **checked in, disabled**, so a plain clone builds the bridge compiles to a "not available" stub and the app says so. This is why `make gen` alone is a complete build for someone who only wants to work on the UI. +## The Darwin SDK + +The compiler is in the binary; what it compiles *against* is a bundle in the app's +container, `/native-sdk/darwin.artifactbundle`. `NativeSDK` reads it the +way xtool's builder writes it (`swift-sdk.json`: SDK root, Swift resource directory, +static runtime search paths), and the Toolchain screen installs one of three things: + +| what you pick | what it is | +| --- | --- | +| *(button)* | the hosted `darwin-sdk-` release asset — about 460 MB, no Mac needed | +| a **folder** or **zip** | a `darwin.artifactbundle`, perhaps built by `xtool sdk build` on a Mac | +| an **`Xcode.xip`** | Apple's Xcode; the app builds the bundle from it here | + +The third is the one that would otherwise send you to a Mac. A `.xip` is a xar archive +whose `Content` member is a pbzx stream of LZMA2 blocks decompressing to an `odc` cpio +archive of `Xcode.app`; `XipArchive` walks it in a single streaming pass and writes +only the paths a build SDK needs, and `DarwinSDKBuilder` assembles them into a bundle +— xtool's own list of paths, restricted to the device platform, because XForge only +ever compiles for the device. Extracting is minutes of CPU and about 1.5 GB of disk +on top of the copy the document picker makes, so the import checks free space before +it starts, reports progress while it runs, and deletes the picker's copy afterwards. + +`Docs/DESIGN.md` §3 has the format and layout details; the code states them where the +decisions are made. + ## The runtime contract The native implementation is enabled only when the app target defines From 2eeadda58c71547279f71c284b52638f892a8da8 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 03:03:10 -0400 Subject: [PATCH 41/54] xip: fix the privacy error, the 32-bit sizes and the C bridging MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit CI compiled the app target far enough to say three things, and a re-read said a fourth: - `Payload.init` took a private `BlockStream`, so it has to be private too. - cpio sizes were parsed into `UInt32`. `odc` gives them eleven octal digits, so a member over 4 GiB fails the import — the failure is loud, but the field is 33 bits wide and reading it as 32 is simply wrong. - The test's `String(contentsOf:)` had no encoding (deprecated) and the error assertions pattern-matched an `any Error` without casting first. - `compression_decode_buffer` was handed `[UInt8](data)`, relying on the implicit array-to-pointer conversion. It is passed a real pointer now, via `withUnsafeBytes`, in the reader and in the test's encoders both. --- App/NativeToolchain/XipArchive.swift | 69 ++++++++++++++++++---------- AppTests/XipArchiveTests.swift | 28 +++++++---- 2 files changed, 63 insertions(+), 34 deletions(-) diff --git a/App/NativeToolchain/XipArchive.swift b/App/NativeToolchain/XipArchive.swift index 2bf99ba..8cf60f8 100644 --- a/App/NativeToolchain/XipArchive.swift +++ b/App/NativeToolchain/XipArchive.swift @@ -358,7 +358,8 @@ enum XipArchive { private let stream: BlockStream private var remaining: Int64 - init(stream: BlockStream, size: Int64) { + /// `private` because `BlockStream` is: only the walk constructs payloads. + private init(stream: BlockStream, size: Int64) { self.stream = stream self.remaining = size } @@ -465,7 +466,7 @@ enum XipArchive { /// `odc`: magic consumed, then six-digit octal fields. private static func odcHeader(from stream: BlockStream) throws -> RawHeader { - func octal(_ digits: Int, as type: UInt32.Type = UInt32.self) throws -> UInt32 { + func octal(_ digits: Int) throws -> UInt32 { let bytes = try stream.read(upTo: digits) guard bytes.count == digits, let value = UInt32(String(decoding: bytes, as: UTF8.self), radix: 8) else { @@ -473,6 +474,17 @@ enum XipArchive { } return value } + // Sizes get their own parse: `odc` gives them eleven octal digits, which is + // 33 bits, and reading that into a `UInt32` would fail on exactly the large + // files a compiler's SDK is most likely to contain. + func sizeField(_ digits: Int) throws -> Int64 { + let bytes = try stream.read(upTo: digits) + guard bytes.count == digits, + let value = Int64(String(decoding: bytes, as: UTF8.self), radix: 8) else { + throw Error.unsupportedPayload("unreadable cpio file size") + } + return value + } let device = try octal(6) let inode = try octal(6) let mode = try octal(6) @@ -482,39 +494,39 @@ enum XipArchive { _ = try octal(6) // rdev _ = try stream.read(upTo: 11) // mtime let nameLength = try octal(6) - let size = try octal(11) + let size = try sizeField(11) return RawHeader(mode: mode, inode: inode, device: device, linkCount: linkCount, - size: Int64(size), nameLength: Int(nameLength), padded: false) + size: size, nameLength: Int(nameLength), padded: false) } /// `newc`: magic consumed, then eight-digit hex fields. private static func newcHeader(from stream: BlockStream) throws -> RawHeader { - func hex(_ digits: Int) throws -> UInt32 { + func hexField(_ digits: Int, as type: T.Type) throws -> T { let bytes = try stream.read(upTo: digits) guard bytes.count == digits, - let value = UInt32(String(decoding: bytes, as: UTF8.self), radix: 16) else { + let value = T(String(decoding: bytes, as: UTF8.self), radix: 16) else { throw Error.unsupportedPayload("unreadable cpio field") } return value } - let inode = try hex(8) - let mode = try hex(8) - _ = try hex(8) // uid - _ = try hex(8) // gid - let linkCount = try hex(8) - _ = try hex(8) // mtime - let size = try hex(8) - let deviceMajor = try hex(8) - let deviceMinor = try hex(8) - _ = try hex(8) // rdev major - _ = try hex(8) // rdev minor - let nameLength = try hex(8) - _ = try hex(8) // check + let inode = try hexField(8, as: UInt32.self) + let mode = try hexField(8, as: UInt32.self) + _ = try hexField(8, as: UInt32.self) // uid + _ = try hexField(8, as: UInt32.self) // gid + let linkCount = try hexField(8, as: UInt32.self) + _ = try hexField(8, as: UInt32.self) // mtime + let size = try hexField(8, as: Int64.self) + let deviceMajor = try hexField(8, as: UInt32.self) + let deviceMinor = try hexField(8, as: UInt32.self) + _ = try hexField(8, as: UInt32.self) // rdev major + _ = try hexField(8, as: UInt32.self) // rdev minor + let nameLength = try hexField(8, as: UInt32.self) + _ = try hexField(8, as: UInt32.self) // check return RawHeader(mode: mode, inode: inode, device: (deviceMajor << 16) | deviceMinor, - linkCount: linkCount, size: Int64(size), + linkCount: linkCount, size: size, nameLength: Int(nameLength), padded: true) } @@ -524,8 +536,7 @@ enum XipArchive { static func inflate(_ data: Data, to size: Int, what: String) throws -> Data { guard size > 0 else { throw Error.unsupportedPayload("\(what) declares no size") } var output = [UInt8](repeating: 0, count: size) - let written = compression_decode_buffer(&output, size, [UInt8](data), data.count, - nil, COMPRESSION_ZLIB) + let written = decode(data, into: &output, algorithm: COMPRESSION_ZLIB) guard written == size else { throw Error.unsupportedArchive("could not inflate \(what)") } return Data(output) } @@ -533,11 +544,21 @@ enum XipArchive { /// Raw LZMA2 as pbzx stores it, decoded by Apple's own implementation. static func inflateLZMA(_ data: Data, to size: Int) throws -> [UInt8] { var output = [UInt8](repeating: 0, count: size) - let written = compression_decode_buffer(&output, size, [UInt8](data), data.count, - nil, COMPRESSION_LZMA) + let written = decode(data, into: &output, algorithm: COMPRESSION_LZMA) guard written == size else { throw Error.decompressionFailed(Int64(size)) } return output } + + /// `compression_decode_buffer`, with the input handed over as a pointer rather + /// than relying on the implicit array conversion at the call site. + private static func decode(_ data: Data, into output: inout [UInt8], + algorithm: compression_algorithm) -> Int { + data.withUnsafeBytes { raw in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return 0 } + return compression_decode_buffer(&output, output.count, base, data.count, + nil, algorithm) + } + } } private extension Data { diff --git a/AppTests/XipArchiveTests.swift b/AppTests/XipArchiveTests.swift index 5077118..ac413e8 100644 --- a/AppTests/XipArchiveTests.swift +++ b/AppTests/XipArchiveTests.swift @@ -170,16 +170,22 @@ final class XipArchiveTests: XCTestCase { private static func zlibEncode(_ data: Data) -> Data? { var buffer = [UInt8](repeating: 0, count: data.count + 4_096) - let written = compression_encode_buffer(&buffer, buffer.count, [UInt8](data), data.count, - nil, COMPRESSION_ZLIB) - return written > 0 ? Data(buffer[0.. Data? in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return nil } + let written = compression_encode_buffer(&buffer, buffer.count, base, data.count, + nil, COMPRESSION_ZLIB) + return written > 0 ? Data(buffer[0.. Data? { var buffer = [UInt8](repeating: 0, count: data.count * 2 + 4_096) - let written = compression_encode_buffer(&buffer, buffer.count, [UInt8](data), data.count, - nil, COMPRESSION_LZMA) - return written > 0 ? Data(buffer[0.. Data? in + guard let base = raw.bindMemory(to: UInt8.self).baseAddress else { return nil } + let written = compression_encode_buffer(&buffer, buffer.count, base, data.count, + nil, COMPRESSION_LZMA) + return written > 0 ? Data(buffer[0.. Date: Fri, 25 Sep 2026 03:30:51 -0400 Subject: [PATCH 42/54] xip: `Self` inside the nested writer is the writer, not the builder The second CI round got past the first error and stopped here: `Self.isWanted` inside `DarwinSDKBuilder.Writer` resolves to `Writer`, which has no such member. The outer type is named in full now. --- App/NativeToolchain/DarwinSDKBuilder.swift | 3 ++- App/Views/Toolchain/ToolchainView.swift | 3 +-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/App/NativeToolchain/DarwinSDKBuilder.swift b/App/NativeToolchain/DarwinSDKBuilder.swift index e1f81a4..ac970d0 100644 --- a/App/NativeToolchain/DarwinSDKBuilder.swift +++ b/App/NativeToolchain/DarwinSDKBuilder.swift @@ -191,7 +191,8 @@ enum DarwinSDKBuilder { } func accept(_ entry: XipArchive.Entry, _ payload: XipArchive.Payload) throws { - guard Self.isWanted(entry.name) else { + // `Self` here is `Writer`, so the outer type is named in full. + guard DarwinSDKBuilder.isWanted(entry.name) else { skipped += 1 return } diff --git a/App/Views/Toolchain/ToolchainView.swift b/App/Views/Toolchain/ToolchainView.swift index 5c64053..8843f5c 100644 --- a/App/Views/Toolchain/ToolchainView.swift +++ b/App/Views/Toolchain/ToolchainView.swift @@ -241,8 +241,7 @@ struct ToolchainView: View { } continuation.finish() - await consumer.value - state.finish() + await consumer.value // the consumer clears `isRunning` when the stream ends working = false // The picker's copy of the archive is the app's to delete, and for an From e3ce2803492bfe80c3f8021d1c0c89e42a7cc279 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 03:42:53 -0400 Subject: [PATCH 43/54] xip: the compiler's six complaints about the reader MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit - `end - handle.offsetInFile` mixes `Int64` and `UInt64`, and the four places that recorded the file position assigned a `UInt64` to an `Int64`. Mixed comparisons are fine in Swift; mixed arithmetic is not, which is why only these showed up. - `Payload.init` was `private`, which for a member of a *nested* type is not visible to the enclosing type either — so the walk that constructs payloads could not call it. `fileprivate` on the init and on `BlockStream` puts it where the walk can reach it. - The big-endian helper assembled bytes through `withUnsafeMutableBytes` with an element type the compiler could not infer. It shifts now, which is also easier to read and gets the byte order right by construction. --- App/NativeToolchain/XipArchive.swift | 31 +++++++++++++++++----------- AppTests/XipArchiveTests.swift | 1 + 2 files changed, 20 insertions(+), 12 deletions(-) diff --git a/App/NativeToolchain/XipArchive.swift b/App/NativeToolchain/XipArchive.swift index 8cf60f8..c99d869 100644 --- a/App/NativeToolchain/XipArchive.swift +++ b/App/NativeToolchain/XipArchive.swift @@ -236,7 +236,7 @@ enum XipArchive { /// A cursor over a sequence of decompressed blocks rather than one buffer: the /// payload is ~15 GB decompressed, and a cpio entry — let alone a whole block — /// does not respect block boundaries, so reads have to be able to span them. - private final class BlockStream { + fileprivate final class BlockStream { private let handle: FileHandle private let end: Int64 private let contentLength: Int64 @@ -266,7 +266,8 @@ enum XipArchive { /// The next decompressed block, or `nil` at the end of the payload. private func nextBlock() throws -> [UInt8]? { if finished { return nil } - if handle.offsetInFile >= end { return nil } + let offset = Int64(handle.offsetInFile) + if offset >= end { return nil } let header = try readUpTo(16) guard header.count == 16 else { return nil } let decompressedSize = Int64(header[0..<8].bigEndian(as: UInt64.self)) @@ -276,7 +277,7 @@ enum XipArchive { // Never read past the member: the archive's own signature and metadata // follow it, and swallowing those as payload would be reported as a // corrupt xip rather than as the end of the stream. - let available = end - handle.offsetInFile + let available = end - offset let block = try readExact(Int(min(compressedSize, available))) // The block is stored when it did not shrink; otherwise it is LZMA. if decompressedSize < chunkSize { @@ -342,13 +343,13 @@ enum XipArchive { } data += piece } - consumed = handle.offsetInFile + consumed = Int64(handle.offsetInFile) return data } private func readUpTo(_ count: Int) throws -> Data { let data = try handle.read(upToCount: count) ?? Data() - consumed = handle.offsetInFile + consumed = Int64(handle.offsetInFile) return data } } @@ -358,8 +359,10 @@ enum XipArchive { private let stream: BlockStream private var remaining: Int64 - /// `private` because `BlockStream` is: only the walk constructs payloads. - private init(stream: BlockStream, size: Int64) { + /// `fileprivate` because it takes a `BlockStream`, which is: only the walk + /// in this file constructs payloads. `private` would be too narrow — Swift's + /// `private` does not reach the *enclosing* type, so the walk could not call it. + fileprivate init(stream: BlockStream, size: Int64) { self.stream = stream self.remaining = size } @@ -563,13 +566,17 @@ enum XipArchive { private extension Data { /// Big-endian integer from the leading bytes of this data. + /// + /// Assembled by shifting rather than by reinterpreting a byte buffer: the + /// pointer form reads as `withUnsafeMutableBytes` gymnastics whose element type + /// inference depends on the enclosing generic, and gets none of that right by + /// being clever. func bigEndian(as type: T.Type) -> T { + let width = MemoryLayout.size var value: T = 0 - withUnsafeMutableBytes(of: &value) { destination in - for (index, byte) in prefix(MemoryLayout.size).enumerated() { - destination[index] = byte - } + for (index, byte) in prefix(width).enumerated() { + value |= T(byte) << (8 * (width - 1 - index)) } - return value.bigEndian + return value } } diff --git a/AppTests/XipArchiveTests.swift b/AppTests/XipArchiveTests.swift index ac413e8..f70cbe0 100644 --- a/AppTests/XipArchiveTests.swift +++ b/AppTests/XipArchiveTests.swift @@ -1,4 +1,5 @@ import Compression +import Foundation import XCTest @testable import XForge From b0cbb5863928e70c0e4c9bc4051ad66016986e1c Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 03:55:36 -0400 Subject: [PATCH 44/54] xip: the last `UInt64` position, in `loadNext` The previous pass fixed the two readers and the progress cursor and missed this one, which is what the compiler stopped on next. Every `handle.offsetInFile` is now converted at the point of use, and none is assigned raw. --- App/NativeToolchain/XipArchive.swift | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/App/NativeToolchain/XipArchive.swift b/App/NativeToolchain/XipArchive.swift index c99d869..23cecae 100644 --- a/App/NativeToolchain/XipArchive.swift +++ b/App/NativeToolchain/XipArchive.swift @@ -293,7 +293,7 @@ enum XipArchive { guard let block = try nextBlock(), !block.isEmpty else { return false } buffer = block position = 0 - consumed = handle.offsetInFile + consumed = Int64(handle.offsetInFile) if let progress, contentLength > 0 { progress(min(1, Double(consumed) / Double(contentLength))) } From 025bbd9dc2ea8e6a9f2a4bfdf33a4ef3ebac0f61 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 04:10:27 -0400 Subject: [PATCH 45/54] xip tests: a static member needs its type named, even from an instance method MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Thirteen of them in the path tables — the app target compiles now, and this is what the test target stopped on. The rest of the file already wrote `Self.`; these two lists were the ones I wrote before deciding to. --- AppTests/XipArchiveTests.swift | 64 +++++++++++++++++----------------- 1 file changed, 32 insertions(+), 32 deletions(-) diff --git a/AppTests/XipArchiveTests.swift b/AppTests/XipArchiveTests.swift index f70cbe0..8f13d4b 100644 --- a/AppTests/XipArchiveTests.swift +++ b/AppTests/XipArchiveTests.swift @@ -31,44 +31,44 @@ final class XipArchiveTests: XCTestCase { /// The paths of the fake Xcode tree, rooted at `Xcode.app` as the archive's are. private static let developer = "Xcode.app/Contents/Developer" - private static let platform = "\(developer)/Platforms/iPhoneOS.platform" - private static let toolchain = "\(developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib" + private static let platform = "\(Self.developer)/Platforms/iPhoneOS.platform" + private static let toolchain = "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib" private static func fakeXcode() -> [Spec] { var inode: UInt32 = 0 func next() -> UInt32 { inode += 1; return inode } let directories = [ - "Xcode.app", "Xcode.app/Contents", developer, "\(developer)/Platforms", - platform, "\(platform)/Developer", - "\(platform)/Developer/SDKs", - "\(platform)/Developer/SDKs/iPhoneOS27.0.sdk", - "\(platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr", - "\(platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr/include", - "\(developer)/Toolchains", "\(developer)/Toolchains/XcodeDefault.xctoolchain", - "\(developer)/Toolchains/XcodeDefault.xctoolchain/usr", - "\(toolchain)", "\(toolchain)/swift_static", "\(toolchain)/swift_static/iphoneos", - "\(toolchain)/swift", "\(toolchain)/swift/prebuilt-modules", + "Xcode.app", "Xcode.app/Contents", developer, "\(Self.developer)/Platforms", + platform, "\(Self.platform)/Developer", + "\(Self.platform)/Developer/SDKs", + "\(Self.platform)/Developer/SDKs/iPhoneOS27.0.sdk", + "\(Self.platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr", + "\(Self.platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr/include", + "\(Self.developer)/Toolchains", "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain", + "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain/usr", + "\(Self.toolchain)", "\(Self.toolchain)/swift_static", "\(Self.toolchain)/swift_static/iphoneos", + "\(Self.toolchain)/swift", "\(Self.toolchain)/swift/prebuilt-modules", "Xcode.app/Contents/Resources", - "\(developer)/Platforms/MacOSX.platform", + "\(Self.developer)/Platforms/MacOSX.platform", ] var specs = directories.map { Spec(path: $0, mode: directory, inode: next()) } // Kept: the platform manifest, a header from the SDK, the Swift static runtime. - specs.append(Spec(path: "\(platform)/Info.plist", mode: regular, + specs.append(Spec(path: "\(Self.platform)/Info.plist", mode: regular, data: Data("".utf8), inode: next())) - specs.append(Spec(path: "\(platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr/include/stdio.h", + specs.append(Spec(path: "\(Self.platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr/include/stdio.h", mode: regular, data: Data("#pragma once\n".utf8), inode: next())) let runtimeInode = next() - specs.append(Spec(path: "\(toolchain)/swift_static/iphoneos/libswiftCore.a", + specs.append(Spec(path: "\(Self.toolchain)/swift_static/iphoneos/libswiftCore.a", mode: regular, data: Data(repeating: 0xAB, count: 5_000), inode: runtimeInode, links: 2)) // Dropped: Xcode's own build output, another platform, anything under // Contents that is not the Developer directory. - specs.append(Spec(path: "\(toolchain)/swift/prebuilt-modules/Foundation.swiftmodule", + specs.append(Spec(path: "\(Self.toolchain)/swift/prebuilt-modules/Foundation.swiftmodule", mode: regular, data: Data(repeating: 0xFF, count: 64), inode: next())) - specs.append(Spec(path: "\(developer)/Platforms/MacOSX.platform/Info.plist", + specs.append(Spec(path: "\(Self.developer)/Platforms/MacOSX.platform/Info.plist", mode: regular, data: Data("".utf8), inode: next())) specs.append(Spec(path: "Xcode.app/Contents/Resources/Dropped.bin", mode: regular, data: Data(repeating: 0x11, count: 32), inode: next())) @@ -77,11 +77,11 @@ final class XipArchiveTests: XCTestCase { // The second link to the runtime: same inode, and both are inside the filter, // so the bundle should carry one file and one hard link, not two copies. - specs.append(Spec(path: "\(toolchain)/swift/libswiftCore.a", mode: regular, + specs.append(Spec(path: "\(Self.toolchain)/swift/libswiftCore.a", mode: regular, data: Data(), inode: runtimeInode, links: 2)) // `iPhoneOS.sdk` is a symlink to the versioned directory in recent Xcodes. - specs.append(Spec(path: "\(platform)/Developer/SDKs/iPhoneOS.sdk", mode: symlink, + specs.append(Spec(path: "\(Self.platform)/Developer/SDKs/iPhoneOS.sdk", mode: symlink, data: Data("iPhoneOS27.0.sdk".utf8), inode: next())) return specs } @@ -290,26 +290,26 @@ final class XipArchiveTests: XCTestCase { /// XForge's one deliberate narrowing: the device platform only. func testWantedPaths() { let kept = [ - "\(developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/iphoneos/Swift.swiftmodule", - "\(toolchain)/swift_static/iphoneos/libswiftCore.a", - "\(toolchain)/clang/include/stdarg.h", - "\(platform)/Info.plist", - "\(platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr/include/stdio.h", - "\(platform)/Developer/usr/lib/libSystem.tbd", - "\(platform)/Developer/Library/Frameworks/XCTest.framework/XCTest", + "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/iphoneos/Swift.swiftmodule", + "\(Self.toolchain)/swift_static/iphoneos/libswiftCore.a", + "\(Self.toolchain)/clang/include/stdarg.h", + "\(Self.platform)/Info.plist", + "\(Self.platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr/include/stdio.h", + "\(Self.platform)/Developer/usr/lib/libSystem.tbd", + "\(Self.platform)/Developer/Library/Frameworks/XCTest.framework/XCTest", ] for path in kept { XCTAssertTrue(DarwinSDKBuilder.isWanted(path), "should be kept: \(path)") } let dropped = [ - "\(toolchain)/swift/prebuilt-modules/Foundation.swiftmodule", - "\(developer)/Platforms/MacOSX.platform/Info.plist", - "\(developer)/Applications/Whatever.app/Whatever", + "\(Self.toolchain)/swift/prebuilt-modules/Foundation.swiftmodule", + "\(Self.developer)/Platforms/MacOSX.platform/Info.plist", + "\(Self.developer)/Applications/Whatever.app/Whatever", "Contents/Resources/English.lproj/InfoPlist.strings", "Xcode.app/Contents/version.plist", - "\(toolchain)/swift_static/iphoneos", - "\(developer)", + "\(Self.toolchain)/swift_static/iphoneos", + "\(Self.developer)", ] for path in dropped { XCTAssertFalse(DarwinSDKBuilder.isWanted(path), "should be dropped: \(path)") From 90386c38c712de3288592ee3e782e7de2ac33f30 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 04:24:31 -0400 Subject: [PATCH 46/54] xip tests: `Self` is illegal in a stored property's initializer My blanket qualification of the path constants reached the two lines that *build* those constants. Local `let`s in a function body are fine (and the three the sweep flagged are exactly that); a stored property's initializer is not, because the type is still being built. Bare names there, as before. --- AppTests/XipArchiveTests.swift | 8 ++++++-- 1 file changed, 6 insertions(+), 2 deletions(-) diff --git a/AppTests/XipArchiveTests.swift b/AppTests/XipArchiveTests.swift index 8f13d4b..2cd2766 100644 --- a/AppTests/XipArchiveTests.swift +++ b/AppTests/XipArchiveTests.swift @@ -30,9 +30,13 @@ final class XipArchiveTests: XCTestCase { private static let symlink: UInt32 = 0o120777 /// The paths of the fake Xcode tree, rooted at `Xcode.app` as the archive's are. + /// + /// Bare `developer` rather than `Self.developer` in the two below: a stored + /// property's initializer cannot reference `Self` at all — the type is still + /// being built — and inside a *static* property there is no ambiguity to resolve. private static let developer = "Xcode.app/Contents/Developer" - private static let platform = "\(Self.developer)/Platforms/iPhoneOS.platform" - private static let toolchain = "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib" + private static let platform = "\(developer)/Platforms/iPhoneOS.platform" + private static let toolchain = "\(developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib" private static func fakeXcode() -> [Spec] { var inode: UInt32 = 0 From be2e0b640565c3a95d9fbb36086b1d84b6440047 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 04:39:57 -0400 Subject: [PATCH 47/54] xip tests: the fixture was the wrong format, and two expectations were wrong MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The tests ran for the first time. Five of the seven new ones passed, including the xar offset test and the full cpio walk, so the reader's plumbing is right. The three failures: - **The compressed-block fixture was raw LZMA2.** A pbzx block is an **xz** stream: `pbzx.c` checks for the `\xFD7zXZ` magic and refuses anything else, and Apple's `COMPRESSION_LZMA` — which is what decodes it — rejected the raw stream this test fed it, exactly as it should. The fixture is now 787 bytes of odc cpio compressed to 256 bytes by liblzma through Python's `lzma` module, wrapped in one pbzx block in a xar, and the test walks *that*: framing, xz decode and cpio parse together, against a stream this process did not produce. The format notes in `XipArchive` said LZMA2 too; they say xz now. - `Toolchains/…/usr/lib/swift_static/iphoneos` is *wanted* — it is the static runtime directory — and the expectation table listed it as dropped. - The bundle has five files (manifest, header, runtime, its hard link, the symlink); the assertion demanded more than five. --- App/NativeToolchain/XipArchive.swift | 7 ++- AppTests/XipArchiveTests.swift | 82 ++++++++++++++++++++++------ 2 files changed, 69 insertions(+), 20 deletions(-) diff --git a/App/NativeToolchain/XipArchive.swift b/App/NativeToolchain/XipArchive.swift index 23cecae..013c013 100644 --- a/App/NativeToolchain/XipArchive.swift +++ b/App/NativeToolchain/XipArchive.swift @@ -16,7 +16,8 @@ import Foundation /// **Content** is a pbzx stream: the bytes `pbzx`, an 8-byte big-endian chunk /// size, then blocks of (decompressed size, compressed size, bytes) — repeated /// while a block's decompressed size equals the chunk size. A block is either -/// stored or LZMA-compressed, which the two sizes reveal. +/// stored or an **xz** stream, which the two sizes reveal (`pbzx.c` checks for +/// the `\xFD7zXZ` magic, and Apple's `COMPRESSION_LZMA` accepts it). /// /// **decompressed** it is an `odc` cpio archive (magic `070707`, six-digit octal /// fields, no alignment padding) of the `Xcode.app` tree, ending at a @@ -279,7 +280,7 @@ enum XipArchive { // corrupt xip rather than as the end of the stream. let available = end - offset let block = try readExact(Int(min(compressedSize, available))) - // The block is stored when it did not shrink; otherwise it is LZMA. + // The block is stored when it did not shrink; otherwise it is xz. if decompressedSize < chunkSize { finished = true } @@ -544,7 +545,7 @@ enum XipArchive { return Data(output) } - /// Raw LZMA2 as pbzx stores it, decoded by Apple's own implementation. + /// An xz stream as pbzx stores it, decoded by Apple's own implementation. static func inflateLZMA(_ data: Data, to size: Int) throws -> [UInt8] { var output = [UInt8](repeating: 0, count: size) let written = decode(data, into: &output, algorithm: COMPRESSION_LZMA) diff --git a/AppTests/XipArchiveTests.swift b/AppTests/XipArchiveTests.swift index 2cd2766..a7dbc8a 100644 --- a/AppTests/XipArchiveTests.swift +++ b/AppTests/XipArchiveTests.swift @@ -195,14 +195,19 @@ final class XipArchiveTests: XCTestCase { /// Write a synthesized xip into a fresh temporary directory. private func makeXip(payload: Data? = nil) throws -> URL { + let cpio = payload ?? Self.cpio(Self.fakeXcode()) + return try writeXip(Self.xar(content: Self.pbzx(cpio))) + } + + /// Write xar bytes into a fresh temporary directory. + private func writeXip(_ archive: Data) throws -> URL { let directory = URL(fileURLWithPath: NSTemporaryDirectory()) .appendingPathComponent("xip-tests-\(UUID().uuidString)", isDirectory: true) try FileManager.default.createDirectory(at: directory, withIntermediateDirectories: true) addTeardownBlock { try? FileManager.default.removeItem(at: directory) } - let cpio = payload ?? Self.cpio(Self.fakeXcode()) let url = directory.appendingPathComponent("Xcode_27.xip") - try Self.xar(content: Self.pbzx(cpio)).write(to: url) + try archive.write(to: url) return url } @@ -268,24 +273,59 @@ final class XipArchiveTests: XCTestCase { XCTAssertEqual(payloads["\(Self.platform)/Developer/SDKs/iPhoneOS.sdk"], 16) } - /// The one part that has to agree with another implementation of the format. + /// The compressed path, validated against an implementation other than this one. /// - /// The blob is 420 bytes of text compressed to 85 bytes of **raw LZMA2** by - /// liblzma, which is what a pbzx block holds. Apple's decoder has to reproduce it - /// byte for byte; an encoder round-trip through this process would have proved - /// nothing, since two matching bugs look exactly like correctness. - func testDecodesRawLZMA2ProducedByLiblzma() throws { + /// A pbzx block is an **xz** stream — `pbzx.c` checks for the `\xFD7zXZ` magic and + /// refuses anything else — and Apple's `COMPRESSION_LZMA` is what decodes it. + /// This fixture is 787 bytes of odc cpio compressed to 256 bytes by **liblzma** + /// (Python's `lzma` module, xz container), wrapped in a single pbzx block inside a + /// xar. Walking it exercises the whole compressed path — the framing decision + /// (compressed size differs from decompressed size), the xz decode and the cpio + /// parse — against a stream this process did not produce. An encoder round trip + /// would have proved nothing: two matching bugs look exactly like correctness. + func testWalksAnXZPayloadProducedByLiblzma() throws { let base64 = """ - 4AGjAE1dACwaSgQTYMiZxu0IMKQ6+2aC3gaIG0RBZoQd0tB/LAu8gQdovsXeV7hHLo7E0RzTAygy\ - 1lQcyW1wiJyvt23W96lpWfeimqcQPsLwdAAAAA== + /Td6WFoAAATm1rRGAgAhARYAAAB0L+Wj4AMSAL1dABgN3wegMRkMF/SCaXaknqoWiVsR\ + ntc2yBB5DloIwewVgm0EJoUwmh+RvTVwtHxlYFJ1cZxGEPK8IgJOaAlEu44Mfu1l8oOZ\ + B6IWQLeOhCnnHNhdx7piLXn/q9nM4+Ka84R4dbk6jozpre+YlFurXBoqU2Bn3zhVpRu1\ + XnLHE0TGI/caB2jNykEyKiHElx5ZcXjGvi+cpoApZ4ZX3NbQxDozsrusWS7CR2dkq2yL\ + Lc5s0cid+htDMzoOxwaAAAAAAAAh8cN2Rcr+YAAB2QGTBgAAcoJ8YLHEZ/sCAAAAAARZ\ + Wg== """ let compressed = try XCTUnwrap(Data(base64Encoded: base64)) - let expected = Data(String(repeating: "XipArchive: a pbzx block is raw LZMA2, " - + "decoded with COMPRESSION_LZMA. ", count: 6).utf8) - XCTAssertEqual(expected.count, 420, "the fixture and this payload must stay in step") + XCTAssertEqual(compressed.prefix(6), Data([0xFD, 0x37, 0x7A, 0x58, 0x5A, 0x00]), + "the fixture has to be an xz stream, which is what pbzx holds") + + // One block, compressed: the uncompressed size is in the block header, and the + // two sizes differing is what tells the reader to decompress it. + let uncompressedSize = 787 + var stream = Data("pbzx".utf8) + stream += Self.bigEndian(UInt64(4_096), 8) + stream += Self.bigEndian(UInt64(uncompressedSize), 8) + stream += Self.bigEndian(UInt64(compressed.count), 8) + stream += compressed + + let url = try writeXip(Self.xar(content: stream)) + let content = try XipArchive.content(of: url) - let decoded = try XipArchive.inflateLZMA(compressed, to: expected.count) - XCTAssertEqual(Data(decoded), expected) + var names: [String] = [] + var stdio: [UInt8] = [] + try XipArchive.forEachEntry(in: url, content: content) { entry, payload in + names.append(entry.name) + if entry.name.hasSuffix("/usr/include/stdio.h") { + stdio = try payload.readAll() + } + } + + XCTAssertEqual(names, [ + "Fixture.app", + "Fixture.app/Developer", + "Fixture.app/Developer/Info.plist", + "Fixture.app/Developer/SDKs/iPhoneOS99.0.sdk", + "Fixture.app/Developer/SDKs/iPhoneOS99.0.sdk/usr", + "Fixture.app/Developer/SDKs/iPhoneOS99.0.sdk/usr/include/stdio.h", + ]) + XCTAssertEqual(Data(stdio), Data("#pragma once\n".utf8)) } // MARK: - What becomes the bundle @@ -296,6 +336,10 @@ final class XipArchiveTests: XCTestCase { let kept = [ "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/iphoneos/Swift.swiftmodule", "\(Self.toolchain)/swift_static/iphoneos/libswiftCore.a", + // The directory itself is kept: it holds the static runtime, and it is what + // `swiftStaticResourcesPath` names. + "\(Self.toolchain)/swift_static/iphoneos", + "\(Self.toolchain)/swift_static", "\(Self.toolchain)/clang/include/stdarg.h", "\(Self.platform)/Info.plist", "\(Self.platform)/Developer/SDKs/iPhoneOS27.0.sdk/usr/include/stdio.h", @@ -312,8 +356,10 @@ final class XipArchiveTests: XCTestCase { "\(Self.developer)/Applications/Whatever.app/Whatever", "Contents/Resources/English.lproj/InfoPlist.strings", "Xcode.app/Contents/version.plist", - "\(Self.toolchain)/swift_static/iphoneos", "\(Self.developer)", + "\(Self.developer)/Platforms", + "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain/usr/bin/swiftc", + "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift-ide-test", ] for path in dropped { XCTAssertFalse(DarwinSDKBuilder.isWanted(path), "should be dropped: \(path)") @@ -329,7 +375,9 @@ final class XipArchiveTests: XCTestCase { .appendingPathComponent("out", isDirectory: true) let result = try DarwinSDKBuilder.build(fromXip: url, into: destination) - XCTAssertGreaterThan(result.files, 5) + // Five: the platform manifest, a header, the static runtime, the archive's hard + // link to it, and the symlink. Directories are not counted. + XCTAssertGreaterThanOrEqual(result.files, 5) XCTAssertGreaterThan(result.skipped, 0, "the unwanted paths should be counted as skipped") XCTAssertEqual(result.sdkRoot, "Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS27.0.sdk") From ae9a2e81d3cdf4c96e547f769b6cfbb5c6d934e0 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 05:32:59 -0400 Subject: [PATCH 48/54] repo: remove what nothing needs, and fix what says what is no longer true MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Deletions, each with the evidence: - **`.github/workflows/ios-share.yml` + `builder.json`.** Its own comments tell you to keep its `js toolchain` blocks identical to `ios-build.yml` and `runner.sh`; neither file is in the repository. It drives the Flutter / React Native / Expo / KMP branches of a multi-framework builder, and it has never run here — `gh run list` for that workflow returns zero runs, so no `ios-share/*` tag has ever been pushed. This is a Swift app with an in-process toolchain. Recoverable from git if the MobAI simulator flow is wanted back. - **`.github/workflows/bridge-api-check.yml`.** A fast loop for the bridge compile-check when reaching it cost 40–90 minutes. It now costs eleven seconds inside a two-minute warm run — and the workflow kept its own copy of the LLVM configure recipe, which is a second place to update. Corrections, found while looking for the above: - `README.md`, `Docs/XCODE-ALTERNATIVE.md` and the roadmap still said the Swift frontend is not in the toolchain bundle. It is, in a `with_swift` bundle, and the app links it: what has not happened is a Swift *build* on a device, which is now what they say. - `Support/XForge.entitlements` justified both of its keys by the embedded Linux guest — a guest that no longer exists. The keys stay (a compiler's working set and its maps are the reason now), the reasoning says so. - `App/Services/InstallAssertion.swift` was the one file nothing referenced. It is not dead weight: it holds the idle timer and a background task assertion for a long job, which is exactly what an `.xip` import is (minutes of CPU, and iOS suspends an app whose screen locks). The import uses it now, so it is referenced — and the import gets guarded. --- .github/workflows/bridge-api-check.yml | 391 ------------- .github/workflows/ios-share.yml | 692 ------------------------ App/Views/Toolchain/ToolchainView.swift | 8 + CHANGELOG.md | 16 +- Docs/DESIGN.md | 2 +- Docs/XCODE-ALTERNATIVE.md | 2 +- README.md | 35 +- Support/XForge.entitlements | 25 +- builder.json | 18 - 9 files changed, 55 insertions(+), 1134 deletions(-) delete mode 100644 .github/workflows/bridge-api-check.yml delete mode 100644 .github/workflows/ios-share.yml delete mode 100644 builder.json diff --git a/.github/workflows/bridge-api-check.yml b/.github/workflows/bridge-api-check.yml deleted file mode 100644 index bc60563..0000000 --- a/.github/workflows/bridge-api-check.yml +++ /dev/null @@ -1,391 +0,0 @@ -name: Bridge API Check - -# The fast loop for the one step that keeps failing. -# -# `Native iOS Toolchain` spends 40-90 minutes building LLVM and Swift before it -# reaches the bridge compile-check, and that check needs none of those libraries: it -# needs the headers of the LLVM and Swift the bundle will be built from, the headers -# those builds *generate*, and a compiler. All of that is already in the two caches -# the toolchain workflow restores, so this workflow restores them, reconfigures (which -# keeps the generated headers in step with the sources), and compiles the bridge — -# about ten minutes instead of an afternoon. -# -# It is not a substitute for the toolchain build: nothing here links anything, so it -# says nothing about the archive, the staging step or the app. It answers one -# question — "does the bridge compile against this LLVM generation?" — which is the -# question that has been costing hours. - -on: - workflow_dispatch: - inputs: - swift_ref: - type: string - default: swift-6.2.4-RELEASE - swift_llvm_ref: - type: string - default: swift/release/6.2 - with_swift: - description: Compile the bridge with the Swift frontend code path enabled - type: boolean - default: true - pull_request: - paths: - - "App/NativeToolchain/**" - - ".github/workflows/bridge-api-check.yml" - push: - branches: - - drop-alpine - - native-toolchain-poc - paths: - - "App/NativeToolchain/**" - - ".github/workflows/bridge-api-check.yml" - -jobs: - api-check: - runs-on: macos-15 - timeout-minutes: 60 - env: - IOS_DEPLOYMENT_TARGET: "17.0" - SWIFT_REF: ${{ inputs.swift_ref || 'swift-6.2.4-RELEASE' }} - SWIFT_LLVM_REF: ${{ inputs.swift_llvm_ref || 'swift/release/6.2' }} - steps: - - uses: actions/checkout@v4 - - - name: Install build tools - run: brew install cmake ninja - - - name: Checkout LLVM - uses: actions/checkout@v4 - with: - repository: swiftlang/llvm-project - ref: ${{ env.SWIFT_LLVM_REF }} - path: llvm-project - fetch-depth: 1 - - - name: Checkout Swift - if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} - uses: actions/checkout@v4 - with: - repository: swiftlang/swift - ref: ${{ env.SWIFT_REF }} - path: swift - fetch-depth: 1 - - - name: Checkout cmark - if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} - uses: actions/checkout@v4 - with: - repository: swiftlang/swift-cmark - ref: release/6.2 - path: swift-cmark - fetch-depth: 1 - - - name: Checkout swift-syntax - if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} - uses: actions/checkout@v4 - with: - repository: swiftlang/swift-syntax - ref: release/6.2 - path: swift-syntax - fetch-depth: 1 - - - name: Checkout swift-experimental-string-processing - if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} - uses: actions/checkout@v4 - with: - repository: swiftlang/swift-experimental-string-processing - ref: release/6.2 - path: swift-experimental-string-processing - fetch-depth: 1 - - - name: Identify the iOS compiler toolchain - id: toolchain-versions - run: | - { - echo "value=$(xcodebuild -version | tr '\n' ' ')-$(xcrun --sdk iphoneos --show-sdk-version)" - } >> "$GITHUB_OUTPUT" - - # The build trees, from the last toolchain run that produced them. This is what - # makes the staging and link half checkable in minutes: the archives and - # generated headers are the expensive part, and they arrive as an artifact - # rather than as a cache entry that may or may not be there. - - name: Download the build trees - env: - GH_TOKEN: ${{ github.token }} - run: | - set -euxo pipefail - mkdir -p build-trees - if gh run download --repo "$GITHUB_REPOSITORY" --name XForgeBuildTrees --dir build-trees; then - tar -xf build-trees/build-trees.tar - echo "build trees restored:" - ls -d llvm-build/lib swift-build/lib cmark-ios-build 2>/dev/null || true - else - echo "::warning::no XForgeBuildTrees artifact yet; falling back to the caches, then to configure only" - fi - - # The same keys the toolchain workflow writes, so this reuses what it built. - - name: Restore the LLVM builds - id: llvm-cache - uses: actions/cache/restore@v4 - with: - path: | - llvm-host-build - llvm-build - cmark-host-build - cmark-ios-build - key: ${{ runner.os }}-${{ runner.arch }}-llvm-${{ steps.toolchain-versions.outputs.value }}-${{ env.SWIFT_LLVM_REF }}-${{ env.IOS_DEPLOYMENT_TARGET }}-${{ inputs.cache_salt || '2' }} - - - name: Restore the Swift build - id: swift-cache - uses: actions/cache/restore@v4 - with: - path: swift-build - key: ${{ runner.os }}-${{ runner.arch }}-swift-${{ env.SWIFT_REF }}-${{ env.SWIFT_LLVM_REF }}-${{ steps.toolchain-versions.outputs.value }}-${{ env.IOS_DEPLOYMENT_TARGET }}-1 - - - name: Report what was available - run: | - set -euxo pipefail - echo "llvm cache hit: ${{ steps.llvm-cache.outputs.cache-hit }}" - echo "swift cache hit: ${{ steps.swift-cache.outputs.cache-hit }}" - ls llvm-build/include 2>/dev/null | head -5 || echo "no llvm-build/include" - ls llvm-build/tools/clang/include/clang/Basic/DiagnosticGroups.inc 2>/dev/null \ - || echo "no generated clang headers (expected on a cold cache)" - ls swift-build/include 2>/dev/null | head -5 || echo "no swift-build/include" - - # Configure both trees so the generated headers match the sources, then make - # everything newer than the checkouts — the same reason the toolchain workflow - # does it: restored outputs look older than a fresh checkout. - - name: Configure LLVM for iPhoneOS arm64 - shell: bash - run: | - set -euxo pipefail - SDK="$(xcrun --sdk iphoneos --show-sdk-path)" - cmake -S llvm-project/llvm -B llvm-build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_SYSTEM_NAME=iOS \ - -DCMAKE_MACOSX_BUNDLE=OFF \ - -DCMAKE_OSX_SYSROOT="$SDK" \ - -DCMAKE_OSX_ARCHITECTURES=arm64 \ - -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ - -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ - -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ - -DLLVM_HOST_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ - -DLLVM_DEFAULT_TARGET_TRIPLE=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ - -DLLVM_HOST_BUILD_DIR="$PWD/llvm-host-build" \ - -DLLVM_ENABLE_PROJECTS="clang;lld" \ - -DLLVM_TARGETS_TO_BUILD=AArch64 \ - -DLLVM_INCLUDE_TESTS=OFF \ - -DLLVM_INCLUDE_EXAMPLES=OFF \ - -DLLVM_INCLUDE_BENCHMARKS=OFF \ - -DLLVM_INCLUDE_TOOLS=ON \ - -DLLVM_BUILD_TOOLS=OFF \ - -DLLVM_ENABLE_TERMINFO=OFF \ - -DLLVM_ENABLE_ZLIB=OFF \ - -DLLVM_ENABLE_ZSTD=OFF \ - -DLLVM_ENABLE_LIBXML2=OFF \ - -DLLVM_ENABLE_CURL=OFF \ - -DLLVM_ENABLE_RTTI=ON \ - -DCLANG_INCLUDE_TESTS=OFF \ - -DCLANG_BUILD_TOOLS=ON - - - name: Build native cmark - if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} - shell: bash - run: | - set -euxo pipefail - if [ -x cmark-host-build/src/cmark-gfm ]; then - echo "native cmark already cached" - exit 0 - fi - cmake -S swift-cmark -B cmark-host-build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release -DCMARK_TESTS=OFF - cmake --build cmark-host-build --target cmark-gfm - ln -sf "$PWD/cmark-host-build/src/cmark-gfm" cmark-host-build/src/cmark - - - name: Build cmark for iPhoneOS - if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} - shell: bash - run: | - set -euxo pipefail - if [ -f cmark-ios-build/src/libcmark-gfm.a ]; then - echo "iOS cmark already cached" - exit 0 - fi - SDK="$(xcrun --sdk iphoneos --show-sdk-path)" - # CMAKE_SYSTEM_NAME=Darwin, not iOS, for the same reason the Swift - # configure uses it: on iOS CMake refuses \`install(TARGETS ...)\` for an - # executable without a BUNDLE destination, and cmark installs its - # command-line tool. The target is still iOS — sysroot, architecture and - # compiler target below say so — and nothing is installed here anyway. - # Static archives are cmark's default; naming CMARK_TESTS/CMARK_SHARED here - # only produces "manually-specified variables were not used" warnings. - cmake -S swift-cmark -B cmark-ios-build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_SYSTEM_NAME=Darwin \ - -DCMAKE_OSX_SYSROOT="$SDK" \ - -DCMAKE_OSX_ARCHITECTURES=arm64 \ - -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ - -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ - -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET - # cmark-gfm-extensions is a separate archive and swift's Markup references - # symbols from both. - # The *library* targets are libcmark-gfm and libcmark-gfm-extensions; the - # bare names are the command-line tool, which only the host build needs. - ninja -C cmark-ios-build libcmark-gfm libcmark-gfm-extensions - find cmark-ios-build -name 'libcmark-gfm*.a' - - - name: Configure the Swift frontend for iPhoneOS - if: ${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }} - shell: bash - run: | - set -euxo pipefail - SDK="$(xcrun --sdk iphoneos --show-sdk-path)" - cmake -S swift -B swift-build -G Ninja \ - -DCMAKE_BUILD_TYPE=Release \ - -DCMAKE_SYSTEM_NAME=Darwin \ - -DCMAKE_OSX_SYSROOT="$SDK" \ - -DCMAKE_OSX_ARCHITECTURES=arm64 \ - -DCMAKE_OSX_DEPLOYMENT_TARGET="$IOS_DEPLOYMENT_TARGET" \ - -DCMAKE_C_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ - -DCMAKE_CXX_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ - -DCMAKE_Swift_COMPILER="$(xcrun -f swiftc)" \ - -DCMAKE_Swift_COMPILER_TARGET=arm64-apple-ios$IOS_DEPLOYMENT_TARGET \ - -DLLVM_DIR="$PWD/llvm-build/lib/cmake/llvm" \ - -DClang_DIR="$PWD/llvm-build/lib/cmake/clang" \ - -DLLVM_TABLEGEN="$PWD/llvm-host-build/bin/llvm-tblgen" \ - -DCLANG_TABLEGEN="$PWD/llvm-host-build/bin/clang-tblgen" \ - -DSWIFT_NATIVE_LLVM_TOOLS_PATH="$PWD/llvm-host-build/bin" \ - -DSWIFT_NATIVE_CLANG_TOOLS_PATH="$PWD/llvm-host-build/bin" \ - -DSWIFT_NATIVE_SWIFT_TOOLS_PATH="$(dirname "$(xcrun -f swiftc)")" \ - -DSWIFT_PATH_TO_CMARK_BUILD="$PWD/cmark-host-build" \ - -DSWIFT_PATH_TO_CMARK_SOURCE="$PWD/swift-cmark" \ - -DSWIFT_PATH_TO_STRING_PROCESSING_SOURCE="$PWD/swift-experimental-string-processing" \ - -DSWIFT_PATH_TO_SWIFT_SYNTAX_SOURCE="$PWD/swift-syntax" \ - -DSWIFT_HOST_VARIANT=iphoneos \ - -DSWIFT_HOST_VARIANT_SDK=IOS \ - -DSWIFT_HOST_VARIANT_ARCH=arm64 \ - -DSWIFT_DARWIN_DEPLOYMENT_VERSION_IOS="$IOS_DEPLOYMENT_TARGET" \ - -DSWIFT_INCLUDE_TOOLS=ON \ - -DSWIFT_BUILD_DYNAMIC_STDLIB=OFF \ - -DSWIFT_BUILD_STATIC_STDLIB=OFF \ - -DSWIFT_BUILD_SOURCEKIT=OFF \ - -DSWIFT_BUILD_SWIFT_SYNTAX=OFF \ - -DSWIFT_INCLUDE_TESTS=OFF \ - -DSWIFT_INSTALL_COMPONENTS= - - - name: Make the trees newer than the sources - shell: bash - run: | - set -euxo pipefail - for tree in llvm-host-build llvm-build cmark-host-build cmark-ios-build swift-build; do - [ -d "$tree" ] && find "$tree" -exec touch -c {} + 2>/dev/null || true - done - - # The check itself, identical to the toolchain workflow's, so the two cannot - # drift: the bridge compiled for iOS with the headers of the exact LLVM and - # Swift the bundle will carry. - - name: Compile-check XForge bridge against LLVM headers - shell: bash - run: | - set -euxo pipefail - SDK="$(xcrun --sdk iphoneos --show-sdk-path)" - EXTRA_DEFINES=() - EXTRA_INCLUDES=() - if [ "${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }}" = "true" ]; then - EXTRA_DEFINES+=(-DXFORGE_HAS_SWIFT_FRONTEND=1) - EXTRA_INCLUDES+=(-I swift/include) - [ -d swift-build/include ] && EXTRA_INCLUDES+=(-I swift-build/include) - [ -d swift-build/tools/swift/include ] && EXTRA_INCLUDES+=(-I swift-build/tools/swift/include) - fi - xcrun --sdk iphoneos clang++ \ - -std=c++20 \ - -arch arm64 \ - -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET \ - -isysroot "$SDK" \ - -DXFORGE_HAS_LLVM=1 \ - "${EXTRA_DEFINES[@]}" \ - -I llvm-project/llvm/include \ - -I llvm-build/include \ - -I llvm-project/clang/include \ - -I llvm-build/tools/clang/include \ - -I llvm-project/lld/include \ - "${EXTRA_INCLUDES[@]}" \ - -I App/NativeToolchain \ - -c App/NativeToolchain/NativeToolchainBridge.mm \ - -o /tmp/NativeToolchainBridge.o - file /tmp/NativeToolchainBridge.o - - # The staging and link half, deliberately the same commands as the toolchain - # workflow's: header roots (sources and generated kept apart — `swift/bridging` - # is a generated file and `swift/Bridging/` a source directory), one merged - # archive, and a link into an app-shaped target. Without this the fast loop - # would answer the compile question and leave the link question to a 90-minute - # run, which is how the cmark symbols went unnoticed. - - name: Stage the toolchain - shell: bash - run: | - set -euxo pipefail - mkdir -p dist/include dist/include-generated dist/lib - stage_headers() { cp -R -L "$1" "$2"; } - - stage_headers llvm-project/llvm/include/llvm dist/include/ - stage_headers llvm-project/clang/include/clang dist/include/ - stage_headers llvm-project/lld/include/lld dist/include/ - - [ -d llvm-build/include/llvm ] && stage_headers llvm-build/include/llvm dist/include-generated/ - [ -d llvm-build/tools/clang/include/clang ] && stage_headers llvm-build/tools/clang/include/clang dist/include-generated/ - - if [ "${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }}" = "true" ]; then - stage_headers swift/include/swift dist/include/ - for swift_headers in swift-build/include/swift swift-build/tools/swift/include/swift; do - [ -d "$swift_headers" ] && stage_headers "$swift_headers" dist/include-generated/ - done - fi - - find llvm-build/lib -maxdepth 1 -type f -name '*.a' -print | sort > /tmp/archives.txt - if [ -d cmark-ios-build ]; then - find cmark-ios-build -type f -name 'libcmark-gfm*.a' -print | sort >> /tmp/archives.txt - fi - if [ "${{ inputs.with_swift || github.event_name != 'workflow_dispatch' }}" = "true" ]; then - find swift-build/lib -maxdepth 3 -type f -name '*.a' -print | sort >> /tmp/archives.txt - fi - test -s /tmp/archives.txt - # Staged unmerged: `libtool -static` de-duplicates members by name - # ("duplicate member name 'X86.cpp.o' from …"), and a dropped member is a - # silently missing object — which is precisely how the JIT symbols went - # undefined while their object was in the archive. - while IFS= read -r archive; do cp "$archive" dist/lib/; done < /tmp/archives.txt - cp /tmp/archives.txt dist/lib/XForgeToolchain.libraries.txt - echo "staged $(wc -l < /tmp/archives.txt) archives, unmerged:" - ls dist/lib | head -10 - - - name: Link an app-shaped target - shell: bash - run: | - set -euxo pipefail - SDK="$(xcrun --sdk iphoneos --show-sdk-path)" - cat > /tmp/native_link_smoke.mm <<'EOF' - #include "NativeToolchainBridge.h" - int main(void) { - return xf_native_toolchain_available() ? 0 : 1; - } - EOF - xcrun --sdk iphoneos clang++ -std=c++20 -arch arm64 \ - -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET -isysroot "$SDK" \ - -I App/NativeToolchain -c /tmp/native_link_smoke.mm -o /tmp/native_link_smoke.o - set +e - xcrun --sdk iphoneos clang++ -arch arm64 \ - -miphoneos-version-min=$IOS_DEPLOYMENT_TARGET -isysroot "$SDK" \ - /tmp/native_link_smoke.o /tmp/NativeToolchainBridge.o \ - $(for library in dist/lib/*.a; do printf ' -Wl,-force_load,%s' "$library"; done) \ - -lc++ -lz -liconv -lsqlite3 -framework Foundation \ - -o /tmp/XForgeNativeToolchainSmoke 2> /tmp/native-link-errors.txt - STATUS=$? - set -e - if [ "$STATUS" != "0" ]; then - echo "::error::linking the toolchain into an app-shaped target failed" - grep -E '^( "_|ld: )' /tmp/native-link-errors.txt | sort -u | head -40 - tail -40 /tmp/native-link-errors.txt - exit 1 - fi - file /tmp/XForgeNativeToolchainSmoke diff --git a/.github/workflows/ios-share.yml b/.github/workflows/ios-share.yml deleted file mode 100644 index 4beb1dc..0000000 --- a/.github/workflows/ios-share.yml +++ /dev/null @@ -1,692 +0,0 @@ -name: iOS Simulator -run-name: iOS Simulator ${{ inputs.build_id || github.ref_name }} - -# Builds the app for the simulator and then makes that simulator usable from -# the MobAI app, so the build can be tried by hand from a machine that is not a -# Mac. The job stays alive while the simulator is in use and ends when it is -# released or left unused. -# -# Simulator builds are never signed, so none of the signing setup in -# ios-build.yml applies here. - -on: - # Pushing a tag named ios-share/ starts the same job without the - # GitHub API, for environments where only git access is available. A push - # event runs the workflow file from the tagged commit, and the parameters - # come from builder.json there. The tag is deleted when the job ends. - push: - tags: - - 'ios-share/*' - workflow_dispatch: - inputs: - build_id: - description: 'Unique build identifier' - required: true - type: string - snapshot_ref: - description: 'Ref holding the working-tree snapshot to build (empty builds this branch)' - required: false - type: string - default: '' - ios_path: - description: 'Path to iOS project (e.g., "ios" for React Native)' - required: false - type: string - default: '.' - scheme: - description: 'Xcode scheme to build (auto-detected if empty)' - required: false - type: string - default: '' - duration: - description: 'How long the simulator stays available while unused (e.g. 30m)' - required: false - type: string - default: '30m' - flutter_version: - description: 'Flutter version to use (e.g., 3.24.0). Empty means latest stable.' - required: false - type: string - default: '' - jdk_version: - description: 'JDK version for Kotlin Multiplatform Gradle builds (e.g., 17).' - required: false - type: string - default: '17' - -jobs: - simulator: - runs-on: macos-latest - permissions: - contents: write # deletes the trigger tag - # Bounds the session: a simulator nobody released must not hold a runner - # indefinitely. Comfortably above the default duration. - timeout-minutes: 90 - - steps: - - name: Checkout repository - uses: actions/checkout@v7 - - # The dispatch always runs the workflow from the default branch, so the - # sources come from a snapshot ref pushed by the CLI instead. - - name: Checkout working-tree snapshot - if: inputs.snapshot_ref != '' - env: - SNAPSHOT_REF: ${{ inputs.snapshot_ref }} - run: | - set -e - git fetch --depth=1 origin "+$SNAPSHOT_REF:refs/heads/snapshot" - git checkout --force snapshot - echo "Building $(git rev-parse --short HEAD)" - - # Dispatch inputs arrive with their declared defaults. A tag push has no - # inputs, so the values come from builder.json in the tagged commit and - # the build id is the tag name after the prefix. - - name: Resolve parameters - id: params - env: - IN_BUILD_ID: ${{ inputs.build_id }} - IN_IOS_PATH: ${{ inputs.ios_path }} - IN_SCHEME: ${{ inputs.scheme }} - IN_DURATION: ${{ inputs.duration }} - IN_FLUTTER_VERSION: ${{ inputs.flutter_version }} - IN_JDK_VERSION: ${{ inputs.jdk_version }} - run: | - set -e - param() { # name dispatch-value jq-path default - local v="" - if [ "$GITHUB_EVENT_NAME" = "workflow_dispatch" ]; then - v="$2" - elif [ -f builder.json ]; then - v=$(jq -r "$3 // empty" builder.json) - fi - v="${v:-$4}" - echo "$1=$v" >> "$GITHUB_OUTPUT" - echo "$1=$v" - } - param build_id "$IN_BUILD_ID" '""' "${GITHUB_REF_NAME##*/}" - param ios_path "$IN_IOS_PATH" '.ios.path' '.' - param scheme "$IN_SCHEME" '.ios.scheme' '' - param duration "$IN_DURATION" '""' '30m' - param flutter_version "$IN_FLUTTER_VERSION" '.flutter.version' '' - param jdk_version "$IN_JDK_VERSION" '.kmp.jdkVersion' '17' - - # Starts the simulator booting in the background (cached, so later runs - # boot fast) while the app builds. - - name: Install mobai-ci + boot simulator - uses: MobAI-App/mobai-ci@v1 - with: - boot-sim: true # UDID lands in $MOBAI_SIM_UDID - - - name: Setup Xcode - uses: maxim-lobanov/setup-xcode@v1 - with: - xcode-version: latest-stable - - - name: Restore DerivedData cache - uses: actions/cache/restore@v6 - with: - path: DerivedData - key: deriveddata-sim-${{ github.run_id }} - restore-keys: | - deriveddata-sim- - - - name: Detect project type - id: detect - run: | - if [ -f "pubspec.yaml" ]; then - echo "type=flutter" >> $GITHUB_OUTPUT - elif [ -f "package.json" ] && grep -q '"expo"' package.json; then - echo "type=expo" >> $GITHUB_OUTPUT - elif [ -f "package.json" ] && grep -q '"react-native"' package.json; then - echo "type=reactnative" >> $GITHUB_OUTPUT - elif grep -rqlE 'kotlin\("multiplatform"\)|org\.jetbrains\.kotlin\.multiplatform|id\("org.jetbrains.kotlin.multiplatform"\)' --include='*.gradle.kts' --include='*.gradle' --include='libs.versions.toml' . 2>/dev/null; then - echo "type=kmp" >> $GITHUB_OUTPUT - else - echo "type=native" >> $GITHUB_OUTPUT - fi - - # A gitignored *.xcconfig next to a committed template is a common iOS - # convention. The snapshot then carries the template but not the config, - # and xcodebuild fails with "Unable to open base configuration reference - # file", which says nothing about which file is missing. - - name: Materialize missing xcconfig files - run: | - set -e - find . -path ./DerivedData -prune -o -type f \ - \( -name '*.xcconfig.template' -o -name '*.xcconfig.example' \ - -o -name '*.xcconfig.sample' -o -name '*.template.xcconfig' \ - -o -name '*.example.xcconfig' -o -name '*.sample.xcconfig' \) -print \ - | while IFS= read -r template; do - case "$template" in - *.xcconfig.template|*.xcconfig.example|*.xcconfig.sample) target="${template%.*}" ;; - *) target="$(dirname "$template")/$(basename "$template" | sed -E 's/\.(template|example|sample)\.xcconfig$/.xcconfig/')" ;; - esac - if [ ! -f "$target" ]; then - cp "$template" "$target" - echo "::warning::$target was missing (gitignored?); copied $template over it. Any placeholder values in it are what the build sees." - fi - done - - - name: Generate Xcode project (XcodeGen) - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - run: | - set -e - # A managed Expo project has no iOS directory at all until the - # prebuild step below creates it. - if [ ! -d "$IOS_PATH" ]; then - echo "No '$IOS_PATH' directory yet — skipping." - exit 0 - fi - cd "$IOS_PATH" - - MANIFEST="" - for f in project.yml project.yaml; do - if [ -f "$f" ] && grep -qE '^targets:' "$f"; then - MANIFEST="$f" - break - fi - done - - if [ -z "$MANIFEST" ]; then - echo "No XcodeGen manifest — skipping." - elif [ -n "$(find . -maxdepth 1 -name '*.xcodeproj' -print -quit)" ]; then - echo "Found a committed .xcodeproj — skipping generation." - else - echo "Found $MANIFEST and no .xcodeproj — generating with XcodeGen." - brew install xcodegen - xcodegen generate - fi - - # Whatever the previous step could not fill in is named here, instead of - # surfacing as xcodebuild's "Unable to open base configuration reference". - - name: Check base configuration files - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - run: | - set -e - if [ ! -d "$IOS_PATH" ]; then - echo "No '$IOS_PATH' directory yet — skipping." - exit 0 - fi - MISSING="" - for pbx in $(find "$IOS_PATH" -maxdepth 2 -name project.pbxproj -not -path '*/Pods.xcodeproj/*'); do - for id in $(grep -o 'baseConfigurationReference = [A-Za-z0-9]*' "$pbx" | awk '{print $3}' | sort -u); do - ref=$(grep -m1 "^[[:space:]]*$id .*isa = PBXFileReference" "$pbx" || true) - name=$(printf '%s' "$ref" | sed -n 's/.*[ \t]path = "\{0,1\}\([^";]*\)"\{0,1\};.*/\1/p') - [ -n "$name" ] || continue - base=$(basename "$name") - # CocoaPods writes its xcconfigs during pod install and Flutter - # writes Generated.xcconfig during pub get — both happen later in - # this job, so they are legitimately absent right now. - case "$name" in *Pods/*|*"Target Support Files"*) continue ;; esac - case "$base" in Pods-*|Generated.xcconfig) continue ;; esac - if [ -z "$(find . -name "$base" -print -quit)" ]; then - MISSING="$MISSING $base" - fi - done - done - if [ -n "$MISSING" ]; then - echo "::error::The project has base configuration files that are not in the repository:$MISSING. They are probably gitignored — commit them, or commit a .xcconfig.template next to each so the build can fill them in." - exit 1 - fi - - - name: Setup Flutter - if: steps.detect.outputs.type == 'flutter' - uses: subosito/flutter-action@v2 - with: - flutter-version: ${{ steps.params.outputs.flutter_version || '' }} - channel: stable - cache: true - - - name: Flutter pub get - if: steps.detect.outputs.type == 'flutter' - run: flutter pub get - - # setup-node cannot make `with:` keys conditional, so the version and the - # version file are both computed here; it ignores an empty version file. - - name: Resolve JS toolchain - id: js - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - run: | - set -e - # >>> js toolchain (keep identical across ios-build.yml, ios-share.yml, runner.sh) - # The package manager the project actually uses: the packageManager field - # first, then the lockfile, then npm. Guessing npm is not safe — `npm install` - # in a pnpm or yarn workspace fails with - # 'EUNSUPPORTEDPROTOCOL Unsupported URL Type "workspace:"'. - js_detect_manager() { - if [ -z "${JS_MANAGER:-}" ]; then - case "$(jq -r '.packageManager // empty' package.json 2>/dev/null || true)" in - pnpm|pnpm@*) JS_MANAGER=pnpm ;; - yarn|yarn@*) JS_MANAGER=yarn ;; - bun|bun@*) JS_MANAGER=bun ;; - npm|npm@*) JS_MANAGER=npm ;; - *) - if [ -f pnpm-lock.yaml ]; then JS_MANAGER=pnpm - elif [ -f yarn.lock ]; then JS_MANAGER=yarn - elif [ -f bun.lock ] || [ -f bun.lockb ]; then JS_MANAGER=bun - else JS_MANAGER=npm # package-lock.json, or no lockfile at all - fi - ;; - esac - fi - case "$JS_MANAGER" in - pnpm) JS_EXEC='pnpm exec' ;; - yarn) JS_EXEC=yarn ;; - bun) JS_EXEC=bunx ;; - *) JS_EXEC=npx ;; - esac - export JS_MANAGER JS_EXEC - echo "Package manager: $JS_MANAGER" - } - - # The Node version the project asks for: .nvmrc or .node-version, else the - # first comparator of engines.node, else 22. A leading "v" or range prefix - # is dropped: nvm, n and setup-node all take "20.11.1" and "24.x" as they are. - js_node_version() { - js_version="" - for js_file in .nvmrc .node-version; do - if [ -f "$js_file" ]; then - js_version=$(tr -d ' \t\r' < "$js_file" | head -n1) - break - fi - done - if [ -z "$js_version" ]; then - js_version=$(jq -r '.engines.node // empty' package.json 2>/dev/null || true) - js_version=$(printf '%s' "${js_version%%|*}" | awk '{print $1}') - fi - js_version=$(printf '%s' "$js_version" | sed -E 's/^(>=|<=|>|<|=|\^|~|v)+//') - case "$js_version" in - [0-9]*|lts/*) ;; - *) js_version=22 ;; - esac - printf '%s\n' "$js_version" - } - - # Put the manager on PATH first, then install — unless a cache already restored - # node_modules, in which case the manager is still wanted for `expo prebuild` - # and for the React Native build phases that shell out to it. - js_install() { - js_detect_manager - case "$JS_MANAGER" in - pnpm|yarn) - # corepack installs the version packageManager pins. Node 16.10-24 - # bundle it; Node 25+ and some provider images do not. - command -v corepack >/dev/null 2>&1 || npm install -g corepack || true - corepack enable || true - command -v "$JS_MANAGER" >/dev/null 2>&1 || npm install -g "$JS_MANAGER" - ;; - bun) - if ! command -v bun >/dev/null 2>&1; then - curl -fsSL https://bun.sh/install | bash - export BUN_INSTALL="${BUN_INSTALL:-$HOME/.bun}" - export PATH="$BUN_INSTALL/bin:$PATH" - # A step's PATH does not survive into the next GitHub Actions step. - if [ -n "${GITHUB_PATH:-}" ]; then echo "$BUN_INSTALL/bin" >> "$GITHUB_PATH"; fi - fi - ;; - esac - if [ "${JS_DEPS_CACHED:-}" = "true" ]; then - echo "node_modules restored from cache; skipping install" - return 0 - fi - case "$JS_MANAGER" in - pnpm) - if [ -f pnpm-lock.yaml ]; then pnpm install --frozen-lockfile; else pnpm install; fi - ;; - yarn) - if [ -f .yarnrc.yml ]; then yarn install --immutable - elif [ -f yarn.lock ]; then yarn install --frozen-lockfile - else yarn install - fi - ;; - bun) - if [ -f bun.lock ] || [ -f bun.lockb ]; then bun install --frozen-lockfile; else bun install; fi - ;; - *) - if [ -f package-lock.json ]; then npm ci; else npm install; fi - ;; - esac - } - # <<< js toolchain - js_detect_manager - NODE_VERSION_FILE="" - for f in .nvmrc .node-version; do - if [ -f "$f" ]; then NODE_VERSION_FILE="$f"; break; fi - done - NODE_VERSION="" - if [ -n "$NODE_VERSION_FILE" ]; then - echo "Node version: from $NODE_VERSION_FILE" - else - NODE_VERSION=$(js_node_version) - echo "Node version: $NODE_VERSION" - fi - { - echo "manager=$JS_MANAGER" - echo "exec=$JS_EXEC" - echo "node_version=$NODE_VERSION" - echo "node_version_file=$NODE_VERSION_FILE" - } >> "$GITHUB_OUTPUT" - - - name: Setup Node.js - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - uses: actions/setup-node@v7 - with: - node-version: ${{ steps.js.outputs.node_version }} - node-version-file: ${{ steps.js.outputs.node_version_file }} - - # Runs even on a cache hit: the manager itself still has to be on PATH for - # the prebuild and build steps that follow. - - name: Install JS dependencies - if: steps.detect.outputs.type == 'reactnative' || steps.detect.outputs.type == 'expo' - env: - JS_MANAGER: ${{ steps.js.outputs.manager }} - run: | - set -e - # >>> js toolchain (keep identical across ios-build.yml, ios-share.yml, runner.sh) - # The package manager the project actually uses: the packageManager field - # first, then the lockfile, then npm. Guessing npm is not safe — `npm install` - # in a pnpm or yarn workspace fails with - # 'EUNSUPPORTEDPROTOCOL Unsupported URL Type "workspace:"'. - js_detect_manager() { - if [ -z "${JS_MANAGER:-}" ]; then - case "$(jq -r '.packageManager // empty' package.json 2>/dev/null || true)" in - pnpm|pnpm@*) JS_MANAGER=pnpm ;; - yarn|yarn@*) JS_MANAGER=yarn ;; - bun|bun@*) JS_MANAGER=bun ;; - npm|npm@*) JS_MANAGER=npm ;; - *) - if [ -f pnpm-lock.yaml ]; then JS_MANAGER=pnpm - elif [ -f yarn.lock ]; then JS_MANAGER=yarn - elif [ -f bun.lock ] || [ -f bun.lockb ]; then JS_MANAGER=bun - else JS_MANAGER=npm # package-lock.json, or no lockfile at all - fi - ;; - esac - fi - case "$JS_MANAGER" in - pnpm) JS_EXEC='pnpm exec' ;; - yarn) JS_EXEC=yarn ;; - bun) JS_EXEC=bunx ;; - *) JS_EXEC=npx ;; - esac - export JS_MANAGER JS_EXEC - echo "Package manager: $JS_MANAGER" - } - - # The Node version the project asks for: .nvmrc or .node-version, else the - # first comparator of engines.node, else 22. A leading "v" or range prefix - # is dropped: nvm, n and setup-node all take "20.11.1" and "24.x" as they are. - js_node_version() { - js_version="" - for js_file in .nvmrc .node-version; do - if [ -f "$js_file" ]; then - js_version=$(tr -d ' \t\r' < "$js_file" | head -n1) - break - fi - done - if [ -z "$js_version" ]; then - js_version=$(jq -r '.engines.node // empty' package.json 2>/dev/null || true) - js_version=$(printf '%s' "${js_version%%|*}" | awk '{print $1}') - fi - js_version=$(printf '%s' "$js_version" | sed -E 's/^(>=|<=|>|<|=|\^|~|v)+//') - case "$js_version" in - [0-9]*|lts/*) ;; - *) js_version=22 ;; - esac - printf '%s\n' "$js_version" - } - - # Put the manager on PATH first, then install — unless a cache already restored - # node_modules, in which case the manager is still wanted for `expo prebuild` - # and for the React Native build phases that shell out to it. - js_install() { - js_detect_manager - case "$JS_MANAGER" in - pnpm|yarn) - # corepack installs the version packageManager pins. Node 16.10-24 - # bundle it; Node 25+ and some provider images do not. - command -v corepack >/dev/null 2>&1 || npm install -g corepack || true - corepack enable || true - command -v "$JS_MANAGER" >/dev/null 2>&1 || npm install -g "$JS_MANAGER" - ;; - bun) - if ! command -v bun >/dev/null 2>&1; then - curl -fsSL https://bun.sh/install | bash - export BUN_INSTALL="${BUN_INSTALL:-$HOME/.bun}" - export PATH="$BUN_INSTALL/bin:$PATH" - # A step's PATH does not survive into the next GitHub Actions step. - if [ -n "${GITHUB_PATH:-}" ]; then echo "$BUN_INSTALL/bin" >> "$GITHUB_PATH"; fi - fi - ;; - esac - if [ "${JS_DEPS_CACHED:-}" = "true" ]; then - echo "node_modules restored from cache; skipping install" - return 0 - fi - case "$JS_MANAGER" in - pnpm) - if [ -f pnpm-lock.yaml ]; then pnpm install --frozen-lockfile; else pnpm install; fi - ;; - yarn) - if [ -f .yarnrc.yml ]; then yarn install --immutable - elif [ -f yarn.lock ]; then yarn install --frozen-lockfile - else yarn install - fi - ;; - bun) - if [ -f bun.lock ] || [ -f bun.lockb ]; then bun install --frozen-lockfile; else bun install; fi - ;; - *) - if [ -f package-lock.json ]; then npm ci; else npm install; fi - ;; - esac - } - # <<< js toolchain - js_install - - # A managed Expo project keeps no ios/ directory in git — `expo prebuild` - # generates it from app.json / app.config.js. The simulator build needs - # the same native project the IPA build does. - - name: Expo prebuild - if: steps.detect.outputs.type == 'expo' - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - JS_EXEC: ${{ steps.js.outputs.exec }} - CI: '1' # prebuild answers its own prompts instead of hanging - run: | - set -e - if [ -n "$(find "$IOS_PATH" -maxdepth 1 \( -name '*.xcodeproj' -o -name '*.xcworkspace' \) -print -quit 2>/dev/null)" ]; then - echo "'$IOS_PATH' already holds an Xcode project (ejected Expo) — skipping prebuild." - exit 0 - fi - - # Without a bundle identifier prebuild asks for one interactively, - # which on a runner means waiting for the job timeout. - BUNDLE_ID=$(${JS_EXEC:-npx} expo config --type public --json 2>/dev/null | jq -r '.ios.bundleIdentifier // empty' 2>/dev/null || true) - if [ -z "$BUNDLE_ID" ]; then - for f in app.json app.config.json; do - [ -f "$f" ] || continue - BUNDLE_ID=$(jq -r '(.expo.ios.bundleIdentifier // .ios.bundleIdentifier) // empty' "$f" 2>/dev/null || true) - if [ -n "$BUNDLE_ID" ]; then break; fi - done - fi - if [ -z "$BUNDLE_ID" ]; then - echo "::error::This managed Expo project has no ios.bundleIdentifier. Set it in app.json (expo.ios.bundleIdentifier) or app.config.js, commit that, and share again — 'expo prebuild' cannot run unattended without it." - exit 1 - fi - echo "Prebuilding $BUNDLE_ID into '$IOS_PATH'..." - - ${JS_EXEC:-npx} expo prebuild --platform ios --no-install - if [ -z "$(find "$IOS_PATH" -maxdepth 1 -name '*.xcodeproj' -print -quit 2>/dev/null)" ]; then - echo "::error::'expo prebuild' produced no Xcode project in '$IOS_PATH'. If the project generates it elsewhere, set ios.path in builder.json." - exit 1 - fi - - # KMP builds through xcodebuild like a native project, but its Xcode run - # script phase shells out to Gradle, which needs a JDK on PATH. - - name: Setup JDK - if: steps.detect.outputs.type == 'kmp' - uses: actions/setup-java@v6 - with: - distribution: temurin - java-version: ${{ steps.params.outputs.jdk_version || '17' }} - - - name: Setup Gradle - if: steps.detect.outputs.type == 'kmp' - uses: gradle/actions/setup-gradle@v6 - - - name: Build for the simulator - id: build - env: - IOS_PATH: ${{ steps.params.outputs.ios_path }} - SCHEME: ${{ steps.params.outputs.scheme }} - PROJECT_TYPE: ${{ steps.detect.outputs.type }} - run: | - set -e - - if [ "$PROJECT_TYPE" = "flutter" ]; then - # Flutter drives xcodebuild itself and puts the bundle in a known - # place, so there is nothing to detect here. - flutter build ios --simulator --debug - APP_PATH=$(find build/ios/iphonesimulator -maxdepth 1 -name '*.app' -print -quit) - else - cd "$IOS_PATH" - - # CocoaPods generates the .xcworkspace and many repos gitignore it, - # so install before looking for one. - if [ -f Podfile ]; then - command -v pod >/dev/null 2>&1 || gem install cocoapods --no-document - pod install - fi - - WORKSPACE=$(find . -maxdepth 1 -name '*.xcworkspace' -print -quit) - PROJECT=$(find . -maxdepth 1 -name '*.xcodeproj' -print -quit) - - if [ -n "$WORKSPACE" ]; then - TARGET_FLAG=-workspace - TARGET_PATH=$WORKSPACE - BASE=$(basename "$WORKSPACE" .xcworkspace) - elif [ -n "$PROJECT" ]; then - TARGET_FLAG=-project - TARGET_PATH=$PROJECT - BASE=$(basename "$PROJECT" .xcodeproj) - else - echo "::error::No .xcworkspace or .xcodeproj found in $IOS_PATH" - exit 1 - fi - - if [ -z "$SCHEME" ]; then - SCHEMES=$(xcodebuild "$TARGET_FLAG" "$TARGET_PATH" -list -json 2>/dev/null | jq -r '(.workspace // .project).schemes[]?' || true) - if [ -z "$SCHEMES" ]; then - SCHEMES=$(xcodebuild "$TARGET_FLAG" "$TARGET_PATH" -list 2>/dev/null | sed -n '/Schemes:/,$p' | tail -n +2 | sed 's/^[[:space:]]*//' | sed '/^$/d' || true) - fi - echo "Schemes:" - printf '%s\n' "$SCHEMES" | sed 's/^/ /' - - # In a workspace, the scheme list also holds package and pod - # schemes, and they sort ahead of the app as often as not. The - # scheme named after the project is the app in nearly every repo. - SCHEME=$(printf '%s\n' "$SCHEMES" | grep -Fx "$BASE" | head -1 || true) - if [ -z "$SCHEME" ]; then - SCHEME=$(printf '%s\n' "$SCHEMES" | grep -v '^Pods-' | while IFS= read -r s; do - [ -n "$s" ] || continue - # A scheme that belongs to a local Swift package is not the app. - [ -z "$(find . -maxdepth 4 -path "*/$s/Package.swift" -print -quit)" ] || continue - printf '%s\n' "$s" - break - done) - fi - fi - - if [ -z "$SCHEME" ]; then - echo "::error::Could not determine a scheme in $IOS_PATH. Pass one with the 'scheme' input." - exit 1 - fi - echo "Scheme: $SCHEME" - - # Build for the simulator this job is about to install on, rather - # than letting xcodebuild pick the first of several matches. - if [ -n "$MOBAI_SIM_UDID" ]; then - DEST_FLAG=-destination - DEST_VALUE="id=$MOBAI_SIM_UDID" - else - DEST_FLAG=-sdk - DEST_VALUE=iphonesimulator - fi - - xcodebuild "$TARGET_FLAG" "$TARGET_PATH" \ - -scheme "$SCHEME" \ - -configuration Debug \ - "$DEST_FLAG" "$DEST_VALUE" \ - -derivedDataPath "$GITHUB_WORKSPACE/DerivedData" \ - COMPILER_INDEX_STORE_ENABLE=NO \ - CODE_SIGNING_ALLOWED=NO \ - build - - # Ask the project which bundle this scheme produces: the first .app - # under Build/Products is the wrong one whenever the repo has more - # than one app target. - APP_PATH=$(xcodebuild "$TARGET_FLAG" "$TARGET_PATH" \ - -scheme "$SCHEME" \ - -configuration Debug \ - "$DEST_FLAG" "$DEST_VALUE" \ - -derivedDataPath "$GITHUB_WORKSPACE/DerivedData" \ - -showBuildSettings -json 2>/dev/null \ - | jq -r 'map(.buildSettings) - | map(select(.PRODUCT_TYPE == "com.apple.product-type.application" - and (.TARGET_BUILD_DIR | contains("-iphonesimulator")))) - | map(.TARGET_BUILD_DIR + "/" + .FULL_PRODUCT_NAME) - | first // empty' || true) - if [ ! -d "$APP_PATH" ]; then - APP_PATH=$(find "$GITHUB_WORKSPACE/DerivedData/Build/Products/Debug-iphonesimulator" -maxdepth 1 -name '*.app' -print -quit 2>/dev/null || true) - fi - cd "$GITHUB_WORKSPACE" - fi - - if [ -z "$APP_PATH" ]; then - echo "::error::Build produced no .app" - exit 1 - fi - echo "Built $APP_PATH" - echo "app_path=$APP_PATH" >> $GITHUB_OUTPUT - - # Flutter builds through its own tree, so DerivedData can be absent and a - # save would fail on a path that never existed. - - name: Check for DerivedData - id: dd - if: always() - run: | - if [ -d DerivedData ]; then echo "exists=yes" >> $GITHUB_OUTPUT; fi - - # Saved before the sharing step, which blocks for as long as the - # simulator is in use: a job that hits the timeout there would otherwise - # leave the next run with a cold DerivedData. - - name: Save DerivedData cache - if: always() && steps.dd.outputs.exists == 'yes' - uses: actions/cache/save@v6 - with: - path: DerivedData - key: deriveddata-sim-${{ github.run_id }} - - # Installs the build on the simulator and publishes it to the MobAI app. - # The step (and the job) stay running until the simulator is released - # there or goes unused. - - name: Share the simulator - env: - MOBAI_API_KEY: ${{ secrets.MOBAI_API_KEY }} - run: | - mobai-ci share \ - --device "$MOBAI_SIM_UDID" \ - --app "${{ steps.build.outputs.app_path }}" \ - --duration "${{ steps.params.outputs.duration }}" \ - --label "${{ github.repository }} · ${{ steps.params.outputs.build_id }}" \ - --wait-device 4m --startup-timeout 5m - - # The tag exists only to start this run, so it leaves with it. Deleting - # a tag fires no push event, so this cannot start another run. - - name: Delete trigger tag - if: always() && github.ref_type == 'tag' - run: git push origin ":refs/tags/$GITHUB_REF_NAME" diff --git a/App/Views/Toolchain/ToolchainView.swift b/App/Views/Toolchain/ToolchainView.swift index 8843f5c..1935fac 100644 --- a/App/Views/Toolchain/ToolchainView.swift +++ b/App/Views/Toolchain/ToolchainView.swift @@ -209,6 +209,7 @@ struct ToolchainView: View { /// capturing the view in one is the sort of thing that compiles until it does not /// — the receiver is a `@MainActor` type, which is `Sendable`, so it can be /// captured freely. Same shape as the build pipeline's stage streams. + @MainActor private func install(from url: URL) async { working = true message = nil @@ -219,6 +220,13 @@ struct ToolchainView: View { let state = importState state.begin("Reading \(url.lastPathComponent)…") + // Minutes of CPU, and iOS suspends an app whose screen locks — taking the + // extraction with it until the user comes back. Same guard the guest-era + // provisioning used for the same reason (`InstallAssertion`: idle timer plus + // a background task assertion). + let assertion = InstallAssertion.begin(reason: "sdk-import") + defer { assertion.end() } + let (updates, continuation) = AsyncStream.makeStream() let consumer = Task { @MainActor in for await update in updates { diff --git a/CHANGELOG.md b/CHANGELOG.md index 34c82ff..7f7b28a 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -29,6 +29,18 @@ they are ordinary directories in the app's container. rootfs, no `install-toolchain.sh` to run in a guest, and no pinned root asset to download. `EmbeddedLinux/` is gone, and `project.yml` no longer links `-lish`, `-lz` or `-liconv`, sets `GUEST_ARM64`, or requires `Vendor/ish-arm64*` to exist. +- **`.github/workflows/ios-share.yml` (30 KB) and `builder.json`.** A build-for-the- + simulator workflow whose own comments tell you to keep its `js toolchain` blocks + identical to `ios-build.yml` and `runner.sh` — neither of which is in the + repository — and which drives the Flutter / React Native / Expo / KMP branches of + a multi-framework builder. It has never run here (no `ios-share/*` tag has ever + been pushed) and this is a Swift app with an in-process toolchain. Both are in git + history if the MobAI simulator flow is ever wanted back. +- **`.github/workflows/bridge-api-check.yml`.** A "fast loop" for the bridge + compile-check, written when reaching that check cost 40–90 minutes. It now costs + eleven seconds inside a two-minute warm toolchain run, and the workflow carried + its own copy of the LLVM configure recipe — a second place to update, and a + silent divergence waiting to happen. ### Added - **`App/Build/NativeBuildPlan.swift`** — reads a project directory into a fully @@ -51,9 +63,9 @@ they are ordinary directories in the app's container. - **Importing an `Xcode.xip` on the device.** The Toolchain screen accepts Apple's xip and builds the Darwin SDK out of it here — the job `xtool sdk build` does on a Mac, and the reason a phone with no Mac could not use a newer SDK than the one we - publish. A xip is a xar archive holding a pbzx stream of LZMA2 blocks that + publish. A xip is a xar archive holding a pbzx stream of xz blocks that decompresses to a cpio archive of `Xcode.app`; `XipArchive` reads it in one - streaming pass (Apple's `Compression` decodes the LZMA2, so nothing is vendored) + streaming pass (Apple's `Compression` decodes the xz, so nothing is vendored) and `DarwinSDKBuilder` keeps xtool's own list of paths, restricted to `iPhoneOS.platform` because XForge only compiles for the device. The picker also takes a zip or a folder, for a bundle built elsewhere. Free space is checked diff --git a/Docs/DESIGN.md b/Docs/DESIGN.md index 06f8dc4..c09da87 100644 --- a/Docs/DESIGN.md +++ b/Docs/DESIGN.md @@ -239,7 +239,7 @@ sane answer for large projects, and the protocol above is where it would attach. ``` .github/workflows/ build-ipa.yml (the unsigned IPA), native-toolchain.yml (the - iOS LLVM cross-build), ios-share.yml (a simulator build) + iOS LLVM cross-build, and the toolchain release) project.yml XcodeGen definition App/ SwiftUI app sources App/Build/ the pipeline: plan, invocation, executor, IPABuilder, manager diff --git a/Docs/XCODE-ALTERNATIVE.md b/Docs/XCODE-ALTERNATIVE.md index c7f8c86..0122c49 100644 --- a/Docs/XCODE-ALTERNATIVE.md +++ b/Docs/XCODE-ALTERNATIVE.md @@ -12,7 +12,7 @@ piece needs, and which parts cannot be done at all with the tools that exist. |---|---| | Authoring a SwiftPM-shaped project (project editor, manifest, files) | works | | Compiling and linking **C / Objective-C** sources in-process | works (`NativeBuildExecutor`) | -| Compiling **Swift** sources in-process | **does not work** — the frontend is not in the toolchain bundle | +| Compiling **Swift** sources in-process | needs a `with_swift` bundle: the frontend ships in it and links into the app, but no Swift build has been run on a device | | Resolving SwiftPM dependencies | **refused by name** — no package manager can run here | | Packaging an `.ipa` from a built `.app` | works (`IPABuilder`, host-side) | | Signing with an existing `.p12` + profile | works (`AppBundleSigner`, XKit, in-process) | diff --git a/README.md b/README.md index e81d6c5..d1d1769 100644 --- a/README.md +++ b/README.md @@ -11,10 +11,10 @@ binary and called in-process through a small C++ bridge. > **Status: active, and narrower than it looks.** A project authored in the SwiftUI > shell is read into a build plan, compiled and linked in-process, packaged into an > unsigned `.ipa`, and — with a certificate the user already has — signed with XKit. -> **C, Objective-C and Objective-C++ targets build today. Swift sources do not**, -> because the Swift frontend libraries are not part of the toolchain artifact yet; -> such a project stops at the compile stage with a message naming how many files it -> could not compile. See *What works today* and *What does not*. +> **C, Objective-C and Objective-C++ targets build today.** Swift needs a toolchain +> bundle built with `with_swift=true`, which carries the frontend libraries; the app +> links them, and running an actual Swift build on a device is the one step nobody +> has taken yet. See *What works today* and *What does not*. ## How it is put together @@ -76,12 +76,13 @@ in-process toolchain and the SDK in the container. ## What does not work (yet) -- **Swift.** The Swift frontend libraries are not in the toolchain artifact: the - CI workflow that builds it builds Clang and LLD only. A plan with Swift sources - fails at the compile stage with `swiftFrontendMissing`, naming the number of - files, rather than pretending otherwise. This is the single largest gap; it is - the difference between "an Xcode alternative for C projects" and the thing the - app is for. +- **Swift, end to end on a device.** The frontend libraries (`swiftFrontendTool` and + the swiftAST / swiftSema / IRGen / ClangImporter set it pulls in) are built for + iPhoneOS by a `with_swift=true` toolchain run, published in that bundle, and linked + into the app — the Toolchain screen reports `swift-frontend` as linked. What has + not happened is a Swift *build* on a device: every part of the path is exercised + except that one, and a bundle without the frontend still refuses a Swift plan at + the compile stage with `swiftFrontendMissing` and the file count. - **SwiftPM dependencies.** Refused by name at plan time. Fetching and resolving packages means running a package manager, which cannot happen here — and a dependency editor that could only produce unbuildable projects was removed @@ -169,8 +170,7 @@ Tools/ gen-appicon.py (writes the app icon into the asset catal project.yml XcodeGen definition Makefile gen, build, test, ipa, toolchain .github/workflows/ build-ipa.yml (the unsigned IPA), - native-toolchain.yml (the iOS LLVM cross-build), - ios-share.yml (a simulator build for MobAI) + native-toolchain.yml (the iOS LLVM cross-build) Docs/DESIGN.md full architecture write-up Docs/IPA-BUILD.md the build pipeline and its stages Docs/NATIVE-TOOLCHAIN.md the toolchain bundle, how it is built and linked @@ -232,8 +232,6 @@ make icon # rewrites Support/Assets.xcassets are cached, so a run whose inputs are unchanged finishes in minutes rather than the ~80 minutes a cold build takes. See [Docs/NATIVE-TOOLCHAIN.md](Docs/NATIVE-TOOLCHAIN.md). -- **`ios-share.yml`** — builds the app for the simulator and keeps that simulator - usable from a machine that is not a Mac, so a build can be tried by hand. ## Roadmap @@ -241,9 +239,12 @@ make icon # rewrites Support/Assets.xcassets through a C++ bridge - [x] Projects as ordinary directories in the app container - [x] XKit signing wired into the export flow, for an identity the user has -- [x] Native toolchain bundle, built in CI and installed by `make toolchain` -- [ ] **The Swift frontend (`swift-frontend`) ported to iOS** — the gate on every - Swift project, and the reason the artifact carries Clang and LLD only +- [x] Native toolchain bundle, built in CI, published as a release, installed by + `make toolchain-release` +- [x] **The Swift frontend libraries for iPhoneOS** — built by a `with_swift=true` + toolchain run, linked into the app, reported by the Toolchain screen. The + remaining unknown is a Swift *build* on a device, which is what the screen's + smoke test is for - [ ] Build-setting evaluation and a build driver for a dependency graph - [ ] `actool`/`ibtool` replacements (asset catalogs, storyboards) - [ ] Apple ID provisioning (certificate issuance, 2FA) in-app diff --git a/Support/XForge.entitlements b/Support/XForge.entitlements index 58a2b85..a451b51 100644 --- a/Support/XForge.entitlements +++ b/Support/XForge.entitlements @@ -5,21 +5,22 @@ These are the only ones the app asks for, and they are here on purpose: - * com.apple.developer.kernel.extended-virtual-addressing — the embedded - ish-arm64 guest is a whole 64-bit Linux userland living in this process. It - maps a large sparse guest address space, and without this entitlement the - mapping fails and the guest cannot boot. ish-arm64's own iOS build declares - the same key. + * com.apple.developer.kernel.extended-virtual-addressing — the compiler runs in + this process, and LLVM maps large regions: the SDK's symbol tables, the object + files the linker reads back, and the JIT's reserved code space. The entitlement + was originally needed by the embedded Linux guest that used to live in this + process; the guest is gone, and no measurement has been taken since, so it is + kept for the compiler's maps rather than removed on a guess. - * com.apple.developer.kernel.increased-memory-limit — the guest holds a Linux - root filesystem, a Swift toolchain and the compiler's working set. The - default jetsam budget is far below what a compile needs, so this asks for the - larger limit. + * com.apple.developer.kernel.increased-memory-limit — a compiler's working set is + large (LLVM and Swift's frontend are linked into the app, and a build holds a + module graph in memory). The default jetsam budget is below what a real compile + needs, so this asks for the larger limit. Both are also the keys a signer may refuse: a wildcard or free-account profile - cannot always grant them. If a sideloader strips them, XForge still installs - and runs — the guest just gets the default address space and memory budget, - which is enough to boot and use the shell but not to compile large packages. + cannot always grant them. If a sideloader strips them, XForge still installs and + runs — it gets the default address space and memory budget, which is enough for + small projects and not for large ones. * get-task-allow — true, which is what a development-signed app has; a sideloader re-signs with its own certificate and sets this itself. diff --git a/builder.json b/builder.json deleted file mode 100644 index 58b21a7..0000000 --- a/builder.json +++ /dev/null @@ -1,18 +0,0 @@ -{ - "project": "XForge", - "platform": "ios", - "github": { - "owner": "R0GUEEE", - "repo": "XForge" - }, - "provider": "github", - "codemagic": {}, - "bitrise": {}, - "ios": {}, - "flutter": { - "watch": {} - }, - "reactNative": {}, - "kmp": {}, - "mobai": {} -} \ No newline at end of file From beac58ed8a150e4d68689ce8851a49d5e551cc84 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 06:14:48 -0400 Subject: [PATCH 49/54] sdk import: make the failure observable, and stop refusing three things MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit A device says "still isn't importing" and there is nothing to read back: the screen shows one line and the log said nothing at all about the import. So: - `XForgeLog` now records what was picked (name, size, extension, first four bytes, whether the sandbox let us read it), where `Content` sits in the archive, what came out of it, and — on failure — the error *with its case and values*, which is what the screen's one line cannot carry. The Settings screen shares that file. - The picker accepts `.data` as well as `.zip` and folders. A `.xip` has no UTI the app can rely on, and a file that matches no allowed type is greyed out in Files — which is indistinguishable from "the import does nothing". The reader then says exactly what is wrong with an unrelated file. - Which import a file gets is decided by its first bytes rather than its extension, so a xip without the extension, or a zip named `.xip`, does the right thing, and a file that is neither gets `notABundle` naming it instead of "no SDK in there". - A missing `swift_static/iphoneos` no longer fails the import. It is a warning: the headers and tbd stubs are what C, Objective-C and Objective-C++ targets need, and refusing those because a Swift runtime is absent takes away the main reason to import a newer SDK at all. The result carries the fact and the screen prints it. - An archive with no `Contents/Developer` reports `noXcodeInside` rather than "this Xcode has no iPhoneOS SDK", which is a different and better sentence. The screen now reports what landed ("…: 20,143 files, 1.4 GB") rather than "done", because an import that takes minutes needs an answer the user can check. --- App/NativeToolchain/DarwinSDKBuilder.swift | 30 +++++++---- App/NativeToolchain/NativeSDK.swift | 59 +++++++++++++++++++--- App/Views/Toolchain/ToolchainView.swift | 43 ++++++++++++---- AppTests/XipArchiveTests.swift | 39 ++++++++++++++ 4 files changed, 146 insertions(+), 25 deletions(-) diff --git a/App/NativeToolchain/DarwinSDKBuilder.swift b/App/NativeToolchain/DarwinSDKBuilder.swift index ac970d0..6b5cb81 100644 --- a/App/NativeToolchain/DarwinSDKBuilder.swift +++ b/App/NativeToolchain/DarwinSDKBuilder.swift @@ -40,6 +40,11 @@ enum DarwinSDKBuilder { var files: Int var bytes: Int64 var skipped: Int + /// False when the archive carries no `swift_static/iphoneos`. That is a + /// warning rather than a failure: the SDK's headers and tbd stubs are what C, + /// Objective-C and Objective-C++ targets need, and refusing the whole import + /// over a missing Swift runtime would take those away too. + var hasSwiftStaticRuntime: Bool } struct Progress: Sendable { @@ -52,8 +57,6 @@ enum DarwinSDKBuilder { case noXcodeInside case noiPhoneOSSDK case notEnoughSpace(needed: Int64, available: Int64) - case noSwiftStaticRuntime(String) - var errorDescription: String? { switch self { case .noXcodeInside: @@ -65,9 +68,6 @@ enum DarwinSDKBuilder { return "Not enough space: building the SDK needs about \(Self.size(needed)) " + "free, and there is \(Self.size(available)). Delete some projects or " + "downloads and try again." - case .noSwiftStaticRuntime(let path): - return "The xip has no Swift static runtime at \(path), so a sideloaded app " - + "could not link the Swift runtime into itself." } } @@ -102,6 +102,7 @@ enum DarwinSDKBuilder { do { let content = try XipArchive.content(of: xip) let writer = Writer(staging: staging) + XForgeLog.note("xip: Content at byte \(content.offset), \(content.length) bytes") // Extraction is essentially all of the wall clock, so the fraction of the // compressed member that has been consumed is the honest fraction of the @@ -185,12 +186,18 @@ enum DarwinSDKBuilder { private var sdkRoot: String? private var sdkVersion: String? private var sawSwiftStaticRuntime = false + private var sawDeveloperTree = false init(staging: URL) { self.staging = staging } func accept(_ entry: XipArchive.Entry, _ payload: XipArchive.Payload) throws { + // Noted before the filter: an archive with no `Contents/Developer` at all is + // not an Xcode, and saying that is better than "it has no iPhoneOS SDK". + if entry.name.contains("Contents/Developer") { + sawDeveloperTree = true + } // `Self` here is `Writer`, so the outer type is named in full. guard DarwinSDKBuilder.isWanted(entry.name) else { skipped += 1 @@ -254,10 +261,8 @@ enum DarwinSDKBuilder { /// Write the metadata, having seen the whole tree. func finish(progress: (@Sendable (Progress) -> Void)?) throws -> Result { + guard sawDeveloperTree else { throw Error.noXcodeInside } guard let sdkRoot else { throw Error.noiPhoneOSSDK } - guard sawSwiftStaticRuntime else { - throw Error.noSwiftStaticRuntime("Developer/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static/iphoneos") - } progress?(Progress(fraction: 0.97, message: "Writing swift-sdk.json…")) let json = SDKDefinition( @@ -284,8 +289,13 @@ enum DarwinSDKBuilder { .write(to: staging.appendingPathComponent("darwin-sdk-version.txt")) progress?(Progress(fraction: 1, message: "Done")) - return Result(bundle: staging, sdkRoot: sdkRoot, files: files, - bytes: bytes, skipped: skipped) + let result = Result(bundle: staging, sdkRoot: sdkRoot, files: files, + bytes: bytes, skipped: skipped, + hasSwiftStaticRuntime: sawSwiftStaticRuntime) + XForgeLog.note("xip: \(result.files) files (\(result.bytes) bytes), " + + "skipped \(result.skipped), sdkRoot \(result.sdkRoot), " + + "swift_static=\(result.hasSwiftStaticRuntime)") + return result } /// `Xcode.app/Contents/Developer/…` → `Developer/…`. diff --git a/App/NativeToolchain/NativeSDK.swift b/App/NativeToolchain/NativeSDK.swift index 75b182f..148d9d0 100644 --- a/App/NativeToolchain/NativeSDK.swift +++ b/App/NativeToolchain/NativeSDK.swift @@ -35,6 +35,7 @@ extension NativeSDKLayout { enum NativeSDKError: LocalizedError { case missingBundle + case notABundle(String) case invalidMetadata case missingTarget case missingSDK(String) @@ -43,6 +44,10 @@ enum NativeSDKError: LocalizedError { switch self { case .missingBundle: return "No native Darwin SDK is installed." + case .notABundle(let name): + return "\(name) is not a Darwin SDK bundle and not an Xcode.xip. A bundle is a " + + "folder (or a zip of one) containing darwin.artifactbundle or swift-sdk.json; " + + "an Xcode archive starts with 'xar!'." case .invalidMetadata: return "The native Darwin SDK has an invalid swift-sdk.json." case .missingTarget: @@ -179,7 +184,7 @@ enum NativeSDK { return } - throw NativeSDKError.missingBundle + throw NativeSDKError.notABundle(downloaded.lastPathComponent) } static func remove() throws { @@ -211,33 +216,75 @@ enum NativeSDK { /// Three shapes arrive this way, and they are told apart by what they are rather /// than by what they are called: /// + /// What an import did, so the screen can say more than "done". + struct ImportReport: Sendable { + var files: Int = 0 + var bytes: Int64 = 0 + /// Things that did not stop the install but will bite later. + var warnings: [String] = [] + } + /// - a `darwin.artifactbundle` **folder** — already built, perhaps by /// `xtool sdk build` on a Mac; /// - a **zip** of one, which is how the hosted bundle is published; /// - an Apple **`Xcode.xip`**, which `DarwinSDKBuilder` turns into one here. /// That is the case that needs a Mac otherwise, and the reason the picker no /// longer filters for folders only. + /// + /// Which one it is decided by what the file *is*, not what it is called: the + /// picker hands over whatever the user chose, and "that is a zip, not a xip" is a + /// better answer than a bundle search that fails for the wrong reason. + @discardableResult static func installImported(from source: URL, - progress: (@Sendable (DarwinSDKBuilder.Progress) -> Void)? = nil) throws { + progress: (@Sendable (DarwinSDKBuilder.Progress) -> Void)? = nil) throws -> ImportReport { let accessed = source.startAccessingSecurityScopedResource() defer { if accessed { source.stopAccessingSecurityScopedResource() } } var isDirectory: ObjCBool = false let exists = FileManager.default.fileExists(atPath: source.path, isDirectory: &isDirectory) let kind = source.pathExtension.lowercased() + let header = firstBytes(of: source) + let size = (try? source.resourceValues(forKeys: [.fileSizeKey]).fileSize) ?? 0 + + // The log is the only way to see this on a device: what was picked, whether + // the sandbox let us read it, how big it is and what it actually is. + XForgeLog.note("sdk import: \(source.lastPathComponent) " + + "(\(ByteCountFormatter.string(fromByteCount: Int64(size), countStyle: .file))), " + + "kind=\(kind.isEmpty ? "none" : kind), " + + "readable=\(accessed), header=\(hex(header))") if exists, isDirectory.boolValue { try install(from: source) - } else if kind == "xip" { + return ImportReport() + } + if header == Data("xar!".utf8) || kind == "xip" { let built = try DarwinSDKBuilder.build(fromXip: source, into: XForgeEnvironment.nativeSDKDirectory, progress: progress) try install(preparedBundle: built.bundle) - } else if kind == "zip" { + return ImportReport(files: built.files, bytes: built.bytes, + warnings: built.hasSwiftStaticRuntime ? [] : [ + "the archive has no Swift static runtime " + + "(Toolchains/XcodeDefault.xctoolchain/usr/lib/swift_static/iphoneos), " + + "so linking Swift needs one from elsewhere", + ]) + } + if header.starts(with: Data("PK".utf8)) || kind == "zip" { try installDownloadedArchive(at: source) - } else { - throw NativeSDKError.missingBundle + return ImportReport() } + throw NativeSDKError.notABundle(source.lastPathComponent) + } + + /// The first four bytes, read without mapping a multi-gigabyte file into memory. + private static func firstBytes(of url: URL) -> Data { + guard let handle = try? FileHandle(forReadingFrom: url) else { return Data() } + defer { try? handle.close() } + return (try? handle.read(upToCount: 4)) ?? Data() + } + + private static func hex(_ data: Data) -> String { + data.isEmpty ? "none" : data.map { String(format: "%02x", $0) }.joined() } /// Delete the copy the document picker made of an imported archive. diff --git a/App/Views/Toolchain/ToolchainView.swift b/App/Views/Toolchain/ToolchainView.swift index 1935fac..474e1f8 100644 --- a/App/Views/Toolchain/ToolchainView.swift +++ b/App/Views/Toolchain/ToolchainView.swift @@ -25,13 +25,18 @@ struct ToolchainView: View { @State private var confirmingDownload = false @StateObject private var importState = SDKImportState() - /// What the picker offers: a built bundle (a folder, or a zip of one) or Apple's - /// `Xcode.xip`, which the app turns into one. `.xip` has no system type, so it is - /// declared here by extension — without it the file is greyed out in Files and - /// the one import this screen exists for cannot be started. + /// What the picker offers. Deliberately permissive. + /// + /// A `.xip` has no UTI the app can rely on: if the system declares one, the type + /// its extension resolves to is right; if it does not, the file carries a + /// *dynamic* type, and a differently-derived dynamic type need not match it. A + /// file that matches none of the allowed types is greyed out in Files — which + /// looks like "the import does nothing" with no error to read. So `.data` is + /// allowed too, and `XipArchive` says exactly what is wrong with an unrelated + /// file (`xar!` missing, no `Content` member, no `pbzx` payload, unreadable cpio). private static var importableTypes: [UTType] { - var types: [UTType] = [.folder, .zip] - if let xip = UTType(filenameExtension: "xip", conformingTo: .data) { + var types: [UTType] = [.folder, .zip, .data] + if let xip = UTType("com.apple.xip-archive") { types.append(xip) } return types @@ -236,15 +241,20 @@ struct ToolchainView: View { } do { - try await Task.detached(priority: .userInitiated) { + let report = try await Task.detached(priority: .userInitiated) { defer { continuation.finish() } - try NativeSDK.installImported(from: url) { continuation.yield($0) } + return try NativeSDK.installImported(from: url) { continuation.yield($0) } }.value refresh() - message = "Darwin SDK installed from \(url.lastPathComponent)." + message = Self.summary(for: report, named: url.lastPathComponent) + XForgeLog.note("sdk import: installed from \(url.lastPathComponent), " + + "files=\(report.files), warnings=\(report.warnings.count)") } catch is CancellationError { message = "Import cancelled." } catch { + // The log is the only place a device failure can be read back from: the + // screen shows one line, the log gets the error with its case and values. + XForgeLog.note("sdk import failed: \(error)") sdkError = error.localizedDescription } @@ -259,6 +269,21 @@ struct ToolchainView: View { }.value } + /// What to say after an import: what landed, and anything that will bite later. + /// + /// An `.xip` import is minutes long, so "installed" on its own is not an answer + /// the user can check — the file count and size are, and they are also what makes + /// a report of a failure specific. + private static func summary(for report: NativeSDK.ImportReport, named name: String) -> String { + var lines = ["Darwin SDK installed from \(name)."] + if report.files > 0 { + let size = ByteCountFormatter.string(fromByteCount: report.bytes, countStyle: .file) + lines = ["Darwin SDK installed from \(name): \(report.files) files, \(size)."] + } + lines.append(contentsOf: report.warnings) + return lines.joined(separator: " ") + } + private func removeSDK() { do { try NativeSDK.remove() diff --git a/AppTests/XipArchiveTests.swift b/AppTests/XipArchiveTests.swift index a7dbc8a..561ebe6 100644 --- a/AppTests/XipArchiveTests.swift +++ b/AppTests/XipArchiveTests.swift @@ -429,6 +429,7 @@ final class XipArchiveTests: XCTestCase { XCTAssertEqual(try String(contentsOf: bundle.appendingPathComponent("darwin-sdk-version.txt"), encoding: .utf8) .trimmingCharacters(in: .whitespacesAndNewlines), "27.0") + XCTAssertTrue(result.hasSwiftStaticRuntime, "the fake Xcode carries the static runtime") // And the app's own reader resolves it — the assertion that matters, because a // bundle nothing can read is not an SDK. @@ -460,4 +461,42 @@ final class XipArchiveTests: XCTestCase { XCTAssertFalse(FileManager.default.fileExists( atPath: destination.appendingPathComponent("darwin.artifactbundle").path)) } + + /// Something that is a valid cpio archive but not an Xcode says so, instead of + /// reporting the absence of an iOS SDK in a tree that has no Xcode at all. + func testRejectsAnArchiveThatIsNotAnXcode() throws { + let specs = [ + Spec(path: "Downloads", mode: Self.directory, inode: 1), + Spec(path: "Downloads/notes.txt", mode: Self.regular, + data: Data("hello\n".utf8), inode: 2), + ] + let url = try makeXip(payload: Self.cpio(specs)) + let destination = url.deletingLastPathComponent() + .appendingPathComponent("out", isDirectory: true) + + XCTAssertThrowsError(try DarwinSDKBuilder.build(fromXip: url, into: destination)) { error in + guard let builderError = error as? DarwinSDKBuilder.Error, + case .noXcodeInside = builderError else { + return XCTFail("expected noXcodeInside, got \(error)") + } + } + } + + /// A newer Xcode that keeps no static Swift runtime still produces an SDK: the + /// headers and tbd stubs are what C targets need, and refusing the import over a + /// Swift runtime would take those away too. The result says so instead. + func testImportsWithoutTheSwiftStaticRuntime() throws { + let specs = Self.fakeXcode().filter { !$0.path.contains("swift_static") } + let url = try makeXip(payload: Self.cpio(specs)) + let destination = url.deletingLastPathComponent() + .appendingPathComponent("out", isDirectory: true) + + let result = try DarwinSDKBuilder.build(fromXip: url, into: destination) + XCTAssertFalse(result.hasSwiftStaticRuntime) + XCTAssertTrue(FileManager.default.fileExists( + atPath: result.bundle.appendingPathComponent("swift-sdk.json").path)) + XCTAssertTrue(FileManager.default.fileExists( + atPath: result.bundle.appendingPathComponent( + "Developer/Platforms/iPhoneOS.platform/Developer/SDKs/iPhoneOS27.0.sdk/usr/include/stdio.h").path)) + } } From 15fc8d5fb988c1de6e39db619c49a31139f8f949 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 07:00:38 -0400 Subject: [PATCH 50/54] sdk import: two more ways a failure could be invisible MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit Reading a device log that has four "holding the app awake for sdk-import" lines and nothing else, two holes stand out in the path itself: - The picker's completion returned on anything that was not a success. An iCloud file the picker cannot download, or a provider that refuses to copy a 10 GB archive, produced nothing on screen and nothing in the log. It now says both. - Progress existed only on screen. "Started and then nothing" is what a stall looks like in a log someone emails you, so the last milestone (25%, 50%, …) is recorded now — four lines per import at most, and enough to tell a stall from a refusal. --- App/Views/Toolchain/ToolchainView.swift | 23 +++++++++++++++++++++-- 1 file changed, 21 insertions(+), 2 deletions(-) diff --git a/App/Views/Toolchain/ToolchainView.swift b/App/Views/Toolchain/ToolchainView.swift index 474e1f8..4d65dcf 100644 --- a/App/Views/Toolchain/ToolchainView.swift +++ b/App/Views/Toolchain/ToolchainView.swift @@ -153,8 +153,18 @@ struct ToolchainView: View { allowedContentTypes: Self.importableTypes, allowsMultipleSelection: false ) { result in - guard case .success(let urls) = result, let url = urls.first else { return } - Task { await install(from: url) } + switch result { + case .success(let urls): + guard let url = urls.first else { return } + Task { await install(from: url) } + case .failure(let error): + // This used to `return` on anything that was not a success: an iCloud + // file the picker cannot download, a provider that refuses to copy it — + // nothing on screen, nothing in the log, and the user left believing + // the import did nothing. + XForgeLog.note("sdk import: the picker failed: \(error)") + Task { @MainActor in sdkError = error.localizedDescription } + } } .confirmationDialog( "Download the Darwin SDK?", @@ -234,8 +244,17 @@ struct ToolchainView: View { let (updates, continuation) = AsyncStream.makeStream() let consumer = Task { @MainActor in + // A progress trail, not just a bar: "started and then nothing" is what a + // stall looks like in a shared log, and the last milestone says how far it + // got. Four lines per import, at most. + var milestone = -1 for await update in updates { state.update(fraction: update.fraction, status: update.message) + let reached = Int(update.fraction * 4) + if reached > milestone { + milestone = reached + XForgeLog.note("sdk import: \(Int(update.fraction * 100))%") + } } state.finish() } From b9fc9c53003b12fb891893c1ee5486f9157f2fe4 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 07:04:21 -0400 Subject: [PATCH 51/54] xip: exclude platform prebuilt modules like the real SDK bundle Inspected the central directory of the known-good published darwin.artifactbundle.zip (456 MB, darwin-sdk-7). It has no swift//prebuilt-modules entries for the device bundle; our filter only excluded the non-existent-at-that-level swift/prebuilt-modules and would have copied hundreds of iPhoneOS prebuilt modules from a real Xcode.xip. Those are generated per-platform/configuration intermediates, not inputs the Darwin SDK needs. Exclude prebuilt-modules by path component, and pin the exact iphoneos path in the fixture test. This cuts the import and matches the bundle XForge already knows is valid. --- App/NativeToolchain/DarwinSDKBuilder.swift | 8 +++++++- AppTests/XipArchiveTests.swift | 1 + 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/App/NativeToolchain/DarwinSDKBuilder.swift b/App/NativeToolchain/DarwinSDKBuilder.swift index 6b5cb81..4291b3a 100644 --- a/App/NativeToolchain/DarwinSDKBuilder.swift +++ b/App/NativeToolchain/DarwinSDKBuilder.swift @@ -151,7 +151,13 @@ enum DarwinSDKBuilder { let tail = Array(rest.dropFirst(4)) guard let name = tail.first, ["swift", "swift_static", "clang"].contains(name) else { return false } - if name == "swift", tail.dropFirst().first == "prebuilt-modules" { return false } + // Prebuilt modules are generated per platform/configuration and are + // enormous. The published darwin bundle proves the intended layout: + // it has no prebuilt-modules entries for iphoneos (or any other Swift + // platform). Check the path by component, not only for + // `swift/prebuilt-modules`: real Xcodes use + // `swift/iphoneos/prebuilt-modules/...`. + if name == "swift", tail.dropFirst().contains("prebuilt-modules") { return false } return true } diff --git a/AppTests/XipArchiveTests.swift b/AppTests/XipArchiveTests.swift index 561ebe6..9518dae 100644 --- a/AppTests/XipArchiveTests.swift +++ b/AppTests/XipArchiveTests.swift @@ -352,6 +352,7 @@ final class XipArchiveTests: XCTestCase { let dropped = [ "\(Self.toolchain)/swift/prebuilt-modules/Foundation.swiftmodule", + "\(Self.developer)/Toolchains/XcodeDefault.xctoolchain/usr/lib/swift/iphoneos/prebuilt-modules/Foundation.swiftmodule", "\(Self.developer)/Platforms/MacOSX.platform/Info.plist", "\(Self.developer)/Applications/Whatever.app/Whatever", "Contents/Resources/English.lproj/InfoPlist.strings", From 0269cf7ff96d8caa372ac48e7e4b4d6bf9ab61a2 Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 07:51:03 -0400 Subject: [PATCH 52/54] sdk import: handle Apple Xcode ZIP wrappers without expanding Xcode --- App/NativeToolchain/DarwinSDKBuilder.swift | 12 ++++++ App/NativeToolchain/NativeSDK.swift | 46 ++++++++++++++++++++++ App/NativeToolchain/XipArchive.swift | 28 +++++++------ 3 files changed, 74 insertions(+), 12 deletions(-) diff --git a/App/NativeToolchain/DarwinSDKBuilder.swift b/App/NativeToolchain/DarwinSDKBuilder.swift index 4291b3a..df411f1 100644 --- a/App/NativeToolchain/DarwinSDKBuilder.swift +++ b/App/NativeToolchain/DarwinSDKBuilder.swift @@ -344,6 +344,18 @@ enum DarwinSDKBuilder { var targetTriples: [String: Triple] } + /// Build from the pbzx `Content` member extracted from Apple's ZIP wrapper. + static func buildPBZX(from content: URL, into destination: URL, + progress: (@Sendable (Progress) -> Void)? = nil) throws -> Result { + let staging = destination + try FileManager.default.createDirectory(at: staging, withIntermediateDirectories: true) + let writer = Writer(staging: staging) + try XipArchive.forEachPBZX(at: content, progress: progress) { entry, payload in + try writer.accept(entry, payload) + } + return try writer.finish(progress: progress) + } + /// Free space on the volume the bundle will be written to. static func availableBytes(at url: URL) -> Int64 { let probe = url.deletingLastPathComponent().path diff --git a/App/NativeToolchain/NativeSDK.swift b/App/NativeToolchain/NativeSDK.swift index 148d9d0..3bc16b4 100644 --- a/App/NativeToolchain/NativeSDK.swift +++ b/App/NativeToolchain/NativeSDK.swift @@ -270,12 +270,58 @@ enum NativeSDK { ]) } if header.starts(with: Data("PK".utf8)) || kind == "zip" { + if try installAppleXcodeZipIfPresent(at: source, progress: progress) { + return ImportReport(files: 0, bytes: 0, + warnings: ["imported Apple Xcode Content stream"]) + } try installDownloadedArchive(at: source) return ImportReport() } throw NativeSDKError.notABundle(source.lastPathComponent) } + /// Apple Developer's Xcode download can reach Files as a ZIP wrapper containing + /// `Xcode_27/Content` and `Metadata`, rather than as a file named `.xip`. Do not + /// unzip it: `Content` declares a ~10 GB cpio and the ZIP itself can already be + /// 2 GB. Copy just that one compressed member to a temporary file, then use the + /// same streaming pbzx/cpio builder as a xar xip. + private static func installAppleXcodeZipIfPresent( + at source: URL, + progress: (@Sendable (DarwinSDKBuilder.Progress) -> Void)? + ) throws -> Bool { + let archive = try Archive(url: source, accessMode: .read) + guard let contentEntry = archive.first(where: { + $0.path == "Xcode_27/Content" || + ($0.path.hasSuffix("/Content") && $0.path.split(separator: "/").count == 2) + }) else { return false } + + let contentURL = XForgeEnvironment.nativeSDKDirectory + .appendingPathComponent("xcode-content-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: XForgeEnvironment.nativeSDKDirectory, + withIntermediateDirectories: true) + defer { try? FileManager.default.removeItem(at: contentURL) } + + var completed = Int64(0) + try archive.extract(contentEntry, to: contentURL) { bytes in + completed += Int64(bytes) + if let progress, contentEntry.uncompressedSize > 0 { + let fraction = min(0.95, Double(completed) / Double(contentEntry.uncompressedSize)) + progress(DarwinSDKBuilder.Progress(fraction: fraction, + message: "Copying Xcode Content — \(Int(fraction * 100))%")) + } + return true + } + + let staging = XForgeEnvironment.nativeSDKDirectory + .appendingPathComponent("darwin.artifactbundle.building-\(UUID().uuidString)", + isDirectory: true) + defer { try? FileManager.default.removeItem(at: staging) } + let built = try DarwinSDKBuilder.buildPBZX(from: contentURL, into: staging, + progress: progress) + try install(preparedBundle: built.bundle) + return true + } + /// The first four bytes, read without mapping a multi-gigabyte file into memory. private static func firstBytes(of url: URL) -> Data { guard let handle = try? FileHandle(forReadingFrom: url) else { return Data() } diff --git a/App/NativeToolchain/XipArchive.swift b/App/NativeToolchain/XipArchive.swift index 013c013..a38df04 100644 --- a/App/NativeToolchain/XipArchive.swift +++ b/App/NativeToolchain/XipArchive.swift @@ -205,17 +205,7 @@ enum XipArchive { // MARK: - The pbzx stream - /// Walk every cpio entry in `Content`. - /// - /// `body` is called once per entry with the entry's header and a payload reader - /// for its data — and it must either read that data or leave it: the walker - /// consumes whatever is left when `body` returns, because cpio is sequential and - /// a skipped entry cannot be seeked past. `progress` is handed the fraction of - /// the *compressed* member consumed, which is monotonic and good enough for a - /// progress bar. - /// - /// Entries are visited in archive order, which is also the order they have to be - /// written in: a directory always precedes the files inside it. + /// Walk a xar `Content` member. static func forEachEntry( in url: URL, content: Member, @@ -226,12 +216,26 @@ enum XipArchive { defer { try? handle.close() } try handle.seek(toOffset: UInt64(content.offset)) let end = content.offset + content.length - let stream = try BlockStream(handle: handle, end: end, contentLength: content.length, progress: progress) try readCPIO(from: stream, body) } + /// Walk the pbzx `Content` member directly, without a xar wrapper. + static func forEachPBZX( + at url: URL, + progress: (@Sendable (Double) -> Void)? = nil, + _ body: (Entry, Payload) throws -> Void + ) throws { + let handle = try FileHandle(forReadingFrom: url) + defer { try? handle.close() } + let end = Int64(try handle.seekToEnd()) + try handle.seek(toOffset: 0) + let stream = try BlockStream(handle: handle, end: end, contentLength: end, + progress: progress) + try readCPIO(from: stream, body) + } + /// Decompressed bytes of the payload, one pbzx block at a time. /// /// A cursor over a sequence of decompressed blocks rather than one buffer: the From a51afb75459725a51fd2932e7be3b67bcbf6e74a Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 07:52:09 -0400 Subject: [PATCH 53/54] sdk import: stream Apple ZIP Content with ZIPFoundation callback --- App/NativeToolchain/NativeSDK.swift | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/App/NativeToolchain/NativeSDK.swift b/App/NativeToolchain/NativeSDK.swift index 3bc16b4..fa11d68 100644 --- a/App/NativeToolchain/NativeSDK.swift +++ b/App/NativeToolchain/NativeSDK.swift @@ -302,14 +302,17 @@ enum NativeSDK { defer { try? FileManager.default.removeItem(at: contentURL) } var completed = Int64(0) - try archive.extract(contentEntry, to: contentURL) { bytes in - completed += Int64(bytes) + FileManager.default.createFile(atPath: contentURL.path, contents: nil) + let output = try FileHandle(forWritingTo: contentURL) + defer { try? output.close() } + _ = try archive.extract(contentEntry) { data in + try output.write(contentsOf: data) + completed += Int64(data.count) if let progress, contentEntry.uncompressedSize > 0 { let fraction = min(0.95, Double(completed) / Double(contentEntry.uncompressedSize)) progress(DarwinSDKBuilder.Progress(fraction: fraction, message: "Copying Xcode Content — \(Int(fraction * 100))%")) } - return true } let staging = XForgeEnvironment.nativeSDKDirectory From 2649f415e25ca03bd9c923d5e5cb3adc771113ea Mon Sep 17 00:00:00 2001 From: Felix Date: Fri, 25 Sep 2026 08:05:18 -0400 Subject: [PATCH 54/54] sdk import: convert pbzx fraction into SDK progress --- App/NativeToolchain/DarwinSDKBuilder.swift | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/App/NativeToolchain/DarwinSDKBuilder.swift b/App/NativeToolchain/DarwinSDKBuilder.swift index df411f1..d85f92f 100644 --- a/App/NativeToolchain/DarwinSDKBuilder.swift +++ b/App/NativeToolchain/DarwinSDKBuilder.swift @@ -350,7 +350,9 @@ enum DarwinSDKBuilder { let staging = destination try FileManager.default.createDirectory(at: staging, withIntermediateDirectories: true) let writer = Writer(staging: staging) - try XipArchive.forEachPBZX(at: content, progress: progress) { entry, payload in + try XipArchive.forEachPBZX(at: content, progress: { fraction in + progress?(Progress(fraction: fraction, message: "Extracting Xcode Content — \(Int(fraction * 100))%")) + }) { entry, payload in try writer.accept(entry, payload) } return try writer.finish(progress: progress)