From 017c07d8557e8aa3436223ccc6b66b7ca3d38c2b Mon Sep 17 00:00:00 2001 From: Eduardo Mozart de Oliveira <2974895+eduardomozart@users.noreply.github.com> Date: Sun, 8 Nov 2026 23:33:19 -0300 Subject: [PATCH 1/3] fix: format multi-hour porting stage durations and ETAs Stage durations of 1 hour or more in the Porting list now format as h:mm:ss instead of minutes overflowing into hundreds of minutes (e.g. 4:12:45 instead of 252:45). Porting ETA estimates of 1 hour or more format with hours and minutes (e.g. 4 hr 13 min). --- .../Views/AddGame/AddGameStepThreeView.swift | 13 +------- .../Models/PortingPresentation.swift | 28 ++++++++++++++-- .../PortingPresentationTests.swift | 33 +++++++++++++++++++ 3 files changed, 59 insertions(+), 15 deletions(-) diff --git a/Sources/NXLibrary/Views/AddGame/AddGameStepThreeView.swift b/Sources/NXLibrary/Views/AddGame/AddGameStepThreeView.swift index f9aaf73..63b5292 100644 --- a/Sources/NXLibrary/Views/AddGame/AddGameStepThreeView.swift +++ b/Sources/NXLibrary/Views/AddGame/AddGameStepThreeView.swift @@ -130,7 +130,7 @@ struct AddGameStepThreeView: View { } Spacer(minLength: canvas.u(12)) if let elapsed = stage.elapsedSeconds { - Text(elapsedText(elapsed)) + Text(PortingETA.elapsed(elapsed)) .seleneFont(14, .regular, design: .monospaced) .foregroundColor(SeleneTheme.ink60) } @@ -167,17 +167,6 @@ struct AddGameStepThreeView: View { } } - private func elapsedText(_ seconds: Double) -> String { - let total = Int(seconds.rounded()) - if total < 60 { return "\(total) s" } - return String(format: "%d:%02d", total / 60, total % 60) - } - - private func formatted(_ seconds: Double) -> String { - let total = Int(seconds.rounded()) - return String(format: "%02d:%02d", total / 60, total % 60) - } - // MARK: - Success /// The game whose new app waits for it to quit, if this porting's did. diff --git a/Sources/NXLibraryCore/Models/PortingPresentation.swift b/Sources/NXLibraryCore/Models/PortingPresentation.swift index aaab074..416d20f 100644 --- a/Sources/NXLibraryCore/Models/PortingPresentation.swift +++ b/Sources/NXLibraryCore/Models/PortingPresentation.swift @@ -17,19 +17,41 @@ public enum PortingETA { switch progress.phase { case .shaders: guard let eta = progress.shaderETA else { return nil } - if let last = lastFinishingSeconds { return "About \(minutes(eta + last)) min left" } - return "Shaders: about \(minutes(eta)) min" + if let last = lastFinishingSeconds { return "About \(duration(eta + last)) left" } + return "Shaders: about \(duration(eta))" case .finishing: guard let last = lastFinishingSeconds else { return "Finishing up" } - return "About \(minutes(max(0, last - progress.finishingElapsed))) min left" + return "About \(duration(max(0, last - progress.finishingElapsed))) left" case .preparing, .done, .failed: return nil } } + /// Stage time in the Porting list: seconds under a minute, m:ss under an hour, h:mm:ss from an hour on. + public static func elapsed(_ seconds: Double) -> String { + let total = max(0, Int(seconds.rounded())) + if total < 60 { return "\(total) s" } + let hours = total / 3600 + let minutes = (total % 3600) / 60 + let secs = total % 60 + if hours > 0 { + return String(format: "%d:%02d:%02d", hours, minutes, secs) + } + return String(format: "%d:%02d", minutes, secs) + } + static func minutes(_ seconds: Double) -> Int { max(1, Int((seconds / 60).rounded())) } + + static func duration(_ seconds: Double) -> String { + let total = minutes(seconds) + let hours = total / 60 + let mins = total % 60 + if hours == 0 { return "\(mins) min" } + if mins == 0 { return "\(hours) hr" } + return "\(hours) hr \(mins) min" + } } /// What a game's art tile shows. diff --git a/Tests/NXLibraryCoreTests/PortingPresentationTests.swift b/Tests/NXLibraryCoreTests/PortingPresentationTests.swift index 4e3cc93..5c2b55d 100644 --- a/Tests/NXLibraryCoreTests/PortingPresentationTests.swift +++ b/Tests/NXLibraryCoreTests/PortingPresentationTests.swift @@ -56,6 +56,39 @@ final class PortingPresentationTests: XCTestCase { XCTAssertEqual(PortingETA.text(finishing, lastFinishingSeconds: nil), "Finishing up") } + func testETAFormatsHoursAndMinutesWhenOverAnHour() { + let oneHour = progress([.stage(k: 5, n: 7, name: "Mac shaders", step: nil), .step("compiling"), + .progress(fraction: 0.1, done: nil, total: nil, unit: nil, eta: 3600)]) + XCTAssertEqual(PortingETA.text(oneHour, lastFinishingSeconds: nil), "Shaders: about 1 hr") + + let longShaders = progress([.stage(k: 5, n: 7, name: "Mac shaders", step: nil), .step("compiling"), + .progress(fraction: 0.5, done: nil, total: nil, unit: nil, eta: 15165)]) + XCTAssertEqual(PortingETA.text(longShaders, lastFinishingSeconds: nil), "Shaders: about 4 hr 13 min") + XCTAssertEqual(PortingETA.text(longShaders, lastFinishingSeconds: 120), "About 4 hr 15 min left") + + let longFinishing = progress([.stage(k: 5, n: 7, name: "Mac shaders", step: nil), + .stageEnd(name: "Mac shaders", ok: true, elapsed: 600, summary: nil), + .stage(k: 6, n: 7, name: "Mac host", step: nil)]) + XCTAssertEqual(PortingETA.text(longFinishing, lastFinishingSeconds: 3600), "About 1 hr left") + XCTAssertEqual(PortingETA.text(longFinishing, lastFinishingSeconds: 3900), "About 1 hr 5 min left") + } + + // MARK: - Stage Elapsed Time + + func testElapsedFormatsSecondsMinutesAndHours() { + XCTAssertEqual(PortingETA.elapsed(0), "0 s") + XCTAssertEqual(PortingETA.elapsed(6), "6 s") + XCTAssertEqual(PortingETA.elapsed(59.4), "59 s") + XCTAssertEqual(PortingETA.elapsed(60), "1:00") + XCTAssertEqual(PortingETA.elapsed(61), "1:01") + XCTAssertEqual(PortingETA.elapsed(397), "6:37") + XCTAssertEqual(PortingETA.elapsed(3599), "59:59") + XCTAssertEqual(PortingETA.elapsed(3600), "1:00:00") + XCTAssertEqual(PortingETA.elapsed(3665), "1:01:05") + // 252 minutes 45 seconds (15165 s) -> 4 hours 12 minutes 45 seconds + XCTAssertEqual(PortingETA.elapsed(15165), "4:12:45") + } + // MARK: - Tile look func testTileFollowsThePortingPhase() { From 4c332d83ac1a9eb9bf93b175519eb6489e32baa6 Mon Sep 17 00:00:00 2001 From: Eduardo Mozart de Oliveira <2974895+eduardomozart@users.noreply.github.com> Date: Mon, 9 Nov 2026 00:11:12 -0300 Subject: [PATCH 2/3] feat: show live elapsed duration on running porting stages Update running stage elapsedSeconds directly on active stages in AppViewModel via a 1-second timer and engine progress events so active actions count up live (0 s, 1 s, ..., 1:00, ...) instead of remaining blank until finished. --- .../NXLibrary/ViewModels/AppViewModel.swift | 44 +++++++++++++++++-- .../Models/PortingProgress.swift | 2 +- .../NXLibraryCore/Services/BuildEvent.swift | 2 +- .../Services/BuildEventParser.swift | 2 +- .../BuildEventParserTests.swift | 6 ++- 5 files changed, 49 insertions(+), 7 deletions(-) diff --git a/Sources/NXLibrary/ViewModels/AppViewModel.swift b/Sources/NXLibrary/ViewModels/AppViewModel.swift index 0d67ae6..b1334d8 100644 --- a/Sources/NXLibrary/ViewModels/AppViewModel.swift +++ b/Sources/NXLibrary/ViewModels/AppViewModel.swift @@ -461,6 +461,8 @@ public final class AppViewModel: ObservableObject { private var environmentCheckGeneration = UUID() private var buildStartedAt: Date? private var buildTempDirectory: URL? + private var stageElapsedTimer: Timer? + private var currentStageStartedAt: Date? public init(settingsStore: AppSettingsStore = AppSettingsStore()) { self.settingsStore = settingsStore @@ -2501,6 +2503,7 @@ public final class AppViewModel: ObservableObject { ) lastPortingCommand = [command.executable.path] + command.arguments + stopStageElapsedTimer() buildStartedAt = Date() buildTempDirectory = tempRoot buildLogURL = logURL @@ -2557,6 +2560,7 @@ public final class AppViewModel: ObservableObject { public func cancelBuild() { guard isBuilding, let buildService else { return } + stopStageElapsedTimer() isCancellingBuild = true currentStageName = "Cancelling build..." logs.append(LogEntry(level: .warning, message: "Cancelling build...")) @@ -2601,6 +2605,8 @@ public final class AppViewModel: ObservableObject { portingProgress?.apply(event) switch event { case .stage(let k, let n, let name, let step): + let now = Date() + currentStageStartedAt = now currentStageName = name stageIndex = k totalStages = n @@ -2612,10 +2618,15 @@ public final class AppViewModel: ObservableObject { if let index = buildStages.firstIndex(where: { $0.id == k }) { buildStages[index].isRunning = true buildStages[index].step = step ?? "" + if buildStages[index].elapsedSeconds == nil { + buildStages[index].elapsedSeconds = 0 + } } else { buildStages.append(BuildStageItem(id: k, name: name, - step: step ?? "", isRunning: true)) + step: step ?? "", isRunning: true, + elapsedSeconds: 0)) } + startStageElapsedTimerIfNeeded() if let step, !step.isEmpty { currentStepName = step } logs.append(LogEntry(level: .step, message: "[\(k)/\(n)] \(name)")) @@ -2626,9 +2637,14 @@ public final class AppViewModel: ObservableObject { } logs.append(LogEntry(level: .info, message: "↳ \(step)")) - case .progress(let fraction, let done, let total, let unit, let eta): + case .progress(let fraction, let done, let total, let unit, let eta, let elapsed): progressFraction = min(max(fraction, 0), 1) etaSeconds = eta + if let elapsed, + let index = buildStages.firstIndex(where: { $0.id == stageIndex && $0.isRunning }) { + buildStages[index].elapsedSeconds = elapsed + currentStageStartedAt = Date().addingTimeInterval(-elapsed) + } if let done, let total, total > 0 { if let unit, !unit.isEmpty, unit.lowercased() != "b", unit.lowercased() != "bytes" { @@ -2650,7 +2666,7 @@ public final class AppViewModel: ObservableObject { buildStages[index].isCompleted = ok buildStages[index].isFailed = !ok buildStages[index].hasWarning = warned - buildStages[index].elapsedSeconds = elapsed + buildStages[index].elapsedSeconds = elapsed ?? buildStages[index].elapsedSeconds buildStages[index].summary = summary } logs.append(LogEntry( @@ -2669,8 +2685,30 @@ public final class AppViewModel: ObservableObject { } } + private func startStageElapsedTimerIfNeeded() { + guard stageElapsedTimer == nil else { return } + stageElapsedTimer = Timer.scheduledTimer(withTimeInterval: 1, repeats: true) { [weak self] _ in + self?.tickStageElapsed() + } + } + + private func stopStageElapsedTimer() { + stageElapsedTimer?.invalidate() + stageElapsedTimer = nil + currentStageStartedAt = nil + } + + private func tickStageElapsed() { + guard let started = currentStageStartedAt, + let index = buildStages.firstIndex(where: { $0.id == stageIndex && $0.isRunning }) else { return } + let now = Date() + let elapsed = max(0, now.timeIntervalSince(started)) + buildStages[index].elapsedSeconds = elapsed + } + private func finishBuild(_ result: BuildResult, gameID: UUID, finalApp: URL, tempDir: URL) { + stopStageElapsedTimer() portingProgress?.finish(success: result.success) isBuilding = false isCancellingBuild = false diff --git a/Sources/NXLibraryCore/Models/PortingProgress.swift b/Sources/NXLibraryCore/Models/PortingProgress.swift index e961e83..7fb86b4 100644 --- a/Sources/NXLibraryCore/Models/PortingProgress.swift +++ b/Sources/NXLibraryCore/Models/PortingProgress.swift @@ -59,7 +59,7 @@ public struct PortingProgress: Equatable { case .step(let title): isUpdatingShaderCache = false stepName = title - case .progress(let fraction, let done, _, _, let eta): + case .progress(let fraction, let done, _, _, let eta, _): guard countsAsShaders(hasCount: done != nil) else { return } phase = .shaders raise(to: Self.shaderShare * min(max(fraction, 0), 1)) diff --git a/Sources/NXLibraryCore/Services/BuildEvent.swift b/Sources/NXLibraryCore/Services/BuildEvent.swift index 4fbfbb9..a8d2f93 100644 --- a/Sources/NXLibraryCore/Services/BuildEvent.swift +++ b/Sources/NXLibraryCore/Services/BuildEvent.swift @@ -1,7 +1,7 @@ public enum BuildEvent: Equatable { case stage(k: Int, n: Int, name: String, step: String?) case step(String) - case progress(fraction: Double, done: Double?, total: Double?, unit: String?, eta: Double?) + case progress(fraction: Double, done: Double?, total: Double?, unit: String?, eta: Double?, elapsed: Double? = nil) /// `warn`: the stage finished without what it makes, though `ok` (a pipeline archive that failed, so the app is /// packed without one). Engines before `warn` existed never send it. case stageEnd(name: String, ok: Bool, elapsed: Double?, summary: String?, warn: Bool = false) diff --git a/Sources/NXLibraryCore/Services/BuildEventParser.swift b/Sources/NXLibraryCore/Services/BuildEventParser.swift index 697700f..b5ec99f 100644 --- a/Sources/NXLibraryCore/Services/BuildEventParser.swift +++ b/Sources/NXLibraryCore/Services/BuildEventParser.swift @@ -33,7 +33,7 @@ public enum BuildEventParser { return wire.step.map { .step($0) } case "progress": return .progress(fraction: wire.frac ?? 0, done: wire.done, - total: wire.total, unit: wire.unit, eta: wire.eta) + total: wire.total, unit: wire.unit, eta: wire.eta, elapsed: wire.elapsed) case "stage_end": return .stageEnd(name: wire.name ?? "", ok: wire.ok ?? true, elapsed: wire.elapsed, summary: wire.summary, warn: wire.warn ?? false) diff --git a/Tests/NXLibraryCoreTests/BuildEventParserTests.swift b/Tests/NXLibraryCoreTests/BuildEventParserTests.swift index ba28e41..3f5b88d 100644 --- a/Tests/NXLibraryCoreTests/BuildEventParserTests.swift +++ b/Tests/NXLibraryCoreTests/BuildEventParserTests.swift @@ -9,7 +9,11 @@ final class BuildEventParserTests: XCTestCase { ) XCTAssertEqual( BuildEventParser.parse(#"{"ev":"progress","frac":0.5,"done":1,"total":2,"unit":"GB","eta":10}"#), - .progress(fraction: 0.5, done: 1, total: 2, unit: "GB", eta: 10) + .progress(fraction: 0.5, done: 1, total: 2, unit: "GB", eta: 10, elapsed: nil) + ) + XCTAssertEqual( + BuildEventParser.parse(#"{"ev":"progress","frac":0.5,"done":1,"total":2,"unit":"GB","eta":10,"elapsed":4.5}"#), + .progress(fraction: 0.5, done: 1, total: 2, unit: "GB", eta: 10, elapsed: 4.5) ) } From ec2addf7d5f55572612d1e6fd98415ccd5a038dc Mon Sep 17 00:00:00 2001 From: Eduardo Mozart de Oliveira <2974895+eduardomozart@users.noreply.github.com> Date: Mon, 9 Nov 2026 01:28:44 -0300 Subject: [PATCH 3/3] feat: show live progress, pipeline counts, and ETA for pipeline archive stage --- .../NXLibrary/ViewModels/AppViewModel.swift | 27 +++ .../Models/PortingPresentation.swift | 1 + .../Models/PortingProgress.swift | 12 + .../Services/PipelineArchiveProgress.swift | 224 ++++++++++++++++++ .../PipelineArchiveProgressTests.swift | 153 ++++++++++++ .../PortingPresentationTests.swift | 6 + .../PortingProgressTests.swift | 23 ++ 7 files changed, 446 insertions(+) create mode 100644 Sources/NXLibraryCore/Services/PipelineArchiveProgress.swift create mode 100644 Tests/NXLibraryCoreTests/PipelineArchiveProgressTests.swift diff --git a/Sources/NXLibrary/ViewModels/AppViewModel.swift b/Sources/NXLibrary/ViewModels/AppViewModel.swift index b1334d8..8bfb160 100644 --- a/Sources/NXLibrary/ViewModels/AppViewModel.swift +++ b/Sources/NXLibrary/ViewModels/AppViewModel.swift @@ -463,6 +463,9 @@ public final class AppViewModel: ObservableObject { private var buildTempDirectory: URL? private var stageElapsedTimer: Timer? private var currentStageStartedAt: Date? + private var currentBuildCacheURL: URL? + private let pipelineArchiveProgress = PipelineArchiveProgress() + private var isPollingPipelineArchive = false public init(settingsStore: AppSettingsStore = AppSettingsStore()) { self.settingsStore = settingsStore @@ -2507,6 +2510,9 @@ public final class AppViewModel: ObservableObject { buildStartedAt = Date() buildTempDirectory = tempRoot buildLogURL = logURL + currentBuildCacheURL = workspace.buildCache + pipelineArchiveProgress.reset() + isPollingPipelineArchive = false buildWizardStep = 3 isBuilding = true isCancellingBuild = false @@ -2696,6 +2702,7 @@ public final class AppViewModel: ObservableObject { stageElapsedTimer?.invalidate() stageElapsedTimer = nil currentStageStartedAt = nil + isPollingPipelineArchive = false } private func tickStageElapsed() { @@ -2704,11 +2711,31 @@ public final class AppViewModel: ObservableObject { let now = Date() let elapsed = max(0, now.timeIntervalSince(started)) buildStages[index].elapsedSeconds = elapsed + + if currentStageName == PortingProgress.pipelineArchiveStageName, + let cacheURL = currentBuildCacheURL, + !isPollingPipelineArchive { + isPollingPipelineArchive = true + let psoWorkDir = cacheURL.appendingPathComponent("pso/mac-work") + let progress = pipelineArchiveProgress + DispatchQueue.global(qos: .utility).async { [weak self] in + let event = progress.poll(psoWorkDirectory: psoWorkDir, stageStartedAt: started, now: Date()) + DispatchQueue.main.async { + guard let self else { return } + self.isPollingPipelineArchive = false + if let event, self.isBuilding, self.currentStageName == PortingProgress.pipelineArchiveStageName { + self.handleBuildEvent(event) + } + } + } + } } private func finishBuild(_ result: BuildResult, gameID: UUID, finalApp: URL, tempDir: URL) { stopStageElapsedTimer() + currentBuildCacheURL = nil + pipelineArchiveProgress.reset() portingProgress?.finish(success: result.success) isBuilding = false isCancellingBuild = false diff --git a/Sources/NXLibraryCore/Models/PortingPresentation.swift b/Sources/NXLibraryCore/Models/PortingPresentation.swift index 416d20f..412f2fc 100644 --- a/Sources/NXLibraryCore/Models/PortingPresentation.swift +++ b/Sources/NXLibraryCore/Models/PortingPresentation.swift @@ -20,6 +20,7 @@ public enum PortingETA { if let last = lastFinishingSeconds { return "About \(duration(eta + last)) left" } return "Shaders: about \(duration(eta))" case .finishing: + if let eta = progress.finishingETA { return "About \(duration(eta)) left" } guard let last = lastFinishingSeconds else { return "Finishing up" } return "About \(duration(max(0, last - progress.finishingElapsed))) left" case .preparing, .done, .failed: diff --git a/Sources/NXLibraryCore/Models/PortingProgress.swift b/Sources/NXLibraryCore/Models/PortingProgress.swift index 7fb86b4..61666dd 100644 --- a/Sources/NXLibraryCore/Models/PortingProgress.swift +++ b/Sources/NXLibraryCore/Models/PortingProgress.swift @@ -15,6 +15,7 @@ public struct PortingProgress: Equatable { } public static let shaderStageName = "Mac shaders" + public static let pipelineArchiveStageName = "Pipeline archive" public static let shaderShare = 0.9 /// The stages after the shaders when the engine has not counted them yet. public static let expectedFinishingStages = 3 @@ -31,6 +32,8 @@ public struct PortingProgress: Equatable { public private(set) var stageName = "" public private(set) var stepName = "" public private(set) var shaderETA: Double? + /// ETA during finishing stages (e.g. Pipeline archive). + public private(set) var finishingETA: Double? /// Cache maintenance has no count or reliable ETA; show its activity instead. public private(set) var isUpdatingShaderCache = false /// Seconds the stages after the shaders took so far. @@ -46,6 +49,7 @@ public struct PortingProgress: Equatable { switch event { case .stage(let k, let n, let name, let stepTitle): isUpdatingShaderCache = false + finishingETA = nil step = k count = n stageName = name @@ -60,11 +64,19 @@ public struct PortingProgress: Equatable { isUpdatingShaderCache = false stepName = title case .progress(let fraction, let done, _, _, let eta, _): + if phase == .finishing, stageName == Self.pipelineArchiveStageName, done != nil { + finishingETA = eta + let stages = max(Self.expectedFinishingStages, count - (shaderStep ?? count)) + let clamped = min(max(fraction, 0), 0.99) + raise(to: Self.shaderShare + (1 - Self.shaderShare) * (Double(finishingEnded) + clamped) / Double(stages)) + return + } guard countsAsShaders(hasCount: done != nil) else { return } phase = .shaders raise(to: Self.shaderShare * min(max(fraction, 0), 1)) shaderETA = eta case .stageEnd(let name, let ok, let elapsed, _, _): + finishingETA = nil if name == Self.shaderStageName { raise(to: Self.shaderShare) phase = .finishing diff --git a/Sources/NXLibraryCore/Services/PipelineArchiveProgress.swift b/Sources/NXLibraryCore/Services/PipelineArchiveProgress.swift new file mode 100644 index 0000000..8ba2378 --- /dev/null +++ b/Sources/NXLibraryCore/Services/PipelineArchiveProgress.swift @@ -0,0 +1,224 @@ +import Foundation + +/// Inspects `/pso/mac-work` while upstream `nxnative` runs `Pipeline archive` +/// to compute live progress, completed pipeline counts, and an ETA without modifying `nxnative`. +public final class PipelineArchiveProgress: @unchecked Sendable { + public struct Counts: Equatable, Sendable { + public let bases: Int + public let computes: Int + public var total: Int { bases + computes } + + public init(bases: Int, computes: Int) { + self.bases = bases + self.computes = computes + } + } + + private let fileManager: FileManager + private var cachedScriptURL: URL? + private var cachedScriptMTime: Date? + private var cachedScriptSize: UInt64? + private var cachedCounts: Counts? + private var maxDone: Int = 0 + + public init(fileManager: FileManager = .default) { + self.fileManager = fileManager + } + + public func reset() { + cachedScriptURL = nil + cachedScriptMTime = nil + cachedScriptSize = nil + cachedCounts = nil + maxDone = 0 + } + + /// Fast occurrence count of ASCII byte needles (e.g. `"vertex_function_descriptor"`) + /// without full JSON AST parsing. + public static func countOccurrences(of needle: String, in data: Data) -> Int { + guard !needle.isEmpty, !data.isEmpty else { return 0 } + let needleBytes = Array(needle.utf8) + let needleLen = needleBytes.count + guard data.count >= needleLen else { return 0 } + + return data.withUnsafeBytes { rawBuffer -> Int in + guard let ptr = rawBuffer.baseAddress?.assumingMemoryBound(to: UInt8.self) else { return 0 } + let total = data.count + var count = 0 + var i = 0 + let limit = total - needleLen + let first = needleBytes[0] + + while i <= limit { + if ptr[i] == first { + var match = true + var j = 1 + while j < needleLen { + if ptr[i + j] != needleBytes[j] { + match = false + break + } + j += 1 + } + if match { + count += 1 + i += needleLen + continue + } + } + i += 1 + } + return count + } + } + + /// Reads descriptor counts from `bases.mtl4-json`. + public static func countDescriptors(in data: Data) -> Counts { + let bases = countOccurrences(of: "\"vertex_function_descriptor\"", in: data) + let computes = countOccurrences(of: "\"compute_function_descriptor\"", in: data) + return Counts(bases: bases, computes: computes) + } + + /// Inspects `psoWorkDirectory` (`/pso/mac-work`) and returns a `.progress` + /// event with completed pipelines, total, and an ETA. + public func poll( + psoWorkDirectory: URL, + stageStartedAt: Date, + now: Date = Date() + ) -> BuildEvent? { + let scriptURL = psoWorkDirectory.appendingPathComponent("bases.mtl4-json") + guard let attributes = try? fileManager.attributesOfItem(atPath: scriptURL.path), + let scriptMTime = attributes[.modificationDate] as? Date, + let scriptSize = (attributes[.size] as? NSNumber)?.uint64Value, + scriptSize > 0 else { + return nil + } + + // Cache counts unless bases.mtl4-json was rewritten + let counts: Counts + if scriptURL == cachedScriptURL, + scriptMTime == cachedScriptMTime, + scriptSize == cachedScriptSize, + let cached = cachedCounts { + counts = cached + } else { + guard let data = try? Data(contentsOf: scriptURL, options: .mappedIfSafe) else { + return nil + } + let fresh = Self.countDescriptors(in: data) + cachedScriptURL = scriptURL + cachedScriptMTime = scriptMTime + cachedScriptSize = scriptSize + cachedCounts = fresh + counts = fresh + } + + let total = counts.total + guard total > 0 else { return nil } + + let referenceTime = min(stageStartedAt, scriptMTime).addingTimeInterval(-2) + var done = 0 + var compiledInRun = 0 + + // Find batches. folder + let batchesDir = (try? fileManager.contentsOfDirectory(at: psoWorkDirectory, includingPropertiesForKeys: nil))? + .first { $0.lastPathComponent.hasPrefix("batches.") } + + if let batchesDir { + let arch = String(batchesDir.lastPathComponent.dropFirst("batches.".count)) + let keyName = "key.\(arch)" + + // 1. Base groups: sequentially compiled (BATCH = 1000) + let baseBatchSize = 1000 + let numBaseBatches = (counts.bases + baseBatchSize - 1) / baseBatchSize + + if numBaseBatches > 0 { + for i in 1...numBaseBatches { + let batchName = String(format: "%05d", i) + let batchDir = batchesDir.appendingPathComponent(batchName) + let thisBatchSize = min(baseBatchSize, counts.bases - (i - 1) * baseBatchSize) + let keyURL = batchDir.appendingPathComponent(keyName) + + if let keyAttrs = try? fileManager.attributesOfItem(atPath: keyURL.path) { + done += thisBatchSize + if let keyMTime = keyAttrs[.modificationDate] as? Date, keyMTime >= referenceTime { + compiledInRun += thisBatchSize + } + } else if fileManager.fileExists(atPath: batchDir.path) { + let activeCount = countTranslateFiles(in: batchDir) + let inBatch = min(thisBatchSize, activeCount) + done += inBatch + compiledInRun += inBatch + break + } else { + break + } + } + } + + // 2. Compute groups: compiled after all bases finish (COMPUTE_BATCH = 16) + if done >= counts.bases && counts.computes > 0 { + let computeBatchSize = 16 + let numComputeBatches = (counts.computes + computeBatchSize - 1) / computeBatchSize + + for j in 1...numComputeBatches { + let batchName = String(format: "c%05d", j) + let batchDir = batchesDir.appendingPathComponent(batchName) + let thisBatchSize = min(computeBatchSize, counts.computes - (j - 1) * computeBatchSize) + let keyURL = batchDir.appendingPathComponent(keyName) + + if let keyAttrs = try? fileManager.attributesOfItem(atPath: keyURL.path) { + done += thisBatchSize + if let keyMTime = keyAttrs[.modificationDate] as? Date, keyMTime >= referenceTime { + compiledInRun += thisBatchSize + } + } else if fileManager.fileExists(atPath: batchDir.path) { + let activeCount = countTranslateFiles(in: batchDir) + let inBatch = min(thisBatchSize, activeCount) + done += inBatch + compiledInRun += inBatch + } + } + } + } else { + // Single-batch fallback directly in psoWorkDirectory + let activeCount = countTranslateFiles(in: psoWorkDirectory) + let inBatch = min(total, activeCount) + done = inBatch + compiledInRun = inBatch + } + + maxDone = max(maxDone, done) + let effectiveDone = min(maxDone, total) + let fraction = Double(effectiveDone) / Double(total) + + // ETA calculation + let elapsed = max(0, now.timeIntervalSince(stageStartedAt)) + let eta: Double? + if compiledInRun > 0, elapsed >= 3, effectiveDone < total { + let rate = Double(compiledInRun) / elapsed + let remaining = Double(total - effectiveDone) + eta = rate > 0 ? (remaining / rate) : nil + } else { + eta = nil + } + + return .progress( + fraction: fraction, + done: Double(effectiveDone), + total: Double(total), + unit: "pipelines", + eta: eta, + elapsed: nil + ) + } + + private func countTranslateFiles(in directory: URL) -> Int { + guard let files = try? fileManager.contentsOfDirectory(atPath: directory.path) else { return 0 } + var count = 0 + for file in files where file.hasPrefix("translate-") { + count += 1 + } + return count + } +} diff --git a/Tests/NXLibraryCoreTests/PipelineArchiveProgressTests.swift b/Tests/NXLibraryCoreTests/PipelineArchiveProgressTests.swift new file mode 100644 index 0000000..680c5a4 --- /dev/null +++ b/Tests/NXLibraryCoreTests/PipelineArchiveProgressTests.swift @@ -0,0 +1,153 @@ +import XCTest +import Foundation +@testable import NXLibraryCore + +final class PipelineArchiveProgressTests: XCTestCase { + private var tempDir: URL! + + override func setUpWithError() throws { + try super.setUpWithError() + tempDir = FileManager.default.temporaryDirectory + .appendingPathComponent("PipelineArchiveProgressTests-\(UUID().uuidString)") + try FileManager.default.createDirectory(at: tempDir, withIntermediateDirectories: true) + } + + override func tearDownWithError() throws { + if let tempDir { + try? FileManager.default.removeItem(at: tempDir) + } + try super.tearDownWithError() + } + + func testCountOccurrences() { + let text = "hello \"vertex_function_descriptor\" world \"vertex_function_descriptor\" end" + let data = Data(text.utf8) + XCTAssertEqual(PipelineArchiveProgress.countOccurrences(of: "\"vertex_function_descriptor\"", in: data), 2) + XCTAssertEqual(PipelineArchiveProgress.countOccurrences(of: "\"compute_function_descriptor\"", in: data), 0) + XCTAssertEqual(PipelineArchiveProgress.countOccurrences(of: "", in: data), 0) + XCTAssertEqual(PipelineArchiveProgress.countOccurrences(of: "test", in: Data()), 0) + } + + func testCountDescriptors() { + let json = """ + { + "pipeline_descriptors": { + "render_pipeline_descriptors": [ + { "vertex_function_descriptor": "fnd:1" }, + { "vertex_function_descriptor": "fnd:2" }, + { "vertex_function_descriptor": "fnd:3" } + ], + "compute_pipeline_descriptors": [ + { "compute_function_descriptor": "fnd:c1" } + ] + } + } + """ + let counts = PipelineArchiveProgress.countDescriptors(in: Data(json.utf8)) + XCTAssertEqual(counts.bases, 3) + XCTAssertEqual(counts.computes, 1) + XCTAssertEqual(counts.total, 4) + } + + func testPollReturnsNilWhenNoBasesJson() { + let scanner = PipelineArchiveProgress() + let event = scanner.poll(psoWorkDirectory: tempDir, stageStartedAt: Date()) + XCTAssertNil(event) + } + + func testPollTracksBatchesAndComputesETA() throws { + let scanner = PipelineArchiveProgress() + + // 2,500 bases and 32 computes = 3 base batches (1000, 1000, 500) + 2 compute batches (16, 16) + var descriptors: [String] = [] + for _ in 0..<2500 { + descriptors.append("\"vertex_function_descriptor\"") + } + for _ in 0..<32 { + descriptors.append("\"compute_function_descriptor\"") + } + let json = "{ " + descriptors.joined(separator: ", ") + " }" + let scriptURL = tempDir.appendingPathComponent("bases.mtl4-json") + try json.write(to: scriptURL, atomically: true, encoding: .utf8) + + let stageStartedAt = Date().addingTimeInterval(-60) // 60s ago + + // Initially no batches folder: active single batch check + let initialEvent = scanner.poll(psoWorkDirectory: tempDir, stageStartedAt: stageStartedAt, now: Date()) + if case .progress(let fraction, let done, let total, let unit, let eta, _) = initialEvent { + XCTAssertEqual(fraction, 0) + XCTAssertEqual(done, 0) + XCTAssertEqual(total, 2532) + XCTAssertEqual(unit, "pipelines") + XCTAssertNil(eta) + } else { + XCTFail("Expected progress event, got \(String(describing: initialEvent))") + } + + // Create batches folder + let batchesDir = tempDir.appendingPathComponent("batches.applegpu_test") + try FileManager.default.createDirectory(at: batchesDir, withIntermediateDirectories: true) + + // Batch 1 (00001) completed + let b1 = batchesDir.appendingPathComponent("00001") + try FileManager.default.createDirectory(at: b1, withIntermediateDirectories: true) + try "key1".write(to: b1.appendingPathComponent("key.applegpu_test"), atomically: true, encoding: .utf8) + + // Batch 2 (00002) in progress with 500 translate- files + let b2 = batchesDir.appendingPathComponent("00002") + try FileManager.default.createDirectory(at: b2, withIntermediateDirectories: true) + for i in 0..<500 { + try "".write(to: b2.appendingPathComponent("translate-\(i)"), atomically: true, encoding: .utf8) + } + + let now = stageStartedAt.addingTimeInterval(30) // 30s elapsed + let activeEvent = scanner.poll(psoWorkDirectory: tempDir, stageStartedAt: stageStartedAt, now: now) + + if case .progress(let fraction, let done, let total, let unit, let eta, _) = activeEvent { + XCTAssertEqual(done, 1500) // 1000 from b1 + 500 from b2 + XCTAssertEqual(total, 2532) + XCTAssertEqual(unit, "pipelines") + XCTAssertEqual(fraction, 1500.0 / 2532.0, accuracy: 1e-6) + // Rate = 1500 / 30 = 50 pipelines/s. Remaining = 1032. ETA = 1032 / 50 = 20.64s + XCTAssertNotNil(eta) + XCTAssertEqual(eta!, 20.64, accuracy: 1.0) + } else { + XCTFail("Expected progress event, got \(String(describing: activeEvent))") + } + } + + func testPollPreservesMonotonicDone() throws { + let scanner = PipelineArchiveProgress() + + var descriptors: [String] = [] + for _ in 0..<1000 { + descriptors.append("\"vertex_function_descriptor\"") + } + let json = "{ " + descriptors.joined(separator: ", ") + " }" + let scriptURL = tempDir.appendingPathComponent("bases.mtl4-json") + try json.write(to: scriptURL, atomically: true, encoding: .utf8) + + let batchesDir = tempDir.appendingPathComponent("batches.applegpu_test") + let b1 = batchesDir.appendingPathComponent("00001") + try FileManager.default.createDirectory(at: b1, withIntermediateDirectories: true) + + // 400 translate files + for i in 0..<400 { + try "".write(to: b1.appendingPathComponent("translate-\(i)"), atomically: true, encoding: .utf8) + } + let event1 = scanner.poll(psoWorkDirectory: tempDir, stageStartedAt: Date(), now: Date()) + if case .progress(_, let done, _, _, _, _) = event1 { + XCTAssertEqual(done, 400) + } + + // Simulate metal-tt cleaning up temporaries before key file is written + for i in 0..<400 { + try? FileManager.default.removeItem(at: b1.appendingPathComponent("translate-\(i)")) + } + + let event2 = scanner.poll(psoWorkDirectory: tempDir, stageStartedAt: Date(), now: Date()) + if case .progress(_, let done, _, _, _, _) = event2 { + XCTAssertEqual(done, 400, "Done count must not drop during temporary cleanup") + } + } +} diff --git a/Tests/NXLibraryCoreTests/PortingPresentationTests.swift b/Tests/NXLibraryCoreTests/PortingPresentationTests.swift index 5c2b55d..b4c400b 100644 --- a/Tests/NXLibraryCoreTests/PortingPresentationTests.swift +++ b/Tests/NXLibraryCoreTests/PortingPresentationTests.swift @@ -54,6 +54,12 @@ final class PortingPresentationTests: XCTestCase { func testETAWhileFinishingCountsDownTheLastTime() { XCTAssertEqual(PortingETA.text(finishing, lastFinishingSeconds: 150), "About 2 min left") XCTAssertEqual(PortingETA.text(finishing, lastFinishingSeconds: nil), "Finishing up") + + // Live ETA during Pipeline archive takes precedence + var pipelineArchive = finishing + pipelineArchive.apply(.progress(fraction: 0.5, done: 1000, total: 2000, unit: "pipelines", eta: 5400)) + XCTAssertEqual(PortingETA.text(pipelineArchive, lastFinishingSeconds: 150), "About 1 hr 30 min left") + XCTAssertEqual(PortingETA.text(pipelineArchive, lastFinishingSeconds: nil), "About 1 hr 30 min left") } func testETAFormatsHoursAndMinutesWhenOverAnHour() { diff --git a/Tests/NXLibraryCoreTests/PortingProgressTests.swift b/Tests/NXLibraryCoreTests/PortingProgressTests.swift index 55e7ae2..f5c1207 100644 --- a/Tests/NXLibraryCoreTests/PortingProgressTests.swift +++ b/Tests/NXLibraryCoreTests/PortingProgressTests.swift @@ -170,6 +170,29 @@ final class PortingProgressTests: XCTestCase { let progress = run(beforeShaders + [.say("removing the shader cache of other chain versions")]) XCTAssertNil(PortingETA.text(progress, lastFinishingSeconds: nil)) } + + func testPipelineArchiveProgressAdvancesOverallAndSetsFinishingETA() { + var progress = run(shaders + [ + .stageEnd(name: "Mac shaders", ok: true, elapsed: 607, summary: nil), + .stage(k: 6, n: 8, name: "Mac host", step: "building"), + .stageEnd(name: "Mac host", ok: true, elapsed: 10, summary: nil), + .stage(k: 7, n: 8, name: "Pipeline archive", step: "applegpu_g13g") + ]) + XCTAssertEqual(progress.phase, .finishing) + XCTAssertEqual(progress.overall, 0.9 + 0.1 / 3, accuracy: 1e-9) // 93.3% + XCTAssertNil(progress.finishingETA) + + // Halfway through Pipeline archive + progress.apply(.progress(fraction: 0.5, done: 1000, total: 2000, unit: "pipelines", eta: 3600)) + XCTAssertEqual(progress.finishingETA, 3600) + // overall should be 0.9 + 0.1 * (1 + 0.5) / 3 = 0.9 + 0.05 = 0.95 (95%) + XCTAssertEqual(progress.overall, 0.95, accuracy: 1e-9) + + // Stage ends: clears finishingETA, overall becomes 0.9 + 0.1 * 2 / 3 = 0.9666... + progress.apply(.stageEnd(name: "Pipeline archive", ok: true, elapsed: 120, summary: nil)) + XCTAssertNil(progress.finishingETA) + XCTAssertEqual(progress.overall, 0.9 + 0.2 / 3, accuracy: 1e-9) + } } final class BuildWarningTests: XCTestCase {