diff --git a/.github/workflows/macos.yml b/.github/workflows/macos.yml index ef7e9ae..473a669 100644 --- a/.github/workflows/macos.yml +++ b/.github/workflows/macos.yml @@ -1,9 +1,9 @@ name: macOS on: push: - paths: ['apps/assets/**', 'apps/macos/**', 'modules/**/*.swift', 'plugins/**', 'build.sh', 'project.yml', 'tools/macos/**', 'tests/**/*.swift', 'scripts/*.sh', '.github/workflows/macos.yml'] + paths: ['apps/assets/**', 'apps/macos/**', 'modules/**/*.swift', 'plugins/**', 'build.sh', 'Package.swift', 'project.yml', 'tools/macos/**', 'tests/**/*.swift', 'scripts/*.sh', '.github/workflows/macos.yml'] pull_request: - paths: ['apps/assets/**', 'apps/macos/**', 'modules/**/*.swift', 'plugins/**', 'build.sh', 'project.yml', 'tools/macos/**', 'tests/**/*.swift', 'scripts/*.sh', '.github/workflows/macos.yml'] + paths: ['apps/assets/**', 'apps/macos/**', 'modules/**/*.swift', 'plugins/**', 'build.sh', 'Package.swift', 'project.yml', 'tools/macos/**', 'tests/**/*.swift', 'scripts/*.sh', '.github/workflows/macos.yml'] workflow_dispatch: permissions: contents: read @@ -17,16 +17,21 @@ jobs: rm -f docs/validation/visualization-host-tests.json docs/validation/visualization-appearance-tests.json zsh scripts/test-visualization-host.sh zsh scripts/test-appearance.sh + - name: Embedded core regression tests + run: swift test - name: Compile complete application without personal signing identity shell: zsh {0} run: | root="$PWD" source scripts/swift-sources.sh mkdir -p build + swift build -c release --product MiucsCore + core_bin="$(swift build -c release --show-bin-path)" swiftc -parse-as-library -swift-version 6 -O \ -framework AppKit -framework SwiftUI -framework ScreenCaptureKit \ -framework CoreMedia -framework CoreAudio -framework Accelerate \ - -o build/Miucs "${app_sources[@]}" + -I "$core_bin/Modules" -L "$core_bin" -lMiucsCore \ + -o build/Miucs "${linked_app_sources[@]}" - uses: actions/upload-artifact@v4 if: always() with: diff --git a/.github/workflows/plugins.yml b/.github/workflows/plugins.yml index 91d7af6..3678131 100644 --- a/.github/workflows/plugins.yml +++ b/.github/workflows/plugins.yml @@ -16,6 +16,7 @@ jobs: with: python-version: '3.12' - run: python -m pip install -r tools/plugins/requirements.txt + - run: python tests/plugins/dependency_download_test.py - run: sh scripts/build-plugins.sh - run: sh scripts/test-plugins.sh - run: zsh scripts/test-plugin-macos.sh @@ -48,6 +49,7 @@ jobs: with: python-version: '3.12' - run: python -m pip install -r tools/plugins/requirements.txt + - run: python tests/plugins/dependency_download_test.py - uses: actions/setup-dotnet@v4 with: dotnet-version: '8.0.x' diff --git a/.gitignore b/.gitignore index 7d90b66..06c9130 100644 --- a/.gitignore +++ b/.gitignore @@ -10,3 +10,7 @@ __pycache__/ # Windows/.NET build products **/bin/ **/obj/ + +# Swift Package Manager +.build/ +.swiftpm/ diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 24f5c73..12cf398 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -21,7 +21,7 @@ open dist/Miucs.app ``` -只修改模块时优先进入相应 `core/` 或 `platforms/macos/`。`scripts/swift-sources.sh` 通过模块目录通配符定义编译分组;增加目录时也要更新 `project.yml`。应用组合点是 `apps/macos/PluginVisualizations.swift`,只注册插件目录中的可视化。 +只修改模块时优先进入相应 `core/` 或 `platforms/macos/`。`MiucsCore` 的模块边界在 `Package.swift`,`build.sh` 和 Xcode 工程均链接该库。`scripts/swift-sources.sh` 保留源码回归测试分组;增加目录时同步维护。嵌入方式见 [核心库指南](docs/embedding.md)。应用组合点是 `apps/macos/PluginVisualizations.swift`,只注册插件目录中的可视化。 离线参考和资源工具需要 Python;应用运行不依赖 Python: @@ -34,6 +34,7 @@ python3 -m pip install -r tests/requirements.txt 以下命令从仓库根目录执行,不改变个人位置或配色偏好。窗口与外观测试使用独立偏好域。 ```sh +swift test # 核心库皮肤、输入、恢复与释放 zsh scripts/test-contracts.sh # 依赖边界、Schema、通道与会话切换 zsh scripts/test-keyboard.sh # 键盘通道、并行输入、权限/休眠/重连和原生事件解码 zsh scripts/test-beat-tracker.sh # 原始 PCM 节拍、半速/倍速、变速、静音和时序 diff --git a/Package.swift b/Package.swift new file mode 100644 index 0000000..670b72f --- /dev/null +++ b/Package.swift @@ -0,0 +1,21 @@ +// swift-tools-version: 6.0 +import PackageDescription + +let package = Package( + name: "Miucs", + platforms: [.macOS(.v14)], + products: [.library(name: "MiucsCore", type: .static, targets: ["MiucsCore"])], + targets: [ + .target( + name: "MiucsCore", path: ".", + exclude: ["apps", "build", "contracts", "dist", "docs", "hardware", "scripts", "tests", + "third_party", "tools", "modules/plugins", "modules/visualizations", + "modules/audio/platforms/windows", "modules/keyboard/platforms/windows", + "build.sh", "project.yml", "README.md", "CONTRIBUTING.md"], + sources: ["modules/audio/core", "modules/audio/platforms/macos", + "modules/keyboard/core", "modules/keyboard/platforms/macos", "modules/hardware/macos"], + resources: [.copy("plugins")] + ), + .testTarget(name: "MiucsCoreTests", dependencies: ["MiucsCore"], path: "tests/core") + ] +) diff --git a/README.md b/README.md index 1f8cc70..e59c860 100644 --- a/README.md +++ b/README.md @@ -52,6 +52,16 @@ macOS 和 Windows 都有独立设置窗口,分为「皮肤」「外观与位 macOS 在程序坞显示 Miucs,点击打开设置;Windows 设置显示在任务栏,关闭时最小化并继续运行,点击任务栏或再次运行应用可恢复设置。菜单栏 / 托盘仅保留打开设置、显示 / 隐藏皮肤、调整位置与大小和退出。 +## USB 硬件屏幕(macOS) + +「设置 → 硬件屏幕」可把当前皮肤通过 USB 发到 ESP32-P4 小屏。使用支持原生桌宠的桌屿固件时,七套内置皮肤上传 Lua 和资源包,由开发板绘制;Mac 发送按键与音频特征,支持实例恢复和连续输入。不支持原生桌宠的设备保留 JPEG 卡片适配。使用 Miucs 屏幕固件时,画面铺到整屏,支持横竖屏及反向切换。 + +将已烧录对应固件的开发板接到原生 USB 数据口,再开启「在硬件屏幕上显示皮肤」。使用支持 CH343 的桌屿固件时,也可以把线接到烧录口,由运行中的桌屿 Mac 应用转发。连接中断后会自动重连。音乐和按键由电脑处理,隐藏桌面皮肤后小屏可继续显示。Miucs 整屏固件的型号、烧录与回退见 [硬件适配说明](hardware/esp32-display/README.md)。 + +## 作为核心库使用 + +本仓库提供 `MiucsCore` Swift Package。桌屿等宿主可以在自己的进程中使用 Miucs 的键盘、音频、皮肤资源和原生桌宠会话,无需另开 Miucs 应用。核心库不启动桌面窗口或 Lua/Skia worker;独立应用仍保留完整桌面功能。仓库依赖、资源打包、权限和生命周期见 [嵌入指南](docs/embedding.md)。 + ## 插件与隐私 应用内置七个 Lua 5.4/Skia 插件,也可安装自己编写的插件。首次构建自动下载并校验运行库依赖。开发和调试见 [插件开发指南](docs/plugin-development.md) 与 [插件工作台](docs/plugin-lab.md)。 diff --git a/apps/macos/App.swift b/apps/macos/App.swift index 7387ffc..237ac4c 100644 --- a/apps/macos/App.swift +++ b/apps/macos/App.swift @@ -1,3 +1,6 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif import AppKit import SwiftUI import Combine @@ -17,6 +20,7 @@ struct MiucsApp: App { @MainActor final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate, ObservableObject { + private static let audioCaptureEnabledKey = "MiucsAudioCapture.enabled" let appearanceStore = VisualizationAppearanceStore() let settings = MiucsSettingsModel() private var settingsWindow: MiucsSettingsWindow? @@ -50,7 +54,8 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate, Observ appearanceStore.saveSelection() configureSettings() let item = NSStatusBar.system.statusItem(withLength: NSStatusItem.variableLength) - trayIcon = AppTrayIcon(item: item, source: keyboardEngine.channel) + trayIcon = AppTrayIcon(item: item, source: keyboardEngine.channel, + animationEnabled: appearanceStore.trayAnimationEnabled) let menu = NSMenu() menu.delegate = self func add(_ title: String, _ selector: Selector, _ key: String) -> NSMenuItem { @@ -69,8 +74,9 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate, Observ PluginHost.shared.start(audio: engine.channel, keyboard: keyboardEngine.channel) createPanelIfNeeded() refreshSkins() + HardwareDisplay.shared.start() showSettings() - Task { await engine.start() } + startAudioIfEnabled() Task { let key = "MiucsKeyboardPermissionRequested.v1" let request = !UserDefaults.standard.bool(forKey: key) @@ -87,11 +93,17 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate, Observ self.syncPanelState() } settings.setOpacity = { [weak self] value in self?.appearanceStore.setOpacity(Int(value.rounded())) } + settings.setTrayAnimationEnabled = { [weak self] enabled in self?.appearanceStore.setTrayAnimationEnabled(enabled) } settings.togglePosition = { [weak self] in self?.togglePositioning() } + settings.audioCaptureEnabled = UserDefaults.standard.object(forKey: Self.audioCaptureEnabledKey) as? Bool ?? true + settings.setAudioCaptureEnabled = { [weak self] enabled in self?.setAudioCaptureEnabled(enabled) } settings.reconnectAudio = { [weak self] in self?.reconnect() } settings.connectKeyboard = { [weak self] in self?.connectKeyboard() } settings.rescan = { [weak self] in self?.rescanSkins() } - settings.reload = { PluginHost.shared.reloadActive() } + settings.reload = { + PluginHost.shared.reloadActive() + HardwareDisplay.shared.reloadSkin() + } settings.openDirectory = { [weak self] in self?.openSkinDirectory() } engine.$status.sink { [weak self] in self?.settings.audioStatus = $0 }.store(in: &subscriptions) keyboardEngine.$state.sink { [weak self] in self?.settings.keyboardState = $0 }.store(in: &subscriptions) @@ -100,6 +112,10 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate, Observ self?.panel?.visualizationHost.applyTransparency(percent) self?.settings.opacity = Double(100 - percent) }.store(in: &subscriptions) + appearanceStore.$trayAnimationEnabled.sink { [weak self] enabled in + self?.settings.trayAnimationEnabled = enabled + self?.trayIcon?.setAnimationEnabled(enabled) + }.store(in: &subscriptions) } private func createPanelIfNeeded() { guard panel == nil, !registry.descriptors.isEmpty else { return } @@ -115,6 +131,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate, Observ } private func syncPanelState() { settings.selectedID = panel?.descriptor.id ?? "" + HardwareDisplay.shared.selectSkin(settings.selectedID) settings.isVisible = panel?.isVisible == true settings.isAdjusting = panel?.isAdjustingPosition == true if let panel { @@ -183,7 +200,24 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate, Observ panel.setAdjustingPosition(!panel.isAdjustingPosition) syncPanelState() } - private func reconnect() { Task { await engine.reconnect() } } + private func setAudioCaptureEnabled(_ enabled: Bool) { + guard settings.audioCaptureEnabled != enabled else { return } + settings.audioCaptureEnabled = enabled + UserDefaults.standard.set(enabled, forKey: Self.audioCaptureEnabledKey) + if enabled { startAudioIfEnabled() } else { engine.stop() } + } + private func startAudioIfEnabled() { + Task { + guard settings.audioCaptureEnabled else { return } + await engine.start() + } + } + private func reconnect() { + Task { + guard settings.audioCaptureEnabled else { return } + await engine.reconnect() + } + } private func connectKeyboard() { Task { await keyboardEngine.start(requestAccess: true) @@ -195,6 +229,7 @@ final class AppDelegate: NSObject, NSApplicationDelegate, NSMenuDelegate, Observ } @objc private func quit() { NSApp.terminate(nil) } func applicationWillTerminate(_ notification: Notification) { + HardwareDisplay.shared.stop() trayIcon?.stop() keyboardEngine.stop() engine.stop() diff --git a/apps/macos/AppTrayIcon.swift b/apps/macos/AppTrayIcon.swift index 991fb88..d5f73bc 100644 --- a/apps/macos/AppTrayIcon.swift +++ b/apps/macos/AppTrayIcon.swift @@ -1,3 +1,6 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif import AppKit /// Application identity and animation are owned by the shell, independently of Lua workers. @@ -5,7 +8,8 @@ import AppKit final class AppTrayIcon { private let item: NSStatusItem private let source: any KeyboardFeatureSource - private var subscription: KeyboardSubscription + private var subscription: KeyboardSubscription? + private var animationEnabled: Bool private var head = TrayHeadAnimation() private var timer: Timer? private var last = ProcessInfo.processInfo.systemUptime @@ -15,12 +19,13 @@ final class AppTrayIcon { private var frames = [Int:NSImage]() private let portrait: NSImage - init(item: NSStatusItem, source: any KeyboardFeatureSource) { - self.item=item;self.source=source;subscription=source.subscribe() + init(item: NSStatusItem, source: any KeyboardFeatureSource, animationEnabled: Bool = true) { + self.item = item + self.source = source + self.animationEnabled = animationEnabled portrait=(Bundle.main.url(forResource:"PixelCatHead",withExtension:"png") ?? Bundle.main.url(forResource:"cat-head",withExtension:"png")).flatMap(NSImage.init(contentsOf:)) ?? NSImage(size:NSSize(width:64,height:64)) NSApp.applicationIconImage=portrait item.button?.setAccessibilityLabel("Miucs") - present(0) let center=NSWorkspace.shared.notificationCenter for (name,reason,paused) in [(NSWorkspace.willSleepNotification,"sleep",true),(NSWorkspace.didWakeNotification,"sleep",false),(NSWorkspace.sessionDidResignActiveNotification,"session",true),(NSWorkspace.sessionDidBecomeActiveNotification,"session",false)] { observers.append(center.addObserver(forName:name,object:nil,queue:.main) { [weak self] _ in @@ -29,18 +34,34 @@ final class AppTrayIcon { } schedule() } + + func setAnimationEnabled(_ enabled: Bool) { + guard animationEnabled != enabled else { return } + animationEnabled = enabled + schedule() + } + private func suspend(_ paused: Bool, reason: String) { if paused { reasons.insert(reason) } else { reasons.remove(reason) } - head.reset();subscription=source.subscribe();last=ProcessInfo.processInfo.systemUptime - present(0);schedule() + schedule() } private func schedule() { - timer?.invalidate();timer=nil - guard reasons.isEmpty else { return } - timer=Timer(timeInterval:1.0/16,repeats:true) { [weak self] _ in MainActor.assumeIsolated { self?.tick() } } - RunLoop.main.add(timer!,forMode:.common) + timer?.invalidate() + timer = nil + subscription = nil + head.reset() + present(0) + guard animationEnabled, reasons.isEmpty else { return } + subscription = source.subscribe() + last = ProcessInfo.processInfo.systemUptime + let timer = Timer(timeInterval: 1.0 / 16, repeats: true) { [weak self] _ in + MainActor.assumeIsolated { self?.tick() } + } + RunLoop.main.add(timer, forMode: .common) + self.timer = timer } private func tick() { + guard let subscription else { return } let now=ProcessInfo.processInfo.systemUptime head.advance(seconds:max(0,now-last),input:subscription.read());last=now present(Int((head.angle * 180 / .pi).rounded())) @@ -64,6 +85,7 @@ final class AppTrayIcon { } func stop() { timer?.invalidate();timer=nil + subscription = nil for observer in observers { NSWorkspace.shared.notificationCenter.removeObserver(observer) };observers=[] } } diff --git a/apps/macos/HardwareDisplay.swift b/apps/macos/HardwareDisplay.swift new file mode 100644 index 0000000..9f8f511 --- /dev/null +++ b/apps/macos/HardwareDisplay.swift @@ -0,0 +1,497 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif +import AppKit +import Combine +import OSLog + +@MainActor +final class HardwareDisplay: ObservableObject { + static let shared = HardwareDisplay() + @Published private(set) var enabled: Bool + @Published private(set) var orientation: HardwareOrientation + @Published private(set) var status = "未启用" + @Published private(set) var detail = "" + @Published private(set) var screen: HardwareScreen? + var connected: Bool { screen != nil } + private let preferences: UserDefaults + private let serial = HardwareSerial() + private var started = false + private var timer: Timer? + private var wakeDeadline = Double.infinity + private var job: Task? + private var generation = UUID() + private var selectedID = "" + private var package: PluginPackage? + private var nextAttempt = 0.0 + private var lastCaptureAt = 0.0 + private var lastCardPixels: Data? + private var lastCardMessageAt = 0.0 + private var sequence: UInt32 = 0 + private var measuredAt = 0.0 + private var measuredFrames = 0 + private var observers = [NSObjectProtocol]() + private var suspended = false + private enum NativePetState { case pending, active, fallback } + private struct NativePetIdentity: Equatable, Sendable { + let size: Int + let crc: UInt32 + + var payload: Data { + var data = Data() + withUnsafeBytes(of: UInt32(size).littleEndian) { data.append(contentsOf: $0) } + withUnsafeBytes(of: crc.littleEndian) { data.append(contentsOf: $0) } + return data + } + } + private var nativePet = NativePetState.pending + private var nativePetIdentity: NativePetIdentity? + private var skinRevision = 0 + private var nativeAudio: FeatureSubscription? + private var nativeKeyboard: KeyboardSubscription? + private var nativeAudioTask: Task? + private var nativeKeyboardTask: Task? + private var nativeAudioPending = false + private var nativeKeyboardPending = false + private var lastNativeInputAt = -Double.infinity + private static let nativeInputInterval = 1.0 / 30 + private let petLog = Logger(subsystem: "com.miucs.hardware", category: "native-pet") + + init(preferences: UserDefaults = .standard) { + self.preferences = preferences + enabled = preferences.bool(forKey: "MiucsHardware.enabled") + orientation = HardwareOrientation(rawValue: preferences.object(forKey: "MiucsHardware.orientation") as? Int ?? 1) ?? .landscape + } + func start() { + guard !started else { return } + started = true + status = enabled ? "等待连接屏幕…" : "未启用" + for name in [NSWorkspace.willSleepNotification, NSWorkspace.didWakeNotification] { + observers.append(NSWorkspace.shared.notificationCenter.addObserver(forName: name, object: nil, queue: .main) { [weak self] _ in + MainActor.assumeIsolated { + guard let self else { return } + self.suspended = name == NSWorkspace.willSleepNotification + self.resetConnection() + } + }) + } + tick() + } + func setEnabled(_ value: Bool) { + enabled = value; preferences.set(value, forKey: "MiucsHardware.enabled") + resetConnection(clearPet: !value) + } + func setOrientation(_ value: HardwareOrientation) { + orientation = value; preferences.set(value.rawValue, forKey: "MiucsHardware.orientation") + } + func selectSkin(_ id: String) { + guard id != selectedID else { return } + selectedID = id + releasePackage() + reloadSkin() + } + func reloadSkin() { + nativePetIdentity = nil + invalidateSkin() + tick() + } + private func invalidateSkin() { + skinRevision &+= 1 + nativePet = .pending + stopNativeInput() + lastNativeInputAt = -.infinity + lastCardPixels = nil + } + private func stopNativeInput() { + nativeAudioTask?.cancel() + nativeAudioTask = nil + nativeKeyboardTask?.cancel() + nativeKeyboardTask = nil + nativeAudioPending = false + nativeKeyboardPending = false + nativeAudio = nil + nativeKeyboard = nil + } + func reconnect() { resetConnection() } + func stop() { + started = false + for observer in observers { NSWorkspace.shared.notificationCenter.removeObserver(observer) } + observers.removeAll() + resetConnection() + } + private func resetConnection(clearPet: Bool = false) { + cancelWake() + generation = UUID(); job?.cancel(); job = nil + let clearedScreen = clearPet ? screen : nil + if clearedScreen != nil { sequence &+= 1 } + let closeSequence = sequence + screen = nil; detail = ""; nextAttempt = 0 + invalidateSkin() + measuredAt = 0; measuredFrames = 0 + releasePackage() + // Keep the close operation in the job slot so a newly opened connection cannot + // race an asynchronous close left over from a previous generation. + let token = generation + job = Task { + guard generation == token else { return } + await serial.close(clearing: clearedScreen, sequence: closeSequence) + finishJob(token) + } + status = !enabled ? "未启用" : suspended ? "电脑休眠,等待恢复" : "等待连接屏幕…" + } + + private func cancelWake() { + timer?.invalidate() + timer = nil + wakeDeadline = .infinity + } + + private func wake(at deadline: Double, now: Double) { + guard deadline < wakeDeadline else { return } + cancelWake() + wakeDeadline = deadline + let interval = max(0.001, deadline - now) + let wake = Timer(timeInterval: interval, repeats: false) { [weak self] _ in + MainActor.assumeIsolated { + guard let self, self.wakeDeadline == deadline else { return } + self.cancelWake() + self.tick() + } + } + if interval > 0.1 { wake.tolerance = min(0.02, interval / 10) } + timer = wake + RunLoop.main.add(wake, forMode: .common) + } + + private func finishJob(_ token: UUID) { + guard generation == token else { return } + job = nil + tick() + } + private func releasePackage() { + guard let package else { return } + package.hardwareActive = false + if package.activeID == nil { package.post("deactivate") } + PluginHost.shared.releaseUnused(package) + PluginHost.shared.updateSchedule() + self.package = nil + } + private func acquirePackage() -> PluginPackage? { + if let package { return package } + guard let metadata = PluginHost.shared.metadata.first(where: { $0["id"] as? String == selectedID }), + let layout = metadata["layout"] as? [String: Double], let w = layout["width"], let h = layout["height"] else { return nil } + let package = PluginHost.shared.package(metadata) + package.hardwareActive = true + if package.activeID == nil { package.post("activate", ["id": selectedID, "width": w, "height": h, "scale": 1]) } + PluginHost.shared.updateSchedule() + self.package = package + return package + } + private func tick() { + guard started, enabled, !suspended, job == nil else { return } + let now = ProcessInfo.processInfo.systemUptime + let token = generation + guard let screen else { + guard now >= nextAttempt else { wake(at: nextAttempt, now: now); return } + cancelWake() + nextAttempt = now + 2 + let ports = HardwarePort.connected() + guard let port = ports.first else { + let waiting = "等待连接屏幕…" + let hint = "请连接小屏。使用 CH343 烧录口时,请保持桌屿运行。" + if status != waiting { status = waiting } + if detail != hint { detail = hint } + wake(at: nextAttempt, now: now) + return + } + status = "正在连接屏幕…" + job = Task { + defer { finishJob(token) } + do { + let screen = try await serial.connect(path: port.path, unixSocket: port.path == HardwarePort.deskIsleRelayPath) + guard generation == token else { return } + self.screen = screen + status = "已连接 · \(screen.card?.title ?? "Miucs 屏幕")" + detail = port.path == HardwarePort.deskIsleRelayPath ? "通过桌屿连接 CH343" : port.path + measuredAt = ProcessInfo.processInfo.systemUptime; measuredFrames = 0 + } catch { + guard generation == token else { return } + status = error.localizedDescription; nextAttempt = ProcessInfo.processInfo.systemUptime + 2 + } + } + return + } + if let card = screen.card, let capability = card.pet, + HardwarePetPack.supports(selectedID, on: capability), nativePet != .fallback { + tickNativePet(now: now, card: card, capability: capability, token: token) + return + } + let captureAt = lastCaptureAt + 1.0 / 20 + guard now >= captureAt else { wake(at: captureAt, now: now); return } + cancelWake() + guard let package = acquirePackage(), + let metadata = PluginHost.shared.metadata.first(where: { $0["id"] as? String == selectedID }), + let layout = metadata["layout"] as? [String: Double], let width = layout["width"], let height = layout["height"] else { + wake(at: now + 2, now: now) + return + } + let orientation = orientation + let scale = screen.fitScale(width: width, height: height, orientation: orientation) + let w = max(1, Int(width * scale)), h = max(1, Int(height * scale)) + lastCaptureAt = now + guard let pixels = package.pixels(width: w, height: h) else { + wake(at: now + 1.0 / 20, now: now) + return + } + let cardAge = now - lastCardMessageAt + let idleCard: DeskIsleCard? + if let card = screen.card, cardAge < card.leaseDuration, pixels == lastCardPixels { + guard cardAge >= card.keepaliveInterval else { + wake(at: now + 1.0 / 20, now: now) + return + } + idleCard = card + } else { + idleCard = nil + } + sequence &+= 1 + let sequence = sequence + job = Task { + defer { finishJob(token) } + do { + let packet: Data? + if let idleCard { + packet = HardwareFrame.keepalive(sequence: sequence, card: idleCard) + } else { + packet = await Task.detached(priority: .userInitiated) { + HardwareFrame.packet(bgra: pixels, width: w, height: h, orientation: orientation, + sequence: sequence, screen: screen) + }.value + } + try Task.checkCancellation() + guard generation == token else { return } + guard let packet else { throw HardwareSerialError.message("当前皮肤无法编码为屏幕画面") } + let sentAt = ProcessInfo.processInfo.systemUptime + let decodeMS = try await serial.send(packet, sequence: sequence, screen: screen) + guard generation == token else { return } + let acceptedAt = ProcessInfo.processInfo.systemUptime + if let card = screen.card { + let leaseExpired = acceptedAt - lastCardMessageAt >= card.leaseDuration + lastCardMessageAt = sentAt + if idleCard != nil { + // If the board may have restored the wallpaper, refresh the JPEG next tick. + if leaseExpired { lastCardPixels = nil } + let idleDetail = "画面静止 · 连接保持中" + if detail != idleDetail { detail = idleDetail } + measuredAt = acceptedAt; measuredFrames = 0 + return + } + lastCardPixels = pixels + } + let displaying = "正在显示 · \(screen.card?.title ?? orientation.title)" + if status != displaying { status = displaying } + measuredFrames += 1 + let elapsed = acceptedAt - measuredAt + if elapsed >= 1 { + detail = String(format: "%.1f 帧/秒 · %.0f KB/帧 · 屏幕处理 %d ms", Double(measuredFrames) / elapsed, Double(packet.count) / 1024, decodeMS) + measuredAt = acceptedAt; measuredFrames = 0 + } + } catch HardwareSerialError.cardChanged(let card) { + guard generation == token else { return } + updateCard(card) + } catch { + guard generation == token else { return } + connectionLost(error) + } + } + } + + private func updateCard(_ card: DeskIsleCard) { + screen = .deskIsle(card) + lastCardPixels = nil + status = nativePet == .active ? "板端绘制 · \(card.title)" : "已连接 · \(card.title)" + detail = "" + measuredAt = ProcessInfo.processInfo.systemUptime; measuredFrames = 0 + } + + private func connectionLost(_ error: Error) { + petLog.info("Screen connection lost: \(error.localizedDescription, privacy: .public)") + screen = nil + invalidateSkin() + releasePackage() + status = error.localizedDescription + detail = "连接恢复后会自动继续。" + nextAttempt = ProcessInfo.processInfo.systemUptime + 2 + } + + private func tickNativePet(now: Double, card: DeskIsleCard, capability: DeskIslePetCapability, token: UUID) { + let interval = capability.version < 2 || nativeAudioPending ? Self.nativeInputInterval : card.keepaliveInterval + let inputAt = lastNativeInputAt + interval + if nativePet == .active, !nativeKeyboardPending, now < inputAt { + wake(at: inputAt, now: now) + return + } + cancelWake() + let screen = HardwareScreen.deskIsle(card) + let revision = skinRevision + job = Task { + defer { finishJob(token) } + guard generation == token, skinRevision == revision else { return } + do { + if nativePet == .pending { + try await syncNativePet(screen: screen, capability: capability, token: token, revision: revision) + } else { + nativeKeyboardPending = false + nativeAudioPending = false + lastNativeInputAt = ProcessInfo.processInfo.systemUptime + let payload = try nativeInput(limit: capability.maxInput) + try await sendPet(.input, payload, screen: screen, capability: capability) + } + } catch HardwareSerialError.cardChanged(let card) { + if generation == token { updateCard(card) } + } catch HardwareSerialError.petRejected(let reason) { + guard generation == token, skinRevision == revision else { return } + if nativePet == .active { + if capability.version >= 3, reason == "6" { + invalidateSkin() + return + } + // Send a fresh snapshot on the next tick if an input packet was corrupted. + if reason == "1" { return } + } + petLog.warning("Board rejected native pet: \(reason)") + nativePet = .fallback + stopNativeInput() + detail = "皮肤已切回 Mac 渲染" + } catch { + if generation == token { connectionLost(error) } + } + } + } + + private func syncNativePet(screen: HardwareScreen, capability: DeskIslePetCapability, token: UUID, revision: Int) async throws { + guard let metadata = PluginHost.shared.metadata.first(where: { $0["id"] as? String == selectedID }), + let path = metadata["directory"] as? String else { + nativePet = .fallback + return + } + let name = metadata["name"] as? String ?? selectedID + let cachedIdentity = nativePetIdentity + if capability.version >= 3, let cachedIdentity { + status = "正在连接桌宠 · \(name)" + if try await resumeNativePet(cachedIdentity, screen: screen, capability: capability, token: token, revision: revision) { return } + guard generation == token, skinRevision == revision else { return } + } + status = "正在同步皮肤 · \(name)" + let bundle: Data + let identity: NativePetIdentity + do { + (bundle, identity) = try await Task.detached(priority: .userInitiated) { + let bundle = try HardwarePetPack.make(directory: URL(fileURLWithPath: path, isDirectory: true), limit: capability.maxPack) + return (bundle, NativePetIdentity(size: bundle.count, crc: HardwareFrame.crc32(bundle))) + }.value + } catch { + if generation == token, skinRevision == revision { + nativePet = .fallback + detail = error.localizedDescription + petLog.warning("Native pet pack failed: \(error.localizedDescription, privacy: .public)") + } + return + } + guard generation == token, skinRevision == revision else { return } + nativePetIdentity = identity + if capability.version >= 3, identity != cachedIdentity { + if try await resumeNativePet(identity, screen: screen, capability: capability, token: token, revision: revision) { return } + guard generation == token, skinRevision == revision else { return } + } + try await sendPet(.begin, identity.payload, screen: screen, capability: capability) + for offset in stride(from: 0, to: bundle.count, by: capability.maxChunk) { + guard generation == token, skinRevision == revision else { return } + var chunk = Data() + withUnsafeBytes(of: UInt32(offset).littleEndian) { chunk.append(contentsOf: $0) } + chunk.append(contentsOf: bundle[offset.. Bool { + do { + try await sendPet(.resume, identity.payload, screen: screen, capability: capability) + } catch HardwareSerialError.petRejected("6") { + return false + } + if generation == token, skinRevision == revision { + activateNativePet(identity, resumed: true, screen: screen, capability: capability, token: token, revision: revision) + } + return true + } + + private func activateNativePet(_ identity: NativePetIdentity, resumed: Bool, screen: HardwareScreen, + capability: DeskIslePetCapability, token: UUID, revision: Int) { + (nativeAudio, nativeKeyboard) = PluginHost.shared.hardwareSubscriptions() + nativePet = .active + if capability.version >= 2, let nativeKeyboard { + let signals = nativeKeyboard.eventSignals() + nativeKeyboardTask = Task { [weak self] in + for await _ in signals { + guard !Task.isCancelled, let self, generation == token, skinRevision == revision else { return } + nativeKeyboardPending = true + tick() + } + } + } + if capability.version >= 2, let nativeAudio { + let signals = nativeAudio.activitySignals() + nativeAudioTask = Task { [weak self] in + for await _ in signals { + guard !Task.isCancelled, let self, generation == token, skinRevision == revision else { return } + nativeAudioPending = true + tick() + } + } + } + releasePackage() + status = "板端绘制 · \(screen.card?.title ?? "桌屿")" + let sync = resumed ? "已恢复连接" : "已同步" + detail = String(format: "皮肤包 %.1f MiB · %@,发送按键与音频特征", Double(identity.size) / 1_048_576, sync) + if resumed { + petLog.info("Screen resumed native Lua pet (\(identity.size) bytes)") + } else { + petLog.info("Screen activated native Lua pet (\(identity.size) bytes)") + } + } + + private func sendPet(_ kind: DeskIslePetMessage, _ payload: Data, screen: HardwareScreen, + capability: DeskIslePetCapability) async throws { + sequence &+= 1 + let sequence = sequence + let packet = HardwareFrame.nativePet(kind, payload: payload, sequence: sequence, id: capability.id) + if kind == .input && capability.version >= 2 { + try await serial.sendPetInput(packet, sequence: sequence, screen: screen) + return + } + if kind == .chunk { + for _ in 0..<2 { + do { + _ = try await serial.send(packet, sequence: sequence, screen: screen, nativePet: true) + return + } catch HardwareSerialError.petRejected("1") { + // A bad CRC leaves the board's upload offset unchanged. + continue + } + } + } + _ = try await serial.send(packet, sequence: sequence, screen: screen, nativePet: true) + } + + private func nativeInput(limit: Int) throws -> Data { + try PetInputSnapshot.encode( + audio: PluginHost.shared.suspended ? nil : nativeAudio?.read(), + keyboard: PluginHost.shared.suspended ? nil : nativeKeyboard?.read(), limit: limit) + } +} diff --git a/apps/macos/HardwarePort.swift b/apps/macos/HardwarePort.swift new file mode 100644 index 0000000..6e053fc --- /dev/null +++ b/apps/macos/HardwarePort.swift @@ -0,0 +1,42 @@ +import Foundation +import Darwin +import IOKit +import IOKit.serial + +struct HardwarePort: Identifiable, Sendable { + var id: String { path } + let path: String + let name: String + static var deskIsleRelayPath: String { + FileManager.default.homeDirectoryForCurrentUser + .appendingPathComponent("Library/Caches/com.deskisle.layout/uart/animation.sock").path + } + + static func connected() -> [HardwarePort] { + var iterator: io_iterator_t = 0 + guard IOServiceGetMatchingServices(kIOMainPortDefault, IOServiceMatching(kIOSerialBSDServiceValue), &iterator) == KERN_SUCCESS else { return [] } + defer { IOObjectRelease(iterator) } + var ports = [HardwarePort]() + while case let service = IOIteratorNext(iterator), service != 0 { + defer { IOObjectRelease(service) } + guard let path = IORegistryEntryCreateCFProperty(service, kIOCalloutDeviceKey as CFString, kCFAllocatorDefault, 0)?.takeRetainedValue() as? String else { continue } + func parentProperty(_ key: String) -> CFTypeRef? { + IORegistryEntrySearchCFProperty(service, kIOServicePlane, key as CFString, kCFAllocatorDefault, + IOOptionBits(kIORegistryIterateRecursively | kIORegistryIterateParents)) + } + let name = parentProperty("USB Product Name") as? String ?? "" + if name == "DeskIsle" { + let interface = (parentProperty("bInterfaceNumber") as? NSNumber)?.intValue + let interfaceName = parentProperty("USB Interface Name") as? String + guard interface != 2, interface != 3, interfaceName != "DeskIsle Layout" else { continue } + } + if name == "Miucs P4 Display" || name == "DeskIsle" { ports.append(HardwarePort(path: path, name: name)) } + } + let relay = deskIsleRelayPath + var info = stat() + if ports.isEmpty, lstat(relay, &info) == 0, info.st_mode & S_IFMT == S_IFSOCK, info.st_uid == getuid() { + ports.append(HardwarePort(path: relay, name: "DeskIsle")) + } + return ports.sorted { $0.path < $1.path } + } +} diff --git a/apps/macos/MiucsSettings.swift b/apps/macos/MiucsSettings.swift index 6d5bb52..af7f966 100644 --- a/apps/macos/MiucsSettings.swift +++ b/apps/macos/MiucsSettings.swift @@ -1,14 +1,18 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif import AppKit import SwiftUI enum SettingsPage: String, CaseIterable, Identifiable { - case skins = "皮肤", appearance = "外观与位置", inputs = "声音与键盘", about = "关于 Miucs" + case skins = "皮肤", appearance = "外观与位置", inputs = "声音与键盘", hardware = "硬件屏幕", about = "关于 Miucs" var id: String { rawValue } var symbol: String { switch self { case .skins: "square.grid.2x2" case .appearance: "slider.horizontal.3" case .inputs: "waveform" + case .hardware: "display" case .about: "info.circle" } } @@ -17,6 +21,7 @@ enum SettingsPage: String, CaseIterable, Identifiable { case .skins: "点击卡片,使用你喜欢的皮肤。" case .appearance: "调整皮肤的大小、透明度和摆放位置。" case .inputs: "连接声音与按键,让皮肤动起来。" + case .hardware: "把当前皮肤显示在 USB 小屏幕上。" case .about: "你敲你的,它蹦它的。" } } @@ -31,7 +36,9 @@ final class MiucsSettingsModel: ObservableObject { @Published var scale = 100.0 @Published var scaleRange = 10.0...200.0 @Published var opacity = 100.0 + @Published var trayAnimationEnabled = true @Published var isAdjusting = false + @Published var audioCaptureEnabled = true @Published var audioStatus = "正在连接系统音频…" @Published var keyboardState = KeyboardSourceState.connecting @Published var errors = [String]() @@ -39,7 +46,9 @@ final class MiucsSettingsModel: ObservableObject { var setVisible: (Bool) -> Void = { _ in } var setScale: (Double) -> Void = { _ in } var setOpacity: (Double) -> Void = { _ in } + var setTrayAnimationEnabled: (Bool) -> Void = { _ in } var togglePosition: () -> Void = {} + var setAudioCaptureEnabled: (Bool) -> Void = { _ in } var reconnectAudio: () -> Void = {} var connectKeyboard: () -> Void = {} var rescan: () -> Void = {} @@ -114,6 +123,7 @@ private struct MiucsSettingsView: View { case .skins: skins case .appearance: appearance case .inputs: inputs + case .hardware: HardwareDisplaySettings() case .about: about } }.frame(maxWidth: .infinity, alignment: .leading) @@ -220,6 +230,12 @@ private struct MiucsSettingsView: View { private var appearance: some View { VStack(alignment: .leading, spacing: 20) { + settingsGroup { + Toggle("菜单栏猫头摇晃", isOn: Binding( + get: { model.trayAnimationEnabled }, set: { model.setTrayAnimationEnabled($0) } + )) + .toggleStyle(.switch) + } if let skin = model.currentSkin { HStack(spacing: 12) { SkinPreview(url: skin.previewURL).frame(width: 76, height: 50) @@ -259,12 +275,19 @@ private struct MiucsSettingsView: View { private var inputs: some View { VStack(alignment: .leading, spacing: 20) { settingsGroup { - HStack { + Toggle(isOn: Binding( + get: { model.audioCaptureEnabled }, set: { model.setAudioCaptureEnabled($0) } + )) { Label("系统音频", systemImage: "speaker.wave.2") + } + .toggleStyle(.switch) + HStack { + Text(model.audioCaptureEnabled ? model.audioStatus : "已关闭") + .font(.system(size: 12)).foregroundStyle(.secondary) Spacer() Button("重新连接", action: model.reconnectAudio) + .disabled(!model.audioCaptureEnabled) } - Text(model.audioStatus).font(.system(size: 12)).foregroundStyle(.secondary) } settingsGroup { HStack { @@ -302,6 +325,40 @@ private struct MiucsSettingsView: View { } } +private struct HardwareDisplaySettings: View { + @ObservedObject private var hardware = HardwareDisplay.shared + var body: some View { + VStack(alignment: .leading, spacing: 20) { + Toggle("在硬件屏幕上显示皮肤", isOn: Binding(get: { hardware.enabled }, set: { hardware.setEnabled($0) })) + .toggleStyle(.switch) + VStack(alignment: .leading, spacing: 14) { + HStack { + Label(hardware.status, systemImage: hardware.connected ? "checkmark.circle.fill" : "cable.connector") + Spacer() + Button("重新连接") { hardware.reconnect() }.disabled(!hardware.enabled) + } + if !hardware.detail.isEmpty { Text(hardware.detail).font(.system(size: 12)).foregroundStyle(.secondary) } + }.padding(20).frame(maxWidth: .infinity, alignment: .leading) + .background(Color(nsColor: .controlBackgroundColor), in: RoundedRectangle(cornerRadius: 10)) + if hardware.screen?.card != nil { + Text("桌屿会把皮肤卡片叠在壁纸上。卡片的位置和方向由桌屿固件决定。") + .font(.system(size: 12)).foregroundStyle(.secondary) + } else { + Picker("摆放方向", selection: Binding(get: { hardware.orientation }, set: { hardware.setOrientation($0) })) { + ForEach(HardwareOrientation.allCases) { orientation in Text(orientation.title).tag(orientation) } + }.pickerStyle(.menu).frame(maxWidth: 340) + Text("方向即时生效。画面会保持比例并居中显示,留出边缘空间。") + .font(.system(size: 12)).foregroundStyle(.secondary) + } + Divider() + Text("选择的皮肤、音乐和按键响应会同步到屏幕;隐藏桌面皮肤后,硬件屏幕仍可继续显示。") + .font(.system(size: 13)).fixedSize(horizontal: false, vertical: true) + Text("用原生 USB 数据口连接已烧录桌屿或 Miucs 屏幕固件的开发板,打开开关即可自动连接。") + .font(.system(size: 12)).foregroundStyle(.secondary).fixedSize(horizontal: false, vertical: true) + } + } +} + private struct SettingsSelectionButtonStyle: ButtonStyle { enum Kind { case navigation, card } let kind: Kind diff --git a/apps/macos/TrayHeadAnimation.swift b/apps/macos/TrayHeadAnimation.swift index 4f90ecd..969ebc1 100644 --- a/apps/macos/TrayHeadAnimation.swift +++ b/apps/macos/TrayHeadAnimation.swift @@ -1,3 +1,6 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif import Foundation /// Typing controls the portrait independently of music. The channel's rate excludes diff --git a/apps/macos/VisualizationAppearanceStore.swift b/apps/macos/VisualizationAppearanceStore.swift index 51830bc..a4b5367 100644 --- a/apps/macos/VisualizationAppearanceStore.swift +++ b/apps/macos/VisualizationAppearanceStore.swift @@ -6,9 +6,11 @@ import Combine final class VisualizationAppearanceStore: ObservableObject { static let preferencesKey = "MiucsVisualizationInk.v1" static let transparencyKey = "MiucsVisualizationTransparency.v1" + static let trayAnimationKey = "MiucsTrayAnimation.enabled" static let opacityPresets = Array(stride(from: 10, through: 100, by: 10)) @Published private(set) var selection: VisualizationInk @Published private(set) var transparency: Int + @Published private(set) var trayAnimationEnabled: Bool var opacity: Int { 100 - transparency } func setOpacity(_ percent: Int) { setTransparency(100 - min(100, max(10, percent))) } private let preferences: UserDefaults @@ -17,6 +19,7 @@ final class VisualizationAppearanceStore: ObservableObject { init(preferences: UserDefaults = .standard) { self.preferences = preferences transparency = min(90, max(0, preferences.object(forKey: Self.transparencyKey) as? Int ?? 0)) + trayAnimationEnabled = preferences.object(forKey: Self.trayAnimationKey) as? Bool ?? true if let raw = preferences.string(forKey: Self.preferencesKey) { selection = VisualizationInk(rawValue: raw) ?? .graphite } else { selection = .graphite } @@ -34,4 +37,10 @@ final class VisualizationAppearanceStore: ObservableObject { preferences.set(value, forKey: Self.transparencyKey) transparency = value } + + func setTrayAnimationEnabled(_ enabled: Bool) { + guard trayAnimationEnabled != enabled else { return } + preferences.set(enabled, forKey: Self.trayAnimationKey) + trayAnimationEnabled = enabled + } } diff --git a/apps/macos/VisualizationHostView.swift b/apps/macos/VisualizationHostView.swift index f7f21e2..46157c3 100644 --- a/apps/macos/VisualizationHostView.swift +++ b/apps/macos/VisualizationHostView.swift @@ -1,3 +1,6 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif import AppKit import QuartzCore diff --git a/apps/macos/VisualizerPanel.swift b/apps/macos/VisualizerPanel.swift index ecaebbb..1383c91 100644 --- a/apps/macos/VisualizerPanel.swift +++ b/apps/macos/VisualizerPanel.swift @@ -1,3 +1,6 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif import AppKit /// A transparent canvas. Skin mode keeps one desktop frame across package switches. diff --git a/build.sh b/build.sh index 95d896a..a57e9fa 100755 --- a/build.sh +++ b/build.sh @@ -12,6 +12,9 @@ fi rm -rf "$app" mkdir -p "$app/Contents/MacOS" +swift build --package-path "$root" -c release --product MiucsCore +core_bin="$(swift build --package-path "$root" -c release --show-bin-path)" + swiftc \ -parse-as-library \ -swift-version 6 \ @@ -24,12 +27,14 @@ swiftc \ -framework CoreAudio \ -framework Accelerate \ -o "$app/Contents/MacOS/Miucs" \ - "${app_sources[@]}" + -I "$core_bin/Modules" -L "$core_bin" -lMiucsCore \ + "${linked_app_sources[@]}" cp "$root/apps/macos/Info.plist" "$app/Contents/Info.plist" mkdir -p "$app/Contents/Resources" cp "$root/apps/assets/cat-head.png" "$app/Contents/Resources/PixelCatHead.png" +ditto "$core_bin/Miucs_MiucsCore.bundle" "$app/Contents/Resources/Miucs_MiucsCore.bundle" cp "$root/apps/assets/Miucs.icns" "$app/Contents/Resources/Miucs.icns" # Ad-hoc signatures change on every build, so macOS drops the screen-recording grant. diff --git a/docs/architecture.md b/docs/architecture.md index 8b79265..6d78a45 100644 --- a/docs/architecture.md +++ b/docs/architecture.md @@ -40,7 +40,9 @@ flowchart LR 通用 API 不出现原生视图、颜色或 Combine 类型;原生扩展单独归入 `platforms/`。音频核心不引用任何角色或窗口对象。macOS 与 Windows 平台实现互不引用。`scripts/check-boundaries.py` 检查当前 Swift 源码的这些约束,`test-contracts.sh` 不链接 Apple UI、采集或 Accelerate 框架。 -这些目录是源码与依赖边界:macOS 由 `swiftc` 分组编译;Windows 由 `apps/windows/Miucs.Windows.csproj` 组合各 Windows 目录的 C# 源码。Lua 插件通过 C ABI 宿主桥接到隔离 worker,不加载第三方原生插件。Windows 的数值、事件、场景和窗口偏好在独立可移植测试中验证,原生平台检查见 `docs/windows-acceptance.md`。 +这些目录也是 Swift Package 的源码边界:`MiucsCore` 编译音频、键盘与 `modules/hardware/macos/`,携带皮肤资源;macOS 独立应用链接同一个核心库,再组合桌面窗口及 Lua/Skia worker。源码回归测试保留 `swiftc` 分组以使用内部测试接缝;Windows 由 `apps/windows/Miucs.Windows.csproj` 组合各 Windows 目录的 C# 源码。Lua 插件通过 C ABI 宿主桥接到隔离 worker,不加载第三方原生插件。Windows 的数值、事件、场景和窗口偏好在独立可移植测试中验证,原生平台检查见 `docs/windows-acceptance.md`。 + +嵌入宿主使用 `NativePetSession` 与注入的 `NativePetTransport`,让开发板执行 Lua 和绘制。核心库不主动发现 USB、不依赖 `PluginHost.shared`。设置、设备所有权与是否启用输入由宿主控制,见 [嵌入指南](embedding.md)。 ## 设置窗口与皮肤位置 diff --git a/docs/embedding.md b/docs/embedding.md new file mode 100644 index 0000000..f3cefaf --- /dev/null +++ b/docs/embedding.md @@ -0,0 +1,71 @@ +# 将 Miucs 嵌入 macOS 应用 + +Miucs 仓库同时提供独立桌面应用和 `MiucsCore` Swift Package。核心库静态链接到宿主进程,包含键盘监听、系统音频分析、输入契约、七套皮肤资源、板端皮肤打包与会话恢复。它不依赖 Skia、`PluginHost`、菜单栏图标或 `miucs-plugin-worker`。最低 macOS 14、Swift 6;Windows 继续使用现有 C# 模块。 + +## 仓库与资源 + +开发时保持两个仓库并列,在宿主 Package.swift 中引用: + +```swift +dependencies: [.package(path: "../miucs")] +// 对应宿主 target: +dependencies: [.product(name: "MiucsCore", package: "miucs")] +``` + +发布构建可以改成仓库 URL 并锁定 tag 或提交。宿主不复制修改 Miucs 源码;库、皮肤和测试由本仓库维护。 + +`swift build -c release --product MiucsCore` 生成静态库和 `Miucs_MiucsCore.bundle`。自行组装 .app 时,把资源包原样复制到 `Contents/Resources/`。Xcode 的包依赖由 Xcode 处理资源。`PetCatalog.scan()` 读取内置皮肤和 `~/Library/Application Support/Miucs/Plugins`,也可注入目录。目录扫描和静态预览不运行 Lua。 + +## 输入与生命周期 + +```swift +import MiucsCore + +@MainActor +final class Companion { + let audio = AudioEngine() + let keyboard = KeyboardEngine() + let pet = NativePetSession() + + func start(skin: PetSkin, transport: any NativePetTransport) async { + pet.start(skin: skin, audio: audio.channel, keyboard: keyboard.channel, transport: transport) + await keyboard.start(requestAccess: false) + // 宿主音频开关启用时,再调用 await audio.start()。 + } + + func stop() async { + pet.stop() + audio.stop() + keyboard.stop() + await pet.waitUntilStopped() + } +} +``` + +键盘与音频权限归宿主应用,应在宿主设置中提供授权入口、开关和状态。宿主提供 `NSAudioCaptureUsageDescription` / `NSScreenCaptureUsageDescription`,使用稳定签名以保持更新后的授权;Miucs 的权限不会直接授予其他 bundle identifier。 + +通道订阅各有独立游标。换肤不重启输入采集;禁用组件或设备断开时宿主可以暂停采集。数据只有物理按键身份和音频特征,不包含输入文字、录音或屏幕图像。关闭或释放会话都会取消订阅、重试、唤醒任务并关闭传输。 + +## USB 所有权 + +宿主实现 `NativePetTransport`,负责设备发现、串口独占与逻辑通道。也可使用 `SerialPetTransport(open:)`:闭包返回已配置为非阻塞的 CDC 描述符或进程内 socket pair,返回后所有权移交给库,库负责关闭。不能返回仍被另一对象读取的同一个描述符。`HardwareIO.wait` 使用 Darwin `select(2)` 同时等待串口和 socket;不要用不支持设备文件的 `poll(2)` 等待串口可读写。 + +桌屿持有 CH343 物理连接,在布局和桌宠通道之间公平切换。内嵌模式直接使用进程内通道,无需另一个 Miucs 应用。独立 Miucs 的 `HardwarePort` 继续在应用层发现设备。宿主应在硬件连接期间保持必要的后台活动,避免隐藏窗口后 App Nap 延误设备回执;不要因此阻止系统正常休眠。 + +会话支持 DESKISLE_PET v1–v3,按设备上限与 32 KiB 中的较小值分块上传,并提供同步进度。v3 先按长度和 CRC 恢复实例,不存在才重传。按键变化立即触发发送;活跃音频合并到约 30 Hz;空闲只有保活。v2/v3 输入有序连续写入,并检查已经到达的 ACK;上传与恢复命令等待对应回执。 + +原生会话不会回退到 Mac 图像渲染。不支持的皮肤或固件返回状态,由宿主提示。独立桌面应用保留 Lua/Skia worker 和旧 JPEG 屏幕适配。 + +## 验证 + +```sh +swift test +zsh scripts/test-contracts.sh +zsh scripts/test-keyboard.sh +zsh scripts/test-audio-recovery.sh +zsh scripts/test-hardware.sh +zsh scripts/test-settings.sh +zsh build.sh +``` + +包测试使用合成输入与模拟传输,不申请系统权限。覆盖七套真实皮肤打包、目录边界、输入限额、按键顺序、快速换肤时的通道所有权、板端实例丢失恢复和会话释放。真实设备联调在宿主应用身份下完成授权并核对设备回执。 diff --git a/docs/validation/audio-recovery-tests.json b/docs/validation/audio-recovery-tests.json index 09b0384..58d79c9 100644 --- a/docs/validation/audio-recovery-tests.json +++ b/docs/validation/audio-recovery-tests.json @@ -28,6 +28,10 @@ "name" : "Stop cancels pending retry and suppresses wake recovery", "passed" : true }, + { + "name" : "Stopping active capture clears the audio input until it is enabled again", + "passed" : true + }, { "name" : "macOS user-stop is respected until explicitly restarted", "passed" : true @@ -83,5 +87,5 @@ ], "failed" : 0, "method" : "injected capture transport, actual retry tasks, workspace notifications and controlled monotonic clock; no machine sleep or device changes", - "passed" : 20 + "passed" : 21 } \ No newline at end of file diff --git a/hardware/esp32-display/.gitignore b/hardware/esp32-display/.gitignore new file mode 100644 index 0000000..55267f8 --- /dev/null +++ b/hardware/esp32-display/.gitignore @@ -0,0 +1,4 @@ +build/ +managed_components/ +sdkconfig +sdkconfig.old diff --git a/hardware/esp32-display/CMakeLists.txt b/hardware/esp32-display/CMakeLists.txt new file mode 100644 index 0000000..00498a3 --- /dev/null +++ b/hardware/esp32-display/CMakeLists.txt @@ -0,0 +1,4 @@ +cmake_minimum_required(VERSION 3.16) +set(COMPONENTS main) +include($ENV{IDF_PATH}/tools/cmake/project.cmake) +project(miucs_display) diff --git a/hardware/esp32-display/OSPTEK-LICENSE.txt b/hardware/esp32-display/OSPTEK-LICENSE.txt new file mode 100644 index 0000000..ac7b169 --- /dev/null +++ b/hardware/esp32-display/OSPTEK-LICENSE.txt @@ -0,0 +1,413 @@ +Copyright (c) 2026 OSPTEK 鱼鹰光电 + +The OSPTEK materials in this repository (documents, drawings, images and +sample code) are licensed under the Creative Commons Attribution 4.0 +International License (CC BY 4.0). You may share and adapt them, including +commercially, as long as you give appropriate credit to OSPTEK. + +Third-party documents included for convenience (such as IC datasheets from +chip vendors) remain the property of their respective owners and are not +covered by this license. + +本仓库中 OSPTEK 的资料(文档、图纸、图片、示例代码)采用 CC BY 4.0 许可: +可以自由转载、修改、商用,但需注明出处 OSPTEK。 +仓库中附带的第三方资料(如芯片厂商的 IC 规格书)版权归原厂所有,不在此许可范围内。 + +----------------------------------------------------------------------- + +Attribution 4.0 International + +======================================================================= + +Creative Commons Corporation ("Creative Commons") is not a law firm and +does not provide legal services or legal advice. Distribution of +Creative Commons public licenses does not create a lawyer-client or +other relationship. Creative Commons makes its licenses and related +information available on an "as-is" basis. Creative Commons gives no +warranties regarding its licenses, any material licensed under their +terms and conditions, or any related information. Creative Commons +disclaims all liability for damages resulting from their use to the +fullest extent possible. + +Using Creative Commons Public Licenses + +Creative Commons public licenses provide a standard set of terms and +conditions that creators and other rights holders may use to share +original works of authorship and other material subject to copyright +and certain other rights specified in the public license below. The +following considerations are for informational purposes only, are not +exhaustive, and do not form part of our licenses. + + Considerations for licensors: Our public licenses are + intended for use by those authorized to give the public + permission to use material in ways otherwise restricted by + copyright and certain other rights. Our licenses are + irrevocable. Licensors should read and understand the terms + and conditions of the license they choose before applying it. + Licensors should also secure all rights necessary before + applying our licenses so that the public can reuse the + material as expected. Licensors should clearly mark any + material not subject to the license. This includes other CC- + licensed material, or material used under an exception or + limitation to copyright. More considerations for licensors: + wiki.creativecommons.org/Considerations_for_licensors + + Considerations for the public: By using one of our public + licenses, a licensor grants the public permission to use the + licensed material under specified terms and conditions. If + the licensor's permission is not necessary for any reason--for + example, because of any applicable exception or limitation to + copyright--then that use is not regulated by the license. Our + licenses grant only permissions under copyright and certain + other rights that a licensor has authority to grant. Use of + the licensed material may still be restricted for other + reasons, including because others have copyright or other + rights in the material. A licensor may make special requests, + such as asking that all changes be marked or described. + Although not required by our licenses, you are encouraged to + respect those requests where reasonable. More considerations + for the public: + wiki.creativecommons.org/Considerations_for_licensees + +======================================================================= + +Creative Commons Attribution 4.0 International Public License + +By exercising the Licensed Rights (defined below), You accept and agree +to be bound by the terms and conditions of this Creative Commons +Attribution 4.0 International Public License ("Public License"). To the +extent this Public License may be interpreted as a contract, You are +granted the Licensed Rights in consideration of Your acceptance of +these terms and conditions, and the Licensor grants You such rights in +consideration of benefits the Licensor receives from making the +Licensed Material available under these terms and conditions. + + +Section 1 -- Definitions. + + a. Adapted Material means material subject to Copyright and Similar + Rights that is derived from or based upon the Licensed Material + and in which the Licensed Material is translated, altered, + arranged, transformed, or otherwise modified in a manner requiring + permission under the Copyright and Similar Rights held by the + Licensor. For purposes of this Public License, where the Licensed + Material is a musical work, performance, or sound recording, + Adapted Material is always produced where the Licensed Material is + synched in timed relation with a moving image. + + b. Adapter's License means the license You apply to Your Copyright + and Similar Rights in Your contributions to Adapted Material in + accordance with the terms and conditions of this Public License. + + c. Copyright and Similar Rights means copyright and/or similar rights + closely related to copyright including, without limitation, + performance, broadcast, sound recording, and Sui Generis Database + Rights, without regard to how the rights are labeled or + categorized. For purposes of this Public License, the rights + specified in Section 2(b)(1)-(2) are not Copyright and Similar + Rights. + + d. Effective Technological Measures means those measures that, in the + absence of proper authority, may not be circumvented under laws + fulfilling obligations under Article 11 of the WIPO Copyright + Treaty adopted on December 20, 1996, and/or similar international + agreements. + + e. Exceptions and Limitations means fair use, fair dealing, and/or + any other exception or limitation to Copyright and Similar Rights + that applies to Your use of the Licensed Material. + + f. Licensed Material means the artistic or literary work, database, + or other material to which the Licensor applied this Public + License. + + g. Licensed Rights means the rights granted to You subject to the + terms and conditions of this Public License, which are limited to + all Copyright and Similar Rights that apply to Your use of the + Licensed Material and that the Licensor has authority to license. + + h. Licensor means the individual(s) or entity(ies) granting rights + under this Public License. + + i. Share means to provide material to the public by any means or + process that requires permission under the Licensed Rights, such + as reproduction, public display, public performance, distribution, + dissemination, communication, or importation, and to make material + available to the public including in ways that members of the + public may access the material from a place and at a time + individually chosen by them. + + j. Sui Generis Database Rights means rights other than copyright + resulting from Directive 96/9/EC of the European Parliament and of + the Council of 11 March 1996 on the legal protection of databases, + as amended and/or succeeded, as well as other essentially + equivalent rights anywhere in the world. + + k. You means the individual or entity exercising the Licensed Rights + under this Public License. Your has a corresponding meaning. + + +Section 2 -- Scope. + + a. License grant. + + 1. Subject to the terms and conditions of this Public License, + the Licensor hereby grants You a worldwide, royalty-free, + non-sublicensable, non-exclusive, irrevocable license to + exercise the Licensed Rights in the Licensed Material to: + + a. reproduce and Share the Licensed Material, in whole or + in part; and + + b. produce, reproduce, and Share Adapted Material. + + 2. Exceptions and Limitations. For the avoidance of doubt, where + Exceptions and Limitations apply to Your use, this Public + License does not apply, and You do not need to comply with + its terms and conditions. + + 3. Term. The term of this Public License is specified in Section + 6(a). + + 4. Media and formats; technical modifications allowed. The + Licensor authorizes You to exercise the Licensed Rights in + all media and formats whether now known or hereafter created, + and to make technical modifications necessary to do so. The + Licensor waives and/or agrees not to assert any right or + authority to forbid You from making technical modifications + necessary to exercise the Licensed Rights, including + technical modifications necessary to circumvent Effective + Technological Measures. For purposes of this Public License, + simply making modifications authorized by this Section 2(a) + (4) never produces Adapted Material. + + 5. Downstream recipients. + + a. Offer from the Licensor -- Licensed Material. Every + recipient of the Licensed Material automatically + receives an offer from the Licensor to exercise the + Licensed Rights under the terms and conditions of this + Public License. + + b. No downstream restrictions. You may not offer or impose + any additional or different terms or conditions on, or + apply any Effective Technological Measures to, the + Licensed Material if doing so restricts exercise of the + Licensed Rights by any recipient of the Licensed + Material. + + 6. No endorsement. Nothing in this Public License constitutes or + may be construed as permission to assert or imply that You + are, or that Your use of the Licensed Material is, connected + with, or sponsored, endorsed, or granted official status by, + the Licensor or others designated to receive attribution as + provided in Section 3(a)(1)(A)(i). + + b. Other rights. + + 1. Moral rights, such as the right of integrity, are not + licensed under this Public License, nor are publicity, + privacy, and/or other similar personality rights; however, to + the extent possible, the Licensor waives and/or agrees not to + assert any such rights held by the Licensor to the limited + extent necessary to allow You to exercise the Licensed + Rights, but not otherwise. + + 2. Patent and trademark rights are not licensed under this + Public License. + + 3. To the extent possible, the Licensor waives any right to + collect royalties from You for the exercise of the Licensed + Rights, whether directly or through a collecting society + under any voluntary or waivable statutory or compulsory + licensing scheme. In all other cases the Licensor expressly + reserves any right to collect such royalties. + + +Section 3 -- License Conditions. + +Your exercise of the Licensed Rights is expressly made subject to the +following conditions. + + a. Attribution. + + 1. If You Share the Licensed Material (including in modified + form), You must: + + a. retain the following if it is supplied by the Licensor + with the Licensed Material: + + i. identification of the creator(s) of the Licensed + Material and any others designated to receive + attribution, in any reasonable manner requested by + the Licensor (including by pseudonym if + designated); + + ii. a copyright notice; + + iii. a notice that refers to this Public License; + + iv. a notice that refers to the disclaimer of + warranties; + + v. a URI or hyperlink to the Licensed Material to the + extent reasonably practicable; + + b. indicate if You modified the Licensed Material and + retain an indication of any previous modifications; and + + c. indicate the Licensed Material is licensed under this + Public License, and include the text of, or the URI or + hyperlink to, this Public License. + + 2. You may satisfy the conditions in Section 3(a)(1) in any + reasonable manner based on the medium, means, and context in + which You Share the Licensed Material. For example, it may be + reasonable to satisfy the conditions by providing a URI or + hyperlink to a resource that includes the required + information. + + 3. If requested by the Licensor, You must remove any of the + information required by Section 3(a)(1)(A) to the extent + reasonably practicable. + + 4. If You Share Adapted Material You produce, the Adapter's + License You apply must not prevent recipients of the Adapted + Material from complying with this Public License. + + +Section 4 -- Sui Generis Database Rights. + +Where the Licensed Rights include Sui Generis Database Rights that +apply to Your use of the Licensed Material: + + a. for the avoidance of doubt, Section 2(a)(1) grants You the right + to extract, reuse, reproduce, and Share all or a substantial + portion of the contents of the database; + + b. if You include all or a substantial portion of the database + contents in a database in which You have Sui Generis Database + Rights, then the database in which You have Sui Generis Database + Rights (but not its individual contents) is Adapted Material; and + + c. You must comply with the conditions in Section 3(a) if You Share + all or a substantial portion of the contents of the database. + +For the avoidance of doubt, this Section 4 supplements and does not +replace Your obligations under this Public License where the Licensed +Rights include other Copyright and Similar Rights. + + +Section 5 -- Disclaimer of Warranties and Limitation of Liability. + + a. UNLESS OTHERWISE SEPARATELY UNDERTAKEN BY THE LICENSOR, TO THE + EXTENT POSSIBLE, THE LICENSOR OFFERS THE LICENSED MATERIAL AS-IS + AND AS-AVAILABLE, AND MAKES NO REPRESENTATIONS OR WARRANTIES OF + ANY KIND CONCERNING THE LICENSED MATERIAL, WHETHER EXPRESS, + IMPLIED, STATUTORY, OR OTHER. THIS INCLUDES, WITHOUT LIMITATION, + WARRANTIES OF TITLE, MERCHANTABILITY, FITNESS FOR A PARTICULAR + PURPOSE, NON-INFRINGEMENT, ABSENCE OF LATENT OR OTHER DEFECTS, + ACCURACY, OR THE PRESENCE OR ABSENCE OF ERRORS, WHETHER OR NOT + KNOWN OR DISCOVERABLE. WHERE DISCLAIMERS OF WARRANTIES ARE NOT + ALLOWED IN FULL OR IN PART, THIS DISCLAIMER MAY NOT APPLY TO YOU. + + b. TO THE EXTENT POSSIBLE, IN NO EVENT WILL THE LICENSOR BE LIABLE + TO YOU ON ANY LEGAL THEORY (INCLUDING, WITHOUT LIMITATION, + NEGLIGENCE) OR OTHERWISE FOR ANY DIRECT, SPECIAL, INDIRECT, + INCIDENTAL, CONSEQUENTIAL, PUNITIVE, EXEMPLARY, OR OTHER LOSSES, + COSTS, EXPENSES, OR DAMAGES ARISING OUT OF THIS PUBLIC LICENSE OR + USE OF THE LICENSED MATERIAL, EVEN IF THE LICENSOR HAS BEEN + ADVISED OF THE POSSIBILITY OF SUCH LOSSES, COSTS, EXPENSES, OR + DAMAGES. WHERE A LIMITATION OF LIABILITY IS NOT ALLOWED IN FULL OR + IN PART, THIS LIMITATION MAY NOT APPLY TO YOU. + + c. The disclaimer of warranties and limitation of liability provided + above shall be interpreted in a manner that, to the extent + possible, most closely approximates an absolute disclaimer and + waiver of all liability. + + +Section 6 -- Term and Termination. + + a. This Public License applies for the term of the Copyright and + Similar Rights licensed here. However, if You fail to comply with + this Public License, then Your rights under this Public License + terminate automatically. + + b. Where Your right to use the Licensed Material has terminated under + Section 6(a), it reinstates: + + 1. automatically as of the date the violation is cured, provided + it is cured within 30 days of Your discovery of the + violation; or + + 2. upon express reinstatement by the Licensor. + + For the avoidance of doubt, this Section 6(b) does not affect any + right the Licensor may have to seek remedies for Your violations + of this Public License. + + c. For the avoidance of doubt, the Licensor may also offer the + Licensed Material under separate terms or conditions or stop + distributing the Licensed Material at any time; however, doing so + will not terminate this Public License. + + d. Sections 1, 5, 6, 7, and 8 survive termination of this Public + License. + + +Section 7 -- Other Terms and Conditions. + + a. The Licensor shall not be bound by any additional or different + terms or conditions communicated by You unless expressly agreed. + + b. Any arrangements, understandings, or agreements regarding the + Licensed Material not stated herein are separate from and + independent of the terms and conditions of this Public License. + + +Section 8 -- Interpretation. + + a. For the avoidance of doubt, this Public License does not, and + shall not be interpreted to, reduce, limit, restrict, or impose + conditions on any use of the Licensed Material that could lawfully + be made without permission under this Public License. + + b. To the extent possible, if any provision of this Public License is + deemed unenforceable, it shall be automatically reformed to the + minimum extent necessary to make it enforceable. If the provision + cannot be reformed, it shall be severed from this Public License + without affecting the enforceability of the remaining terms and + conditions. + + c. No term or condition of this Public License will be waived and no + failure to comply consented to unless expressly agreed to by the + Licensor. + + d. Nothing in this Public License constitutes or may be interpreted + as a limitation upon, or waiver of, any privileges and immunities + that apply to the Licensor or You, including from the legal + processes of any jurisdiction or authority. + + +======================================================================= + +Creative Commons is not a party to its public +licenses. Notwithstanding, Creative Commons may elect to apply one of +its public licenses to material it publishes and in those instances +will be considered the “Licensor.” The text of the Creative Commons +public licenses is dedicated to the public domain under the CC0 Public +Domain Dedication. Except for the limited purpose of indicating that +material is shared under a Creative Commons public license or as +otherwise permitted by the Creative Commons policies published at +creativecommons.org/policies, Creative Commons does not authorize the +use of the trademark "Creative Commons" or any other trademark or logo +of Creative Commons without its prior written consent including, +without limitation, in connection with any unauthorized modifications +to any of its public licenses or any other arrangements, +understandings, or agreements concerning use of licensed material. For +the avoidance of doubt, this paragraph does not form part of the +public licenses. + +Creative Commons may be contacted at creativecommons.org. + diff --git a/hardware/esp32-display/README.md b/hardware/esp32-display/README.md new file mode 100644 index 0000000..8f69863 --- /dev/null +++ b/hardware/esp32-display/README.md @@ -0,0 +1,96 @@ +# Miucs USB 屏幕适配 + +本文描述本目录的独立 Miucs **整屏 JPEG 固件**,与桌屿的多组件/Lua 固件是两个目标。桌屿板端皮肤由 `MiucsCore` 与桌屿固件配合运行,见 [嵌入指南](../../docs/embedding.md)。使用核心库不需要给桌屿设备刷本目录固件;下文的 1 MiB 检查仅适用于旧整屏固件布局。 + +面向已经点亮的 **OSPTEK ESP32-P4C5 V1.3 / P4 rev 3.2 + YDP500BT004-V1(720×1280、JD9365)**。macOS 继续运行原来的 Lua/Skia 皮肤、音频分析和键盘输入;ESP32 接收 JPEG 帧并显示,不在板上运行 Lua/Skia。 + +## 使用 + +1. 构建 macOS 应用:在仓库根目录执行 `zsh build.sh`,启动 `dist/Miucs.app`。 +2. 首次通过板上的 **USB TO UART / CH343 烧录口**更新应用固件。保留已验证的屏幕排线方向和现有 bootloader。 +3. 将数据线改接板上的 **原生 USB 2.0 Type-C 数据口**。Type-C / USB-A 选择开关应选择 Type-C;CH343 口不传输动画。 +4. 在「设置 → 硬件屏幕」打开开关。应用只自动识别产品名为 `Miucs P4 Display` 的设备,通过握手后开始传图。 +5. 在设置里选择竖屏、横屏或各自反向。无需重新烧录。当前皮肤同步到屏幕;隐藏桌面皮肤后,硬件仍继续运行。 + +画面保持比例,浅色背景,居中并保留至少 16 像素边距。发送上限为 20 帧/秒,实际帧率由渲染、USB 和屏幕回执决定,设置中显示实测值。未连接时等待自动重连;连接中断时屏幕保留最后一帧。USB 传输只包含渲染好的皮肤画面。 + +## 构建固件 + +使用 ESP-IDF **v5.5.6**;组件版本固定在 `dependencies.lock`。ESP-IDF 和工具链不放入仓库。 + +```sh +export IDF_PATH=/absolute/path/to/esp-idf +export IDF_TOOLS_PATH=/absolute/path/to/esp-idf-tools +zsh scripts/build-hardware.sh +``` + +输出为 `hardware/esp32-display/build/miucs_display.bin`。此固件的芯片最低版本为 P4 rev 3.1,不适用于早期 P4 芯片。 + +本板先前启动测试使用了 `esp-idf-v5.5.6-osptek-startup.patch`,针对 rev 3.2 在 bootloader 调频前增加 5 ms 等待。这是本板的临时启动修正,**本次更新只写应用分区,不覆盖已经可用的 bootloader**。如重新构建并写入启动程序,需单独核对该补丁,不能把普通开发板的全量烧录命令直接当作本板验收结果。 + +## 烧录与回退 + +在已安装 `esptool` 的 Python 环境运行: + +```sh +python scripts/hardware/flash.py --port /dev/cu.YOUR_CH343_PORT +``` + +脚本先读取分区表,确认 factory 应用在 `0x10000`、容量 1 MiB,再备份当前应用,最后写入新固件。备份位于 `build/hardware-backups/<时间>/`,包含原应用、分区表和 SHA-256 记录。脚本限定 ESP32-P4 和已验证的分区布局,布局不匹配时停止。 + +回退到备份应用: + +```sh +python -m esptool --chip esp32p4 --port /dev/cu.YOUR_CH343_PORT --baud 115200 --no-stub \ + write-flash 0x10000 build/hardware-backups/TIMESTAMP/previous-app.bin +``` + +## 传输协议 v1 + +设备通过原生 USB High-Speed CDC-ACM 枚举,产品名 `Miucs P4 Display`,日志仍走 UART。原生 CDC 的 line-coding 波特率不限制 USB 数据传输速率。没有安装屏幕固件时,应用不会尝试向其他串口发送图片。 + +空闲设备每秒发送:`MIUCS_DISPLAY 1 720 1280 262144\n`。 + +主机发送 24 字节小端头,后接 JPEG: + +| 偏移 | 字段 | 约束 | +| --- | --- | --- | +| 0 | magic,4 字节 | `MIUC` | +| 4 | version,u8 | 1 | +| 5 | type,u8 | 1 = JPEG | +| 6 | reserved,u16 | 0 | +| 8 | sequence,u32 | 回执关联 | +| 12 | payload length,u32 | 4…262144 | +| 16 | payload CRC-32,u32 | IEEE / zlib,初始与最终异或 0xffffffff | +| 20 | width,u16 | 720 | +| 22 | height,u16 | 1280 | + +方向在主机旋转合成时处理,因此 JPEG 总是 720×1280。固件校验尺寸及 CRC,用硬件 JPEG 直接解码到 RGB565 双缓冲,省去整帧 CPU 颜色转换。每次启动的首帧会与 BGR888 参考路径逐像素校验字节顺序,之后释放参考缓冲。USB 接收和端点缓冲均为 8 KiB,CRC 使用按字节查表。等待 DMA 边界后回执:`MIUCS_ACK <处理毫秒>\n`,状态 0 表示显示提交完成。 + +只有一帧处于发送/等待回执阶段,不缓存过期动画。主机遇到写入失败、3 秒超时或错误回执会关闭连接并重试。设备丢弃断线及超过 2 秒未接收完整的包,并能跳过无效头重新同步。 + +## 验证范围 + +```sh +zsh scripts/test-hardware.sh +zsh scripts/test-plugin-macos.sh +zsh scripts/test-settings.sh +``` + +已覆盖:实际 Swift 串口代码与伪终端之间的分片握手/回执、超时和断线;C 接收解析器的分片/噪声/超限/CRC/重连(ASan、UBSan);四种方向的 JPEG 输出;七个真实皮肤导出;隐藏桌面后的 worker 生命周期;热重载后导出缓存失效;设置窗口和原有皮肤切换回归。 + +**实机验证(2026-11-09)**:已备份原应用并烧录本版,保留 bootloader;原生 USB 枚举、七种皮肤 JPEG 解码、错误 CRC 拒绝及断开后的握手重连通过。使用同一组七张测试图片连续发送 20 秒,完成 330 帧(16.46 帧/秒),平均板端显示处理 35.8 ms。首帧 RGB565 与参考路径的 921,600 像素校验误差为 0;日志未出现 panic 或任务栈溢出。此测试使用预编码图片,不代表所有皮肤的实时动画帧率,也不是长时间耐久测试。实际应用中「摇曳露营」测得约 14.5 帧/秒、167 KiB/帧,板端处理约 34 ms;音乐及键盘输入已连接,用户实测确认动画明显顺滑、可以接受。 + +实机探测使用生产传输代码。先关闭应用的硬件开关以释放串口,再运行: + +```sh +zsh scripts/probe-hardware.sh --seconds 20 --output build/hardware-live-probe.json +``` + +测试完成后重新打开硬件开关即可恢复实时动画。 当前版本不转发触摸事件,也不将外接键盘插到开发板上;键盘与音乐来自电脑现有输入链路。Windows 宿主尚未接入此传输功能。 + +## 来源 + +- [OSPTEK 屏幕资料及示例](https://github.com/osptek/display-tft-5inch-720x1280):`main/jd9365_init_cmds.h` 和 DSI 初始化时序源自厂家示例,沿用 CC BY 4.0,版权归 OSPTEK 鱼鹰光电;许可副本见 `OSPTEK-LICENSE.txt`。本适配增加了 USB 帧传输、双缓冲和动态 JPEG 显示。 +- [OSPTEK 开发板资料](https://github.com/osptek/devboard-esp32p4c5_module)。 +- [Espressif USB Device Stack](https://docs.espressif.com/projects/esp-usb/en/latest/esp32p4/usb_device.html)。 diff --git a/hardware/esp32-display/dependencies.lock b/hardware/esp32-display/dependencies.lock new file mode 100644 index 0000000..0e2d5d8 --- /dev/null +++ b/hardware/esp32-display/dependencies.lock @@ -0,0 +1,74 @@ +dependencies: + espressif/cmake_utilities: + component_hash: 351350613ceafba240b761b4ea991e0f231ac7a9f59a9ee901f751bddc0bb18f + dependencies: + - name: idf + require: private + version: '>=4.1' + source: + registry_url: https://components.espressif.com + type: service + version: 0.5.3 + espressif/esp_lcd_jd9365: + component_hash: ca0517ca0aaccac53e1eb572fbfc8262086401875a4823862b8bca475843db1a + dependencies: + - name: espressif/cmake_utilities + registry_url: https://components.espressif.com + require: private + version: 0.* + - name: idf + require: private + version: '>=5.4' + source: + registry_url: https://components.espressif.com/ + type: service + targets: + - esp32p4 + version: 2.0.2 + espressif/esp_tinyusb: + component_hash: cfb066b239957ebfba5f7a7764e534b3000d3b400fd42d11bedb6ee5e6eaf3c9 + dependencies: + - name: idf + require: private + version: '>=5.0' + - name: espressif/tinyusb + registry_url: https://components.espressif.com + require: public + version: '>=0.17.0~2' + source: + registry_url: https://components.espressif.com/ + type: service + targets: + - esp32s2 + - esp32s3 + - esp32p4 + - esp32h4 + - esp32s31 + version: 2.4.0 + espressif/tinyusb: + component_hash: e7f993ae9b1b0eb8779c04dd26b5f8900108a19e5e4534b19946e18f1edd8895 + dependencies: + - name: idf + require: private + version: '>=5.0' + source: + registry_url: https://components.espressif.com + type: service + targets: + - esp32s2 + - esp32s3 + - esp32p4 + - esp32h4 + - esp32s31 + version: 0.21.0~4 + idf: + source: + type: idf + version: 5.5.6 +direct_dependencies: +- espressif/esp_lcd_jd9365 +- espressif/esp_tinyusb +- idf +manifest_hash: 2f526e1dab830710b90d796a9c702a39fcb6ba28b2c27332b261dbdcd1568589 +target: esp32p4 +version: 2.0.0 diff --git a/hardware/esp32-display/esp-idf-v5.5.6-osptek-startup.patch b/hardware/esp32-display/esp-idf-v5.5.6-osptek-startup.patch new file mode 100644 index 0000000..552ced2 --- /dev/null +++ b/hardware/esp32-display/esp-idf-v5.5.6-osptek-startup.patch @@ -0,0 +1,18 @@ +diff --git a/components/esp_hw_support/port/esp32p4/rtc_clk_init.c b/components/esp_hw_support/port/esp32p4/rtc_clk_init.c +index f82d414a..572fc0b3 100644 +--- a/components/esp_hw_support/port/esp32p4/rtc_clk_init.c ++++ b/components/esp_hw_support/port/esp32p4/rtc_clk_init.c +@@ -100,6 +100,13 @@ void rtc_clk_init(rtc_clk_config_t cfg) + ESP_HW_LOGE(TAG, "invalid CPU frequency value"); + abort(); + } ++#if defined(BOOTLOADER_BUILD) && CONFIG_ESP32P4_REV_MIN_FULL >= 300 ++ // Diagnostic settling interval for the OSPTEK rev3.2 board under test. ++ // Keep this scoped to the bootloader; this is not an upstream IDF fix. ++ if (efuse_hal_chip_revision() == 302) { ++ esp_rom_delay_us(5000); ++ } ++#endif + rtc_clk_cpu_freq_set_config(&new_config); + + /* Re-calculate the ccount to make time calculation correct. */ diff --git a/hardware/esp32-display/main/CMakeLists.txt b/hardware/esp32-display/main/CMakeLists.txt new file mode 100644 index 0000000..d790eb2 --- /dev/null +++ b/hardware/esp32-display/main/CMakeLists.txt @@ -0,0 +1,3 @@ +idf_component_register(SRCS "display.c" "stream.c" "protocol.c" INCLUDE_DIRS "." + REQUIRES esp_lcd esp_driver_gpio esp_hw_support esp_timer esp_psram + esp_driver_jpeg esp_tinyusb) diff --git a/hardware/esp32-display/main/display.c b/hardware/esp32-display/main/display.c new file mode 100644 index 0000000..4eacf5d --- /dev/null +++ b/hardware/esp32-display/main/display.c @@ -0,0 +1,160 @@ +/* Display timing and initialization adapted from OSPTEK's CC-BY-4.0 examples. + * https://github.com/osptek/display-tft-5inch-720x1280 + * Verified wiring: OSPTEK P4C5 V1.3 + YDP500BT004-V1. + */ +#include +#include +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "freertos/semphr.h" +#include "driver/gpio.h" +#include "driver/jpeg_decode.h" +#include "esp_log.h" +#include "esp_ldo_regulator.h" +#include "esp_lcd_panel_ops.h" +#include "esp_lcd_mipi_dsi.h" +#include "esp_lcd_jd9365.h" +#include "jd9365_init_cmds.h" +#include "protocol.h" +#include "display.h" + +#define WIDTH MIUCS_WIDTH +#define HEIGHT MIUCS_HEIGHT +#define LCD_RESET GPIO_NUM_5 +#define BACKLIGHT GPIO_NUM_20 +static const char *TAG = "MIUCS_DISPLAY"; +static esp_lcd_panel_handle_t panel; +static uint16_t *frames[2]; +static unsigned back = 1; +static SemaphoreHandle_t frame_done; +static jpeg_decoder_handle_t decoder; +static uint8_t *decoded; +static size_t decoded_capacity; +// IDF's BGR choice gives native little-endian RGB565 words for this LCD. +// Verified against the RGB888 reference across all 921,600 pixels on hardware. +static const jpeg_dec_rgb_element_order_t pixel_order = JPEG_DEC_RGB_ELEMENT_ORDER_BGR; + +static bool on_frame_done(esp_lcd_panel_handle_t p, esp_lcd_dpi_panel_event_data_t *event, void *context) +{ + BaseType_t wake = pdFALSE; + xSemaphoreGiveFromISR(frame_done, &wake); + return wake == pdTRUE; +} + +#define DISPLAY_TRY(expr) do { esp_err_t error = (expr); if (error != ESP_OK) { ESP_LOGE(TAG, "DISPLAY_FAILED step=%s error=%s", #expr, esp_err_to_name(error)); return error; } } while (0) + +esp_err_t miucs_display_init(void) +{ + gpio_config_t bl = {.pin_bit_mask = 1ULL << BACKLIGHT, .mode = GPIO_MODE_OUTPUT}; + DISPLAY_TRY(gpio_config(&bl)); + DISPLAY_TRY(gpio_set_level(BACKLIGHT, 0)); + frame_done = xSemaphoreCreateBinary(); + if (!frame_done) return ESP_ERR_NO_MEM; + esp_ldo_channel_handle_t phy_power; + esp_ldo_channel_config_t ldo_cfg = {.chan_id = 3, .voltage_mv = 2500}; + DISPLAY_TRY(esp_ldo_acquire_channel(&ldo_cfg, &phy_power)); + esp_lcd_dsi_bus_handle_t bus; + esp_lcd_dsi_bus_config_t bus_cfg = { + .bus_id = 0, .num_data_lanes = 2, + .phy_clk_src = 0, .lane_bit_rate_mbps = 870, + }; + DISPLAY_TRY(esp_lcd_new_dsi_bus(&bus_cfg, &bus)); + esp_lcd_panel_io_handle_t io; + esp_lcd_dbi_io_config_t io_cfg = {.virtual_channel = 0, .lcd_cmd_bits = 8, .lcd_param_bits = 8}; + DISPLAY_TRY(esp_lcd_new_panel_io_dbi(bus, &io_cfg, &io)); + esp_lcd_dpi_panel_config_t dpi_cfg = { + .dpi_clk_src = MIPI_DSI_DPI_CLK_SRC_DEFAULT, .dpi_clock_freq_mhz = 63, + .virtual_channel = 0, .pixel_format = LCD_COLOR_PIXEL_FORMAT_RGB565, .num_fbs = 2, + .video_timing = { + .h_size = WIDTH, .v_size = HEIGHT, + .hsync_back_porch = 42, .hsync_pulse_width = 8, .hsync_front_porch = 42, + .vsync_back_porch = 12, .vsync_pulse_width = 4, .vsync_front_porch = 24, + }, + .flags.use_dma2d = false, + }; + jd9365_vendor_config_t vendor = { + .init_cmds = lcd_init_cmds, .init_cmds_size = lcd_init_cmds_size, + .mipi_config = {.dsi_bus = bus, .dpi_config = &dpi_cfg, .lane_num = 2}, + }; + esp_lcd_panel_dev_config_t panel_cfg = { + .reset_gpio_num = LCD_RESET, .rgb_ele_order = LCD_RGB_ELEMENT_ORDER_RGB, + .bits_per_pixel = 16, .vendor_config = &vendor, + }; + DISPLAY_TRY(esp_lcd_new_panel_jd9365(io, &panel_cfg, &panel)); + DISPLAY_TRY(esp_lcd_panel_reset(panel)); + DISPLAY_TRY(esp_lcd_panel_init(panel)); + DISPLAY_TRY(esp_lcd_dpi_panel_get_frame_buffer(panel, 2, (void **)&frames[0], (void **)&frames[1])); + // A quiet light background while waiting for the desktop application. + for (unsigned i = 0; i < WIDTH * HEIGHT; ++i) frames[0][i] = frames[1][i] = 0xf79e; + esp_lcd_dpi_panel_event_callbacks_t callbacks = {.on_frame_buf_complete = on_frame_done}; + DISPLAY_TRY(esp_lcd_dpi_panel_register_event_callbacks(panel, &callbacks, NULL)); + DISPLAY_TRY(esp_lcd_panel_draw_bitmap(panel, 0, 0, WIDTH, HEIGHT, frames[0])); + DISPLAY_TRY(esp_lcd_panel_disp_on_off(panel, true)); + DISPLAY_TRY(gpio_set_level(BACKLIGHT, 1)); + jpeg_decode_memory_alloc_cfg_t memory = {.buffer_direction = JPEG_DEC_ALLOC_OUTPUT_BUFFER}; + decoded = jpeg_alloc_decoder_mem(WIDTH * HEIGHT * 3, &memory, &decoded_capacity); + if (!decoded) return ESP_ERR_NO_MEM; + jpeg_decode_engine_cfg_t engine = {.timeout_ms = 1000}; + DISPLAY_TRY(jpeg_new_decoder_engine(&engine, &decoder)); + ESP_LOGI(TAG, "DISPLAY_READY %dx%d", WIDTH, HEIGHT); + return ESP_OK; +} + +esp_err_t miucs_display_jpeg(const uint8_t *data, size_t length) +{ + jpeg_decode_picture_info_t info; + esp_err_t result = jpeg_decoder_get_info(data, length, &info); + if (result != ESP_OK) return result; + if (info.width != WIDTH || info.height != HEIGHT) return ESP_ERR_INVALID_SIZE; + jpeg_decode_cfg_t config = { + .output_format = JPEG_DECODE_OUT_FORMAT_RGB565, + .rgb_order = pixel_order, + .conv_std = JPEG_YUV_RGB_CONV_STD_BT601, + }; + uint32_t size = 0; + uint16_t *frame = frames[back]; + if (decoded) { + // On the first frame, verify the DMA byte order against the already-tested + // RGB888 path. Subsequent frames decode directly into the LCD back buffer. + jpeg_decode_cfg_t reference = config; + reference.output_format = JPEG_DECODE_OUT_FORMAT_RGB888; + reference.rgb_order = JPEG_DEC_RGB_ELEMENT_ORDER_BGR; + result = jpeg_decoder_process(decoder, &reference, data, length, decoded, decoded_capacity, &size); + if (result != ESP_OK) return result; + if (size != WIDTH * HEIGHT * 3) return ESP_ERR_INVALID_SIZE; + } + result = jpeg_decoder_process(decoder, &config, data, length, (uint8_t *)frame, WIDTH * HEIGHT * 2, &size); + if (result != ESP_OK) return result; + if (size != WIDTH * HEIGHT * 2) return ESP_ERR_INVALID_SIZE; + if (decoded) { + uint32_t error = 0; + for (unsigned i = 0; i < WIDTH * HEIGHT; ++i) { + const uint8_t *p = decoded + i * 3; + uint16_t v = frame[i]; + error += abs((int)(v >> 11) - (p[2] >> 3)); + error += abs((int)((v >> 5) & 63) - (p[1] >> 2)); + error += abs((int)(v & 31) - (p[0] >> 3)); + } + ESP_LOGI(TAG, "RGB565_VERIFY error=%u pixels=%u", (unsigned)error, WIDTH * HEIGHT); + if (error > WIDTH * HEIGHT * 3) return ESP_ERR_INVALID_STATE; + free(decoded); + decoded = NULL; + ESP_LOGI(TAG, "RGB565_DIRECT_READY byte_order=%d", pixel_order); + } + while (xSemaphoreTake(frame_done, 0) == pdTRUE) {} + result = esp_lcd_panel_draw_bitmap(panel, 0, 0, WIDTH, HEIGHT, frame); + if (result != ESP_OK) return result; + // Wait two DMA boundaries, including the possible interrupt racing draw_bitmap. + // The former front buffer can then be reused without writing into an active scan. + for (unsigned i = 0; i < 2; ++i) { + if (xSemaphoreTake(frame_done, pdMS_TO_TICKS(150)) != pdTRUE) return ESP_ERR_TIMEOUT; + } + back ^= 1; + return ESP_OK; +} + +void app_main(void) +{ + ESP_ERROR_CHECK(miucs_display_init()); + miucs_stream_start(); +} diff --git a/hardware/esp32-display/main/display.h b/hardware/esp32-display/main/display.h new file mode 100644 index 0000000..e864184 --- /dev/null +++ b/hardware/esp32-display/main/display.h @@ -0,0 +1,8 @@ +#pragma once +#include +#include +#include "esp_err.h" + +esp_err_t miucs_display_init(void); +esp_err_t miucs_display_jpeg(const uint8_t *data, size_t length); +void miucs_stream_start(void); diff --git a/hardware/esp32-display/main/idf_component.yml b/hardware/esp32-display/main/idf_component.yml new file mode 100644 index 0000000..e9405e5 --- /dev/null +++ b/hardware/esp32-display/main/idf_component.yml @@ -0,0 +1,4 @@ +dependencies: + idf: ">=5.5.6,<6.0.0" + espressif/esp_lcd_jd9365: "2.0.*" + espressif/esp_tinyusb: "^2.0.0" diff --git a/hardware/esp32-display/main/jd9365_init_cmds.h b/hardware/esp32-display/main/jd9365_init_cmds.h new file mode 100644 index 0000000..70b95e9 --- /dev/null +++ b/hardware/esp32-display/main/jd9365_init_cmds.h @@ -0,0 +1,241 @@ +#ifndef JD9365_INIT_CMDS_H +#define JD9365_INIT_CMDS_H + +#include "esp_lcd_jd9365.h" + +#ifdef __cplusplus +extern "C" { +#endif + +// ==================== JD9365 初始化命令数组 ==================== +static const jd9365_lcd_init_cmd_t lcd_init_cmds[] = { +// {cmd, { data }, data_size, delay_ms} + +// Page0 解锁密码页 +{0xE0, (uint8_t []){0x00}, 1, 0}, + +// PASSWORD +{0xE1, (uint8_t []){0x93}, 1, 0}, +{0xE2, (uint8_t []){0x65}, 1, 0}, +{0xE3, (uint8_t []){0xF8}, 1, 0}, + +/* 设置双通道 (2-Lane) */ +{0x80, (uint8_t []){0x01}, 1, 0}, + +// Page1 +{0xE0, (uint8_t []){0x01}, 1, 0}, +{0x00, (uint8_t []){0x00}, 1, 0}, +{0x01, (uint8_t []){0x55}, 1, 0}, +{0x03, (uint8_t []){0x00}, 1, 0}, +{0x04, (uint8_t []){0x55}, 1, 0}, +{0x17, (uint8_t []){0x00}, 1, 0}, +{0x18, (uint8_t []){0xAF}, 1, 0}, +{0x19, (uint8_t []){0x00}, 1, 0}, +{0x1A, (uint8_t []){0x00}, 1, 0}, +{0x1B, (uint8_t []){0xAF}, 1, 0}, +{0x1C, (uint8_t []){0x00}, 1, 0}, +{0x1F, (uint8_t []){0x7E}, 1, 0}, +{0x22, (uint8_t []){0x4E}, 1, 0}, +{0x23, (uint8_t []){0x02}, 1, 0}, +{0x24, (uint8_t []){0xFE}, 1, 0}, +{0x25, (uint8_t []){0x20}, 1, 0}, +{0x27, (uint8_t []){0x22}, 1, 0}, +{0x35, (uint8_t []){0x23}, 1, 0}, +{0x37, (uint8_t []){0x09}, 1, 0}, +{0x38, (uint8_t []){0x04}, 1, 0}, +{0x40, (uint8_t []){0x44}, 1, 0}, +{0x41, (uint8_t []){0xA0}, 1, 0}, +{0x42, (uint8_t []){0x81}, 1, 0}, +{0x43, (uint8_t []){0x18}, 1, 0}, +{0x44, (uint8_t []){0x1C}, 1, 0}, +{0x45, (uint8_t []){0x10}, 1, 0}, +{0x55, (uint8_t []){0x02}, 1, 0}, +{0x57, (uint8_t []){0x69}, 1, 0}, +{0x5A, (uint8_t []){0x1B}, 1, 0}, +{0x5B, (uint8_t []){0x1B}, 1, 0}, +// Gamma 2.2 +{0x5D, (uint8_t []){0x7F}, 1, 0}, +{0x5E, (uint8_t []){0x66}, 1, 0}, +{0x5F, (uint8_t []){0x57}, 1, 0}, +{0x60, (uint8_t []){0x4B}, 1, 0}, +{0x61, (uint8_t []){0x47}, 1, 0}, +{0x62, (uint8_t []){0x39}, 1, 0}, +{0x63, (uint8_t []){0x3D}, 1, 0}, +{0x64, (uint8_t []){0x27}, 1, 0}, +{0x65, (uint8_t []){0x40}, 1, 0}, +{0x66, (uint8_t []){0x3F}, 1, 0}, +{0x67, (uint8_t []){0x40}, 1, 0}, +{0x68, (uint8_t []){0x5F}, 1, 0}, +{0x69, (uint8_t []){0x4E}, 1, 0}, +{0x6A, (uint8_t []){0x56}, 1, 0}, +{0x6B, (uint8_t []){0x49}, 1, 0}, +{0x6C, (uint8_t []){0x44}, 1, 0}, +{0x6D, (uint8_t []){0x35}, 1, 0}, +{0x6E, (uint8_t []){0x22}, 1, 0}, +{0x6F, (uint8_t []){0x07}, 1, 0}, +{0x70, (uint8_t []){0x7F}, 1, 0}, +{0x71, (uint8_t []){0x66}, 1, 0}, +{0x72, (uint8_t []){0x57}, 1, 0}, +{0x73, (uint8_t []){0x4B}, 1, 0}, +{0x74, (uint8_t []){0x47}, 1, 0}, +{0x75, (uint8_t []){0x39}, 1, 0}, +{0x76, (uint8_t []){0x3D}, 1, 0}, +{0x77, (uint8_t []){0x27}, 1, 0}, +{0x78, (uint8_t []){0x40}, 1, 0}, +{0x79, (uint8_t []){0x3F}, 1, 0}, +{0x7A, (uint8_t []){0x40}, 1, 0}, +{0x7B, (uint8_t []){0x5F}, 1, 0}, +{0x7C, (uint8_t []){0x4E}, 1, 0}, +{0x7D, (uint8_t []){0x56}, 1, 0}, +{0x7E, (uint8_t []){0x49}, 1, 0}, +{0x7F, (uint8_t []){0x44}, 1, 0}, +{0x80, (uint8_t []){0x35}, 1, 0}, +{0x81, (uint8_t []){0x22}, 1, 0}, +{0x82, (uint8_t []){0x07}, 1, 0}, + +// Page2 GIP配置 +{0xE0, (uint8_t []){0x02}, 1, 0}, +{0x00, (uint8_t []){0x5F}, 1, 0}, +{0x01, (uint8_t []){0x5F}, 1, 0}, +{0x02, (uint8_t []){0x42}, 1, 0}, +{0x03, (uint8_t []){0x40}, 1, 0}, +{0x04, (uint8_t []){0x48}, 1, 0}, +{0x05, (uint8_t []){0x4A}, 1, 0}, +{0x06, (uint8_t []){0x44}, 1, 0}, +{0x07, (uint8_t []){0x46}, 1, 0}, +{0x08, (uint8_t []){0x5F}, 1, 0}, +{0x09, (uint8_t []){0x5F}, 1, 0}, +{0x0A, (uint8_t []){0x5F}, 1, 0}, +{0x0B, (uint8_t []){0x5F}, 1, 0}, +{0x0C, (uint8_t []){0x5F}, 1, 0}, +{0x0D, (uint8_t []){0x5F}, 1, 0}, +{0x0E, (uint8_t []){0x5F}, 1, 0}, +{0x0F, (uint8_t []){0x5F}, 1, 0}, +{0x10, (uint8_t []){0x5F}, 1, 0}, +{0x11, (uint8_t []){0x5F}, 1, 0}, +{0x12, (uint8_t []){0x5F}, 1, 0}, +{0x13, (uint8_t []){0x5F}, 1, 0}, +{0x14, (uint8_t []){0x5E}, 1, 0}, +{0x15, (uint8_t []){0x5E}, 1, 0}, +{0x16, (uint8_t []){0x5F}, 1, 0}, +{0x17, (uint8_t []){0x5F}, 1, 0}, +{0x18, (uint8_t []){0x43}, 1, 0}, +{0x19, (uint8_t []){0x41}, 1, 0}, +{0x1A, (uint8_t []){0x49}, 1, 0}, +{0x1B, (uint8_t []){0x4B}, 1, 0}, +{0x1C, (uint8_t []){0x45}, 1, 0}, +{0x1D, (uint8_t []){0x47}, 1, 0}, +{0x1E, (uint8_t []){0x5F}, 1, 0}, +{0x1F, (uint8_t []){0x5F}, 1, 0}, +{0x20, (uint8_t []){0x5F}, 1, 0}, +{0x21, (uint8_t []){0x5F}, 1, 0}, +{0x22, (uint8_t []){0x5F}, 1, 0}, +{0x23, (uint8_t []){0x5F}, 1, 0}, +{0x24, (uint8_t []){0x5F}, 1, 0}, +{0x25, (uint8_t []){0x5F}, 1, 0}, +{0x26, (uint8_t []){0x5F}, 1, 0}, +{0x27, (uint8_t []){0x5F}, 1, 0}, +{0x28, (uint8_t []){0x5F}, 1, 0}, +{0x29, (uint8_t []){0x5F}, 1, 0}, +{0x2A, (uint8_t []){0x5E}, 1, 0}, +{0x2B, (uint8_t []){0x5E}, 1, 0}, +{0x2C, (uint8_t []){0x1F}, 1, 0}, +{0x2D, (uint8_t []){0x1E}, 1, 0}, +{0x2E, (uint8_t []){0x01}, 1, 0}, +{0x2F, (uint8_t []){0x03}, 1, 0}, +{0x30, (uint8_t []){0x07}, 1, 0}, +{0x31, (uint8_t []){0x05}, 1, 0}, +{0x32, (uint8_t []){0x0B}, 1, 0}, +{0x33, (uint8_t []){0x09}, 1, 0}, +{0x34, (uint8_t []){0x1F}, 1, 0}, +{0x35, (uint8_t []){0x1F}, 1, 0}, +{0x36, (uint8_t []){0x1F}, 1, 0}, +{0x37, (uint8_t []){0x1F}, 1, 0}, +{0x38, (uint8_t []){0x1F}, 1, 0}, +{0x39, (uint8_t []){0x1F}, 1, 0}, +{0x3A, (uint8_t []){0x1F}, 1, 0}, +{0x3B, (uint8_t []){0x1F}, 1, 0}, +{0x3C, (uint8_t []){0x1F}, 1, 0}, +{0x3D, (uint8_t []){0x1F}, 1, 0}, +{0x3E, (uint8_t []){0x1F}, 1, 0}, +{0x3F, (uint8_t []){0x1F}, 1, 0}, +{0x40, (uint8_t []){0x1E}, 1, 0}, +{0x41, (uint8_t []){0x1F}, 1, 0}, +{0x42, (uint8_t []){0x1F}, 1, 0}, +{0x43, (uint8_t []){0x1E}, 1, 0}, +{0x44, (uint8_t []){0x00}, 1, 0}, +{0x45, (uint8_t []){0x02}, 1, 0}, +{0x46, (uint8_t []){0x06}, 1, 0}, +{0x47, (uint8_t []){0x04}, 1, 0}, +{0x48, (uint8_t []){0x0A}, 1, 0}, +{0x49, (uint8_t []){0x08}, 1, 0}, +{0x4A, (uint8_t []){0x1F}, 1, 0}, +{0x4B, (uint8_t []){0x1F}, 1, 0}, +{0x4C, (uint8_t []){0x12}, 1, 0}, +{0x4D, (uint8_t []){0x1F}, 1, 0}, +{0x4E, (uint8_t []){0x1F}, 1, 0}, +{0x4F, (uint8_t []){0x1F}, 1, 0}, +{0x50, (uint8_t []){0x1F}, 1, 0}, +{0x51, (uint8_t []){0x1F}, 1, 0}, +{0x52, (uint8_t []){0x1F}, 1, 0}, +{0x53, (uint8_t []){0x1F}, 1, 0}, +{0x54, (uint8_t []){0x1F}, 1, 0}, +{0x55, (uint8_t []){0x1F}, 1, 0}, +{0x56, (uint8_t []){0x1E}, 1, 0}, +{0x57, (uint8_t []){0x1F}, 1, 0}, +// GIP Timing +{0x58, (uint8_t []){0x00}, 1, 0}, +{0x59, (uint8_t []){0x00}, 1, 0}, +{0x5A, (uint8_t []){0x00}, 1, 0}, +{0x5B, (uint8_t []){0x30}, 1, 0}, +{0x5C, (uint8_t []){0x14}, 1, 0}, +{0x5D, (uint8_t []){0x30}, 1, 0}, +{0x5E, (uint8_t []){0x01}, 1, 0}, +{0x5F, (uint8_t []){0x02}, 1, 0}, +{0x60, (uint8_t []){0x00}, 1, 0}, +{0x61, (uint8_t []){0x01}, 1, 0}, +{0x62, (uint8_t []){0x02}, 1, 0}, +{0x63, (uint8_t []){0x03}, 1, 0}, +{0x64, (uint8_t []){0x64}, 1, 0}, +{0x65, (uint8_t []){0x00}, 1, 0}, +{0x66, (uint8_t []){0x00}, 1, 0}, +{0x67, (uint8_t []){0x73}, 1, 0}, +{0x68, (uint8_t []){0x16}, 1, 0}, +{0x69, (uint8_t []){0x1C}, 1, 0}, +{0x6A, (uint8_t []){0x68}, 1, 0}, +{0x6B, (uint8_t []){0x08}, 1, 0}, +{0x77, (uint8_t []){0xDD}, 1, 0}, +{0x79, (uint8_t []){0x0E}, 1, 0}, +{0x7A, (uint8_t []){0x00}, 1, 0}, +{0x7D, (uint8_t []){0x1F}, 1, 0}, +{0x7E, (uint8_t []){0x37}, 1, 0}, + +{0xE0, (uint8_t []){0x04}, 1, 0}, +{0x04, (uint8_t []){0x01}, 1, 0}, +{0x09, (uint8_t []){0x10}, 1, 0}, +{0x0E, (uint8_t []){0x2A}, 1, 0}, +{0x0F, (uint8_t []){0x87}, 1, 0}, + +// Page5 +{0xE0, (uint8_t []){0x05}, 1, 0}, +{0x15, (uint8_t []){0x1B}, 1, 0}, + +// 切回Page0 + 解锁后寄存器 +{0xE0, (uint8_t []){0x00}, 1, 0}, +{0xE6, (uint8_t []){0x02}, 1, 0}, +{0xE7, (uint8_t []){0x1F}, 1, 0}, + +// 休眠退出+延时120ms +{0x11, (uint8_t []){0x00}, 0, 120}, +// 开显示+延时5ms +{0x29, (uint8_t []){0x00}, 0, 5}, +// {0x35, (uint8_t []){0x00}, 1, 0}, +}; + +static const size_t lcd_init_cmds_size = sizeof(lcd_init_cmds) / sizeof(jd9365_lcd_init_cmd_t); + +#ifdef __cplusplus +} +#endif + +#endif // JD9365_INIT_CMDS_H \ No newline at end of file diff --git a/hardware/esp32-display/main/protocol.c b/hardware/esp32-display/main/protocol.c new file mode 100644 index 0000000..e5c3910 --- /dev/null +++ b/hardware/esp32-display/main/protocol.c @@ -0,0 +1,87 @@ +#include "protocol.h" +#include + +static uint32_t le32(const uint8_t *p) +{ + return (uint32_t)p[0] | (uint32_t)p[1] << 8 | (uint32_t)p[2] << 16 | (uint32_t)p[3] << 24; +} + +uint32_t miucs_crc32(const uint8_t *data, size_t size) +{ + // Byte-wise IEEE CRC avoids eight bit rounds for every USB payload byte. + static const uint32_t table[256] = { + 0x00000000u, 0x77073096u, 0xee0e612cu, 0x990951bau, 0x076dc419u, 0x706af48fu, 0xe963a535u, 0x9e6495a3u, + 0x0edb8832u, 0x79dcb8a4u, 0xe0d5e91eu, 0x97d2d988u, 0x09b64c2bu, 0x7eb17cbdu, 0xe7b82d07u, 0x90bf1d91u, + 0x1db71064u, 0x6ab020f2u, 0xf3b97148u, 0x84be41deu, 0x1adad47du, 0x6ddde4ebu, 0xf4d4b551u, 0x83d385c7u, + 0x136c9856u, 0x646ba8c0u, 0xfd62f97au, 0x8a65c9ecu, 0x14015c4fu, 0x63066cd9u, 0xfa0f3d63u, 0x8d080df5u, + 0x3b6e20c8u, 0x4c69105eu, 0xd56041e4u, 0xa2677172u, 0x3c03e4d1u, 0x4b04d447u, 0xd20d85fdu, 0xa50ab56bu, + 0x35b5a8fau, 0x42b2986cu, 0xdbbbc9d6u, 0xacbcf940u, 0x32d86ce3u, 0x45df5c75u, 0xdcd60dcfu, 0xabd13d59u, + 0x26d930acu, 0x51de003au, 0xc8d75180u, 0xbfd06116u, 0x21b4f4b5u, 0x56b3c423u, 0xcfba9599u, 0xb8bda50fu, + 0x2802b89eu, 0x5f058808u, 0xc60cd9b2u, 0xb10be924u, 0x2f6f7c87u, 0x58684c11u, 0xc1611dabu, 0xb6662d3du, + 0x76dc4190u, 0x01db7106u, 0x98d220bcu, 0xefd5102au, 0x71b18589u, 0x06b6b51fu, 0x9fbfe4a5u, 0xe8b8d433u, + 0x7807c9a2u, 0x0f00f934u, 0x9609a88eu, 0xe10e9818u, 0x7f6a0dbbu, 0x086d3d2du, 0x91646c97u, 0xe6635c01u, + 0x6b6b51f4u, 0x1c6c6162u, 0x856530d8u, 0xf262004eu, 0x6c0695edu, 0x1b01a57bu, 0x8208f4c1u, 0xf50fc457u, + 0x65b0d9c6u, 0x12b7e950u, 0x8bbeb8eau, 0xfcb9887cu, 0x62dd1ddfu, 0x15da2d49u, 0x8cd37cf3u, 0xfbd44c65u, + 0x4db26158u, 0x3ab551ceu, 0xa3bc0074u, 0xd4bb30e2u, 0x4adfa541u, 0x3dd895d7u, 0xa4d1c46du, 0xd3d6f4fbu, + 0x4369e96au, 0x346ed9fcu, 0xad678846u, 0xda60b8d0u, 0x44042d73u, 0x33031de5u, 0xaa0a4c5fu, 0xdd0d7cc9u, + 0x5005713cu, 0x270241aau, 0xbe0b1010u, 0xc90c2086u, 0x5768b525u, 0x206f85b3u, 0xb966d409u, 0xce61e49fu, + 0x5edef90eu, 0x29d9c998u, 0xb0d09822u, 0xc7d7a8b4u, 0x59b33d17u, 0x2eb40d81u, 0xb7bd5c3bu, 0xc0ba6cadu, + 0xedb88320u, 0x9abfb3b6u, 0x03b6e20cu, 0x74b1d29au, 0xead54739u, 0x9dd277afu, 0x04db2615u, 0x73dc1683u, + 0xe3630b12u, 0x94643b84u, 0x0d6d6a3eu, 0x7a6a5aa8u, 0xe40ecf0bu, 0x9309ff9du, 0x0a00ae27u, 0x7d079eb1u, + 0xf00f9344u, 0x8708a3d2u, 0x1e01f268u, 0x6906c2feu, 0xf762575du, 0x806567cbu, 0x196c3671u, 0x6e6b06e7u, + 0xfed41b76u, 0x89d32be0u, 0x10da7a5au, 0x67dd4accu, 0xf9b9df6fu, 0x8ebeeff9u, 0x17b7be43u, 0x60b08ed5u, + 0xd6d6a3e8u, 0xa1d1937eu, 0x38d8c2c4u, 0x4fdff252u, 0xd1bb67f1u, 0xa6bc5767u, 0x3fb506ddu, 0x48b2364bu, + 0xd80d2bdau, 0xaf0a1b4cu, 0x36034af6u, 0x41047a60u, 0xdf60efc3u, 0xa867df55u, 0x316e8eefu, 0x4669be79u, + 0xcb61b38cu, 0xbc66831au, 0x256fd2a0u, 0x5268e236u, 0xcc0c7795u, 0xbb0b4703u, 0x220216b9u, 0x5505262fu, + 0xc5ba3bbeu, 0xb2bd0b28u, 0x2bb45a92u, 0x5cb36a04u, 0xc2d7ffa7u, 0xb5d0cf31u, 0x2cd99e8bu, 0x5bdeae1du, + 0x9b64c2b0u, 0xec63f226u, 0x756aa39cu, 0x026d930au, 0x9c0906a9u, 0xeb0e363fu, 0x72076785u, 0x05005713u, + 0x95bf4a82u, 0xe2b87a14u, 0x7bb12baeu, 0x0cb61b38u, 0x92d28e9bu, 0xe5d5be0du, 0x7cdcefb7u, 0x0bdbdf21u, + 0x86d3d2d4u, 0xf1d4e242u, 0x68ddb3f8u, 0x1fda836eu, 0x81be16cdu, 0xf6b9265bu, 0x6fb077e1u, 0x18b74777u, + 0x88085ae6u, 0xff0f6a70u, 0x66063bcau, 0x11010b5cu, 0x8f659effu, 0xf862ae69u, 0x616bffd3u, 0x166ccf45u, + 0xa00ae278u, 0xd70dd2eeu, 0x4e048354u, 0x3903b3c2u, 0xa7672661u, 0xd06016f7u, 0x4969474du, 0x3e6e77dbu, + 0xaed16a4au, 0xd9d65adcu, 0x40df0b66u, 0x37d83bf0u, 0xa9bcae53u, 0xdebb9ec5u, 0x47b2cf7fu, 0x30b5ffe9u, + 0xbdbdf21cu, 0xcabac28au, 0x53b39330u, 0x24b4a3a6u, 0xbad03605u, 0xcdd70693u, 0x54de5729u, 0x23d967bfu, + 0xb3667a2eu, 0xc4614ab8u, 0x5d681b02u, 0x2a6f2b94u, 0xb40bbe37u, 0xc30c8ea1u, 0x5a05df1bu, 0x2d02ef8du, + }; + uint32_t crc = UINT32_MAX; + while (size--) crc = (crc >> 8) ^ table[(crc ^ *data++) & 0xff]; + return ~crc; +} + +void miucs_parser_reset(miucs_parser *p) +{ + p->header_used = p->payload_used = p->payload_size = 0; +} + +void miucs_parser_feed(miucs_parser *p, const uint8_t *data, size_t size) +{ + while (size) { + if (p->payload_size) { + size_t n = p->payload_size - p->payload_used; + if (n > size) n = size; + memcpy(p->payload + p->payload_used, data, n); + p->payload_used += n; data += n; size -= n; + if (p->payload_used == p->payload_size) { + bool valid = miucs_crc32(p->payload, p->payload_size) == p->expected_crc; + p->on_frame(p->context, p->sequence, p->payload, p->payload_size, valid); + miucs_parser_reset(p); + } + continue; + } + p->header[p->header_used++] = *data++; --size; + if (p->header_used < MIUCS_HEADER_SIZE) continue; + const uint8_t *h = p->header; + uint32_t length = le32(h + 12); + bool valid = memcmp(h, "MIUC", 4) == 0 && h[4] == 1 && h[5] == 1 && + h[6] == 0 && h[7] == 0 && length >= 4 && length <= MIUCS_MAX_JPEG && + (h[20] | h[21] << 8) == MIUCS_WIDTH && (h[22] | h[23] << 8) == MIUCS_HEIGHT; + if (valid) { + p->sequence = le32(h + 8); p->expected_crc = le32(h + 16); + p->payload_size = length; p->payload_used = 0; + } else { + // Sliding synchronization survives boot logs, bad headers and reconnects. + memmove(p->header, p->header + 1, MIUCS_HEADER_SIZE - 1); + p->header_used = MIUCS_HEADER_SIZE - 1; + } + } +} diff --git a/hardware/esp32-display/main/protocol.h b/hardware/esp32-display/main/protocol.h new file mode 100644 index 0000000..a59119b --- /dev/null +++ b/hardware/esp32-display/main/protocol.h @@ -0,0 +1,24 @@ +#pragma once +#include +#include +#include + +#define MIUCS_HEADER_SIZE 24 +#define MIUCS_MAX_JPEG (256 * 1024) +#define MIUCS_WIDTH 720 +#define MIUCS_HEIGHT 1280 + +typedef void (*miucs_frame_fn)(void *context, uint32_t sequence, + const uint8_t *jpeg, size_t length, bool crc_valid); +typedef struct { + uint8_t header[MIUCS_HEADER_SIZE]; + size_t header_used, payload_used, payload_size; + uint32_t sequence, expected_crc; + uint8_t *payload; + miucs_frame_fn on_frame; + void *context; +} miucs_parser; + +uint32_t miucs_crc32(const uint8_t *data, size_t size); +void miucs_parser_reset(miucs_parser *parser); +void miucs_parser_feed(miucs_parser *parser, const uint8_t *data, size_t size); diff --git a/hardware/esp32-display/main/stream.c b/hardware/esp32-display/main/stream.c new file mode 100644 index 0000000..33d6859 --- /dev/null +++ b/hardware/esp32-display/main/stream.c @@ -0,0 +1,93 @@ +#include +#include +#include "freertos/FreeRTOS.h" +#include "freertos/task.h" +#include "driver/jpeg_decode.h" +#include "esp_log.h" +#include "esp_timer.h" +#include "tinyusb.h" +#include "tinyusb_default_config.h" +#include "tinyusb_cdc_acm.h" +#include "tusb.h" +#include "protocol.h" +#include "display.h" + +static TaskHandle_t receiver; +static int64_t last_frame; +static unsigned frames; + +static void reply(const char *text, size_t size) +{ + if (!tud_cdc_n_connected(TINYUSB_CDC_ACM_0)) return; + tinyusb_cdcacm_write_queue(TINYUSB_CDC_ACM_0, (const uint8_t *)text, size); + tinyusb_cdcacm_write_flush(TINYUSB_CDC_ACM_0, pdMS_TO_TICKS(50)); +} + +static void receive_callback(int port, cdcacm_event_t *event) +{ + if (receiver) xTaskNotifyGive(receiver); +} + +static void display_frame(void *context, uint32_t sequence, const uint8_t *jpeg, + size_t length, bool crc_valid) +{ + int64_t started = esp_timer_get_time(); + esp_err_t result = crc_valid ? miucs_display_jpeg(jpeg, length) : ESP_ERR_INVALID_CRC; + char response[96]; + int n = snprintf(response, sizeof(response), "MIUCS_ACK %" PRIu32 " %d %" PRId64 "\n", + sequence, result, (esp_timer_get_time() - started) / 1000); + reply(response, n); + if (result == ESP_OK) { + last_frame = esp_timer_get_time(); + if (++frames % 100 == 1) ESP_LOGI("MIUCS", "FRAME_OK sequence=%" PRIu32 " bytes=%u frames=%u stack_free=%u", sequence, (unsigned)length, frames, (unsigned)uxTaskGetStackHighWaterMark(NULL)); + } else { + ESP_LOGW("MIUCS", "FRAME_REJECTED sequence=%" PRIu32 " error=%s", sequence, esp_err_to_name(result)); + } +} + +static void stream_task(void *arg) +{ + jpeg_decode_memory_alloc_cfg_t config = {.buffer_direction = JPEG_DEC_ALLOC_INPUT_BUFFER}; + size_t capacity; + miucs_parser parser = {.payload = jpeg_alloc_decoder_mem(MIUCS_MAX_JPEG, &config, &capacity), .on_frame = display_frame}; + if (!parser.payload) abort(); + // This task calls the JPEG driver and logging from inside the parser callback. + // Keep the USB scratch buffer in BSS rather than consuming 4 KiB of that stack. + static uint8_t chunk[4096]; + int64_t last_rx = 0, last_hello = 0; + for (;;) { + ulTaskNotifyTake(pdTRUE, pdMS_TO_TICKS(20)); + int64_t now = esp_timer_get_time(); + if (!tud_cdc_n_connected(TINYUSB_CDC_ACM_0)) { + miucs_parser_reset(&parser); + continue; + } + // Discard a partial packet after cable loss; the next header starts cleanly. + if ((parser.header_used || parser.payload_size) && now - last_rx > 2000000) miucs_parser_reset(&parser); + size_t n = 0; + do { + if (tinyusb_cdcacm_read(TINYUSB_CDC_ACM_0, chunk, sizeof(chunk), &n) != ESP_OK) break; + if (n) { + last_rx = esp_timer_get_time(); + miucs_parser_feed(&parser, chunk, n); + } + } while (n); + now = esp_timer_get_time(); + if (now - last_frame > 1500000 && now - last_hello > 1000000) { + const char hello[] = "MIUCS_DISPLAY 1 720 1280 262144\n"; + reply(hello, sizeof(hello) - 1); + last_hello = now; + } + } +} + +void miucs_stream_start(void) +{ + const tinyusb_config_t config = TINYUSB_DEFAULT_CONFIG(); + ESP_ERROR_CHECK(tinyusb_driver_install(&config)); + tinyusb_config_cdcacm_t cdc = {.cdc_port = TINYUSB_CDC_ACM_0, .callback_rx = receive_callback}; + ESP_ERROR_CHECK(tinyusb_cdcacm_init(&cdc)); + BaseType_t ok = xTaskCreate(stream_task, "miucs_stream", 8192, NULL, 5, &receiver); + if (ok != pdPASS) abort(); + ESP_LOGI("MIUCS", "USB_STREAM_READY protocol=1 max_jpeg=%u", MIUCS_MAX_JPEG); +} diff --git a/hardware/esp32-display/sdkconfig.defaults b/hardware/esp32-display/sdkconfig.defaults new file mode 100644 index 0000000..ae52d78 --- /dev/null +++ b/hardware/esp32-display/sdkconfig.defaults @@ -0,0 +1,22 @@ +CONFIG_IDF_TARGET="esp32p4" +CONFIG_ESP32P4_REV_MIN_301=y +CONFIG_ESPTOOLPY_FLASHSIZE_16MB=y +CONFIG_ESPTOOLPY_FLASHMODE_DIO=y +CONFIG_ESPTOOLPY_FLASHFREQ_40M=y +CONFIG_ESP_CONSOLE_UART_DEFAULT=y +CONFIG_ESP_CONSOLE_UART_BAUDRATE=115200 +CONFIG_COMPILER_OPTIMIZATION_PERF=y +CONFIG_SPIRAM=y +CONFIG_SPIRAM_SPEED_200M=y +CONFIG_SPIRAM_MEMTEST=y +CONFIG_CACHE_L2_CACHE_256KB=y +CONFIG_CACHE_L2_CACHE_LINE_128B=y +CONFIG_ESP_MAIN_TASK_STACK_SIZE=8192 +CONFIG_FREERTOS_HZ=1000 +CONFIG_TINYUSB_CDC_ENABLED=y +CONFIG_TINYUSB_CDC_COUNT=1 +CONFIG_TINYUSB_CDC_RX_BUFSIZE=8192 +CONFIG_TINYUSB_CDC_EP_BUFSIZE=8192 +CONFIG_TINYUSB_CDC_TX_BUFSIZE=512 +CONFIG_TINYUSB_DESC_MANUFACTURER_STRING="Miucs" +CONFIG_TINYUSB_DESC_PRODUCT_STRING="Miucs P4 Display" diff --git a/modules/audio/core/AudioFeatures.swift b/modules/audio/core/AudioFeatures.swift index 586d9b0..646d4ba 100644 --- a/modules/audio/core/AudioFeatures.swift +++ b/modules/audio/core/AudioFeatures.swift @@ -1,41 +1,41 @@ import Foundation -struct AudioFeatures: Codable, Sendable { - var time: Double - var sampleRate: Double - var channelCount: Int - var rms: Double - var rmsDBFS: Double - var peak: Double - var crestDB: Double? - var envelope: Double - var isSilent: Bool - var silenceDuration: Double - var lowEnergy: Double - var midEnergy: Double - var highEnergy: Double - var lowShare: Double? - var midShare: Double? - var highShare: Double? - var centroidHz: Double? - var bandwidthHz: Double? - var flatness: Double? - var spectralFlux: Double - var onsetThreshold: Double - var isOnset: Bool - var balance: Double? - var sideEnergyRatio: Double? - var correlation: Double? - var displayBands: [Float] +public struct AudioFeatures: Codable, Sendable { + public var time: Double + public var sampleRate: Double + public var channelCount: Int + public var rms: Double + public var rmsDBFS: Double + public var peak: Double + public var crestDB: Double? + public var envelope: Double + public var isSilent: Bool + public var silenceDuration: Double + public var lowEnergy: Double + public var midEnergy: Double + public var highEnergy: Double + public var lowShare: Double? + public var midShare: Double? + public var highShare: Double? + public var centroidHz: Double? + public var bandwidthHz: Double? + public var flatness: Double? + public var spectralFlux: Double + public var onsetThreshold: Double + public var isOnset: Bool + public var balance: Double? + public var sideEnergyRatio: Double? + public var correlation: Double? + public var displayBands: [Float] /// Optional extension; absent in older recordings and synthetic inputs. - var beat: BeatFrame? = nil + public var beat: BeatFrame? = nil } -enum AudioAnalysisError: Error, CustomStringConvertible { +public enum AudioAnalysisError: Error, CustomStringConvertible { case invalidInput(String) case fftUnavailable - var description: String { + public var description: String { switch self { case .invalidInput(let reason): return reason case .fftUnavailable: return "Cannot create the audio FFT" diff --git a/modules/audio/core/BeatFrame.swift b/modules/audio/core/BeatFrame.swift index 4d4f507..bc217ab 100644 --- a/modules/audio/core/BeatFrame.swift +++ b/modules/audio/core/BeatFrame.swift @@ -1,22 +1,22 @@ import Foundation /// Causal pulse tracking on the original mix. Confidence is not a probability. -struct BeatFrame: Codable, Sendable { - let time: Double - let bpm: Double? +public struct BeatFrame: Codable, Sendable { + public let time: Double + public let bpm: Double? /// Fraction of a beat since the most recent metrical pulse, in 0..<1. - let phase: Double? - let confidence: Double - let strength: Double + public let phase: Double? + public let confidence: Double + public let strength: Double /// A newly due pulse in the PCM timeline; nil on all intervening windows. - let beatTime: Double? + public let beatTime: Double? } -struct AudioBeatEvent: Codable, Sendable { - let sequence: UInt64 - let time: Double - let receivedAt: Double - let strength: Double - let bpm: Double - let confidence: Double +public struct AudioBeatEvent: Codable, Sendable { + public let sequence: UInt64 + public let time: Double + public let receivedAt: Double + public let strength: Double + public let bpm: Double + public let confidence: Double } diff --git a/modules/audio/core/FeatureChannel.swift b/modules/audio/core/FeatureChannel.swift index 08bc0b9..76461ce 100644 --- a/modules/audio/core/FeatureChannel.swift +++ b/modules/audio/core/FeatureChannel.swift @@ -1,44 +1,44 @@ import Foundation -enum AudioSourceState: String, Codable, Sendable { +public enum AudioSourceState: String, Codable, Sendable { case stopped, connecting, running, recovering, suspended, permissionRequired, userStopped } /// Value contract v1. PCM window time and host-monotonic receipt time are distinct. -struct AudioFeatureFrame: Codable, Sendable { - var version = 1 - let sessionID: UUID - let sequence: UInt64 - let receivedAt: Double - let audio: AudioFeatures - let melody: MelodyFrame? +public struct AudioFeatureFrame: Codable, Sendable { + public var version = 1 + public let sessionID: UUID + public let sequence: UInt64 + public let receivedAt: Double + public let audio: AudioFeatures + public let melody: MelodyFrame? } -struct AudioOnsetEvent: Codable, Sendable { - let sequence: UInt64 - let time: Double - let receivedAt: Double - let strength: Double +public struct AudioOnsetEvent: Codable, Sendable { + public let sequence: UInt64 + public let time: Double + public let receivedAt: Double + public let strength: Double } -struct FeatureInput: Codable, Sendable { - var version = 1 - let sessionID: UUID - let state: AudioSourceState - let frame: AudioFeatureFrame? - let frameAge: Double? - let onsets: [AudioOnsetEvent] - let spectralFluxPeak: Double +public struct FeatureInput: Codable, Sendable { + public var version = 1 + public let sessionID: UUID + public let state: AudioSourceState + public let frame: AudioFeatureFrame? + public let frameAge: Double? + public let onsets: [AudioOnsetEvent] + public let spectralFluxPeak: Double /// Missing in v1 recordings made before beat tracking was added. - var beats: [AudioBeatEvent]? = nil + public var beats: [AudioBeatEvent]? = nil } -protocol AudioFeatureSource: Sendable { +public protocol AudioFeatureSource: Sendable { func subscribe() -> FeatureSubscription } /// A consumer owns its cursor; one consumer cannot drain another's events. -final class FeatureSubscription: @unchecked Sendable { +public final class FeatureSubscription: @unchecked Sendable { private let channel: FeatureChannel private let lock = NSLock() private var sessionID: UUID @@ -46,20 +46,22 @@ final class FeatureSubscription: @unchecked Sendable { fileprivate init(channel: FeatureChannel, sessionID: UUID, sequence: UInt64) { self.channel = channel; self.sessionID = sessionID; self.sequence = sequence } - func read() -> FeatureInput { + public func read() -> FeatureInput { lock.lock(); defer { lock.unlock() } let input = channel.read(sessionID: sessionID, after: sequence) sessionID = input.sessionID sequence = input.frame?.sequence ?? 0 return input } + + public func activitySignals() -> AsyncStream { channel.activitySignals() } } /// Short-lock UI handoff: one snapshot, 64 onsets, and a recent flux maximum deque. -/// No callback runs under this lock and no UI work is enqueued per audio frame. -final class FeatureChannel: AudioFeatureSource, @unchecked Sendable { - static let eventCapacity = 64 - static let eventLifetime = 0.250 +/// Consumers can opt in to wakeups for active audio and capture state changes. +public final class FeatureChannel: AudioFeatureSource, @unchecked Sendable { + public static let eventCapacity = 64 + public static let eventLifetime = 0.250 private struct FluxSample { let sequence: UInt64; let receivedAt: Double; let value: Double } private let lock = NSLock() private let now: @Sendable () -> Double @@ -75,28 +77,33 @@ final class FeatureChannel: AudioFeatureSource, @unchecked Sendable { private var lastFrameAt: Double? private var recordFrames = false private var recorded = [AudioFeatureFrame]() + private var activityObservers = [UUID: AsyncStream.Continuation]() - init(now: @escaping @Sendable () -> Double = { ProcessInfo.processInfo.systemUptime }) { self.now = now } + public init(now: @escaping @Sendable () -> Double = { ProcessInfo.processInfo.systemUptime }) { self.now = now } - func reset(sessionID: UUID, state: AudioSourceState) { - lock.lock(); defer { lock.unlock() } + public func reset(sessionID: UUID, state: AudioSourceState) { + lock.lock() self.sessionID = sessionID; self.state = state; sequence = 0 latestFrame = nil; onsets.removeAll(keepingCapacity: true); flux.removeAll(keepingCapacity: true) firstFrameAt = nil; lastFrameAt = nil beats.removeAll(keepingCapacity: true); lastBeatTime = nil + unlockAndSignalActivity() } - func setState(_ state: AudioSourceState) { - lock.lock(); defer { lock.unlock() } + public func setState(_ state: AudioSourceState) { + lock.lock() + guard self.state != state else { lock.unlock(); return } self.state = state if state != .running { beats.removeAll(keepingCapacity: true) } + unlockAndSignalActivity() } @discardableResult - func publish(audio: AudioFeatures, melody: MelodyFrame?, sessionID: UUID) -> Bool { + public func publish(audio: AudioFeatures, melody: MelodyFrame?, sessionID: UUID) -> Bool { let time = now() - lock.lock(); defer { lock.unlock() } - guard sessionID == self.sessionID, state == .running else { return false } + lock.lock() + guard sessionID == self.sessionID, state == .running else { lock.unlock(); return false } + let signalActivity = !audio.isSilent || latestFrame?.audio.isSilent != true if let previous = latestFrame?.audio, previous.sampleRate != audio.sampleRate || previous.channelCount != audio.channelCount || audio.time < previous.time { beats.removeAll(keepingCapacity: true); lastBeatTime = nil @@ -140,14 +147,37 @@ final class FeatureChannel: AudioFeatureSource, @unchecked Sendable { recorded.append(frame) if recorded.count > 10_000 { recorded.removeFirst(recorded.count - 10_000) } } + if signalActivity { unlockAndSignalActivity() } else { lock.unlock() } return true } - func subscribe() -> FeatureSubscription { + private func unlockAndSignalActivity() { + let observers = activityObservers.values + lock.unlock() + for observer in observers { observer.yield() } + } + + public func subscribe() -> FeatureSubscription { lock.lock(); defer { lock.unlock() } return FeatureSubscription(channel: self, sessionID: sessionID, sequence: sequence) } + fileprivate func activitySignals() -> AsyncStream { + let id = UUID() + let (stream, continuation) = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + continuation.onTermination = { [weak self] _ in self?.removeActivityObserver(id) } + lock.lock() + activityObservers[id] = continuation + lock.unlock() + continuation.yield() + return stream + } + + private func removeActivityObserver(_ id: UUID) { + lock.lock(); defer { lock.unlock() } + activityObservers.removeValue(forKey: id) + } + fileprivate func read(sessionID: UUID, after cursor: UInt64) -> FeatureInput { let time = now() let cutoff = time - Self.eventLifetime @@ -162,24 +192,24 @@ final class FeatureChannel: AudioFeatureSource, @unchecked Sendable { onsets: events, spectralFluxPeak: peak, beats: beats.filter { $0.sequence > after }) } - var snapshot: AudioFeatureFrame? { + public var snapshot: AudioFeatureFrame? { lock.lock(); defer { lock.unlock() }; return latestFrame } - var health: (first: Double?, last: Double?) { + public var health: (first: Double?, last: Double?) { lock.lock(); defer { lock.unlock() }; return (firstFrameAt, lastFrameAt) } - var currentState: AudioSourceState { + public var currentState: AudioSourceState { lock.lock(); defer { lock.unlock() }; return state } - func startRecording() { + public func startRecording() { lock.lock(); defer { lock.unlock() }; recorded.removeAll(); recordFrames = true } - func stopRecording() -> [AudioFeatureFrame] { + public func stopRecording() -> [AudioFeatureFrame] { lock.lock(); defer { lock.unlock() }; recordFrames = false defer { recorded.removeAll() }; return recorded } - var recordedFrameCount: Int { + public var recordedFrameCount: Int { lock.lock(); defer { lock.unlock() }; return recorded.count } } diff --git a/modules/audio/core/MelodyFrame.swift b/modules/audio/core/MelodyFrame.swift index 0c42f2a..8142dc7 100644 --- a/modules/audio/core/MelodyFrame.swift +++ b/modules/audio/core/MelodyFrame.swift @@ -1,11 +1,11 @@ import Foundation -struct MelodyFrame: Codable, Sendable { +public struct MelodyFrame: Codable, Sendable { /// Center of the analysis window, in the same PCM timeline as AudioFeatures. - var time: Double - var pitchHz: Double? + public var time: Double + public var pitchHz: Double? /// Estimator-specific reliability, not a calibrated voice probability. - var confidence: Double - var isMelodic: Bool - var midi: Double? { pitchHz.map { 69 + 12 * log2($0 / 440) } } + public var confidence: Double + public var isMelodic: Bool + public var midi: Double? { pitchHz.map { 69 + 12 * log2($0 / 440) } } } diff --git a/modules/audio/platforms/macos/AudioEngine.swift b/modules/audio/platforms/macos/AudioEngine.swift index eba364e..10c72a5 100644 --- a/modules/audio/platforms/macos/AudioEngine.swift +++ b/modules/audio/platforms/macos/AudioEngine.swift @@ -27,18 +27,18 @@ struct AudioRecoveryTiming { var permissionPoll = 5.0 } -enum AudioCaptureLifecycleEvent { +public enum AudioCaptureLifecycleEvent { case sleep, wake, screensSleep, screensWake, configurationChanged } /// Owns one capture generation. Old callbacks and cancelled attempts cannot revive it. @MainActor -final class AudioEngine: NSObject, ObservableObject { - let channel: FeatureChannel - var features: AudioFeatures? { channel.snapshot?.audio } - var melody: MelodyFrame? { channel.snapshot?.melody } - @Published private(set) var status = "正在连接系统音频…" - @Published private(set) var isRunning = false +public final class AudioEngine: NSObject, ObservableObject { + public let channel: FeatureChannel + public var features: AudioFeatures? { channel.snapshot?.audio } + public var melody: MelodyFrame? { channel.snapshot?.melody } + @Published public private(set) var status = "正在连接系统音频…" + @Published public private(set) var isRunning = false private let factory: AudioCaptureFactory? private let timing: AudioRecoveryTiming @@ -65,6 +65,8 @@ final class AudioEngine: NSObject, ObservableObject { private var expectedAudioSince: Double? private var suspended: Bool { systemSleeping || screensSleeping } + public override convenience init() { self.init(factory: nil) } + init(factory: AudioCaptureFactory? = nil, timing: AudioRecoveryTiming = AudioRecoveryTiming(), monitorsSystem: Bool = true, now: @escaping @Sendable () -> Double = { ProcessInfo.processInfo.systemUptime }, @@ -79,21 +81,21 @@ final class AudioEngine: NSObject, ObservableObject { super.init() } - func start(excludingExistingApplications: Bool = false) async { + public func start(excludingExistingApplications: Bool = false) async { guard !wanted else { return } wanted = true; excludesExistingApplications = excludingExistingApplications installObservers() await connect() } - func reconnect() async { + public func reconnect() async { wanted = true; failures = 0; waitingForPermission = false installObservers() invalidateCapture() await connect() } - func stop() { + public func stop() { wanted = false invalidateCapture() observers.removeAll(); outputObserver = nil @@ -102,7 +104,7 @@ final class AudioEngine: NSObject, ObservableObject { setStatus(.stopped, "已暂停") } - func handleLifecycle(_ event: AudioCaptureLifecycleEvent) { + public func handleLifecycle(_ event: AudioCaptureLifecycleEvent) { guard wanted else { return } let wasSuspended = suspended switch event { @@ -187,7 +189,7 @@ final class AudioEngine: NSObject, ObservableObject { let nsError = error as NSError if nsError.domain == SCStreamErrorDomain && nsError.code == SCStreamError.Code.userStopped.rawValue { stop() // Respect an explicit stop from macOS's capture controls. - setStatus(.userStopped, "系统已停止音频捕获 · 可从菜单栏重新连接") + setStatus(.userStopped, "系统已停止音频捕获") } else if nsError.domain == SCStreamErrorDomain && nsError.code == SCStreamError.Code.userDeclined.rawValue { waitingForPermission = true recover("等待音频捕获授权", delay: timing.permissionPoll) @@ -318,6 +320,7 @@ private final class ScreenAudioCapture: NSObject, AudioCaptureSession, SCStreamD onFrame: @escaping @Sendable (AudioFeatures, MelodyFrame?) -> Void, onFailure: @escaping @Sendable (Error) -> Void) async throws -> ScreenAudioCapture { let content = try await SCShareableContent.excludingDesktopWindows(false, onScreenWindowsOnly: !excludingExistingApplications) + try Task.checkCancellation() guard let display = content.displays.first else { throw NSError(domain: SCStreamErrorDomain, code: SCStreamError.Code.noCaptureSource.rawValue) } @@ -334,19 +337,21 @@ private final class ScreenAudioCapture: NSObject, AudioCaptureSession, SCStreamD capture.output = try await Task.detached(priority: .userInitiated) { try StreamOutput(onFrame: onFrame, onFailure: onFailure) }.value + try Task.checkCancellation() capture.stream = SCStream(filter: filter, configuration: config, delegate: capture) try capture.stream.addStreamOutput(capture.output, type: .audio, sampleHandlerQueue: capture.output.queue) return capture } - func start() async throws { + public func start() async throws { try await stream.startCapture() hasStarted = true if stopRequested { await stop() } } - func stop() async { + public func stop() async { stopRequested = true + output.stopAcceptingAudio() guard hasStarted else { return } hasStarted = false try? await stream.stopCapture() @@ -383,6 +388,8 @@ private final class StreamOutput: NSObject, SCStreamOutput, @unchecked Sendable let queue = DispatchQueue(label: "miucs.audio", qos: .userInteractive) private let onFrame: @Sendable (AudioFeatures, MelodyFrame?) -> Void private let onFailure: @Sendable (Error) -> Void + private let inputLock = NSLock() + private var acceptsAudio = true private var consecutiveFailures = 0 private var reportedFailure = false private let analysis: AudioAnalysisPipeline @@ -396,12 +403,16 @@ private final class StreamOutput: NSObject, SCStreamOutput, @unchecked Sendable AudioEngine.trace("melody analysis: predominant mixture, harmonic salience + reliability + contour continuity") } + func stopAcceptingAudio() { + inputLock.withLock { acceptsAudio = false } + } + func stream( _ stream: SCStream, didOutputSampleBuffer sampleBuffer: CMSampleBuffer, of type: SCStreamOutputType ) { - guard type == .audio, sampleBuffer.isValid else { return } + guard type == .audio, inputLock.withLock({ acceptsAudio }), sampleBuffer.isValid else { return } do { let block = try AudioPCMDecoder.decode(sampleBuffer) for frame in try analysis.process(block) { diff --git a/modules/hardware/macos/HardwareFrame.swift b/modules/hardware/macos/HardwareFrame.swift new file mode 100644 index 0000000..d1b052b --- /dev/null +++ b/modules/hardware/macos/HardwareFrame.swift @@ -0,0 +1,243 @@ +import Foundation +import CoreGraphics +import ImageIO +import UniformTypeIdentifiers + +public enum HardwareOrientation: Int, CaseIterable, Identifiable, Sendable { + case portrait = 0, landscape = 1, portraitReverse = 2, landscapeReverse = 3 + public var id: Int { rawValue } + public var title: String { + switch self { + case .portrait: "竖屏" + case .landscape: "横屏" + case .portraitReverse: "竖屏 · 反向" + case .landscapeReverse: "横屏 · 反向" + } + } + public var width: Int { rawValue % 2 == 0 ? 720 : 1280 } + public var height: Int { rawValue % 2 == 0 ? 1280 : 720 } +} + +public struct DeskIslePetCapability: Equatable, Sendable { + public let version: Int + public let id: UInt16 + public let maxPack: Int + public let maxChunk: Int + public let maxInput: Int + + public init?(_ line: String) { + let fields = line.split(separator: " ") + guard fields.count == 6, fields[0] == "DESKISLE_PET", + let version = Int(fields[1]), (1...3).contains(version), + let id = UInt16(fields[2]), id > 0, + let pack = Int(fields[3]), (4...(version == 1 ? 6 : 8) * 1024 * 1024).contains(pack), + let chunk = Int(fields[4]), (1024...96 * 1024).contains(chunk), + let input = Int(fields[5]), (4096...16 * 1024).contains(input) else { return nil } + self.version = version + self.id = id + maxPack = pack + maxChunk = chunk + maxInput = input + } +} + +public enum DeskIslePetMessage: UInt8, Sendable { + case begin = 8, chunk, commit, input + case resume = 14, clear +} + +public struct DeskIsleCard: Equatable, Sendable { + public let id: UInt16 + public let width: UInt16 + public let height: UInt16 + public let maxJPEG: Int + public let leaseMS: UInt32 + public let supportsAlpha: Bool + public let pet: DeskIslePetCapability? + + public init?(announcement: String, alphaCardID: UInt16? = nil, pet: DeskIslePetCapability? = nil) { + let fields = announcement.split(whereSeparator: \.isWhitespace) + // A card must fit within the 720 × 1280 panel. + guard fields.count == 8, fields[0] == "DESKISLE", fields[1] == "1", fields[5] == "JPEG", + let id = UInt16(fields[2]), id > 0, + let width = UInt16(fields[3]), (1...720).contains(width), + let height = UInt16(fields[4]), (1...1280).contains(height), + let maxJPEG = UInt32(fields[6]), maxJPEG >= 4, + let leaseMS = UInt32(fields[7]), leaseMS > 0 else { return nil } + self.id = id + self.width = width + self.height = height + self.maxJPEG = Int(maxJPEG) + self.leaseMS = leaseMS + supportsAlpha = id == alphaCardID + self.pet = id == pet?.id ? pet : nil + } + + public var title: String { "桌屿 \(width)×\(height)" } + public var leaseDuration: Double { Double(leaseMS) / 1000 } + public var keepaliveInterval: Double { min(1, leaseDuration / 2) } +} + +public enum HardwareScreen: Equatable, Sendable { + case miucs + case deskIsle(DeskIsleCard) + + public var card: DeskIsleCard? { + if case let .deskIsle(card) = self { return card } + return nil + } + + public func fitScale(width: Double, height: Double, orientation: HardwareOrientation) -> Double { + let viewportWidth = card.map { Int($0.width) } ?? orientation.width + let viewportHeight = card.map { Int($0.height) } ?? orientation.height + let padding = card == nil ? 32 : 16 + return min(Double(max(1, viewportWidth - padding)) / width, + Double(max(1, viewportHeight - padding)) / height) + } +} + +public enum HardwareFrame { + public static let maxJPEG = 262_144 + private static let crcTable: [UInt32] = (0..<256).map { index in + var value = UInt32(index) + for _ in 0..<8 { value = (value >> 1) ^ ((value & 1) == 1 ? 0xedb88320 : 0) } + return value + } + public static func crc32(_ bytes: Data) -> UInt32 { + var value = UInt32.max + for byte in bytes { value = (value >> 8) ^ crcTable[Int((value ^ UInt32(byte)) & 0xff)] } + return ~value + } + public static func packet(jpeg: Data, sequence: UInt32, screen: HardwareScreen = .miucs) -> Data? { + guard (4...(screen.card?.maxJPEG ?? maxJPEG)).contains(jpeg.count) else { return nil } + if let card = screen.card { + guard jpeg.starts(with: [0xff, 0xd8]) else { return nil } + return packet(magic: "ISLE", kind: 1, cardID: card.id, sequence: sequence, + payload: jpeg, width: card.width, height: card.height) + } + return packet(magic: "MIUC", kind: 1, cardID: 0, sequence: sequence, payload: jpeg, width: 720, height: 1280) + } + public static func packet(bgra: Data, width: Int, height: Int, orientation: HardwareOrientation, + sequence: UInt32, screen: HardwareScreen) -> Data? { + guard let card = screen.card, card.supportsAlpha else { + return jpeg(bgra: bgra, width: width, height: height, orientation: orientation, screen: screen) + .flatMap { packet(jpeg: $0, sequence: sequence, screen: screen) } + } + guard let canvas = bitmap(bgra: bgra, width: width, height: height, orientation: orientation, + screen: screen, transparent: true), let base = canvas.data else { return nil } + let rendered = Data(bytes: base, count: canvas.bytesPerRow * canvas.height) + // Ignore the alpha byte for JPEG so its channels retain their premultiplied values. + let info = CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.noneSkipFirst.rawValue + guard let provider = CGDataProvider(data: rendered as CFData), + let color = CGImage(width: canvas.width, height: canvas.height, bitsPerComponent: 8, bitsPerPixel: 32, + bytesPerRow: canvas.bytesPerRow, space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGBitmapInfo(rawValue: info), provider: provider, decode: nil, + shouldInterpolate: false, intent: .defaultIntent), + let jpeg = encodeJPEG(color, screen: screen) else { return nil } + var payload = jpeg + payload.append(alphaMask(bgra: rendered)) + // Keep the JPEG at the DMA-aligned start of the payload. + withUnsafeBytes(of: UInt32(jpeg.count).littleEndian) { payload.append(contentsOf: $0) } + return packet(magic: "ISLE", kind: 5, cardID: card.id, sequence: sequence, + payload: payload, width: card.width, height: card.height) + } + public static func keepalive(sequence: UInt32, card: DeskIsleCard) -> Data { + packet(magic: "ISLE", kind: 2, cardID: card.id, sequence: sequence, payload: Data(), width: 0, height: 0) + } + public static func nativePet(_ kind: DeskIslePetMessage, payload: Data, sequence: UInt32, id: UInt16) -> Data { + packet(magic: "ISLE", kind: kind.rawValue, cardID: id, sequence: sequence, payload: payload, width: 0, height: 0) + } + private static func packet(magic: String, kind: UInt8, cardID: UInt16, sequence: UInt32, + payload: Data, width: UInt16, height: UInt16) -> Data { + var result = Data(magic.utf8) + result.append(contentsOf: [1, kind]) + withUnsafeBytes(of: cardID.littleEndian) { result.append(contentsOf: $0) } + for value in [sequence, UInt32(payload.count), crc32(payload)] { + withUnsafeBytes(of: value.littleEndian) { result.append(contentsOf: $0) } + } + for dimension in [width, height] { + withUnsafeBytes(of: dimension.littleEndian) { result.append(contentsOf: $0) } + } + result.append(payload) + return result + } + + private static func alphaMask(bgra: Data) -> Data { + let pixelCount = bgra.count / 4 + var raw = Data(count: pixelCount) + raw.withUnsafeMutableBytes { (alpha: UnsafeMutableRawBufferPointer) in + bgra.withUnsafeBytes { (pixels: UnsafeRawBufferPointer) in + for i in 0.. Data in + var runs = Data() + var start = 0 + while start < pixelCount { + let value = alpha[start] + let limit = min(start + 255, pixelCount) + var end = start + 1 + while end < limit && alpha[end] == value { end += 1 } + if runs.count + 2 >= pixelCount { return raw } + runs.append(UInt8(end - start)) + runs.append(value) + start = end + } + return runs + } + } + + public static func image(bgra: Data, width: Int, height: Int, orientation: HardwareOrientation, + screen: HardwareScreen = .miucs) -> CGImage? { + bitmap(bgra: bgra, width: width, height: height, orientation: orientation, screen: screen, transparent: false)? + .makeImage() + } + private static func bitmap(bgra: Data, width: Int, height: Int, orientation: HardwareOrientation, + screen: HardwareScreen, transparent: Bool) -> CGContext? { + guard width > 0, height > 0, bgra.count == width * height * 4, + let provider = CGDataProvider(data: bgra as CFData) else { return nil } + let outputWidth = screen.card.map { Int($0.width) } ?? 720 + let outputHeight = screen.card.map { Int($0.height) } ?? 1280 + let space = CGColorSpaceCreateDeviceRGB() + let info = CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.premultipliedFirst.rawValue + guard let source = CGImage(width: width, height: height, bitsPerComponent: 8, bitsPerPixel: 32, + bytesPerRow: width * 4, space: space, bitmapInfo: CGBitmapInfo(rawValue: info), + provider: provider, decode: nil, shouldInterpolate: true, intent: .defaultIntent), + let context = CGContext(data: nil, width: outputWidth, height: outputHeight, bitsPerComponent: 8, + bytesPerRow: outputWidth * 4, space: space, bitmapInfo: info) else { return nil } + let bounds = CGRect(x: 0, y: 0, width: outputWidth, height: outputHeight) + if transparent { + context.clear(bounds) + } else { + context.setFillColor(CGColor(red: 0.965, green: 0.965, blue: 0.95, alpha: 1)) + context.fill(bounds) + } + context.translateBy(x: CGFloat(outputWidth) / 2, y: CGFloat(outputHeight) / 2) + if screen.card == nil { context.rotate(by: CGFloat(orientation.rawValue) * .pi / 2) } + let scale = screen.fitScale(width: Double(width), height: Double(height), orientation: orientation) + let w = Double(width) * scale, h = Double(height) * scale + context.interpolationQuality = .high + context.draw(source, in: CGRect(x: -w / 2, y: -h / 2, width: w, height: h)) + return context + } + public static func jpeg(bgra: Data, width: Int, height: Int, orientation: HardwareOrientation, + screen: HardwareScreen = .miucs) -> Data? { + guard let image = image(bgra: bgra, width: width, height: height, orientation: orientation, screen: screen) else { return nil } + return encodeJPEG(image, screen: screen) + } + private static func encodeJPEG(_ image: CGImage, screen: HardwareScreen) -> Data? { + let qualities = screen.card == nil ? [0.65, 0.50, 0.35] : [0.65, 0.50, 0.35, 0.20, 0.05] + for quality in qualities { + let bytes = NSMutableData() + guard let destination = CGImageDestinationCreateWithData(bytes, UTType.jpeg.identifier as CFString, 1, nil) else { return nil } + let properties: [CFString: Any] = [ + kCGImageDestinationLossyCompressionQuality: quality, + kCGImagePropertyJFIFDictionary: [kCGImagePropertyJFIFIsProgressive: false], + ] + CGImageDestinationAddImage(destination, image, properties as CFDictionary) + guard CGImageDestinationFinalize(destination) else { return nil } + if bytes.length <= (screen.card?.maxJPEG ?? maxJPEG) { return bytes as Data } + } + return nil + } +} diff --git a/modules/hardware/macos/HardwareIO.swift b/modules/hardware/macos/HardwareIO.swift new file mode 100644 index 0000000..8c24f1c --- /dev/null +++ b/modules/hardware/macos/HardwareIO.swift @@ -0,0 +1,45 @@ +import Darwin + +/// Darwin's poll(2) does not support device files reliably. Use select(2) for +/// the small sets of serial/socket descriptors used by the hardware transports. +public enum HardwareIO { + public static func wait(_ descriptors: inout [pollfd], milliseconds: Int32) -> Int32 { + guard milliseconds >= 0 else { errno = EINVAL; return -1 } + var reads = fd_set(), writes = fd_set(), errors = fd_set() + var highest: Int32 = -1 + for index in descriptors.indices { + descriptors[index].revents = 0 + let descriptor = descriptors[index] + guard descriptor.fd >= 0 else { continue } + guard descriptor.fd < FD_SETSIZE else { errno = EINVAL; return -1 } + highest = max(highest, descriptor.fd) + if descriptor.events & Int16(POLLIN) != 0 { insert(descriptor.fd, into: &reads) } + if descriptor.events & Int16(POLLOUT) != 0 { insert(descriptor.fd, into: &writes) } + insert(descriptor.fd, into: &errors) + } + var timeout = timeval(tv_sec: Int(milliseconds / 1000), tv_usec: Int32(milliseconds % 1000) * 1000) + let ready = select(highest + 1, &reads, &writes, &errors, &timeout) + guard ready > 0 else { return ready } + var count: Int32 = 0 + for index in descriptors.indices where descriptors[index].fd >= 0 { + let fd = descriptors[index].fd + if contains(fd, in: &reads) { descriptors[index].revents |= Int16(POLLIN) } + if contains(fd, in: &writes) { descriptors[index].revents |= Int16(POLLOUT) } + if contains(fd, in: &errors) { descriptors[index].revents |= Int16(POLLERR) } + if descriptors[index].revents != 0 { count += 1 } + } + return count + } + + // FD_SET/FD_ISSET are C macros. Darwin fd_set consists of 32-bit masks. + private static func insert(_ fd: Int32, into set: inout fd_set) { + withUnsafeMutableBytes(of: &set) { + $0.bindMemory(to: UInt32.self)[Int(fd) / 32] |= UInt32(1) << (UInt32(fd) % 32) + } + } + private static func contains(_ fd: Int32, in set: inout fd_set) -> Bool { + withUnsafeBytes(of: &set) { + $0.bindMemory(to: UInt32.self)[Int(fd) / 32] & (UInt32(1) << (UInt32(fd) % 32)) != 0 + } + } +} diff --git a/modules/hardware/macos/HardwarePetPack.swift b/modules/hardware/macos/HardwarePetPack.swift new file mode 100644 index 0000000..3dbf6cf --- /dev/null +++ b/modules/hardware/macos/HardwarePetPack.swift @@ -0,0 +1,132 @@ +import CoreGraphics +import CoreText +import Foundation +import ImageIO + +public enum HardwarePetPack { + public static let nativeIDs: Set = [ + "heiji-kazuha", "inuyasha", "muffin-puppies", "niulai-keyboard", + "pixel-keyboard", "yuru-camp", "zelda-guardian" + ] + + public static func supports(_ id: String, on capability: DeskIslePetCapability) -> Bool { + capability.version == 1 ? id == "yuru-camp" : nativeIDs.contains(id) + } + + public static func make(directory: URL, limit: Int) throws -> Data { + let fileKeys: [URLResourceKey] = [.isRegularFileKey, .isSymbolicLinkKey, .fileSizeKey] + guard let enumerator = FileManager.default.enumerator(at: directory, includingPropertiesForKeys: fileKeys, + options: [.skipsHiddenFiles]) else { + throw HardwareSerialError.message("无法读取桌宠皮肤") + } + var files: [(name: String, kind: UInt8, url: URL)] = [] + for case let url as URL in enumerator { + let values = try url.resourceValues(forKeys: Set(fileKeys)) + if values.isSymbolicLink == true { + enumerator.skipDescendants() + continue + } + guard values.isRegularFile == true else { continue } + let name = String(url.path.dropFirst(directory.path.count + 1)) + let kind: UInt8 + switch (name, url.pathExtension.lowercased()) { + case ("manifest.json", _): kind = 4 + case (_, "lua"): kind = 1 + case (_, "png") where name.hasPrefix("assets/"): kind = 2 + case (_, "ttf") where name.hasPrefix("assets/"): kind = 3 + default: continue + } + guard !name.isEmpty, name.utf8.count < 80, name.utf8.allSatisfy({ (32...126).contains($0) }), + files.count < 32, let size = values.fileSize, (1...12 * 1024 * 1024).contains(size) else { + throw HardwareSerialError.message("桌宠皮肤文件超过小屏限制") + } + files.append((name, kind, url)) + } + guard files.contains(where: { $0.kind == 4 }) else { + throw HardwareSerialError.message("找不到桌宠皮肤描述") + } + var pack = Data("PET1".utf8) + for file in files.sorted(by: { $0.name < $1.name }) { + let bytes: Data + switch file.kind { + case 2: bytes = try atlas(file.url) + case 3: bytes = try glyphs(file.url) + default: bytes = try Data(contentsOf: file.url) + } + guard pack.count + 8 + file.name.utf8.count + bytes.count <= limit else { + throw HardwareSerialError.message("桌宠皮肤包超过小屏限制") + } + pack.append(contentsOf: [file.kind, UInt8(file.name.utf8.count), 0, 0]) + withUnsafeBytes(of: UInt32(bytes.count).littleEndian) { pack.append(contentsOf: $0) } + pack.append(contentsOf: file.name.utf8) + pack.append(bytes) + } + return pack + } + + private static func atlas(_ url: URL) throws -> Data { + guard let source = CGImageSourceCreateWithURL(url as CFURL, nil), + let properties = CGImageSourceCopyPropertiesAtIndex(source, 0, nil) as? [CFString: Any], + let originalWidth = properties[kCGImagePropertyPixelWidth] as? Int, + let originalHeight = properties[kCGImagePropertyPixelHeight] as? Int, + (1...4096).contains(originalWidth), (1...4096).contains(originalHeight), + let image = CGImageSourceCreateImageAtIndex(source, 0, nil) else { + throw HardwareSerialError.message("无法读取桌宠图集") + } + let divisor = max(originalWidth, originalHeight) <= 256 ? 1 : 2 + let width = (originalWidth + divisor - 1) / divisor + let height = (originalHeight + divisor - 1) / divisor + var pixels = Data(count: width * height * 4) + let bitmapInfo = CGBitmapInfo.byteOrder32Big.rawValue | CGImageAlphaInfo.premultipliedLast.rawValue + let rendered = pixels.withUnsafeMutableBytes { buffer -> Bool in + guard let context = CGContext(data: buffer.baseAddress, width: width, height: height, + bitsPerComponent: 8, bytesPerRow: width * 4, + space: CGColorSpace(name: CGColorSpace.sRGB)!, + bitmapInfo: bitmapInfo) else { return false } + context.interpolationQuality = .high + context.setBlendMode(.copy) + context.draw(image, in: CGRect(x: 0, y: 0, width: width, height: height)) + return true + } + guard rendered else { throw HardwareSerialError.message("无法转换桌宠图集") } + var bytes = Data() + for value in [originalWidth, originalHeight, width, height] { + withUnsafeBytes(of: UInt16(value).littleEndian) { bytes.append(contentsOf: $0) } + } + bytes.append(pixels) + return bytes + } + + private static func glyphs(_ url: URL) throws -> Data { + guard let provider = CGDataProvider(url: url as CFURL), let graphicsFont = CGFont(provider) else { + throw HardwareSerialError.message("无法读取桌宠字体") + } + let font = CTFontCreateWithGraphicsFont(graphicsFont, 24, nil, nil) + let width = 32, height = 36 + var bytes = Data() + for ascii in 32...126 { + var character = UniChar(ascii), glyph: CGGlyph = 0 + guard CTFontGetGlyphsForCharacters(font, &character, &glyph, 1) else { + throw HardwareSerialError.message("桌宠字体缺少键帽字形") + } + var advance = CGSize.zero + CTFontGetAdvancesForGlyphs(font, .horizontal, &glyph, &advance, 1) + var pixels = Data(count: width * height) + let rendered = pixels.withUnsafeMutableBytes { buffer -> Bool in + guard let context = CGContext(data: buffer.baseAddress, width: width, height: height, + bitsPerComponent: 8, bytesPerRow: width, + space: CGColorSpaceCreateDeviceGray(), + bitmapInfo: CGImageAlphaInfo.none.rawValue) else { return false } + context.setFillColor(gray: 1, alpha: 1) + context.setShouldSmoothFonts(false) + var position = CGPoint(x: 2, y: 8) + CTFontDrawGlyphs(font, &glyph, &position, 1, context) + return true + } + guard rendered else { throw HardwareSerialError.message("无法转换桌宠字形") } + bytes.append(UInt8(clamping: Int(advance.width.rounded()))) + bytes.append(pixels) + } + return bytes + } +} diff --git a/modules/hardware/macos/HardwareSerial.swift b/modules/hardware/macos/HardwareSerial.swift new file mode 100644 index 0000000..65a4579 --- /dev/null +++ b/modules/hardware/macos/HardwareSerial.swift @@ -0,0 +1,270 @@ +import Foundation +import Darwin +import IOKit +import IOKit.serial + + +public enum HardwareSerialError: LocalizedError { + case message(String) + case cardChanged(DeskIsleCard) + case petRejected(String) + + public var errorDescription: String? { + switch self { + case .message(let text): text + case .cardChanged: "桌屿卡片规格已更新" + case .petRejected(let code): "小屏未能运行桌宠皮肤(\(code))" + } + } +} + +/// Owns the descriptor off the UI thread and serializes all writes. +public actor HardwareSerial { + private static let responseTimeout = 3.0 + private var descriptor: Int32 = -1 + private var input = Data() + private var screen: HardwareScreen? + private var alphaCardID: UInt16? + private var pet: DeskIslePetCapability? + private var pendingPetInput: (first: UInt32, last: UInt32, deadline: Double)? + + public init() {} + deinit { if descriptor >= 0 { Darwin.close(descriptor) } } + + public func close() { + if descriptor >= 0 { Darwin.close(descriptor) } + descriptor = -1; input.removeAll(keepingCapacity: true) + screen = nil + alphaCardID = nil + pet = nil + pendingPetInput = nil + } + + public func close(clearing expected: HardwareScreen?, sequence: UInt32) { + guard !Task.isCancelled else { return } + defer { close() } + if let expected, let pet = expected.card?.pet, pet.version >= 3 { + let packet = HardwareFrame.nativePet(.clear, payload: Data(), sequence: sequence, id: pet.id) + _ = try? send(packet, sequence: sequence, screen: expected, nativePet: true) + } + } + + @discardableResult + public func connect(path: String, unixSocket: Bool = false) throws -> HardwareScreen { + close() + do { + if unixSocket { + try connectRelay(path: path) + } else { + try connectSerial(path: path) + } + return try handshake() + } catch { close(); throw error } + } + + /// Takes ownership of a host-configured native CDC or in-process channel descriptor. + @discardableResult + public func connect(takingOwnershipOf fd: Int32) throws -> HardwareScreen { + close() + descriptor = fd + do { return try handshake() } catch { close(); throw error } + } + + private func handshake() throws -> HardwareScreen { + let deadline = ProcessInfo.processInfo.systemUptime + 3.5 + while ProcessInfo.processInfo.systemUptime < deadline { + let line = try readLine(deadline: deadline) + if line == "MIUCS_DISPLAY 1 720 1280 262144" { + screen = .miucs + return .miucs + } + if let card = cardAnnouncement(line) { + screen = .deskIsle(card) + return .deskIsle(card) + } + } + throw HardwareSerialError.message("没有收到屏幕握手,请检查固件和 USB 接口") + } + + private func connectSerial(path: String) throws { + descriptor = Darwin.open(path, O_RDWR | O_NOCTTY | O_NONBLOCK | O_CLOEXEC) + guard descriptor >= 0 else { throw HardwareSerialError.message("无法打开屏幕:\(String(cString: strerror(errno)))") } + guard ioctl(descriptor, TIOCEXCL) == 0 else { throw HardwareSerialError.message("屏幕端口正在被其他程序使用") } + var config = termios() + guard tcgetattr(descriptor, &config) == 0 else { throw HardwareSerialError.message("无法读取屏幕端口配置") } + cfmakeraw(&config) + config.c_cflag |= tcflag_t(CLOCAL | CREAD) + config.c_cflag &= ~tcflag_t(HUPCL) + cfsetspeed(&config, speed_t(B115200)) // CDC line coding; native USB transfer is not baud limited. + guard tcsetattr(descriptor, TCSANOW, &config) == 0 else { throw HardwareSerialError.message("无法设置屏幕端口") } + } + + private func connectRelay(path: String) throws { + let unavailable = HardwareSerialError.message("桌屿的 CH343 连接暂不可用") + var address = sockaddr_un() + let bytes = Array(path.utf8CString) + guard bytes.count <= MemoryLayout.size(ofValue: address.sun_path) else { throw unavailable } + let length = socklen_t(MemoryLayout.size) + address.sun_len = UInt8(length) + address.sun_family = sa_family_t(AF_UNIX) + withUnsafeMutableBytes(of: &address.sun_path) { destination in + bytes.withUnsafeBytes { destination.copyBytes(from: $0) } + } + descriptor = socket(AF_UNIX, SOCK_STREAM, 0) + guard descriptor >= 0 else { throw unavailable } + var enabled: Int32 = 1 + guard fcntl(descriptor, F_SETFD, FD_CLOEXEC) == 0, fcntl(descriptor, F_SETFL, O_NONBLOCK) == 0, + setsockopt(descriptor, SOL_SOCKET, SO_NOSIGPIPE, &enabled, socklen_t(MemoryLayout.size)) == 0 else { + throw unavailable + } + let result = withUnsafePointer(to: &address) { + $0.withMemoryRebound(to: sockaddr.self, capacity: 1) { Darwin.connect(descriptor, $0, length) } + } + var uid: uid_t = 0 + var gid: gid_t = 0 + guard result == 0, getpeereid(descriptor, &uid, &gid) == 0, uid == getuid() else { throw unavailable } + } + public func send(_ packet: Data, sequence: UInt32, screen expected: HardwareScreen = .miucs, + nativePet: Bool = false) throws -> Int { + guard descriptor >= 0, let screen else { throw HardwareSerialError.message("屏幕已断开") } + // This command starts a new operation. Ignore replies from the previous input stream. + pendingPetInput = nil + do { + if screen.card != nil { + // Announcements may arrive between frames, including while the card is idle. + try readInput() + while let line = popLine() { updateCard(from: line) } + } + try checkScreen(expected) + let deadline = ProcessInfo.processInfo.systemUptime + Self.responseTimeout + try writePacket(packet, deadline: deadline) + while true { + let line = try readLine(deadline: deadline) + updateCard(from: line) + let fields = line.split(separator: " ") + let prefix = screen.card == nil ? "MIUCS_ACK" : "DESKISLE_ACK" + guard fields.count == 4, fields[0] == prefix, UInt32(fields[1]) == sequence else { continue } + try checkScreen(expected) + if nativePet && fields[2] != "0" { throw HardwareSerialError.petRejected(String(fields[2])) } + guard fields[2] == "0", let ms = Int(fields[3]) else { throw HardwareSerialError.message("屏幕未能显示此帧(\(fields[2]))") } + return ms + } + } catch HardwareSerialError.cardChanged(let card) { + throw HardwareSerialError.cardChanged(card) + } catch HardwareSerialError.petRejected(let code) { + throw HardwareSerialError.petRejected(code) + } catch { close(); throw error } + } + + public func sendPetInput(_ packet: Data, sequence: UInt32, screen expected: HardwareScreen) throws { + guard descriptor >= 0 else { throw HardwareSerialError.message("屏幕已断开") } + do { + // Drain whatever has already arrived. A missing reply leaves the writer free to continue. + try readInput() + var rejection: String? + while let line = popLine() { + updateCard(from: line) + guard let pending = pendingPetInput, line.hasPrefix("DESKISLE_ACK ") else { continue } + let fields = line.split(separator: " ") + guard fields.count == 4, let ack = UInt32(fields[1]), Int(fields[3]) != nil, + ack &- pending.first <= pending.last &- pending.first else { continue } + if fields[2] != "0", rejection == nil { rejection = String(fields[2]) } + pendingPetInput = ack == pending.last ? nil : + (ack &+ 1, pending.last, ProcessInfo.processInfo.systemUptime + Self.responseTimeout) + } + try checkScreen(expected) + if let rejection { throw HardwareSerialError.petRejected(rejection) } + + let now = ProcessInfo.processInfo.systemUptime + if let pending = pendingPetInput, now >= pending.deadline { + throw HardwareSerialError.message("屏幕响应超时,正在重新连接") + } + try writePacket(packet, deadline: now + Self.responseTimeout) + if pendingPetInput == nil { + pendingPetInput = (sequence, sequence, now + Self.responseTimeout) + } else { + pendingPetInput?.last = sequence + } + } catch HardwareSerialError.cardChanged(let card) { + pendingPetInput = nil + throw HardwareSerialError.cardChanged(card) + } catch HardwareSerialError.petRejected(let code) { + pendingPetInput = nil + throw HardwareSerialError.petRejected(code) + } catch { close(); throw error } + } + + private func writePacket(_ packet: Data, deadline: Double) throws { + var offset = 0 + try packet.withUnsafeBytes { bytes in + while offset < bytes.count { + try wait(POLLOUT, deadline: deadline) + let n = Darwin.write(descriptor, bytes.baseAddress!.advanced(by: offset), bytes.count - offset) + if n > 0 { offset += n } + else if n < 0 && errno != EAGAIN && errno != EINTR { throw HardwareSerialError.message("USB 发送中断") } + } + } + } + + private func updateCard(from line: String) { + guard screen?.card != nil, let card = cardAnnouncement(line) else { return } + screen = .deskIsle(card) + } + private func cardAnnouncement(_ line: String) -> DeskIsleCard? { + if line.hasPrefix("DESKISLE_ALPHA ") { + let fields = line.split(separator: " ") + if fields.count == 3, fields[1] == "1", let id = UInt16(fields[2]), id > 0 { + alphaCardID = id + } + return nil + } + if let capability = DeskIslePetCapability(line) { + if capability.version >= (pet?.version ?? 0) { pet = capability } + return nil + } + return DeskIsleCard(announcement: line, alphaCardID: alphaCardID, pet: pet) + } + private func checkScreen(_ expected: HardwareScreen) throws { + guard screen != expected else { return } + if let card = screen?.card { throw HardwareSerialError.cardChanged(card) } + throw HardwareSerialError.message("屏幕协议已变化,正在重新连接") + } + private func wait(_ events: Int32, deadline: Double) throws { + while true { + try Task.checkCancellation() + guard descriptor >= 0 else { throw HardwareSerialError.message("屏幕已断开") } + guard ProcessInfo.processInfo.systemUptime < deadline else { throw HardwareSerialError.message("屏幕响应超时,正在重新连接") } + var descriptors = [pollfd(fd: descriptor, events: Int16(events), revents: 0)] + let result = HardwareIO.wait(&descriptors, milliseconds: 20) + if result < 0 && errno != EINTR { throw HardwareSerialError.message("USB 连接错误") } + if result > 0 { + if descriptors[0].revents & Int16(POLLERR | POLLHUP | POLLNVAL) != 0 { throw HardwareSerialError.message("屏幕已断开") } + if descriptors[0].revents & Int16(events) != 0 { return } + } + } + } + private func readLine(deadline: Double) throws -> String { + while true { + if let line = popLine() { return line } + try wait(POLLIN, deadline: deadline) + try readInput() + } + } + private func popLine() -> String? { + guard let end = input.firstIndex(of: 10) else { return nil } + let line = String(decoding: input[.. 0 { + input.append(contentsOf: buffer.prefix(n)) + if input.count > 8192 { throw HardwareSerialError.message("屏幕返回了无效数据") } + } else if n == 0 { throw HardwareSerialError.message("屏幕已断开") } + else if n < 0 && errno != EAGAIN && errno != EINTR { + throw HardwareSerialError.message("USB 接收中断") + } + } +} diff --git a/modules/hardware/macos/NativePetSession.swift b/modules/hardware/macos/NativePetSession.swift new file mode 100644 index 0000000..b2f1595 --- /dev/null +++ b/modules/hardware/macos/NativePetSession.swift @@ -0,0 +1,203 @@ +import Foundation +import Combine +import OSLog + +/// Native-only pet lifecycle. It never creates a desktop window, render timer or plugin worker. +@MainActor +public final class NativePetSession: ObservableObject { + @Published public private(set) var status = "未启用" + @Published public private(set) var active = false + private struct Configuration { + let skin: PetSkin + let audio: any AudioFeatureSource + let keyboard: any KeyboardFeatureSource + let transport: any NativePetTransport + } + private struct Identity { + let length: Int + let crc: UInt32 + var payload: Data { + var bytes = Data() + for value in [UInt32(length), crc] { + withUnsafeBytes(of: value.littleEndian) { bytes.append(contentsOf: $0) } + } + return bytes + } + } + private var desired: Configuration? + private var revision = 0 + private var task: Task? + private static let log = Logger(subsystem: "com.miucs.core", category: "native-pet") + + public init() {} + deinit { task?.cancel() } + + public func start(skin: PetSkin, audio: any AudioFeatureSource, keyboard: any KeyboardFeatureSource, + transport: any NativePetTransport) { + desired = Configuration(skin: skin, audio: audio, keyboard: keyboard, transport: transport) + revision &+= 1 + task?.cancel() + if task == nil { launch() } + } + + public func stop() { + desired = nil + revision &+= 1 + task?.cancel() + active = false + status = "未启用" + } + + /// Waits until in-flight work has relinquished the channel; useful for host shutdown/tests. + public func waitUntilStopped() async { await task?.value } + + private func launch() { + guard let configuration = desired, task == nil else { return } + let token = revision + task = Task { [weak self] in + var identity: Identity? + while !Task.isCancelled, token == self?.revision { + do { + self?.active = false + self?.status = "正在连接桌宠…" + let card = try await configuration.transport.connect() + try Task.checkCancellation() + guard let capability = card.pet, HardwarePetPack.supports(configuration.skin.id, on: capability) else { + throw HardwareSerialError.message("当前固件不支持这个皮肤") + } + self?.status = "正在同步皮肤 · \(configuration.skin.name)" + try await Self.synchronize(configuration, card: card, identity: &identity) { [weak self] percent in + guard self?.revision == token else { return } + self?.status = "正在同步皮肤 · \(configuration.skin.name) · \(percent)%" + } + try Task.checkCancellation() + self?.active = true + self?.status = "板端绘制 · \(configuration.skin.name)" + try await Self.stream(configuration, card: card) + } catch is CancellationError { + break + } catch { + if Task.isCancelled { break } + self?.active = false + self?.status = error.localizedDescription + Self.log.info("Pet connection will retry: \(error.localizedDescription, privacy: .public)") + await configuration.transport.close(clear: false) + do { try await Task.sleep(for: .seconds(2)) } catch { break } + } + } + // On a skin switch preserve the last board frame until its replacement is ready. + // Removing/disabling the component clears explicitly after cancellation has unwound. + let clear = self?.desired == nil + await Task.detached { await configuration.transport.close(clear: clear) }.value + guard let self else { return } + self.active = false + self.task = nil + if self.desired != nil { self.launch() } + } + } + + private static func synchronize(_ configuration: Configuration, card: DeskIsleCard, identity: inout Identity?, + progress: (Int) -> Void) async throws { + let capability = card.pet! + if let identity, try await resume(identity, card: card, transport: configuration.transport) { return } + let skin = configuration.skin + let pack = try await Task.detached(priority: .utility) { + try HardwarePetPack.make(directory: skin.directory, limit: capability.maxPack) + }.value + try Task.checkCancellation() + let next = Identity(length: pack.count, crc: HardwareFrame.crc32(pack)) + if identity == nil, try await resume(next, card: card, transport: configuration.transport) { + identity = next + return + } + try await configuration.transport.send(.begin, payload: next.payload, card: card) + // maxChunk is a limit, not a required size. Smaller commands leave room + // for other components on a shared UART and bound cancellation latency. + let chunkSize = min(capability.maxChunk, 32 * 1024) + var lastPercent = -1 + for offset in stride(from: 0, to: pack.count, by: chunkSize) { + try Task.checkCancellation() + var chunk = Data() + withUnsafeBytes(of: UInt32(offset).littleEndian) { chunk.append(contentsOf: $0) } + let end = min(offset + chunkSize, pack.count) + chunk.append(pack[offset.. Bool { + guard card.pet!.version >= 3 else { return false } + do { + try await transport.send(.resume, payload: identity.payload, card: card) + log.info("Screen resumed native Lua pet (\(identity.length) bytes)") + return true + } catch HardwareSerialError.petRejected("6") { return false } + } + + private final class WakeState { + var keyboard = true + var audio = true + } + + private static func stream(_ configuration: Configuration, card: DeskIsleCard) async throws { + let audio = configuration.audio.subscribe() + let keyboard = configuration.keyboard.subscribe() + let state = WakeState() + let (signals, continuation) = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + let keys = Task { + for await _ in keyboard.eventSignals() { + guard !Task.isCancelled else { return } + state.keyboard = true; continuation.yield() + } + } + let sound = Task { + for await _ in audio.activitySignals() { + guard !Task.isCancelled else { return } + state.audio = true; continuation.yield() + } + } + let keepalive = Task { + while !Task.isCancelled { + continuation.yield() + do { try await Task.sleep(for: .seconds(card.keepaliveInterval)) } catch { return } + } + } + var audioWake: Task? + defer { + keys.cancel(); sound.cancel(); keepalive.cancel(); audioWake?.cancel() + continuation.finish() + } + var lastSent = -Double.infinity + for await _ in signals { + try Task.checkCancellation() + let now = ProcessInfo.processInfo.systemUptime + let interval = card.pet!.version == 1 || state.audio ? 1.0 / 30 : card.keepaliveInterval + let delay = lastSent + interval - now + if !state.keyboard, delay > 0 { + if state.audio, audioWake == nil { + audioWake = Task { + do { try await Task.sleep(for: .seconds(delay)) } catch { return } + continuation.yield() + } + } + continue + } + audioWake?.cancel(); audioWake = nil + state.keyboard = false; state.audio = false + let payload = try PetInputSnapshot.encode(audio: audio.read(), keyboard: keyboard.read(), limit: card.pet!.maxInput) + do { try await configuration.transport.send(.input, payload: payload, card: card) } + catch HardwareSerialError.petRejected("1") { /* Next snapshot refreshes state after a corrupt packet. */ } + lastSent = now + } + try Task.checkCancellation() + } +} diff --git a/modules/hardware/macos/NativePetTransport.swift b/modules/hardware/macos/NativePetTransport.swift new file mode 100644 index 0000000..f362647 --- /dev/null +++ b/modules/hardware/macos/NativePetTransport.swift @@ -0,0 +1,71 @@ +import Foundation + +/// The embedding host owns discovery and the USB connection. MiucsCore only consumes a logical channel. +public protocol NativePetTransport: Sendable { + func connect() async throws -> DeskIsleCard + func send(_ kind: DeskIslePetMessage, payload: Data, card: DeskIsleCard) async throws + func close(clear: Bool) async +} + +public actor SerialPetTransport: NativePetTransport { + private let serial = HardwareSerial() + private let open: @Sendable () async throws -> Int32 + private var sequence = UInt32.random(in: 0...UInt32.max) + private var screen: HardwareScreen? + + public init(open: @escaping @Sendable () async throws -> Int32) { self.open = open } + + public func connect() async throws -> DeskIsleCard { + let fd = try await open() + let connected = try await serial.connect(takingOwnershipOf: fd) + guard let card = connected.card, card.pet != nil else { + await serial.close() + throw HardwareSerialError.message("小屏未提供板端桌宠,请更新固件") + } + screen = connected + return card + } + + public func send(_ kind: DeskIslePetMessage, payload: Data, card: DeskIsleCard) async throws { + guard let capability = card.pet else { throw HardwareSerialError.message("小屏不支持板端桌宠") } + sequence &+= 1 + let sequence = sequence + let packet = HardwareFrame.nativePet(kind, payload: payload, sequence: sequence, id: capability.id) + if kind == .input, capability.version >= 2 { + try await serial.sendPetInput(packet, sequence: sequence, screen: .deskIsle(card)) + } else { + _ = try await serial.send(packet, sequence: sequence, screen: .deskIsle(card), nativePet: true) + } + } + + public func close(clear: Bool) async { + let current = screen + screen = nil + if clear, let current { + sequence &+= 1 + await serial.close(clearing: current, sequence: sequence) + } else { await serial.close() } + } +} + +public enum PetInputSnapshot { + public static func encode(audio: FeatureInput?, keyboard: KeyboardInput?, limit: Int, + now: Double = ProcessInfo.processInfo.systemUptime) throws -> Data { + var sound = audio.map(PluginInput.audio) ?? [:] + var keys = keyboard.map(PluginInput.keyboard) ?? [:] + func encode() throws -> Data { + try JSONSerialization.data(withJSONObject: ["now": now, "audio": sound, "keyboard": keys]) + } + var payload = try encode() + if payload.count > limit { + keys["droppedEventCount"] = (keys["droppedEventCount"] as? UInt64 ?? 0) + + UInt64((keys["events"] as? [Any])?.count ?? 0) + sound["droppedBeatCount"] = (sound["beats"] as? [Any])?.count ?? 0 + sound["droppedOnsetCount"] = (sound["onsets"] as? [Any])?.count ?? 0 + keys["events"] = []; sound["onsets"] = []; sound["beats"] = [] + payload = try encode() + } + guard payload.count <= limit else { throw HardwareSerialError.message("桌宠输入超过小屏限制") } + return payload + } +} diff --git a/modules/hardware/macos/PetCatalog.swift b/modules/hardware/macos/PetCatalog.swift new file mode 100644 index 0000000..3bfb7cb --- /dev/null +++ b/modules/hardware/macos/PetCatalog.swift @@ -0,0 +1,74 @@ +import Foundation + +public struct PetSkin: Identifiable, Equatable, Sendable { + public let id: String + public let name: String + public let directory: URL + public let inputs: Set + public let preview: URL? +} + +/// Reads manifests and still previews without loading Lua, Skia or a worker process. +public enum PetCatalog { + public static var bundledDirectory: URL { + #if SWIFT_PACKAGE + // Hosts copy the SwiftPM resource bundle into Contents/Resources when making an .app. + if let resources = Bundle.main.resourceURL { + let installed = resources.appendingPathComponent("Miucs_MiucsCore.bundle/plugins") + if FileManager.default.fileExists(atPath: installed.path) { return installed } + } + return Bundle.module.bundleURL.appendingPathComponent("plugins") + #else + return (Bundle.main.resourceURL ?? URL(fileURLWithPath: ".")).appendingPathComponent("Plugins") + #endif + } + + public static var userDirectory: URL { + FileManager.default.homeDirectoryForCurrentUser.appendingPathComponent("Library/Application Support/Miucs/Plugins") + } + + public static func scan(builtin: URL = bundledDirectory, user: URL? = userDirectory) -> [PetSkin] { + var skins = [String: PetSkin]() + for root in [builtin, user].compactMap({ $0 }) { + let folders = (try? FileManager.default.contentsOfDirectory(at: root, includingPropertiesForKeys: nil, + options: [.skipsHiddenFiles])) ?? [] + for folder in folders.sorted(by: { $0.path < $1.path }) { + if let skin = read(folder) { skins[skin.id] = skin } + } + } + return skins.values.sorted { + if ($0.id == "pixel-keyboard") != ($1.id == "pixel-keyboard") { return $0.id == "pixel-keyboard" } + return $0.id < $1.id + } + } + + private static func read(_ folder: URL) -> PetSkin? { + let fm = FileManager.default + guard let manifest = safeFile("manifest.json", in: folder) else { return nil } + guard let values = try? folder.resourceValues(forKeys: [.isDirectoryKey, .isSymbolicLinkKey]), + values.isDirectory == true, values.isSymbolicLink != true, + let size = try? manifest.resourceValues(forKeys: [.fileSizeKey]).fileSize, size <= 65_536, + let data = try? Data(contentsOf: manifest), + let json = try? JSONSerialization.jsonObject(with: data) as? [String: Any], + let api = json["api"] as? Int, api == 1, + let id = json["id"] as? String, HardwarePetPack.nativeIDs.contains(id), + let name = json["name"] as? String, !name.isEmpty, name.count <= 128, + let entry = json["entry"] as? String, + let inputs = json["inputs"] as? [String], Set(inputs).isSubset(of: ["audio", "keyboard"]), + safeFile(entry, in: folder) != nil else { return nil } + let preview = ["preview.png", "preview.jpg", "preview.jpeg"].compactMap { safeFile($0, in: folder) }.first + guard fm.fileExists(atPath: manifest.path) else { return nil } + return PetSkin(id: id, name: name, directory: folder, inputs: Set(inputs), preview: preview) + } + + private static func safeFile(_ name: String, in folder: URL) -> URL? { + guard !name.isEmpty, !name.hasPrefix("/"), !name.split(separator: "/").contains("..") else { return nil } + let url = folder.appendingPathComponent(name) + let resolved = url.resolvingSymlinksInPath().standardizedFileURL + let root = folder.resolvingSymlinksInPath().standardizedFileURL.path + "/" + guard resolved.path.hasPrefix(root), + let values = try? url.resourceValues(forKeys: [.isRegularFileKey, .isSymbolicLinkKey]), + values.isRegularFile == true, values.isSymbolicLink != true else { return nil } + return url + } +} diff --git a/modules/hardware/macos/PluginInput.swift b/modules/hardware/macos/PluginInput.swift new file mode 100644 index 0000000..7576232 --- /dev/null +++ b/modules/hardware/macos/PluginInput.swift @@ -0,0 +1,58 @@ +import Foundation + +public enum PluginInput { + public static func audio(_ input: FeatureInput) -> [String: Any] { + var result: [String: Any] = ["version": 1, "sessionID": input.sessionID.uuidString, "state": input.state.rawValue, + "frameAge": input.frameAge as Any? ?? NSNull(), "spectralFluxPeak": input.spectralFluxPeak, + "onsets": input.onsets.map { ["sequence": $0.sequence, "time": $0.time, "receivedAt": $0.receivedAt, "strength": $0.strength] as [String: Any] }, + "beats": (input.beats ?? []).map { ["sequence": $0.sequence, "time": $0.time, "receivedAt": $0.receivedAt, "strength": $0.strength, "bpm": $0.bpm, "confidence": $0.confidence] as [String: Any] }] + if let frame = input.frame { + let a = frame.audio + var values: [String: Any] = [ + "time": a.time, + "sampleRate": a.sampleRate, + "channelCount": a.channelCount, + "rms": a.rms, + "rmsDBFS": a.rmsDBFS, + "peak": a.peak, + "crestDB": a.crestDB as Any? ?? NSNull(), + "envelope": a.envelope, + "isSilent": a.isSilent, + "silenceDuration": a.silenceDuration, + "lowEnergy": a.lowEnergy, + "midEnergy": a.midEnergy, + "highEnergy": a.highEnergy, + "lowShare": a.lowShare as Any? ?? NSNull(), + "midShare": a.midShare as Any? ?? NSNull(), + "highShare": a.highShare as Any? ?? NSNull(), + "centroidHz": a.centroidHz as Any? ?? NSNull(), + "bandwidthHz": a.bandwidthHz as Any? ?? NSNull(), + "flatness": a.flatness as Any? ?? NSNull(), + "spectralFlux": a.spectralFlux, + "onsetThreshold": a.onsetThreshold, + "isOnset": a.isOnset, + "balance": a.balance as Any? ?? NSNull(), + "sideEnergyRatio": a.sideEnergyRatio as Any? ?? NSNull(), + "correlation": a.correlation as Any? ?? NSNull(), + "displayBands": a.displayBands + ] + if let b = a.beat { values["beat"] = ["time": b.time, "bpm": b.bpm as Any? ?? NSNull(), "phase": b.phase as Any? ?? NSNull(), "confidence": b.confidence, "strength": b.strength, "beatTime": b.beatTime as Any? ?? NSNull()] } + var f: [String: Any] = ["version": 1, "sessionID": frame.sessionID.uuidString, "sequence": frame.sequence, "receivedAt": frame.receivedAt, "audio": values] + if let m = frame.melody { f["melody"] = ["time": m.time, "pitchHz": m.pitchHz as Any? ?? NSNull(), "confidence": m.confidence, "isMelodic": m.isMelodic] } + result["frame"] = f + } + return result + } + public static func keyboard(_ input: KeyboardInput) -> [String: Any] { + ["version": 1, "sessionID": input.sessionID.uuidString, "sequence": input.sequence, + "state": input.state.rawValue, "sampledAt": input.sampledAt, "modifiers": input.modifiers.rawValue, + "pressesPerSecond": input.pressesPerSecond, "idleDuration": input.idleDuration as Any? ?? NSNull(), + "droppedEventCount": input.droppedEventCount, + "heldKeys": input.heldKeys.map { ["key": MacPhysicalKeys.names[$0.keyCode] as Any? ?? NSNull(), "duration": $0.duration] as [String: Any] }, + "events": input.events.map { e -> [String: Any] in + ["sequence": e.sequence, "receivedAt": e.receivedAt, "kind": e.kind.rawValue, + "key": MacPhysicalKeys.names[e.keyCode] as Any? ?? NSNull(), "modifiers": e.modifiers.rawValue, + "heldDuration": e.heldDuration as Any? ?? NSNull(), "modifierIsPressed": e.modifierIsPressed as Any? ?? NSNull()] + }] + } +} diff --git a/modules/keyboard/core/KeyboardChannel.swift b/modules/keyboard/core/KeyboardChannel.swift index f4e4289..a25e3b3 100644 --- a/modules/keyboard/core/KeyboardChannel.swift +++ b/modules/keyboard/core/KeyboardChannel.swift @@ -1,6 +1,6 @@ import Foundation -final class KeyboardSubscription: @unchecked Sendable { +public final class KeyboardSubscription: @unchecked Sendable { private let channel: KeyboardChannel private let lock = NSLock() private var sessionID: UUID @@ -10,19 +10,21 @@ final class KeyboardSubscription: @unchecked Sendable { self.channel = channel; self.sessionID = sessionID; self.sequence = sequence } - func read() -> KeyboardInput { + public func read() -> KeyboardInput { lock.lock(); defer { lock.unlock() } let input = channel.read(sessionID: sessionID, after: sequence) sessionID = input.sessionID; sequence = input.sequence return input } + + public func eventSignals() -> AsyncStream { channel.eventSignals() } } -/// Bounded in-memory handoff. Capture never calls UI code or queues a task per key. -final class KeyboardChannel: KeyboardFeatureSource, @unchecked Sendable { - static let eventCapacity = 128 - static let eventLifetime = 0.250 - static let rateCapacity = 256 +/// Bounded in-memory handoff. A signal wakes the reader, and the channel preserves event order. +public final class KeyboardChannel: KeyboardFeatureSource, @unchecked Sendable { + public static let eventCapacity = 128 + public static let eventLifetime = 0.250 + public static let rateCapacity = 256 private let lock = NSLock() private let now: @Sendable () -> Double private var sessionID = UUID() @@ -33,31 +35,41 @@ final class KeyboardChannel: KeyboardFeatureSource, @unchecked Sendable { private var presses = [Double]() private var events = [KeyboardInputEvent]() private var lastActivity: Double? + private var eventObservers = [UUID: AsyncStream.Continuation]() - init(now: @escaping @Sendable () -> Double = { ProcessInfo.processInfo.systemUptime }) { + public init(now: @escaping @Sendable () -> Double = { ProcessInfo.processInfo.systemUptime }) { self.now = now } - func reset(sessionID: UUID, state: KeyboardSourceState) { - lock.lock(); defer { lock.unlock() } + public func reset(sessionID: UUID, state: KeyboardSourceState) { + lock.lock() self.sessionID = sessionID; self.state = state; sequence = 0 modifiers = []; held.removeAll(keepingCapacity: true) presses.removeAll(keepingCapacity: true); events.removeAll(keepingCapacity: true) lastActivity = nil + unlockAndSignalEvents() } - func setRunning(sessionID: UUID) { - lock.lock(); defer { lock.unlock() } - if self.sessionID == sessionID { state = .running } + public func setRunning(sessionID: UUID) { + lock.lock() + guard self.sessionID == sessionID, state != .running else { lock.unlock(); return } + state = .running + unlockAndSignalEvents() } @discardableResult - func publish(_ event: KeyboardRawEvent, sessionID: UUID) -> Bool { - lock.lock(); defer { lock.unlock() } - guard self.sessionID == sessionID, state == .running else { return false } + public func publish(_ event: KeyboardRawEvent, sessionID: UUID) -> Bool { + lock.lock() + guard self.sessionID == sessionID, state == .running else { + lock.unlock() + return false + } // Sample under the lock, so concurrent callbacks cannot reorder receipt time. let time = now() - guard time.isFinite else { return false } + guard time.isFinite else { + lock.unlock() + return false + } prune(at: time) var kind = event.kind var duration: Double? @@ -82,14 +94,38 @@ final class KeyboardChannel: KeyboardFeatureSource, @unchecked Sendable { keyCode: event.keyCode, modifiers: event.modifiers, heldDuration: duration, modifierIsPressed: event.modifierIsPressed)) if events.count > Self.eventCapacity { events.removeFirst(events.count - Self.eventCapacity) } + unlockAndSignalEvents() return true } - func subscribe() -> KeyboardSubscription { + private func unlockAndSignalEvents() { + let observers = eventObservers.values + lock.unlock() + for observer in observers { observer.yield() } + } + + public func subscribe() -> KeyboardSubscription { lock.lock(); defer { lock.unlock() } return KeyboardSubscription(channel: self, sessionID: sessionID, sequence: sequence) } + fileprivate func eventSignals() -> AsyncStream { + let id = UUID() + let (stream, continuation) = AsyncStream.makeStream(bufferingPolicy: .bufferingNewest(1)) + continuation.onTermination = { [weak self] _ in self?.removeEventObserver(id) } + lock.lock() + eventObservers[id] = continuation + lock.unlock() + // Include events that arrived between subscribing and registering for wakeups. + continuation.yield() + return stream + } + + private func removeEventObserver(_ id: UUID) { + lock.lock(); defer { lock.unlock() } + eventObservers.removeValue(forKey: id) + } + fileprivate func read(sessionID: UUID, after cursor: UInt64) -> KeyboardInput { lock.lock(); defer { lock.unlock() } let time = now() diff --git a/modules/keyboard/core/KeyboardInput.swift b/modules/keyboard/core/KeyboardInput.swift index 9112ce4..dff4b3b 100644 --- a/modules/keyboard/core/KeyboardInput.swift +++ b/modules/keyboard/core/KeyboardInput.swift @@ -1,65 +1,66 @@ import Foundation -enum KeyboardSourceState: String, Codable, Sendable { +public enum KeyboardSourceState: String, Codable, Sendable { case stopped, connecting, running, permissionRequired, suspended, secureInput, recovering } /// Portable modifier bits; keyCode is the platform's physical key identifier (macOS virtual key code). -struct KeyboardModifiers: OptionSet, Codable, Sendable, Equatable { - let rawValue: UInt8 - static let shift = Self(rawValue: 1 << 0) - static let control = Self(rawValue: 1 << 1) - static let option = Self(rawValue: 1 << 2) - static let command = Self(rawValue: 1 << 3) - static let capsLock = Self(rawValue: 1 << 4) - static let function = Self(rawValue: 1 << 5) +public struct KeyboardModifiers: OptionSet, Codable, Sendable, Equatable { + public let rawValue: UInt8 + public init(rawValue: UInt8) { self.rawValue = rawValue } + public static let shift = Self(rawValue: 1 << 0) + public static let control = Self(rawValue: 1 << 1) + public static let option = Self(rawValue: 1 << 2) + public static let command = Self(rawValue: 1 << 3) + public static let capsLock = Self(rawValue: 1 << 4) + public static let function = Self(rawValue: 1 << 5) } -enum KeyboardEventKind: String, Codable, Sendable { +public enum KeyboardEventKind: String, Codable, Sendable { case pressed, released, repeated, modifiersChanged } /// No characters, text composition, application identity or clipboard content crosses this boundary. -struct KeyboardRawEvent: Sendable { - let kind: KeyboardEventKind - let keyCode: UInt16 - let modifiers: KeyboardModifiers +public struct KeyboardRawEvent: Sendable { + public let kind: KeyboardEventKind + public let keyCode: UInt16 + public let modifiers: KeyboardModifiers /// Physical modifier transition when the platform can resolve its side; Caps Lock is its lock state. - var modifierIsPressed: Bool? = nil + public var modifierIsPressed: Bool? = nil } -struct KeyboardInputEvent: Codable, Sendable { - let sequence: UInt64 - let receivedAt: Double - let kind: KeyboardEventKind - let keyCode: UInt16 - let modifiers: KeyboardModifiers +public struct KeyboardInputEvent: Codable, Sendable { + public let sequence: UInt64 + public let receivedAt: Double + public let kind: KeyboardEventKind + public let keyCode: UInt16 + public let modifiers: KeyboardModifiers /// Known only when this channel observed the corresponding press. - let heldDuration: Double? - var modifierIsPressed: Bool? = nil + public let heldDuration: Double? + public var modifierIsPressed: Bool? = nil } -struct KeyboardHeldKey: Codable, Sendable { - let keyCode: UInt16 - let duration: Double +public struct KeyboardHeldKey: Codable, Sendable { + public let keyCode: UInt16 + public let duration: Double } -struct KeyboardInput: Codable, Sendable { - var version = 1 - let sessionID: UUID - let sequence: UInt64 - let state: KeyboardSourceState - let sampledAt: Double - let modifiers: KeyboardModifiers - let heldKeys: [KeyboardHeldKey] +public struct KeyboardInput: Codable, Sendable { + public var version = 1 + public let sessionID: UUID + public let sequence: UInt64 + public let state: KeyboardSourceState + public let sampledAt: Double + public let modifiers: KeyboardModifiers + public let heldKeys: [KeyboardHeldKey] /// Non-repeat, non-modifier presses in the last second, capped at the channel's rate capacity. - let pressesPerSecond: Double - let idleDuration: Double? - let events: [KeyboardInputEvent] + public let pressesPerSecond: Double + public let idleDuration: Double? + public let events: [KeyboardInputEvent] /// Number of unread events lost to age/capacity for this reader, never replayed later. - let droppedEventCount: UInt64 + public let droppedEventCount: UInt64 } -protocol KeyboardFeatureSource: Sendable { +public protocol KeyboardFeatureSource: Sendable { func subscribe() -> KeyboardSubscription } diff --git a/modules/keyboard/platforms/macos/KeyboardEngine.swift b/modules/keyboard/platforms/macos/KeyboardEngine.swift index a50bceb..2ea9e3d 100644 --- a/modules/keyboard/platforms/macos/KeyboardEngine.swift +++ b/modules/keyboard/platforms/macos/KeyboardEngine.swift @@ -20,9 +20,9 @@ typealias KeyboardCaptureFactory = @Sendable ( /// Keyboard lifetime is independent of audio capture and of the currently visible module. @MainActor -final class KeyboardEngine: ObservableObject { - let channel: KeyboardChannel - @Published private(set) var state = KeyboardSourceState.stopped +public final class KeyboardEngine: ObservableObject { + public let channel: KeyboardChannel + @Published public private(set) var state = KeyboardSourceState.stopped private let factory: KeyboardCaptureFactory private let environment: @MainActor () async -> KeyboardEnvironment private let requestPermission: @MainActor () -> Bool @@ -42,6 +42,8 @@ final class KeyboardEngine: ObservableObject { private var failures = 0 private var retryAt = 0.0 + public convenience init() { self.init(monitorsSystem: true) } + init(factory: @escaping KeyboardCaptureFactory = { KeyboardEventTap(onEvent: $0, onInterruption: $1) }, environment: @escaping @MainActor () async -> KeyboardEnvironment = { await KeyboardEnvironment.current() }, requestPermission: @escaping @MainActor () -> Bool = { CGRequestListenEventAccess() }, @@ -54,7 +56,7 @@ final class KeyboardEngine: ObservableObject { deinit { polling?.cancel(); capture?.stop() } - func start(requestAccess: Bool = false) async { + public func start(requestAccess: Bool = false) async { if !wanted { wanted = true; lifecycle = UUID(); installObservers() polling = Task { [weak self, pollInterval] in @@ -68,7 +70,7 @@ final class KeyboardEngine: ObservableObject { await refresh(requestAccess: requestAccess) } - func stop() { + public func stop() { wanted = false; lifecycle = UUID() checking = nil; pendingPermissionRequest = false polling?.cancel(); polling = nil; observers.removeAll() diff --git a/modules/plugins/platforms/macos/PluginMessagePack.swift b/modules/plugins/platforms/macos/PluginMessagePack.swift index 0c447da..6087e3e 100644 --- a/modules/plugins/platforms/macos/PluginMessagePack.swift +++ b/modules/plugins/platforms/macos/PluginMessagePack.swift @@ -32,61 +32,3 @@ enum PluginMessagePack { append(value); return data } } - -@MainActor -enum PluginInput { - static func audio(_ input: FeatureInput) -> [String: Any] { - var result: [String: Any] = ["version": 1, "sessionID": input.sessionID.uuidString, "state": input.state.rawValue, - "frameAge": input.frameAge as Any? ?? NSNull(), "spectralFluxPeak": input.spectralFluxPeak, - "onsets": input.onsets.map { ["sequence": $0.sequence, "time": $0.time, "receivedAt": $0.receivedAt, "strength": $0.strength] as [String: Any] }, - "beats": (input.beats ?? []).map { ["sequence": $0.sequence, "time": $0.time, "receivedAt": $0.receivedAt, "strength": $0.strength, "bpm": $0.bpm, "confidence": $0.confidence] as [String: Any] }] - if let frame = input.frame { - let a = frame.audio - var values: [String: Any] = [ - "time": a.time, - "sampleRate": a.sampleRate, - "channelCount": a.channelCount, - "rms": a.rms, - "rmsDBFS": a.rmsDBFS, - "peak": a.peak, - "crestDB": a.crestDB as Any? ?? NSNull(), - "envelope": a.envelope, - "isSilent": a.isSilent, - "silenceDuration": a.silenceDuration, - "lowEnergy": a.lowEnergy, - "midEnergy": a.midEnergy, - "highEnergy": a.highEnergy, - "lowShare": a.lowShare as Any? ?? NSNull(), - "midShare": a.midShare as Any? ?? NSNull(), - "highShare": a.highShare as Any? ?? NSNull(), - "centroidHz": a.centroidHz as Any? ?? NSNull(), - "bandwidthHz": a.bandwidthHz as Any? ?? NSNull(), - "flatness": a.flatness as Any? ?? NSNull(), - "spectralFlux": a.spectralFlux, - "onsetThreshold": a.onsetThreshold, - "isOnset": a.isOnset, - "balance": a.balance as Any? ?? NSNull(), - "sideEnergyRatio": a.sideEnergyRatio as Any? ?? NSNull(), - "correlation": a.correlation as Any? ?? NSNull(), - "displayBands": a.displayBands - ] - if let b = a.beat { values["beat"] = ["time": b.time, "bpm": b.bpm as Any? ?? NSNull(), "phase": b.phase as Any? ?? NSNull(), "confidence": b.confidence, "strength": b.strength, "beatTime": b.beatTime as Any? ?? NSNull()] } - var f: [String: Any] = ["version": 1, "sessionID": frame.sessionID.uuidString, "sequence": frame.sequence, "receivedAt": frame.receivedAt, "audio": values] - if let m = frame.melody { f["melody"] = ["time": m.time, "pitchHz": m.pitchHz as Any? ?? NSNull(), "confidence": m.confidence, "isMelodic": m.isMelodic] } - result["frame"] = f - } - return result - } - static func keyboard(_ input: KeyboardInput) -> [String: Any] { - ["version": 1, "sessionID": input.sessionID.uuidString, "sequence": input.sequence, - "state": input.state.rawValue, "sampledAt": input.sampledAt, "modifiers": input.modifiers.rawValue, - "pressesPerSecond": input.pressesPerSecond, "idleDuration": input.idleDuration as Any? ?? NSNull(), - "droppedEventCount": input.droppedEventCount, - "heldKeys": input.heldKeys.map { ["key": MacPhysicalKeys.names[$0.keyCode] as Any? ?? NSNull(), "duration": $0.duration] as [String: Any] }, - "events": input.events.map { e -> [String: Any] in - ["sequence": e.sequence, "receivedAt": e.receivedAt, "kind": e.kind.rawValue, - "key": MacPhysicalKeys.names[e.keyCode] as Any? ?? NSNull(), "modifiers": e.modifiers.rawValue, - "heldDuration": e.heldDuration as Any? ?? NSNull(), "modifierIsPressed": e.modifierIsPressed as Any? ?? NSNull()] - }] - } -} diff --git a/modules/plugins/platforms/macos/PluginSession.swift b/modules/plugins/platforms/macos/PluginSession.swift index 708a655..d2c8ee7 100644 --- a/modules/plugins/platforms/macos/PluginSession.swift +++ b/modules/plugins/platforms/macos/PluginSession.swift @@ -1,3 +1,6 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif import AppKit import QuartzCore import Combine @@ -8,6 +11,9 @@ final class PluginPackage { let id: String var handle: UnsafeMutableRawPointer? var activeID: String? + var hardwareActive = false + var needsUpdates: Bool { activeID != nil || hardwareActive } + private var exportSurface: UnsafeMutableRawPointer? private var viewport: [String:Any] = [:] var appearance: [String: Any] = ["ink": "#3d3d3d"] var reloadError = "" @@ -27,6 +33,20 @@ final class PluginPackage { let bytes = PluginMessagePack.encode(["op": op, "value": value]) _ = bytes.withUnsafeBytes { api.post?(handle,$0.bindMemory(to: UInt8.self).baseAddress!,bytes.count) } } + func pixels(width: Int, height: Int) -> Data? { + guard let handle else { return nil } + if exportSurface == nil { exportSurface = id.withCString { api.createSurface?(handle, nil, $0) } } + guard let exportSurface else { return nil } + var result = Data(count: width * height * 4) + let valid = result.withUnsafeMutableBytes { + api.pixels?(exportSurface, UInt32(width), UInt32(height), $0.bindMemory(to: UInt8.self).baseAddress!, $0.count) == 1 + } + return valid ? result : nil + } + private func releaseExportSurface() { + if let exportSurface { api.destroySurface?(exportSurface) } + exportSurface = nil + } func checkFiles() { let url = URL(fileURLWithPath: directory) var files=[URL]() @@ -51,7 +71,7 @@ final class PluginPackage { let bytes = PluginMessagePack.encode(["op": op, "value": value]) _ = bytes.withUnsafeBytes { api.post?(candidate,$0.bindMemory(to: UInt8.self).baseAddress!,bytes.count) } } - send(activeID == nil ? "deactivate" : "activate", viewport) + send(needsUpdates ? "activate" : "deactivate", viewport) send("appearanceChanged", appearance) send("update", ["now":reloadStarted,"deltaTime":0]) } @@ -72,12 +92,14 @@ final class PluginPackage { if !valid { reloadError=api.failure(nil);api.close?(candidate);self.candidate=nil;return } } for view in views.allObjects { view.releaseSurface() } + releaseExportSurface() let previous=handle;handle=candidate;self.candidate=nil if let previous { api.close?(previous) } } func close() { if let candidate { api.close?(candidate) };candidate=nil for view in views.allObjects { view.releaseSurface() } + releaseExportSurface() if let handle { api.close?(handle) }; handle = nil } } @@ -94,7 +116,7 @@ final class PluginHost: ObservableObject { private var keyboard: KeyboardSubscription? private var timer: Timer? private var last = 0.0 - private var suspended = false + private(set) var suspended = false private var suspensionReasons = Set() private var registeredIDs = Set() private var observers = [NSObjectProtocol]() @@ -131,6 +153,9 @@ final class PluginHost: ObservableObject { observers.append(NSWorkspace.shared.notificationCenter.addObserver(forName: name, object: nil, queue: .main) { [weak self] _ in MainActor.assumeIsolated { self?.setSuspended(false, reason: name == NSWorkspace.didWakeNotification ? "sleep" : "session") } }) } } + func hardwareSubscriptions() -> (FeatureSubscription?, KeyboardSubscription?) { + (audioSource?.subscribe(), keyboardSource?.subscribe()) + } private func setSuspended(_ value: Bool, reason: String) { if value { suspensionReasons.insert(reason) } else { suspensionReasons.remove(reason) } suspended = !suspensionReasons.isEmpty;updateSchedule(); last = ProcessInfo.processInfo.systemUptime @@ -140,7 +165,7 @@ final class PluginHost: ObservableObject { } } func updateSchedule() { - let needed = !suspended && packages.values.contains { $0.activeID != nil || $0.isReloading } + let needed = !suspended && packages.values.contains { $0.needsUpdates || $0.isReloading } guard needed else { timer?.invalidate();timer=nil;return } guard timer == nil else { return } last=ProcessInfo.processInfo.systemUptime @@ -163,7 +188,7 @@ final class PluginHost: ObservableObject { package.pollReload() if !package.reloadError.isEmpty, !errors.contains(package.reloadError) { errors.append(package.reloadError) } } - for package in packages.values where package.activeID != nil { + for package in packages.values where package.needsUpdates { package.post("update",frame) let error = PluginNative.shared.failure(package.handle) if !error.isEmpty, !errors.contains(error) { errors.append(error); fputs("Plugin: \(error)\n",stderr) } @@ -171,7 +196,7 @@ final class PluginHost: ObservableObject { updateSchedule() } func releaseUnused(_ package: PluginPackage) { - guard package.activeID == nil else { return } + guard !package.needsUpdates else { return } package.close();packages.removeValue(forKey:package.id) } func reloadActive() { errors=[];for package in packages.values { package.requestReload() };updateSchedule() } @@ -231,7 +256,7 @@ final class PluginSession: MacVisualizationSession { package.appearance=["ink":String(format:"#%02x%02x%02x",Int(ink.red*255),Int(ink.green*255),Int(ink.blue*255))] package.post("appearanceChanged",package.appearance) } - func deactivate() { if package.activeID==descriptor.id { package.activeID=nil;package.post("deactivate");PluginHost.shared.updateSchedule() } } + func deactivate() { if package.activeID==descriptor.id { package.activeID=nil;if !package.hardwareActive { package.post("deactivate") };PluginHost.shared.updateSchedule() } } func dispose() { deactivate();canvas.releaseSurface();PluginHost.shared.releaseUnused(package) } func presentSettings() {} private func viewportValue(_ v: VisualizationViewport)->[String:Any] { ["id":descriptor.id,"width":v.width,"height":v.height,"scale":v.scale] } diff --git a/modules/visualizations/api/VisualizationModule.swift b/modules/visualizations/api/VisualizationModule.swift index a46e642..defdbe1 100644 --- a/modules/visualizations/api/VisualizationModule.swift +++ b/modules/visualizations/api/VisualizationModule.swift @@ -1,3 +1,6 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif import Foundation struct VisualizationLayout: Equatable, Sendable { diff --git a/project.yml b/project.yml index 06825f9..ade5b58 100644 --- a/project.yml +++ b/project.yml @@ -12,17 +12,16 @@ settings: CODE_SIGN_STYLE: Automatic ENABLE_HARDENED_RUNTIME: YES CODE_SIGN_ENTITLEMENTS: apps/macos/Miucs.entitlements +packages: + MiucsCore: + path: . targets: Miucs: type: application platform: macOS sources: - path: apps/macos - - path: modules/audio/core - - path: modules/audio/platforms/macos - - path: modules/keyboard/core - path: modules/plugins/platforms/macos - - path: modules/keyboard/platforms/macos - path: modules/visualizations/api excludes: - platforms/** @@ -33,6 +32,9 @@ targets: buildPhase: resources - path: tools/macos/AudioDiagnostics.swift - path: tools/macos/InputDiagnostics.swift + dependencies: + - package: MiucsCore + product: MiucsCore settings: base: PRODUCT_BUNDLE_IDENTIFIER: com.miucs.app diff --git a/scripts/build-hardware.sh b/scripts/build-hardware.sh new file mode 100644 index 0000000..8f8f959 --- /dev/null +++ b/scripts/build-hardware.sh @@ -0,0 +1,9 @@ +#!/bin/zsh +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd)" +if [[ -z "${IDF_PATH:-}" || ! -f "$IDF_PATH/export.sh" ]]; then + print -u2 "Set IDF_PATH to ESP-IDF v5.5.6 and IDF_TOOLS_PATH if tools are installed outside ~/.espressif." + exit 1 +fi +source "$IDF_PATH/export.sh" >/dev/null +idf.py -C "$root/hardware/esp32-display" build diff --git a/scripts/check-boundaries.py b/scripts/check-boundaries.py index b3480c5..d65e66a 100644 --- a/scripts/check-boundaries.py +++ b/scripts/check-boundaries.py @@ -10,7 +10,7 @@ for path in shared: code = re.sub(r"//[^\n]*|/\*[\s\S]*?\*/", "", path.read_text()) imports = re.findall(r"^import\s+(\w+)", code, re.M) - if set(imports) - {"Foundation"}: + if set(imports) - {"Foundation", "MiucsCore"}: failures.append(f"{path.relative_to(root)}: shared import {imports}") if re.search(native_names, code): failures.append(f"{path.relative_to(root)}: native type in shared source") diff --git a/scripts/hardware/flash.py b/scripts/hardware/flash.py new file mode 100644 index 0000000..b2f8045 --- /dev/null +++ b/scripts/hardware/flash.py @@ -0,0 +1,55 @@ +#!/usr/bin/env python3 +"""Back up and replace the factory app, keeping this board's working bootloader.""" +import argparse +from datetime import datetime +import hashlib +import json +from pathlib import Path +import struct +import subprocess +import sys + + +def main(): + root = Path(__file__).resolve().parents[2] + parser = argparse.ArgumentParser(description=__doc__) + parser.add_argument("--port", required=True, help="CH343 programming port, not the Miucs display USB port") + parser.add_argument("--firmware", type=Path, default=root / "hardware/esp32-display/build/miucs_display.bin") + parser.add_argument("--backup-dir", type=Path, default=root / "build/hardware-backups") + args = parser.parse_args() + firmware = args.firmware.resolve() + content = firmware.read_bytes() + if not content or content[0] != 0xE9: + raise SystemExit("Not an ESP application image") + backup = args.backup_dir / datetime.now().strftime("%Y%m%d-%H%M%S-%f") + backup.mkdir(parents=True, exist_ok=False) + command = [sys.executable, "-m", "esptool", "--chip", "esp32p4", "--port", args.port, + "--baud", "115200", "--no-stub"] + + def run(*arguments): + subprocess.run(command + list(arguments), check=True) + + table_file = backup / "partition-table.bin" + run("read-flash", "0x8000", "0x1000", str(table_file)) + table = table_file.read_bytes() + factory = [] + for offset in range(0, len(table), 32): + magic, kind, subtype, address, size, label, flags = struct.unpack(" factory[0][1]: + raise SystemExit("Firmware exceeds factory partition; nothing written") + run("read-flash", "0x10000", "0x100000", str(backup / "previous-app.bin")) + manifest = {"port": args.port, "firmware": str(firmware), "sha256": hashlib.sha256(content).hexdigest(), + "app_offset": "0x10000", "previous_app": str(backup / "previous-app.bin"), "flashed": False} + (backup / "manifest.json").write_text(json.dumps(manifest, indent=2)) + run("write-flash", "0x10000", str(firmware)) + manifest["flashed"] = True + (backup / "manifest.json").write_text(json.dumps(manifest, indent=2)) + print(f"Flashed and verified. Previous application: {backup / 'previous-app.bin'}") + + +if __name__ == "__main__": + main() diff --git a/scripts/prepare-plugin-deps.py b/scripts/prepare-plugin-deps.py index 39f8d66..b7ff36c 100644 --- a/scripts/prepare-plugin-deps.py +++ b/scripts/prepare-plugin-deps.py @@ -1,6 +1,6 @@ #!/usr/bin/env python3 """Fetch pinned Lua, JSON and matching Skia headers/static libraries; verify before extraction.""" -import argparse, hashlib, pathlib, platform, tarfile, urllib.request, tempfile, shutil +import argparse, hashlib, http.client, pathlib, platform, tarfile, urllib.error, urllib.request, tempfile, shutil, time ROOT=pathlib.Path(__file__).resolve().parents[1] DEST=ROOT/'build/plugin-deps' SKIA='0746a1afaf750159e1a6aef29992980f4e293585' @@ -9,14 +9,52 @@ 'mac-arm64':('aarch64-apple-darwin-ganesh-jpegd-jpege-metal-pdf-textlayout','34ffe920f5a5cd4838e7ffe4cf6af48529c87bc0d74462de3e49a5715b35591a'), 'win-x64':('x86_64-pc-windows-msvc-d3d-ganesh-jpegd-jpege-pdf-textlayout','6fd83e1a78729c5bb3eb9d331e999d3ba88ad13e99bf8370362ea29b81aef495'), 'win-arm64':('aarch64-pc-windows-msvc-d3d-ganesh-jpegd-jpege-pdf-textlayout','b11178ef3f64140a3774b055385f07202597cfe2ffcdfe578fec603b2fb94086')} + +def download_once(file, url, digest): + temporary = None + try: + # Close the file before replacing it; Windows cannot rename an open file. + with tempfile.NamedTemporaryFile(dir=file.parent, prefix=file.name+'.', suffix='.download', delete=False) as output: + temporary = pathlib.Path(output.name) + checksum = hashlib.sha256() + size = 0 + with urllib.request.urlopen(url, timeout=30) as response: + while chunk := response.read(1024 * 1024): + output.write(chunk) + checksum.update(chunk) + size += len(chunk) + expected = response.headers.get('Content-Length') + if expected is not None and size < int(expected): + raise http.client.IncompleteRead(b'', int(expected) - size) + if checksum.hexdigest() != digest: + raise RuntimeError('SHA-256 mismatch: '+file.name) + temporary.replace(file) + finally: + if temporary is not None: + temporary.unlink(missing_ok=True) + +def download(file, url, digest): + attempts = 4 + for attempt in range(attempts): + try: + download_once(file, url, digest) + return + except (urllib.error.URLError, ConnectionError, TimeoutError, http.client.IncompleteRead) as error: + # Retry transient transport/server failures, not missing files or a + # checksum mismatch. TLS verification and pinned digests stay enabled. + if isinstance(error, urllib.error.HTTPError) and error.code not in (408, 429) and not 500 <= error.code < 600: + raise + if attempt + 1 == attempts: + raise RuntimeError(f'Download failed after {attempts} attempts: {file.name}') from error + delay = 2 ** attempt + print(f'Download interrupted for {file.name} ({error}); retrying in {delay}s [{attempt+2}/{attempts}]', flush=True) + time.sleep(delay) + def fetch(name,url,digest,folder=None): file=DEST/name if not file.exists() or hashlib.sha256(file.read_bytes()).hexdigest()!=digest: print('Downloading',name,flush=True) - temp=file.with_suffix('.download') - urllib.request.urlretrieve(url,temp) - if hashlib.sha256(temp.read_bytes()).hexdigest()!=digest:temp.unlink();raise RuntimeError('SHA-256 mismatch: '+name) - temp.replace(file) + download(file,url,digest) marker=DEST/(folder+'.verified' if folder else name+'.verified') if folder and (not (DEST/folder).exists() or not marker.exists() or marker.read_text()!=digest): with tarfile.open(file) as archive: diff --git a/scripts/probe-hardware.sh b/scripts/probe-hardware.sh new file mode 100644 index 0000000..a85744b --- /dev/null +++ b/scripts/probe-hardware.sh @@ -0,0 +1,9 @@ +#!/bin/zsh +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd)" +cd "$root" +mkdir -p build/hardware-tests +swiftc -swift-version 6 -parse-as-library -O \ + modules/hardware/macos/HardwareFrame.swift modules/hardware/macos/HardwareSerial.swift modules/hardware/macos/HardwareIO.swift apps/macos/HardwarePort.swift tools/macos/HardwareProbe.swift \ + -o build/hardware-tests/hardware-probe +build/hardware-tests/hardware-probe "$@" diff --git a/scripts/swift-sources.sh b/scripts/swift-sources.sh index ea444a1..1b9d616 100644 --- a/scripts/swift-sources.sh +++ b/scripts/swift-sources.sh @@ -9,8 +9,13 @@ fft_macos=("$root/modules/audio/platforms/macos/AccelerateSpectrumTransform.swif pcm_macos=("$root/modules/audio/platforms/macos/AudioPCMDecoder.swift") visualization_core=("$root"/modules/visualizations/api/*.swift) visualization_macos=("$root"/modules/visualizations/api/platforms/macos/*.swift "$root"/modules/plugins/platforms/macos/*.swift) +hardware_core=("$root"/modules/hardware/macos/*.swift) +core_sources=("${audio_core[@]}" "${audio_macos[@]}" "${keyboard_core[@]}" "${keyboard_macos[@]}" "${hardware_core[@]}") app_macos=("$root"/apps/macos/*.swift) host_macos=(${app_macos:#*/App.swift}) tool_libraries=("$root/tools/macos/AudioDiagnostics.swift" "$root/tools/macos/InputDiagnostics.swift") -native_libraries=("${audio_core[@]}" "${audio_macos[@]}" "${keyboard_core[@]}" "${keyboard_macos[@]}" "${visualization_core[@]}" "${visualization_macos[@]}" "${host_macos[@]}" "${tool_libraries[@]}") +native_libraries=("${audio_core[@]}" "${audio_macos[@]}" "${keyboard_core[@]}" "${keyboard_macos[@]}" "${hardware_core[@]}" "${visualization_core[@]}" "${visualization_macos[@]}" "${host_macos[@]}" "${tool_libraries[@]}") app_sources=("${native_libraries[@]}" "$root/apps/macos/App.swift") + +# App builds link MiucsCore; source-based regression suites above retain internal test seams. +linked_app_sources=("${visualization_core[@]}" "${visualization_macos[@]}" "${app_macos[@]}" "${tool_libraries[@]}") diff --git a/scripts/test-hardware.sh b/scripts/test-hardware.sh new file mode 100644 index 0000000..33f2e19 --- /dev/null +++ b/scripts/test-hardware.sh @@ -0,0 +1,14 @@ +#!/bin/zsh +set -euo pipefail +root="$(cd "$(dirname "$0")/.." && pwd)" +cd "$root" +mkdir -p build/hardware-tests +clang -std=c11 -Wall -Wextra -Wno-unused-parameter -fsanitize=address,undefined \ + -I hardware/esp32-display/main hardware/esp32-display/main/protocol.c tests/hardware/protocol_test.c \ + -o build/hardware-tests/protocol-tests +build/hardware-tests/protocol-tests +swiftc -swift-version 6 -parse-as-library -O \ + modules/hardware/macos/HardwareFrame.swift modules/hardware/macos/HardwareSerial.swift modules/hardware/macos/HardwareIO.swift apps/macos/HardwarePort.swift tests/hardware/TransportTests.swift \ + -o build/hardware-tests/transport-tests +build/hardware-tests/transport-tests +python3 tests/hardware/transport_test.py diff --git a/tests/core/HardwareIOTests.swift b/tests/core/HardwareIOTests.swift new file mode 100644 index 0000000..911a78a --- /dev/null +++ b/tests/core/HardwareIOTests.swift @@ -0,0 +1,38 @@ +import Darwin +import XCTest +@testable import MiucsCore + +final class HardwareIOTests: XCTestCase { + func testMultiplexesReadWriteAndEOFWithHighDescriptors() throws { + var pair: [Int32] = [-1, -1] + XCTAssertEqual(socketpair(AF_UNIX, SOCK_STREAM, 0, &pair), 0) + let reader = fcntl(pair[0], F_DUPFD, 96) + XCTAssertGreaterThanOrEqual(reader, 96) + close(pair[0]) + var writer = pair[1] + defer { close(reader); if writer >= 0 { close(writer) } } + var descriptors = [pollfd(fd: reader, events: Int16(POLLIN), revents: 0), + pollfd(fd: writer, events: Int16(POLLOUT), revents: 0), + pollfd(fd: -1, events: Int16(POLLIN), revents: 0)] + XCTAssertEqual(HardwareIO.wait(&descriptors, milliseconds: 0), 1) + XCTAssertEqual(descriptors[0].revents, 0) + XCTAssertEqual(descriptors[1].revents, Int16(POLLOUT)) + var byte: UInt8 = 42 + XCTAssertEqual(write(writer, &byte, 1), 1) + XCTAssertEqual(HardwareIO.wait(&descriptors, milliseconds: 20), 2) + XCTAssertEqual(descriptors[0].revents, Int16(POLLIN)) + XCTAssertEqual(read(reader, &byte, 1), 1) + XCTAssertEqual(byte, 42) + descriptors.removeLast(2) + XCTAssertEqual(HardwareIO.wait(&descriptors, milliseconds: 10), 0) + close(writer); writer = -1 + XCTAssertEqual(HardwareIO.wait(&descriptors, milliseconds: 20), 1) + XCTAssertEqual(read(reader, &byte, 1), 0, "EOF must wake the transport instead of spinning") + } + + func testRejectsOutOfBoundsDescriptors() { + var descriptors = [pollfd(fd: FD_SETSIZE, events: Int16(POLLIN), revents: 0)] + XCTAssertEqual(HardwareIO.wait(&descriptors, milliseconds: 0), -1) + XCTAssertEqual(errno, EINVAL) + } +} diff --git a/tests/core/NativePetTests.swift b/tests/core/NativePetTests.swift new file mode 100644 index 0000000..10a2c73 --- /dev/null +++ b/tests/core/NativePetTests.swift @@ -0,0 +1,203 @@ +import XCTest +@testable import MiucsCore + +private actor TestTransport: NativePetTransport { + let card = DeskIsleCard(announcement: "DESKISLE 1 1 512 288 JPEG 98304 3000", + pet: DeskIslePetCapability("DESKISLE_PET 3 1 8388608 98304 16384"))! + var present = false + var messages: [(DeskIslePetMessage, Data)] = [] + var opens = 0 + var maximumOpen = 0 + var connected = false + var rejectNextInput = false + var clearCount = 0 + + func connect() async throws -> DeskIsleCard { + opens += 1 + maximumOpen = max(maximumOpen, connected ? 2 : 1) + connected = true + return card + } + + func send(_ kind: DeskIslePetMessage, payload: Data, card: DeskIsleCard) async throws { + try Task.checkCancellation() + try await Task.sleep(for: .milliseconds(2)) + messages.append((kind, payload)) + switch kind { + case .resume: if !present { throw HardwareSerialError.petRejected("6") } + case .commit: present = true + case .input: + if rejectNextInput { rejectNextInput = false; present = false; throw HardwareSerialError.petRejected("6") } + default: break + } + } + + func close(clear: Bool) { + connected = false + if clear { present = false; clearCount += 1 } + } + func loseBoard() { rejectNextInput = true } + func count(_ kind: DeskIslePetMessage) -> Int { messages.filter { $0.0 == kind }.count } + func uploadedPack() throws -> Data { + var result = Data() + for (_, payload) in messages.filter({ $0.0 == .chunk }) { + XCTAssertLessThanOrEqual(payload.count, 32 * 1024 + 4) + let offset = payload.withUnsafeBytes { Int(UInt32(littleEndian: $0.loadUnaligned(as: UInt32.self))) } + XCTAssertEqual(offset, result.count) + result.append(payload.dropFirst(4)) + } + return result + } + func snapshots() throws -> [[String: Any]] { + try messages.filter { $0.0 == .input }.map { try JSONSerialization.jsonObject(with: $0.1) as! [String: Any] } + } + func keyboardSequences() throws -> [UInt64] { + try snapshots().flatMap { + (($0["keyboard"] as? [String: Any])?["events"] as? [[String: Any]] ?? []).compactMap { $0["sequence"] as? UInt64 } + } + } +} + +final class NativePetTests: XCTestCase { + func testBundledCatalogAndAllNativePacks() throws { + let skins = PetCatalog.scan(user: nil) + XCTAssertEqual(Set(skins.map(\.id)), HardwarePetPack.nativeIDs) + XCTAssertEqual(skins.first?.id, "pixel-keyboard") + for skin in skins { + let bytes = try HardwarePetPack.make(directory: skin.directory, limit: 8 * 1024 * 1024) + XCTAssertEqual(String(decoding: bytes.prefix(4), as: UTF8.self), "PET1") + var offset = 4, names = Set() + while offset < bytes.count { + XCTAssertLessThanOrEqual(offset + 8, bytes.count) + let n = Int(bytes[offset + 1]) + let size = bytes.withUnsafeBytes { Int(UInt32(littleEndian: $0.loadUnaligned(fromByteOffset: offset + 4, as: UInt32.self))) } + XCTAssertLessThanOrEqual(offset + 8 + n + size, bytes.count) + let name = String(decoding: bytes[(offset + 8)..<(offset + 8 + n)], as: UTF8.self) + XCTAssertTrue(names.insert(name).inserted) + offset += 8 + n + size + } + XCTAssertEqual(offset, bytes.count) + XCTAssertTrue(names.contains("manifest.json")) + XCTAssertTrue(names.contains("main.lua")) + } + } + + func testCatalogRejectsEscapingEntryAndSymlinkManifest() throws { + let root = FileManager.default.temporaryDirectory.appendingPathComponent(UUID().uuidString) + defer { try? FileManager.default.removeItem(at: root) } + let skin = root.appendingPathComponent("test") + try FileManager.default.createDirectory(at: skin, withIntermediateDirectories: true) + let manifest = skin.appendingPathComponent("manifest.json") + let json = "{\"id\":\"pixel-keyboard\",\"name\":\"Unsafe\",\"api\":1,\"entry\":\"../main.lua\",\"inputs\":[\"keyboard\"]}" + try Data(json.utf8).write(to: manifest) + XCTAssertTrue(PetCatalog.scan(builtin: root, user: nil).isEmpty) + try FileManager.default.removeItem(at: manifest) + let outside = root.appendingPathComponent("manifest.json") + try Data(json.utf8).write(to: outside) + try FileManager.default.createSymbolicLink(at: manifest, withDestinationURL: outside) + XCTAssertTrue(PetCatalog.scan(builtin: root, user: nil).isEmpty) + } + + func testInputBurstIsBoundedAndReportsDiscardedEvents() throws { + let channel = KeyboardChannel() + let id = UUID() + channel.reset(sessionID: id, state: .running) + let subscription = channel.subscribe() + for _ in 0..<64 { + channel.publish(KeyboardRawEvent(kind: .pressed, keyCode: 0, modifiers: []), sessionID: id) + channel.publish(KeyboardRawEvent(kind: .released, keyCode: 0, modifiers: []), sessionID: id) + } + let payload = try PetInputSnapshot.encode(audio: nil, keyboard: subscription.read(), limit: 4096) + XCTAssertLessThanOrEqual(payload.count, 4096) + let json = try JSONSerialization.jsonObject(with: payload) as! [String: Any] + let keys = json["keyboard"] as! [String: Any] + XCTAssertEqual((keys["events"] as? [Any])?.count, 0) + XCTAssertEqual(keys["droppedEventCount"] as? Int, 128) + XCTAssertEqual((keys["heldKeys"] as? [Any])?.count, 0) + } + + @MainActor + func testSessionPreservesKeyOrderAndClosesWithoutRendering() async throws { + let transport = TestTransport(), session = NativePetSession() + let audio = FeatureChannel(), keys = KeyboardChannel() + let id = UUID() + keys.reset(sessionID: id, state: .running) + let skin = PetCatalog.scan(user: nil).first! + session.start(skin: skin, audio: audio, keyboard: keys, transport: transport) + try await wait { let count = await transport.count(.input); return session.active && count > 0 } + let uploaded = try await transport.uploadedPack() + XCTAssertEqual(uploaded, try HardwarePetPack.make(directory: skin.directory, limit: 8 * 1024 * 1024)) + for _ in 0..<12 { + keys.publish(KeyboardRawEvent(kind: .pressed, keyCode: 0, modifiers: []), sessionID: id) + keys.publish(KeyboardRawEvent(kind: .released, keyCode: 0, modifiers: []), sessionID: id) + } + try await wait { (try? await transport.keyboardSequences().count) == 24 } + let sequences = try await transport.keyboardSequences() + XCTAssertEqual(sequences, Array(1...24)) + let before = await transport.count(.input) + try await Task.sleep(for: .milliseconds(120)) + let after = await transport.count(.input) + XCTAssertLessThanOrEqual(after - before, 1, "Idle native rendering must not produce a Mac frame loop") + session.stop() + await session.waitUntilStopped() + let cleared = await transport.clearCount + let connected = await transport.connected + XCTAssertEqual(cleared, 1) + XCTAssertFalse(connected) + XCTAssertFalse(session.active) + } + + @MainActor + func testRapidSkinChangesSerializeTransportOwnership() async throws { + let transport = TestTransport(), session = NativePetSession() + let skins = PetCatalog.scan(user: nil), audio = FeatureChannel(), keys = KeyboardChannel() + session.start(skin: skins[0], audio: audio, keyboard: keys, transport: transport) + try await Task.sleep(for: .milliseconds(10)) + for skin in skins.suffix(3) { session.start(skin: skin, audio: audio, keyboard: keys, transport: transport) } + try await wait { session.active && session.status.contains(skins.last!.name) } + let maximum = await transport.maximumOpen + XCTAssertEqual(maximum, 1) + session.stop(); await session.waitUntilStopped() + } + + @MainActor + func testBoardResetReuploadsThenRecovers() async throws { + let transport = TestTransport(), session = NativePetSession() + let keys = KeyboardChannel(), audio = FeatureChannel(), id = UUID() + keys.reset(sessionID: id, state: .running) + session.start(skin: PetCatalog.scan(user: nil).first!, audio: audio, keyboard: keys, transport: transport) + try await wait { let count = await transport.count(.input); return session.active && count > 0 } + await transport.loseBoard() + keys.publish(KeyboardRawEvent(kind: .pressed, keyCode: 0, modifiers: []), sessionID: id) + try await wait { (await transport.count(.commit)) >= 2 && session.active } + let opens = await transport.opens + XCTAssertGreaterThanOrEqual(opens, 2) + session.stop(); await session.waitUntilStopped() + } + + @MainActor + func testReleasingSessionStopsItsTasksAndClosesTransport() async throws { + let transport = TestTransport(), audio = FeatureChannel(), keys = KeyboardChannel() + var session: NativePetSession? = NativePetSession() + weak var weakSession = session + session?.start(skin: PetCatalog.scan(user: nil).first!, audio: audio, keyboard: keys, transport: transport) + try await wait { session?.active == true } + session = nil + XCTAssertNil(weakSession) + try await wait { await transport.clearCount == 1 } + let connected = await transport.connected + XCTAssertFalse(connected) + } + + @MainActor + private func wait(_ ready: () async -> Bool) async throws { + let deadline = ProcessInfo.processInfo.systemUptime + 8 + while !(await ready()) { + guard ProcessInfo.processInfo.systemUptime < deadline else { + XCTFail("Native pet state did not converge") + throw CocoaError(.userCancelled) + } + try await Task.sleep(for: .milliseconds(10)) + } + } +} diff --git a/tests/hardware/TransportTests.swift b/tests/hardware/TransportTests.swift new file mode 100644 index 0000000..19df45d --- /dev/null +++ b/tests/hardware/TransportTests.swift @@ -0,0 +1,161 @@ +import Foundation +import CoreGraphics +import ImageIO + +@main +enum TransportTests { + static func main() async throws { + if CommandLine.arguments.count == 3 { + let port = CommandLine.arguments[1], mode = CommandLine.arguments[2] + let serial = HardwareSerial() + let screen = try await serial.connect(path: port) + if mode == "desk-isle" { + try await testDeskIsle(serial: serial, screen: screen) + await serial.close() + return + } + if mode == "desk-isle-input" { + try await testPetInput(serial: serial, screen: screen) + await serial.close() + return + } + if mode == "desk-isle-resume" { + try await testPetResume(serial: serial, screen: screen) + return + } + precondition(screen == .miucs) + let payload = Data([0xff, 0xd8, 0x12, 0x34, 0xff, 0xd9]) + let packet = HardwareFrame.packet(jpeg: payload, sequence: 42)! + do { + let ms = try await serial.send(packet, sequence: 42) + precondition(mode == "ok" && ms == 17) + print("PASS fragmented handshake, packet bytes, ignored stale ACK and matching ACK") + } catch { + guard mode == "timeout" || mode == "disconnect" else { throw error } + print("PASS \(mode): \(error.localizedDescription)") + } + await serial.close() + return + } + precondition(HardwareFrame.crc32(Data("123456789".utf8)) == 0xcbf43926) + let maximum = Data((0.. Data { + var pixels = Data() + for _ in 0..<36 { + for x in 0..<72 { + let bgra: [UInt8] = x < 24 ? [0, 0, 0, 0] : x < 48 ? [255, 255, 255, 255] : [0, 0, 128, 128] + pixels.append(contentsOf: bgra) + } + } + return HardwareFrame.packet(bgra: pixels, width: 72, height: 36, orientation: .landscapeReverse, + sequence: sequence, screen: screen)! + } + + let firstMS = try await serial.send(frame(screen: screen, sequence: 42), sequence: 42, screen: screen) + precondition(firstMS == 17) + let idleMS = try await serial.send(HardwareFrame.keepalive(sequence: 43, card: card), sequence: 43, screen: screen) + precondition(idleMS == 0) + + let updated: HardwareScreen + do { + _ = try await serial.send(frame(screen: screen, sequence: 44), sequence: 44, screen: screen) + fatalError("Sent a frame with the old dimensions") + } catch HardwareSerialError.cardChanged(let card) { + precondition(card.id == 9 && card.width == 384 && card.height == 224 && card.maxJPEG == 16_384) + precondition(card.keepaliveInterval == 0.6 && card.supportsAlpha) + precondition(card.pet?.version == 2 && card.pet?.maxPack == 8 * 1024 * 1024) + updated = .deskIsle(card) + } + let alphaPacket = frame(screen: updated, sequence: 45) + let jpegLength = alphaPacket.withUnsafeBytes { + Int(UInt32(littleEndian: $0.loadUnaligned(fromByteOffset: alphaPacket.count - 4, as: UInt32.self))) + } + let jpeg = alphaPacket.subdata(in: 24..<(24 + jpegLength)) + let source = CGImageSourceCreateWithData(jpeg as CFData, nil)! + let image = CGImageSourceCreateImageAtIndex(source, 0, nil)! + let canvas = CGContext(data: nil, width: image.width, height: image.height, bitsPerComponent: 8, + bytesPerRow: image.width * 4, space: CGColorSpaceCreateDeviceRGB(), + bitmapInfo: CGBitmapInfo.byteOrder32Little.rawValue | CGImageAlphaInfo.noneSkipFirst.rawValue)! + canvas.draw(image, in: CGRect(x: 0, y: 0, width: image.width, height: image.height)) + let row = canvas.data!.assumingMemoryBound(to: UInt8.self).advanced(by: image.height / 2 * canvas.bytesPerRow) + precondition(row[image.width / 6 * 4 + 2] < 8) + precondition(row[image.width / 2 * 4] > 247 && row[image.width / 2 * 4 + 2] > 247) + precondition((120...136).contains(row[(image.width * 5 / 6) * 4 + 2])) + let updatedMS = try await serial.send(alphaPacket, sequence: 45, screen: updated) + precondition(updatedMS == 19) + print("PASS DeskIsle legacy frames, keepalive, matching ACK, alpha and native pet negotiation") + } + + private static func testPetInput(serial: HardwareSerial, screen: HardwareScreen) async throws { + let card = screen.card! + func send(_ sequence: UInt32, _ payload: String) async throws { + let packet = HardwareFrame.nativePet(.input, payload: Data(payload.utf8), sequence: sequence, id: card.id) + try await serial.sendPetInput(packet, sequence: sequence, screen: screen) + } + + try await send(41, #"{"event":"pressed"}"#) + try await send(42, #"{"event":"released"}"#) + let keepalive = HardwareFrame.keepalive(sequence: 43, card: card) + let processing = try await serial.send(keepalive, sequence: 43, screen: screen) + precondition(processing == 17) + + try await send(UInt32.max, "{}") + try await send(0, "{}") + do { + for sequence: UInt32 in 1...25 { + try await Task.sleep(for: .milliseconds(20)) + try await send(sequence, "{}") + } + fatalError("Missed input rejection") + } catch HardwareSerialError.petRejected("5") { + print("PASS ordered native input before ACKs, matching control reply and input rejection across wraparound") + } + } + + private static func testPetResume(serial: HardwareSerial, screen: HardwareScreen) async throws { + let card = screen.card! + precondition(card.pet?.version == 3 && card.pet?.maxPack == 8 * 1024 * 1024) + let identity = Data([0x39, 0x30, 0, 0, 0x78, 0x56, 0x34, 0x12]) + let miss = HardwareFrame.nativePet(.resume, payload: identity, sequence: 51, id: card.id) + do { + _ = try await serial.send(miss, sequence: 51, screen: screen, nativePet: true) + fatalError("Missed missing-skin status") + } catch HardwareSerialError.petRejected("6") {} + let hit = HardwareFrame.nativePet(.resume, payload: identity, sequence: 52, id: card.id) + let processing = try await serial.send(hit, sequence: 52, screen: screen, nativePet: true) + precondition(processing == 2) + await serial.close(clearing: screen, sequence: 53) + print("PASS native pet v3 negotiation, missing skin, resume and clear") + } +} diff --git a/tests/hardware/protocol_test.c b/tests/hardware/protocol_test.c new file mode 100644 index 0000000..de345ed --- /dev/null +++ b/tests/hardware/protocol_test.c @@ -0,0 +1,59 @@ +#include "protocol.h" +#include +#include +#include +#include + +static unsigned calls, good; +static void received(void *unused, uint32_t sequence, const uint8_t *jpeg, size_t size, bool valid) +{ + assert(sequence == 0x01020304); + assert(size == 37); + if (valid) { assert(jpeg[0] == 0xff && jpeg[36] == 0xd9); ++good; } + ++calls; +} +static void le32(uint8_t *into, uint32_t value) +{ + for (unsigned i = 0; i < 4; ++i) into[i] = (uint8_t)(value >> (i * 8)); +} +int main(void) +{ + assert(miucs_crc32((const uint8_t *)"123456789", 9) == 0xcbf43926); + // Independent zlib vector at the maximum accepted JPEG payload size. + uint8_t *maximum = malloc(MIUCS_MAX_JPEG); + assert(maximum); + for (size_t i = 0; i < MIUCS_MAX_JPEG; ++i) maximum[i] = (uint8_t)i; + assert(miucs_crc32(maximum, MIUCS_MAX_JPEG) == 0xc790bff6u); + free(maximum); + uint8_t packet[24 + 37] = {'M','I','U','C',1,1,0,0}; + le32(packet + 8, 0x01020304); le32(packet + 12, 37); + packet[20] = 0xd0; packet[21] = 2; packet[23] = 5; + packet[24] = 0xff; packet[60] = 0xd9; + le32(packet + 16, miucs_crc32(packet + 24, 37)); + miucs_parser parser = {.payload = malloc(MIUCS_MAX_JPEG), .on_frame = received}; + assert(parser.payload); + // Every split boundary, including a one-byte-at-a-time USB stream. + for (size_t chunk = 1; chunk <= sizeof(packet); ++chunk) { + miucs_parser_reset(&parser); unsigned before = calls; + miucs_parser_feed(&parser, (const uint8_t *)"garbageMIUC", 11); + for (size_t i = 0; i < sizeof(packet); i += chunk) + miucs_parser_feed(&parser, packet + i, chunk < sizeof(packet) - i ? chunk : sizeof(packet) - i); + assert(calls == before + 1 && calls == good); + } + // An oversized/corrupt header must not allocate or consume the next valid frame. + uint8_t invalid[24]; memcpy(invalid, packet, 24); le32(invalid + 12, UINT32_MAX); + miucs_parser_feed(&parser, invalid, sizeof(invalid)); + miucs_parser_feed(&parser, packet, sizeof(packet)); + assert(calls == 62 && good == 62); + packet[30] ^= 1; + miucs_parser_feed(&parser, packet, sizeof(packet)); + assert(calls == 63 && good == 62); + packet[30] ^= 1; + miucs_parser_feed(&parser, packet, 40); + miucs_parser_reset(&parser); // A cable loss discards the partial JPEG. + miucs_parser_feed(&parser, packet, sizeof(packet)); + miucs_parser_feed(&parser, packet, sizeof(packet)); + assert(calls == 65 && good == 64); + free(parser.payload); + puts("PASS USB frame parser: fragmentation, garbage, oversize, CRC, partial reconnect, consecutive frames"); +} diff --git a/tests/hardware/transport_test.py b/tests/hardware/transport_test.py new file mode 100644 index 0000000..e3fdf08 --- /dev/null +++ b/tests/hardware/transport_test.py @@ -0,0 +1,179 @@ +"""Exercise the actual Swift serial transport against a pseudo-terminal device.""" +import os +import pty +import select +import struct +import subprocess +import time +import zlib + + +def run(mode): + master, slave = pty.openpty() + process = subprocess.Popen(["build/hardware-tests/transport-tests", os.ttyname(slave), mode], + stdout=subprocess.PIPE, stderr=subprocess.STDOUT, text=True) + try: + time.sleep(0.15) + for part in [b"MIUCS_DIS", b"PLAY 1 720 1280 262144", b"\n"]: + os.write(master, part) + time.sleep(0.02) + received = bytearray() + deadline = time.monotonic() + 5 + while len(received) < 30 and time.monotonic() < deadline: + if select.select([master], [], [], 0.1)[0]: + received.extend(os.read(master, 4096)) + assert len(received) == 30, received + magic, version, kind, flags, sequence, length, crc, width, height = struct.unpack("<4sBBHIIIHH", received[:24]) + assert (magic, version, kind, flags, sequence, length, width, height) == (b"MIUC", 1, 1, 0, 42, 6, 720, 1280) + assert zlib.crc32(received[24:]) == crc + if mode == "ok": + os.write(master, b"MIUCS_ACK 41 0 9\nMIUCS_ACK 42 ") + time.sleep(0.03) + os.write(master, b"0 17\n") + elif mode == "disconnect": + os.close(master) + master = -1 + output, _ = process.communicate(timeout=6) + assert process.returncode == 0, output + print(output.strip()) + finally: + if process.poll() is None: + process.kill() + process.wait() + if master >= 0: + os.close(master) + os.close(slave) + + +def read_exact(port, count): + result = bytearray() + deadline = time.monotonic() + 5 + while len(result) < count and time.monotonic() < deadline: + if select.select([port], [], [], 0.1)[0]: + result.extend(os.read(port, count - len(result))) + assert len(result) == count, (len(result), count) + return result + + +def baseline_size(jpeg): + assert jpeg[:2] == b"\xff\xd8" + offset = 2 + while offset + 4 <= len(jpeg) and jpeg[offset] == 0xFF: + marker = jpeg[offset + 1] + if marker == 0xC0: + height, width = struct.unpack_from(">HH", jpeg, offset + 5) + return width, height + if marker == 0xDA: + break + offset += 2 + struct.unpack_from(">H", jpeg, offset + 2)[0] + raise AssertionError("Expected a baseline JPEG SOF0 marker") + + +def expect_desk_isle_frame(port, slot, sequence, width, height, limit, alpha=False): + header = read_exact(port, 24) + magic, version, kind, card, seq, size, crc, w, h = struct.unpack("<4sBBHIIIHH", header) + assert (magic, version, kind, card, seq, w, h) == (b"ISLE", 1, 5 if alpha else 1, slot, sequence, width, height) + assert 4 <= size <= limit + (width * height + 4 if alpha else 0) + payload = read_exact(port, size) + assert zlib.crc32(payload) == crc + jpeg = payload + if alpha: + jpeg_length = struct.unpack_from(" 0 && input.read().frame == nil, + "Capture did not stop or restarted on its own") + + await engine.start() + h.captures[1].emit() + try require(h.requests == 2 && input.read().frame != nil, "Explicit restart did not resume input") + } await check("macOS user-stop is respected until explicitly restarted") { let h = CaptureHarness(), engine = h.engine(); defer { engine.stop() } await engine.start(); h.captures[0].fail(.userStopped) diff --git a/tests/platforms/macos/SettingsTests.swift b/tests/platforms/macos/SettingsTests.swift index 79f7a45..e889cdf 100644 --- a/tests/platforms/macos/SettingsTests.swift +++ b/tests/platforms/macos/SettingsTests.swift @@ -81,8 +81,8 @@ enum SettingsTests { let window = controller.window! let view = window.contentView! check("Configuration is a regular, key-capable window independent from the desktop panel", window.canBecomeKey && !panel.canBecomeKey) - check("Only skins, appearance, inputs and about are navigation destinations", SettingsPage.allCases.map(\.rawValue) == ["皮肤", "外观与位置", "声音与键盘", "关于 Miucs"]) - for (page, name) in [(SettingsPage.skins, "skins"), (.appearance, "appearance"), (.inputs, "inputs")] { + check("Skin, appearance, input, hardware and about settings are available", SettingsPage.allCases.map(\.rawValue) == ["皮肤", "外观与位置", "声音与键盘", "硬件屏幕", "关于 Miucs"]) + for (page, name) in [(SettingsPage.skins, "skins"), (.appearance, "appearance"), (.inputs, "inputs"), (.hardware, "hardware")] { model.page = page await Task.yield() try await Task.sleep(for: .milliseconds(180)) diff --git a/tests/plugins/dependency_download_test.py b/tests/plugins/dependency_download_test.py new file mode 100644 index 0000000..1776cc9 --- /dev/null +++ b/tests/plugins/dependency_download_test.py @@ -0,0 +1,107 @@ +#!/usr/bin/env python3 +"""Offline regressions for dependency downloads interrupted on CI runners.""" +import hashlib +import importlib.util +import io +import pathlib +import tempfile +import unittest +import urllib.error +from unittest.mock import patch + +ROOT = pathlib.Path(__file__).resolve().parents[2] +spec = importlib.util.spec_from_file_location('plugin_deps', ROOT/'scripts/prepare-plugin-deps.py') +deps = importlib.util.module_from_spec(spec) +spec.loader.exec_module(deps) +PAYLOAD = b'verified dependency bytes' +DIGEST = hashlib.sha256(PAYLOAD).hexdigest() +URL = 'https://example.invalid/dependency' + + +class Response(io.BytesIO): + def __init__(self, data=PAYLOAD, length=None): + super().__init__(data) + self.headers = {'Content-Length': str(len(data) if length is None else length)} + + +class ResetAfterPartialRead(Response): + def read(self, size=-1): + if self.tell(): + raise ConnectionResetError('connection reset during transfer') + return super().read(3) + + +class DependencyDownloadTests(unittest.TestCase): + def setUp(self): + directory = tempfile.TemporaryDirectory() + self.addCleanup(directory.cleanup) + self.root = pathlib.Path(directory.name) + self.file = self.root/'dependency.tar.gz' + target = patch.object(deps, 'DEST', self.root) + target.start() + self.addCleanup(target.stop) + sleeper = patch.object(deps.time, 'sleep', side_effect=self.assert_no_partial) + self.sleep = sleeper.start() + self.addCleanup(sleeper.stop) + + def assert_no_partial(self, *_): + self.assertEqual(list(self.root.glob('*.download')), []) + + def fetch(self): + deps.fetch(self.file.name, URL, DIGEST) + + def test_connection_reset_and_partial_transfer_retry_without_publishing(self): + responses = [urllib.error.URLError(ConnectionResetError('TLS connection reset')), + ResetAfterPartialRead(), Response()] + with patch.object(deps.urllib.request, 'urlopen', side_effect=responses) as request: + self.fetch() + self.assertEqual(self.file.read_bytes(), PAYLOAD) + self.assertEqual(request.call_count, 3) + self.assertTrue(all(call.kwargs == {'timeout': 30} for call in request.call_args_list)) + self.assertEqual([call.args[0] for call in self.sleep.call_args_list], [1, 2]) + self.assert_no_partial() + + def test_short_response_and_retryable_http_error_recover(self): + for failure in (Response(b'partial', length=100), urllib.error.HTTPError(URL, 503, 'unavailable', {}, None)): + with self.subTest(failure=failure), patch.object(deps.urllib.request, 'urlopen', side_effect=[failure, Response()]): + deps.download(self.file, URL, DIGEST) + self.assertEqual(self.file.read_bytes(), PAYLOAD) + self.assert_no_partial() + + def test_retries_are_bounded_and_no_incomplete_file_is_cached(self): + with patch.object(deps.urllib.request, 'urlopen', side_effect=TimeoutError('timed out')) as request: + with self.assertRaisesRegex(RuntimeError, 'after 4 attempts'): + self.fetch() + self.assertEqual(request.call_count, 4) + self.assertEqual(self.sleep.call_count, 3) + self.assertFalse(self.file.exists()) + self.assert_no_partial() + + def test_checksum_mismatch_preserves_existing_file_without_retry(self): + self.file.write_bytes(b'existing cache') + with patch.object(deps.urllib.request, 'urlopen', return_value=Response(b'corrupt')) as request: + with self.assertRaisesRegex(RuntimeError, 'SHA-256 mismatch'): + self.fetch() + request.assert_called_once() + self.sleep.assert_not_called() + self.assertEqual(self.file.read_bytes(), b'existing cache') + self.assert_no_partial() + + def test_missing_file_fails_without_retry(self): + with patch.object(deps.urllib.request, 'urlopen', side_effect=urllib.error.HTTPError(URL, 404, 'missing', {}, None)) as request: + with self.assertRaises(urllib.error.HTTPError): + self.fetch() + request.assert_called_once() + self.sleep.assert_not_called() + self.assert_no_partial() + + def test_verified_cache_skips_network(self): + self.file.write_bytes(PAYLOAD) + with patch.object(deps.urllib.request, 'urlopen') as request: + self.fetch() + request.assert_not_called() + self.assertEqual(self.file.read_bytes(), PAYLOAD) + + +if __name__ == '__main__': + unittest.main() diff --git a/tests/plugins/mac_reload.swift b/tests/plugins/mac_reload.swift index 2967568..f710b9f 100644 --- a/tests/plugins/mac_reload.swift +++ b/tests/plugins/mac_reload.swift @@ -26,6 +26,7 @@ import AppKit try wait { api.ready?(package.handle)==1 } package.activeID="test";package.post("activate",["id":"test","width":100,"height":100,"scale":2]) try wait { (api.completed?(package.handle) ?? 0)>=2 } + precondition(package.pixels(width: 16, height: 16)?[2] == 255, "Hardware export initially renders red") let original=package.handle try source("this is invalid Lua") package.requestReload();try wait { !package.reloadError.isEmpty } @@ -33,6 +34,7 @@ import AppKit try source(script("#0000ff")) package.requestReload();try wait { package.handle != original } precondition(package.reloadError.isEmpty) + precondition(package.pixels(width: 16, height: 16)?[0] == 255, "Reload invalidates the hardware export surface before closing the old worker") let surface="test".withCString {api.createSurface?(package.handle,nil,$0)}! defer { api.destroySurface?(surface) } var pixels=[UInt8](repeating:0,count:16*16*4) @@ -76,16 +78,35 @@ import AppKit let tray=AppTrayIcon(item:item,source:keys) defer {tray.stop();NSStatusBar.system.removeStatusItem(item);host.stop()} precondition(item.button?.image != nil,"Application logo exists without any plugin") + let card = DeskIsleCard(announcement: "DESKISLE 1 1 512 288 JPEG 98304 3000")! for id in ids.sorted() { let selected=registry.makeSession(id:id)! precondition(selected is PluginSession,"Every catalog entry must use the plugin runtime") selected.activate(viewport:.desktop) precondition(Set(host.packages.keys)==Set([id]),"Only selected plugin owns a worker") + let plugin = selected as! PluginSession + try wait { (api.completed?(plugin.package.handle) ?? 0) >= 2 } + guard let pixels = plugin.package.pixels(width: 800, height: 460), + let jpeg = HardwareFrame.jpeg(bgra: pixels, width: 800, height: 460, orientation: .landscape) else { + fatalError("Hardware export failed for \(id)") + } + precondition(HardwareFrame.jpeg(bgra: pixels, width: 800, height: 460, orientation: .landscape, + screen: .deskIsle(card)) != nil, "DeskIsle card export failed for \(id)") + let output = URL(fileURLWithPath: FileManager.default.currentDirectoryPath).appendingPathComponent("build/hardware-previews") + try FileManager.default.createDirectory(at: output, withIntermediateDirectories: true) + try jpeg.write(to: output.appendingPathComponent(id + ".jpg")) + plugin.package.hardwareActive = true + selected.deactivate() + precondition(plugin.package.needsUpdates, "Hiding the desktop must keep hardware animation active") selected.dispose() + precondition(host.packages[id] === plugin.package, "Hardware holds the worker after the desktop view is disposed") + plugin.package.hardwareActive = false + host.releaseUnused(plugin.package) precondition(host.packages.isEmpty,"Switching/disposal releases the old worker") precondition(item.button?.image != nil,"Plugin disposal preserves application tray") } print("PASS independent plugin selection and application tray lifetime") + print("PASS hardware export for all seven skins, hidden desktop lifetime and reload invalidation") print("PASS Mac plugin reload: syntax failure preserves live scene; successful candidate preserves viewport and swaps rendered output") } } diff --git a/tools/macos/AudioDiagnostics.swift b/tools/macos/AudioDiagnostics.swift index 39677b0..60cf346 100644 --- a/tools/macos/AudioDiagnostics.swift +++ b/tools/macos/AudioDiagnostics.swift @@ -1,3 +1,6 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif import AppKit import Foundation diff --git a/tools/macos/AudioFileProbe.swift b/tools/macos/AudioFileProbe.swift index 8fa1a9f..d7b723c 100644 --- a/tools/macos/AudioFileProbe.swift +++ b/tools/macos/AudioFileProbe.swift @@ -1,3 +1,6 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif import AVFoundation import Foundation diff --git a/tools/macos/HardwareProbe.swift b/tools/macos/HardwareProbe.swift new file mode 100644 index 0000000..d307b71 --- /dev/null +++ b/tools/macos/HardwareProbe.swift @@ -0,0 +1,82 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif +import Foundation + +/// Explicit, bounded hardware test. Uses the production USB discovery and transport. +@main +enum HardwareProbe { + static func main() async { + let args = CommandLine.arguments + func argument(_ flag: String, _ fallback: String) -> String { + guard let index = args.firstIndex(of: flag), index + 1 < args.count else { return fallback } + return args[index + 1] + } + let ports = HardwarePort.connected().filter { $0.name == "Miucs P4 Display" } + if args.contains("--list") { + for port in ports { print("\(port.path)\t\(port.name)") } + return + } + let output = argument("--output", "build/hardware-live-probe.json") + let seconds = min(60, max(1, Double(argument("--seconds", "15")) ?? 15)) + let path = argument("--port", ports.first?.path ?? "") + let folder = URL(fileURLWithPath: argument("--frames", "build/hardware-previews")) + let serial = HardwareSerial() + var report: [String: Any] = ["port": path, "passed": false, "secondsRequested": seconds] + do { + guard !path.isEmpty else { throw HardwareSerialError.message("没有检测到 Miucs P4 Display USB 设备") } + let files = try FileManager.default.contentsOfDirectory(at: folder, includingPropertiesForKeys: nil) + .filter { $0.pathExtension == "jpg" }.sorted { $0.path < $1.path } + guard !files.isEmpty else { throw HardwareSerialError.message("没有测试 JPEG") } + let frames = try files.map { try Data(contentsOf: $0) } + guard try await serial.connect(path: path) == .miucs else { + throw HardwareSerialError.message("此探测程序需要 Miucs 整屏固件") + } + let start = ProcessInfo.processInfo.systemUptime + var sequence: UInt32 = 0 + var bytes = 0, decodeMS = [Int](), roundTripMS = [Double]() + while ProcessInfo.processInfo.systemUptime - start < seconds { + let index = Int(sequence) % frames.count + sequence += 1 + guard let packet = HardwareFrame.packet(jpeg: frames[index], sequence: sequence) else { + throw HardwareSerialError.message("测试图片超过帧大小限制") + } + let sent = ProcessInfo.processInfo.systemUptime + decodeMS.append(try await serial.send(packet, sequence: sequence)) + roundTripMS.append((ProcessInfo.processInfo.systemUptime - sent) * 1000) + bytes += packet.count + if sequence <= frames.count { print("FRAME_OK \(files[index].lastPathComponent) \(decodeMS.last!) ms") } + } + let elapsed = ProcessInfo.processInfo.systemUptime - start + report["frames"] = sequence + report["elapsedSeconds"] = elapsed + report["framesPerSecond"] = Double(sequence) / elapsed + report["bytesPerSecond"] = Double(bytes) / elapsed + report["averageDisplayMS"] = Double(decodeMS.reduce(0, +)) / Double(decodeMS.count) + report["averageRoundTripMS"] = roundTripMS.reduce(0, +) / Double(roundTripMS.count) + report["maxRoundTripMS"] = roundTripMS.max() + report["skins"] = files.map { $0.deletingPathExtension().lastPathComponent } + // Verify a corrupted payload is rejected, then reconnect using the normal handshake. + var corrupted = HardwareFrame.packet(jpeg: frames[0], sequence: sequence + 1)! + corrupted[corrupted.count - 1] ^= 1 + do { + _ = try await serial.send(corrupted, sequence: sequence + 1) + throw HardwareSerialError.message("损坏帧未被拒绝") + } catch { + guard error.localizedDescription.contains("屏幕未能显示此帧") else { throw error } + report["crcRejected"] = true + } + try await serial.connect(path: path) + _ = try await serial.send(HardwareFrame.packet(jpeg: frames[0], sequence: sequence + 2)!, sequence: sequence + 2) + report["reconnectPassed"] = true + report["passed"] = true + } catch { report["error"] = error.localizedDescription } + await serial.close() + do { + let data = try JSONSerialization.data(withJSONObject: report, options: [.prettyPrinted, .sortedKeys]) + try data.write(to: URL(fileURLWithPath: output)) + print(String(decoding: data, as: UTF8.self)) + } catch { fputs("Could not write hardware report: \(error)\n", stderr) } + if report["passed"] as? Bool != true { exit(1) } + } +} diff --git a/tools/macos/InputDiagnostics.swift b/tools/macos/InputDiagnostics.swift index 4defdc0..b116c07 100644 --- a/tools/macos/InputDiagnostics.swift +++ b/tools/macos/InputDiagnostics.swift @@ -1,3 +1,6 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif import AppKit /// Explicit, aggregate-only diagnostics: never exports key codes, held keys or individual timestamps. diff --git a/tools/macos/MelodyFileProbe.swift b/tools/macos/MelodyFileProbe.swift index 1edc445..bc4d642 100644 --- a/tools/macos/MelodyFileProbe.swift +++ b/tools/macos/MelodyFileProbe.swift @@ -1,3 +1,6 @@ +#if canImport(MiucsCore) +import MiucsCore +#endif import AVFoundation import Foundation