From 68fddf0193e61917319aef00eca044891216572b Mon Sep 17 00:00:00 2001 From: ranjeshj Date: Wed, 17 Jun 2026 23:12:55 -0700 Subject: [PATCH] fix(canvas): repair fullscreen WebView bridge Send fullscreen shortcut messages as object-root WebView bridge payloads, guard bridge parsing against non-object roots, and use shared fullscreen message constants. Co-authored-by: Copilot <223556219+Copilot@users.noreply.github.com> --- src/OpenClaw.Shared/WebBridgeMessage.cs | 9 ++ .../Windows/A2UICanvasWindow.xaml.cs | 49 ++++++++++- .../Windows/CanvasWindow.xaml.cs | 87 ++++++++++++++++++- .../WebBridgeMessageTests.cs | 12 +++ 4 files changed, 153 insertions(+), 4 deletions(-) diff --git a/src/OpenClaw.Shared/WebBridgeMessage.cs b/src/OpenClaw.Shared/WebBridgeMessage.cs index 64d9308c4..48246c365 100644 --- a/src/OpenClaw.Shared/WebBridgeMessage.cs +++ b/src/OpenClaw.Shared/WebBridgeMessage.cs @@ -46,6 +46,12 @@ public WebBridgeMessage(string type, string? payloadJson = null) /// Sent SPA→native when the SPA is fully initialised and ready for messages. public const string TypeReady = "ready"; + /// Sent SPA→native to toggle the owning canvas window fullscreen state. + public const string TypeFullscreenToggle = "fullscreen-toggle"; + + /// Sent SPA→native to exit fullscreen on the owning canvas window. + public const string TypeFullscreenExit = "fullscreen-exit"; + // ── parsing ──────────────────────────────────────────────────────────── /// @@ -62,6 +68,9 @@ public WebBridgeMessage(string type, string? payloadJson = null) using var doc = JsonDocument.Parse(json); var root = doc.RootElement; + if (root.ValueKind != JsonValueKind.Object) + return null; + if (!root.TryGetProperty("type", out var typeEl) || typeEl.ValueKind != JsonValueKind.String) return null; diff --git a/src/OpenClaw.Tray.WinUI/Windows/A2UICanvasWindow.xaml.cs b/src/OpenClaw.Tray.WinUI/Windows/A2UICanvasWindow.xaml.cs index 4188ab231..e6f27e964 100644 --- a/src/OpenClaw.Tray.WinUI/Windows/A2UICanvasWindow.xaml.cs +++ b/src/OpenClaw.Tray.WinUI/Windows/A2UICanvasWindow.xaml.cs @@ -5,8 +5,10 @@ using System.Text.Json.Nodes; using System.Threading.Tasks; using Microsoft.UI.Dispatching; +using Microsoft.UI.Windowing; using Microsoft.UI.Xaml; using Microsoft.UI.Xaml.Controls; +using Microsoft.UI.Xaml.Input; using Microsoft.UI.Xaml.Media.Imaging; using OpenClaw.Shared; using OpenClawTray.A2UI.Actions; @@ -16,6 +18,7 @@ using WinUIEx; using Windows.Graphics.Imaging; using Windows.Storage.Streams; +using Windows.System; namespace OpenClawTray.Windows; @@ -42,9 +45,10 @@ public sealed partial class A2UICanvasWindow : WindowEx private const uint SWP_NOSIZE = 0x0001; private const uint SWP_SHOWWINDOW = 0x0040; - private readonly DispatcherQueue _dispatcher; + private readonly Microsoft.UI.Dispatching.DispatcherQueue _dispatcher; private readonly A2UIRouter _router; private readonly DataModelStore _dataModel; + private bool _isFullScreen; public bool IsClosed { get; private set; } @@ -61,7 +65,7 @@ public A2UICanvasWindow(IActionSink actions, MediaResolver media, IOpenClawLogge Title = OpenClawTray.Helpers.LocalizationHelper.GetString("A2UI_CanvasTitle"); WaitingForContentText.Text = OpenClawTray.Helpers.LocalizationHelper.GetString("Canvas_WaitingForContent"); - _dispatcher = DispatcherQueue.GetForCurrentThread(); + _dispatcher = Microsoft.UI.Dispatching.DispatcherQueue.GetForCurrentThread(); _dataModel = new DataModelStore(_dispatcher); var registry = ComponentRendererRegistry.BuildDefault(media); _router = new A2UIRouter(_dispatcher, _dataModel, registry, actions, logger); @@ -69,6 +73,22 @@ public A2UICanvasWindow(IActionSink actions, MediaResolver media, IOpenClawLogge _router.SurfaceCreated += OnSurfaceCreated; _router.SurfaceDeleted += OnSurfaceDeleted; + // F11 toggles borderless fullscreen; Escape exits it. + var f11Accel = new KeyboardAccelerator { Key = VirtualKey.F11 }; + f11Accel.Invoked += (_, args) => + { + args.Handled = true; + ToggleFullScreen(); + }; + var escAccel = new KeyboardAccelerator { Key = VirtualKey.Escape }; + escAccel.Invoked += (_, args) => + { + if (_isFullScreen) { args.Handled = true; ExitFullScreen(); } + }; + RootGrid.KeyboardAccelerators.Add(f11Accel); + RootGrid.KeyboardAccelerators.Add(escAccel); + RootGrid.KeyboardAcceleratorPlacementMode = KeyboardAcceleratorPlacementMode.Hidden; + // Explicit teardown: unsubscribe router events and reset surfaces so // the router's component subscriptions don't outlive the window. The // router holds back-references via event delegates; without this, a @@ -77,6 +97,7 @@ public A2UICanvasWindow(IActionSink actions, MediaResolver media, IOpenClawLogge Closed += (_, _) => { IsClosed = true; + ExitFullScreen(); try { _router.SurfaceCreated -= OnSurfaceCreated; } catch (Exception ex) { OpenClawTray.Services.Logger.Debug($"A2UICanvasWindow: unsubscribe SurfaceCreated failed: {ex.Message}"); } try { _router.SurfaceDeleted -= OnSurfaceDeleted; } @@ -338,6 +359,30 @@ public string GetStateSnapshot() return snapshot.ToJsonString(); } + // ── Fullscreen ────────────────────────────────────────────────────────── + + /// Toggle borderless fullscreen on F11. + public void ToggleFullScreen() + { + if (_isFullScreen) ExitFullScreen(); + else EnterFullScreen(); + } + + private void EnterFullScreen() + { + _isFullScreen = true; + try { AppWindow.SetPresenter(AppWindowPresenterKind.FullScreen); } + catch (Exception ex) { OpenClawTray.Services.Logger.Debug($"A2UICanvasWindow: EnterFullScreen failed: {ex.Message}"); } + } + + private void ExitFullScreen() + { + if (!_isFullScreen) return; + _isFullScreen = false; + try { AppWindow.SetPresenter(AppWindowPresenterKind.Default); } + catch (Exception ex) { OpenClawTray.Services.Logger.Debug($"A2UICanvasWindow: ExitFullScreen failed: {ex.Message}"); } + } + public void BringToFront(bool keepTopMost = false) { try diff --git a/src/OpenClaw.Tray.WinUI/Windows/CanvasWindow.xaml.cs b/src/OpenClaw.Tray.WinUI/Windows/CanvasWindow.xaml.cs index 02f086663..1eceb824b 100644 --- a/src/OpenClaw.Tray.WinUI/Windows/CanvasWindow.xaml.cs +++ b/src/OpenClaw.Tray.WinUI/Windows/CanvasWindow.xaml.cs @@ -5,8 +5,10 @@ using System.Threading.Tasks; using System.Runtime.InteropServices; using Microsoft.UI.Dispatching; +using Microsoft.UI.Windowing; using Microsoft.UI.Xaml; using Microsoft.UI.Xaml.Automation; +using Microsoft.UI.Xaml.Input; using Microsoft.Web.WebView2.Core; using OpenClaw.Shared; using OpenClawTray.Helpers; @@ -14,6 +16,7 @@ using WinUIEx; using Windows.Foundation; using Windows.Storage.Streams; +using Windows.System; namespace OpenClawTray.Windows; @@ -39,6 +42,7 @@ public sealed partial class CanvasWindow : WindowEx private const uint SWP_SHOWWINDOW = 0x0040; private bool _isWebViewInitialized; + private bool _isFullScreen; private string? _pendingUrl; private string? _pendingHtml; private readonly TaskCompletionSource _webViewReadyTcs = new(TaskCreationOptions.RunContinuationsAsynchronously); @@ -50,7 +54,7 @@ public sealed partial class CanvasWindow : WindowEx private FileSystemWatcher? _canvasWatcher; private long _lastReloadTicks = 0; - private readonly DispatcherQueue? _dispatcherQueue; + private readonly Microsoft.UI.Dispatching.DispatcherQueue? _dispatcherQueue; private TypedEventHandler? _webMessageReceivedHandler; private TypedEventHandler? _webResourceRequestedHandler; private string? _webResourceRequestedFilter; @@ -235,7 +239,30 @@ public CanvasWindow() this.SetIcon("Assets\\openclaw.ico"); _dispatcherQueue = DispatcherQueue; this.Closed += OnWindowClosed; - + + // F11 toggles borderless fullscreen; Escape exits it. + // KeyboardAccelerators on the root content get first-class handling + // when focus is in the XAML tree (title bar, toolbar, etc.). + // When focus is inside the WebView2, F11/Escape are also intercepted + // via an injected content script that posts bridge messages. + if (this.Content is FrameworkElement contentRoot) + { + var f11Accel = new KeyboardAccelerator { Key = VirtualKey.F11 }; + f11Accel.Invoked += (_, args) => + { + args.Handled = true; + ToggleFullScreen(); + }; + var escAccel = new KeyboardAccelerator { Key = VirtualKey.Escape }; + escAccel.Invoked += (_, args) => + { + if (_isFullScreen) { args.Handled = true; ExitFullScreen(); } + }; + contentRoot.KeyboardAccelerators.Add(f11Accel); + contentRoot.KeyboardAccelerators.Add(escAccel); + contentRoot.KeyboardAcceleratorPlacementMode = KeyboardAcceleratorPlacementMode.Hidden; + } + // Initialize WebView2 InitializeWebViewAsync(); } @@ -284,6 +311,24 @@ private async Task InitializeWebViewCoreAsync() CanvasWebView.CoreWebView2.Settings.IsStatusBarEnabled = false; CanvasWebView.CoreWebView2.Settings.AreDevToolsEnabled = false; + // Inject F11/Escape fullscreen bridge: intercepts these keys when + // WebView2 content has focus and routes them as bridge messages so + // the native window can toggle its presenter the same way the XAML + // keyboard accelerators do when focus is in the title bar. + await CanvasWebView.CoreWebView2.AddScriptToExecuteOnDocumentCreatedAsync(""" + (function () { + document.addEventListener('keydown', function (e) { + if (!window.chrome || !window.chrome.webview) return; + if (e.key === 'F11') { + e.preventDefault(); + window.chrome.webview.postMessage({ type: 'fullscreen-toggle' }); + } else if (e.key === 'Escape') { + window.chrome.webview.postMessage({ type: 'fullscreen-exit' }); + } + }, true); + })(); + """); + // Wire the bidirectional native↔SPA bridge // SPA → native: window.chrome.webview.postMessage({ type, payload }) _webMessageReceivedHandler = (s, e) => @@ -297,6 +342,19 @@ private async Task InitializeWebViewCoreAsync() var msg = WebBridgeMessage.TryParse(e.WebMessageAsJson); if (msg != null) { + // Fullscreen control messages are handled natively and not + // forwarded to external bridge subscribers. + if (msg.Type == WebBridgeMessage.TypeFullscreenToggle) + { + _dispatcherQueue?.TryEnqueue(ToggleFullScreen); + return; + } + if (msg.Type == WebBridgeMessage.TypeFullscreenExit) + { + _dispatcherQueue?.TryEnqueue(ExitFullScreen); + return; + } + Logger.Debug($"[Canvas] bridge message from SPA, type={SanitizeBridgeLogValue(msg.Type)}"); BridgeMessageReceived?.Invoke(this, msg); } @@ -473,6 +531,7 @@ private void OnCanvasFileChanged(object sender, FileSystemEventArgs e) private void OnWindowClosed(object sender, WindowEventArgs args) { IsClosed = true; + ExitFullScreen(); _gatewayToken = null; if (CanvasWebView.CoreWebView2 != null) @@ -659,6 +718,30 @@ public void BringToFront(bool keepTopMost) } } + // ── Fullscreen ────────────────────────────────────────────────────────── + + /// Toggle borderless fullscreen on F11. + public void ToggleFullScreen() + { + if (_isFullScreen) ExitFullScreen(); + else EnterFullScreen(); + } + + private void EnterFullScreen() + { + _isFullScreen = true; + try { AppWindow.SetPresenter(AppWindowPresenterKind.FullScreen); } + catch (Exception ex) { Logger.Debug($"[Canvas] EnterFullScreen failed: {ex.Message}"); } + } + + private void ExitFullScreen() + { + if (!_isFullScreen) return; + _isFullScreen = false; + try { AppWindow.SetPresenter(AppWindowPresenterKind.Default); } + catch (Exception ex) { Logger.Debug($"[Canvas] ExitFullScreen failed: {ex.Message}"); } + } + public async Task EnsureA2UIHostAsync(string url) { await EnsureWebViewReadyAsync(); diff --git a/tests/OpenClaw.Shared.Tests/WebBridgeMessageTests.cs b/tests/OpenClaw.Shared.Tests/WebBridgeMessageTests.cs index f99cc9ad8..3cd2e3cca 100644 --- a/tests/OpenClaw.Shared.Tests/WebBridgeMessageTests.cs +++ b/tests/OpenClaw.Shared.Tests/WebBridgeMessageTests.cs @@ -53,6 +53,16 @@ public void TryParse_InvalidJson_ReturnsNull() Assert.Null(WebBridgeMessage.TryParse("{bad json")); } + [Fact] + public void TryParse_StringRoot_ReturnsNull() + { + var jsonStringRoot = """ + "{\"type\":\"fullscreen-toggle\"}" + """; + + Assert.Null(WebBridgeMessage.TryParse(jsonStringRoot)); + } + [Fact] public void TryParse_TypeIsNotString_ReturnsNull() { @@ -129,6 +139,8 @@ public void ToJson_PassedPayloadOverridesStoredPayloadJson() [InlineData(WebBridgeMessage.TypeVoiceStart)] [InlineData(WebBridgeMessage.TypeVoiceStop)] [InlineData(WebBridgeMessage.TypeReady)] + [InlineData(WebBridgeMessage.TypeFullscreenToggle)] + [InlineData(WebBridgeMessage.TypeFullscreenExit)] public void RoundTrip_WellKnownTypes_PreserveType(string type) { var original = new WebBridgeMessage(type);