macOS port: fix freezes, unclickable controls and a crash found while running the lattice-attack dialogs

CrypTool/RSABloemerMayDlg.cpp

@@ -446,6 +446,10 @@ BOOL CRSABloemerMayDlg::OnSetCursor(CWnd* pWnd, UINT nHitTest, UINT message)
 }
 
 
+// Set while the dialog itself updates the delta / bit length fields, so their change
+// handlers (which clear N, e and d after a user edit) do not fire on those updates.
+static bool g_programmaticUpdate = false;
+
 void CRSABloemerMayDlg::checkStartButton()
 {
 	NTLExpPars myPars;
@@ -457,7 +461,9 @@ void CRSABloemerMayDlg::checkStartButton()
 		m_delta= doubleToCString(log(d)/log(N));
 	m_BitsOfN.Format("%d",NumBits(N));
 	GetDlgItem(IDC_BUTTONSTART)->EnableWindow(N>0 && e>0 && m_m>3);
+	g_programmaticUpdate = true;
 	UpdateData(false);
+	g_programmaticUpdate = false;
 }
 
 void CRSABloemerMayDlg::OnButtondefault() 
@@ -536,6 +542,7 @@ void CRSABloemerMayDlg::OnChangeEditd()
 
 void CRSABloemerMayDlg::OnChangeEditbitsofn() 
 {
+	if (g_programmaticUpdate) return;
 	UpdateData();
 	m_N="";
 	m_e="";
@@ -545,6 +552,7 @@ void CRSABloemerMayDlg::OnChangeEditbitsofn()
 
 void CRSABloemerMayDlg::OnChangeEditdelta() 
 {
+	if (g_programmaticUpdate) return;
 	UpdateData();
 	m_d="";
 	if(m_choice==1){

port/mfcwx/src/controls.cpp

@@ -191,10 +191,11 @@ void SetControlText(wxWindow* w, const wxString& text) {
     case ControlKind::RichEdit:
         if (auto* t = AsText(w)) {
             wxString v = ToEditText(*st, text);
+            // Only notify when the text really changes: dialogs such as the stereotyped-message
+            // attack set sibling edits from their change handlers, which otherwise recurse
+            // until the stack overflows.
             if (t->GetValue() != v)
                 t->SetValue(v);
-            else
-                t->SetValue(v);
             Extra<EditExtra>(*st).modified = false;
         }
         return;
@@ -1135,6 +1136,12 @@ wxWindow* CreateControl(wxWindow* parent, const char* cls, const char* text, int
         // Statics are never in the tab order.
         st.style &= ~WS_TABSTOP;
     }
+    if (kind == ControlKind::GroupBox) {
+        // Win32 group boxes are transparent to the mouse, but rc files often declare them after the
+        // controls they enclose. A wxStaticBox created later would sit on top and swallow clicks
+        // (e.g. the ciphertext edit of the stereotyped-message dialog), so keep it at the bottom.
+        w->Lower();
+    }
     if (kind == ControlKind::Slider) {
         Extra<RangeExtra>(st).maxValue = 100;
     }

port/mfcwx/src/docview.cpp

@@ -60,6 +60,12 @@ class MfcFrame : public wxFrame {
     }
 
     static void UpdateCommandUI(wxWindow* frame, wxUpdateUIEvent& e) {
+        // The tab control / notebook of the MDI client get wx-generated ids that may fall into the
+        // command id range; auto-disabling them as "unhandled commands" made tabs unclickable.
+        if (wxWindow* w = wxDynamicCast(e.GetEventObject(), wxWindow)) {
+            if (wxDynamicCast(w, wxAuiTabCtrl) || wxDynamicCast(w, wxAuiNotebook))
+                return;
+        }
         CWnd* p = PermanentWnd(frame);
         if (!p || !IsCommandWxId(e.GetId()) || !p->IsKindOf(RUNTIME_CLASS(CFrameWnd))) {
             if (e.GetId() == wxID_ABOUT || e.GetId() == wxID_EXIT)

port/mfcwx/src/internal.h

@@ -146,6 +146,7 @@ bool IsMainThread();
 // Runs fn on the main thread and waits for it (Windows SendMessage semantics across threads).
 void RunOnMainThread(const std::function<void()>& fn);
 void PostToMainThread(std::function<void()> fn);
+void PumpMainThreadTasks();
 
 // Converts a window rectangle in dialog units to pixels for the given dialog window.
 wxRect DialogUnitsToPixels(wxWindow* dialog, int x, int y, int cx, int cy);

port/mfcwx/src/kernel.cpp

@@ -584,8 +584,11 @@ HANDLE GetCurrentProcess(void) { return kCurrentProcess; }
 
 HANDLE GetCurrentThread(void) { return kCurrentThread; }
 
+namespace mfcwx { void PumpMainThreadTasks(); }
+
 void Sleep(DWORD ms) {
     CheckSuspend();
+    mfcwx::PumpMainThreadTasks();
     if (ms == 0) {
         sched_yield();
     } else if (ms == INFINITE) {

port/mfcwx/src/wnd.cpp

@@ -50,6 +50,14 @@ void RunOnMainThread(const std::function<void()>& fn) {
         std::rethrow_exception(error);
 }
 
+// Sleep() on the main thread must keep serving RunOnMainThread requests from worker threads,
+// otherwise a worker holding a lock while waiting for the main thread deadlocks with the main
+// thread spinning in a Sleep loop (e.g. the factorisation progress dialog).
+void PumpMainThreadTasks() {
+    if (IsMainThread() && wxTheApp)
+        wxTheApp->ProcessPendingEvents();
+}
+
 void PostToMainThread(std::function<void()> fn) {
     if (wxTheApp)
         wxTheApp->CallAfter(std::move(fn));