Skip to content
Open
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
13 changes: 13 additions & 0 deletions CMakeLists.txt
Original file line number Diff line number Diff line change
Expand Up @@ -1396,6 +1396,19 @@ if(BUILD_TESTING)
add_executable(ARMixRecoveryTests tests/direct_restore_backoff_test.cpp)
target_compile_features(ARMixRecoveryTests PRIVATE cxx_std_20)
add_test(NAME BoundedDirectDeviceRecovery COMMAND ARMixRecoveryTests)
# N-MOD-19: native JUCE CONTROL coordinator tested with fake device/driver
# status and real diagnostics, but never opening hardware or rendering PCM.
juce_add_console_app(ARMixNativeDirectRestoreTests
PRODUCT_NAME "ArSonMix Native Direct Restore Coordinator Contract")
target_sources(ARMixNativeDirectRestoreTests PRIVATE
tests/native_direct_restore_coordinator_test.cpp
src/NativeDirectRestoreCoordinator.h)
target_compile_features(ARMixNativeDirectRestoreTests PRIVATE cxx_std_20)
target_link_libraries(ARMixNativeDirectRestoreTests PRIVATE
juce::juce_core juce::juce_recommended_config_flags)
add_test(NAME NativeDirectRestoreBoundedRetryAndFaultEvidence
COMMAND ARMixNativeDirectRestoreTests)
add_dependencies(ARMixMixerRoutingTests ARMixNativeDirectRestoreTests)
add_executable(ARMixClockSizingTests tests/clock_bridge_sizing_test.cpp)
target_compile_features(ARMixClockSizingTests PRIVATE cxx_std_20)
add_test(NAME AdaptiveClockPeriodHeadroom COMMAND ARMixClockSizingTests)
Expand Down
104 changes: 17 additions & 87 deletions src/Main.cpp
Original file line number Diff line number Diff line change
Expand Up @@ -19,6 +19,7 @@
#include "../engine/core/DearVrPolarPolicy.h"
#include "DiagnosticsRecorder.h"
#include "../engine/device/DirectRestoreBackoff.h"
#include "NativeDirectRestoreCoordinator.h"
#include "../engine/device/AudioSetupRecoveryPolicy.h"
#include "../engine/device/AsioRestartRecoveryGate.h"
#include "../engine/device/PeriodCapabilityEvidence.h"
Expand Down Expand Up @@ -928,95 +929,24 @@ class ConsoleWindow final : public juce::DocumentWindow, private juce::Timer, pr
return report;
}

void annotateRecovery (juce::var& state) const
{
auto* object = state.getDynamicObject();
if (object == nullptr) return;
const auto snapshot = directRestore_.snapshot();
using State = armix::device::DirectRestoreBackoff::State;
const char* label = snapshot.state == State::idle ? "IDLE"
: snapshot.state == State::waiting ? "WAITING" : "EXHAUSTED";
object->setProperty ("directRestoreState", label);
object->setProperty ("directRestoreAttempts", static_cast<int> (snapshot.attempts));
object->setProperty ("directRestoreNextMs",
static_cast<double> (directRestore_.remainingMs (monotonicMilliseconds())));
object->setProperty ("directRestoreGeneration", static_cast<double> (snapshot.generation));
object->setProperty ("directRestoreError", lastDirectRestoreError_);
object->setProperty ("directRestorePending", engine_.hasPendingDirectRestore());
void annotateRecovery (juce::var& state) const {
armix::device::NativeDirectRestoreCoordinator::annotate(
state,directRestore_,lastDirectRestoreError_,
engine_.hasPendingDirectRestore(),monotonicMilliseconds());
}

// Device reopens happen on the JUCE message thread, never the RT callback.
// Restoring DIRECT never silently re-enables a faulting bridged route.
void serviceDirectRestore()
{
using Gate = armix::device::DirectRestoreBackoff;
const auto now = monotonicMilliseconds();
// Independent drivers can stop callbacks without emitting JUCE error.
// Read existing atomic progress on message thread before failback gate.
engine_.pollIndependentClockHealth (now);
auto snapshot = directRestore_.snapshot();
if (snapshot.state == Gate::State::idle &&
(engine_.independentClockFaulted() ||
(!engine_.independentClockActive() && engine_.hasPendingDirectRestore())))
{
const auto generation = directRestore_.begin (now);
diagnostics_.record ("DIRECT_RESTORE_BEGIN",
juce::String ("Generation ") + juce::String (static_cast<juce::int64> (generation)));
snapshot = directRestore_.snapshot();
}
if (!directRestore_.due (snapshot.generation, now)) return;

const bool wasIndependent = engine_.independentClockActive();
const auto stateBeforeRestore = engine_.status();
const auto faultStatus = juce::JSON::toString (stateBeforeRestore, false);
// DiagnosticsRecorder bounds event details to 950 characters. Keep
// the independent stream fault source and durations FIRST, before the
// potentially long full status JSON, so field reports retain evidence
// after closing the adapter resets its watchdog.
const auto watchdogSummary = juce::String ("format=")
+ stateBeforeRestore.getProperty ("bridgeNegotiationFault", "NONE").toString()
+ " watchdog="
+ stateBeforeRestore.getProperty ("bridgeWatchdogFault", "NONE").toString()
+ " captureSilentMs="
+ stateBeforeRestore.getProperty ("bridgeCaptureSilentMs", 0).toString()
+ " renderSilentMs="
+ stateBeforeRestore.getProperty ("bridgeRenderSilentMs", 0).toString()
+ " continuity="
+ stateBeforeRestore.getProperty ("bridgeContinuityFault", "NONE").toString()
+ " lossBp="
+ stateBeforeRestore.getProperty ("bridgeContinuityLossBp", 0).toString()
+ " badWindows="
+ stateBeforeRestore.getProperty ("bridgeContinuityBadWindows", 0).toString();
const auto error = engine_.setIndependentClockEnabled (false);
if (error.isEmpty())
{
directRestore_.recovered (snapshot.generation);
lastDirectRestoreError_.clear();
deviceSettingsDirty_ = true;
diagnostics_.record ("SMART_CLOCK_RECOVERY",
juce::String ("DIRECT_RESTORED after ") + juce::String (snapshot.attempts + 1) +
" attempt(s) | " + watchdogSummary + " | " + faultStatus);
if (engine_.isOutputMuted())
emitNotice ("DIRECT device reopened. Output remains muted for safety; unmute only when ready.", true);
else
emitNotice (wasIndependent
? "Independent device fault: safe DIRECT audio restored."
: "DIRECT audio device restored after a bounded retry.", true);
return;
}

directRestore_.failed (snapshot.generation, now);
lastDirectRestoreError_ = error;
const auto next = directRestore_.snapshot();
diagnostics_.record ("SMART_CLOCK_RECOVERY",
juce::String ("RESTORE_FAIL ") + juce::String (next.attempts) + "/" +
juce::String (static_cast<int> (Gate::maxAttempts)) + ": " + error +
" | " + watchdogSummary + " | " + faultStatus);
if (next.state == Gate::State::exhausted)
emitNotice ("Automatic DIRECT recovery exhausted; audio remains safe/muted. "
"Check AUDIO SETUP or retry DIRECT manually. " + error, true);
else if (next.attempts == 1)
emitNotice ("Device interrupted. DIRECT restore is retrying with bounded backoff.", true);
// One control owner; recovery policy, telemetry and bounded retries live
// in the dedicated Native coordinator. No change to ASIO callback path.
void serviceDirectRestore() {
armix::device::NativeDirectRestoreCoordinator::service(
directRestore_,lastDirectRestoreError_,deviceSettingsDirty_,
monotonicMilliseconds(),engine_,
Comment thread
masarray marked this conversation as resolved.
Resolved
[this](const juce::String& code,const juce::String& detail) {
diagnostics_.record(code,detail);
},
[this](const juce::String& message,bool error) {
emitNotice(message,error);
});
}

void emitNotice (const juce::String& message, bool error = false)
Expand Down
105 changes: 105 additions & 0 deletions src/NativeDirectRestoreCoordinator.h
Original file line number Diff line number Diff line change
@@ -0,0 +1,105 @@
// SPDX-License-Identifier: AGPL-3.0-only
#pragma once
#include "../engine/device/DirectRestoreBackoff.h"
#include <juce_core/juce_core.h>

// N-MOD-19: ONE control-thread DIRECT restore coordinator. Does not own a
// device, callback, PCM renderer, driver binding, or recovery timer. The
// existing LiveAudioEngine still exclusively owns JUCE device transactions.
// The caller supplies diagnostics and operator notification sinks.
namespace armix::device {
class NativeDirectRestoreCoordinator final {
public:
using Gate=DirectRestoreBackoff;
static void annotate(juce::var& state,const Gate& gate,
const juce::String& previousError,
bool pending,std::uint64_t nowMs) {
auto* obj=state.getDynamicObject();
if(!obj)return;
const auto snapshot=gate.snapshot();
const char* stateName=snapshot.state==Gate::State::idle?"IDLE":
snapshot.state==Gate::State::waiting?"WAITING":"EXHAUSTED";
obj->setProperty("directRestoreState",stateName);
obj->setProperty("directRestoreAttempts",
static_cast<int>(snapshot.attempts));
obj->setProperty("directRestoreNextMs",
static_cast<double>(gate.remainingMs(nowMs)));
obj->setProperty("directRestoreGeneration",
static_cast<double>(snapshot.generation));
obj->setProperty("directRestoreError",previousError);
obj->setProperty("directRestorePending",pending);
}

template<class Engine,class Diagnostic,class Notice>
static void service(Gate& gate,juce::String& previousError,
bool& settingsDirty,std::uint64_t nowMs,
Engine& engine,Diagnostic&& record,Notice&& notify) {
// Control thread only. Polling observes independent callback atomic
// progress but never executes or blocks inside either audio callback.
engine.pollIndependentClockHealth(nowMs);
auto snapshot=gate.snapshot();
if(snapshot.state==Gate::State::idle &&
(engine.independentClockFaulted() ||
(!engine.independentClockActive() &&
engine.hasPendingDirectRestore()))) {
const auto generation=gate.begin(nowMs);
record("DIRECT_RESTORE_BEGIN",
juce::String("Generation ") +
juce::String(static_cast<juce::int64>(generation)));
snapshot=gate.snapshot();
}
if(!gate.due(snapshot.generation,nowMs))return;

const bool wasIndependent=engine.independentClockActive();
const auto stateBeforeRestore=engine.status();
const auto faultStatus=juce::JSON::toString(stateBeforeRestore,false);
// Keep field diagnostics FIRST: recorder truncation must not erase
// stream watchdog evidence when the device is disposed.
const auto watchdogSummary=juce::String("format=")
+ stateBeforeRestore.getProperty("bridgeNegotiationFault","NONE").toString()
+ " watchdog="
+ stateBeforeRestore.getProperty("bridgeWatchdogFault","NONE").toString()
+ " captureSilentMs="
+ stateBeforeRestore.getProperty("bridgeCaptureSilentMs",0).toString()
+ " renderSilentMs="
+ stateBeforeRestore.getProperty("bridgeRenderSilentMs",0).toString()
+ " continuity="
+ stateBeforeRestore.getProperty("bridgeContinuityFault","NONE").toString()
+ " lossBp="
+ stateBeforeRestore.getProperty("bridgeContinuityLossBp",0).toString()
+ " badWindows="
+ stateBeforeRestore.getProperty("bridgeContinuityBadWindows",0).toString();
// The actual restore call remains on the one existing engine owner.
const auto error=engine.setIndependentClockEnabled(false);
if(error.isEmpty()) {
gate.recovered(snapshot.generation);
previousError.clear();
settingsDirty=true;
record("SMART_CLOCK_RECOVERY",
juce::String("DIRECT_RESTORED after ") +
juce::String(snapshot.attempts+1) +
" attempt(s) | " +watchdogSummary+" | "+faultStatus);
if(engine.isOutputMuted())
notify("DIRECT device reopened. Output remains muted for safety; unmute only when ready.",true);
else
notify(wasIndependent?
"Independent device fault: safe DIRECT audio restored.":
"DIRECT audio device restored after a bounded retry.",true);
return;
}
gate.failed(snapshot.generation,nowMs);
previousError=error;
const auto next=gate.snapshot();
record("SMART_CLOCK_RECOVERY",
juce::String("RESTORE_FAIL ") +
juce::String(next.attempts) + "/" +
juce::String(static_cast<int>(Gate::maxAttempts)) +
": " +error+" | "+watchdogSummary+" | "+faultStatus);
if(next.state==Gate::State::exhausted)
notify("Automatic DIRECT recovery exhausted; audio remains safe/muted. "
"Check AUDIO SETUP or retry DIRECT manually. "+error,true);
else if(next.attempts==1)
notify("Device interrupted. DIRECT restore is retrying with bounded backoff.",true);
}
};
} // namespace armix::device
32 changes: 21 additions & 11 deletions tests/direct_restore_recovery_contract_test.mjs
Original file line number Diff line number Diff line change
Expand Up @@ -7,6 +7,7 @@ const read=p=>readFileSync(p,'utf8');
const main=read('src/Main.cpp');
const engine=read('src/LiveAudioEngine.h');
const gate=read('engine/device/DirectRestoreBackoff.h');
const coordinator=read('src/NativeDirectRestoreCoordinator.h');
const cmake=read('CMakeLists.txt');
const workflow=read('.github/workflows/p1-windows.yml');

Expand All @@ -21,8 +22,10 @@ test('actual device recoveries occur on message thread, not inside audio callbac
assert.ok(graphAck>timer);
assert.match(main.slice(timer,graphAck),/serviceDirectRestore\(\)/);
assert.match(main.slice(runner,main.indexOf('void emitNotice',runner)),
/engine_\.setIndependentClockEnabled \(false\)/);
assert.match(main,/engine_\.independentClockFaulted\(\)/);
/NativeDirectRestoreCoordinator::service/);
assert.match(coordinator,/engine\.setIndependentClockEnabled\(false\)/);
assert.match(coordinator,/engine\.independentClockFaulted\(\)/);
assert.doesNotMatch(coordinator,/audioDeviceIOCallbackWithContext/);
assert.match(engine,/hasPendingDirectRestore\(\) const noexcept/);
assert.match(engine,/verifyRestoredDirectDevice \(\*savedDirectSetup_\)/);
assert.match(engine,/verifyRestoredDirectDevice \(original\)/);
Expand All @@ -49,18 +52,23 @@ test('output-only SAME-device WASAPI recovery never invents or requires micropho
assert.match(main,/No verified stereo playback device; audio remains gated/);
});
test('restore retry is bounded and disabled on success, explicit direct and setup paths',()=>{
assert.match(main,/directRestore_\.failed \(snapshot\.generation, now\)/);
assert.match(main,/directRestore_\.recovered \(snapshot\.generation\)/);
assert.match(coordinator,/gate\.failed\(snapshot\.generation,nowMs\)/);
assert.match(coordinator,/gate\.recovered\(snapshot\.generation\)/);
assert.match(main,/directRestore_\.cancel\(\)/);
assert.match(main,/independentClockActive\(\) \|\| engine_\.hasPendingDirectRestore\(\)/);
assert.match(main,/directRestoreAttempts/);
assert.match(main,/directRestoreNextMs/);
assert.match(main,/directRestoreError/);
assert.match(main,/DIRECT_RESTORE_BEGIN/);
assert.match(main,/SMART_CLOCK_RECOVERY/);
assert.match(coordinator,/engine\.independentClockActive\(\)/);
assert.match(coordinator,/engine\.hasPendingDirectRestore\(\)/);
assert.match(coordinator,/directRestoreAttempts/);
assert.match(coordinator,/directRestoreNextMs/);
assert.match(coordinator,/directRestoreError/);
assert.match(coordinator,/DIRECT_RESTORE_BEGIN/);
assert.match(coordinator,/SMART_CLOCK_RECOVERY/);
assert.match(coordinator,/bridgeCaptureSilentMs/);
assert.match(coordinator,/bridgeWatchdogFault/);
assert.match(main,/annotateRecovery \(status\)/);
assert.match(main,/annotateRecovery \(state\)/);
assert.match(cmake,/add_test\(NAME BoundedDirectDeviceRecovery COMMAND ARMixRecoveryTests\)/);
assert.match(cmake,/add_test\(NAME NativeDirectRestoreBoundedRetryAndFaultEvidence/);
assert.match(main,/NativeDirectRestoreCoordinator::annotate/);
assert.match(workflow,/direct_restore_recovery_contract_test.mjs/);
});
test('verified native FAT package preserves actual beta live-bridge provenance, not older offline-only claim',()=>{
Expand All @@ -77,7 +85,9 @@ test('verified native FAT package preserves actual beta live-bridge provenance,
});

test('healthy independent sessions never arm restoration; setup can replace missing devices',()=>{
assert.match(main,/\(!engine_\.independentClockActive\(\) && engine_\.hasPendingDirectRestore\(\)\)/);
assert.match(coordinator,/!engine\.independentClockActive\(\) &&/);
assert.match(coordinator,/engine\.hasPendingDirectRestore\(\)/);
assert.doesNotMatch(coordinator,/setIndependentClockEnabled\(true\)/);
assert.match(engine,/bool abandonPendingDirectRestoreForSetup\(\)/);
assert.match(engine,/deviceManager_\.addAudioCallback \(this\)/);
assert.match(main,/DIRECT_RESTORE_ABANDON/);
Expand Down
Loading
Loading