diff --git a/.cargo/clippy.toml b/.cargo/clippy.toml new file mode 100644 index 0000000..490a51c --- /dev/null +++ b/.cargo/clippy.toml @@ -0,0 +1,39 @@ +# Settings for clippy, found through CLIPPY_CONF_DIR in config.toml. + +# Time comes from the clock of the Ark connection, so the calls it replaces are +# banned. The first group is darkbio-clock's recommended list, crossbeam's +# entries included. The second covers crypto's functions that read the system +# time themselves, and the third chrono's reads of the wall time. +disallowed-methods = [ + { path = "std::time::Instant::now", reason = "use Clock::now" }, + { path = "std::time::Instant::elapsed", reason = "use Clock::elapsed" }, + { path = "std::time::SystemTime::now", reason = "use Clock::system_time" }, + { path = "std::time::SystemTime::elapsed", reason = "use Clock::system_time" }, + { path = "std::thread::sleep", reason = "use Clock::sleep" }, + { path = "std::thread::park_timeout", reason = "use a clock condvar" }, + { path = "std::sync::Condvar::wait_timeout", reason = "use darkbio_clock::sync::Condvar" }, + { path = "std::sync::Condvar::wait_timeout_while", reason = "use darkbio_clock::sync::Condvar" }, + { path = "std::sync::mpsc::Receiver::recv_timeout", reason = "use a crossbeam receiver with Clock::recv_timeout" }, + { path = "crossbeam_channel::after", reason = "use Clock::after" }, + { path = "crossbeam_channel::at", reason = "use Clock::at" }, + { path = "crossbeam_channel::tick", reason = "arm Clock::at per period" }, + { path = "crossbeam_channel::Receiver::recv_timeout", reason = "use Clock::recv_timeout" }, + { path = "crossbeam_channel::Receiver::recv_deadline", reason = "use Clock::recv_deadline" }, + { path = "crossbeam_channel::Sender::send_timeout", reason = "select against a clock timer" }, + { path = "crossbeam_channel::Sender::send_deadline", reason = "select against a clock timer" }, + { path = "crossbeam_channel::Select::select_timeout", reason = "select against a clock timer" }, + { path = "crossbeam_channel::Select::select_deadline", reason = "select against a clock timer" }, + { path = "crossbeam_channel::Select::ready_timeout", reason = "select against a clock timer" }, + { path = "crossbeam_channel::Select::ready_deadline", reason = "select against a clock timer" }, + + { path = "darkbio_crypto::cose::sign", reason = "use cose::sign_at with the clock's wall time" }, + { path = "darkbio_crypto::cose::sign_detached", reason = "use cose::sign_detached_at with the clock's wall time" }, + { path = "darkbio_crypto::cose::verify", reason = "use cose::verify_at with the clock's wall time" }, + { path = "darkbio_crypto::cose::verify_detached", reason = "use cose::verify_detached_at with the clock's wall time" }, + { path = "darkbio_crypto::cose::seal", reason = "use cose::seal_at with the clock's wall time" }, + { path = "darkbio_crypto::cose::open", reason = "use cose::open_at with the clock's wall time" }, + { path = "darkbio_crypto::cwt::issue", reason = "use cwt::issue_at with the clock's wall time" }, + + { path = "chrono::Utc::now", reason = "convert Clock::system_time" }, + { path = "chrono::Local::now", reason = "convert Clock::system_time" }, +] diff --git a/.cargo/config.toml b/.cargo/config.toml new file mode 100644 index 0000000..7b2f125 --- /dev/null +++ b/.cargo/config.toml @@ -0,0 +1,6 @@ +# Settings for every cargo command run inside this repository. + +[env] +# Clippy only searches the crate directory and its parents for its settings, +# so point it at the ones kept here. +CLIPPY_CONF_DIR = { value = ".cargo", relative = true } diff --git a/Cargo.lock b/Cargo.lock index d66e239..cfe1412 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -319,6 +319,21 @@ dependencies = [ "libc", ] +[[package]] +name = "crossbeam-channel" +version = "0.5.17" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "98b0cc327b5bc766e7fda9c9260cc0fa81b43a8e240440422dff70788e3f9ef1" +dependencies = [ + "crossbeam-utils", +] + +[[package]] +name = "crossbeam-utils" +version = "0.8.23" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "a31eee39dddec8330830986fcd7625edb5a24ec90ea038215273bbc3adb08ac6" + [[package]] name = "crypto-common" version = "0.1.7" @@ -386,6 +401,7 @@ dependencies = [ "clap", "clap_complete", "console", + "darkbio-clock", "darkbio-crypto", "darkbio-trust", "darkbio-wire", @@ -407,6 +423,15 @@ dependencies = [ "windows-sys 0.61.2", ] +[[package]] +name = "darkbio-clock" +version = "0.3.1" +source = "registry+https://github.com/rust-lang/crates.io-index" +checksum = "b5722bbf6e80f8681f36b6d0f0e6f5c81ba719fe6e7773b6e66ef8ba9d8e8c62" +dependencies = [ + "crossbeam-channel", +] + [[package]] name = "darkbio-cobs" version = "1.0.0" @@ -419,14 +444,14 @@ dependencies = [ [[package]] name = "darkbio-crypto" -version = "0.18.2" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "dbbc26cef72218bb30779a4269d2f5be84bf20575848a63a6044a786b9e332a1" +checksum = "030dcbb25b2b423e3ffddbf27b834815353a86aad76e828d852be3581a8e0b15" dependencies = [ "base64", "chacha20poly1305", "darkbio-crypto-cbor-derive", - "der 0.8.1", + "der 0.8.2", "ed25519-dalek", "getrandom 0.2.17", "getrandom 0.4.3", @@ -445,20 +470,20 @@ dependencies = [ [[package]] name = "darkbio-crypto-cbor-derive" -version = "0.18.2" +version = "0.19.1" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "0e0905e0df57aae0ba154f635c11a8e6ea39f79aba99edfdf2c8114fc31ad250" +checksum = "a4a03fa7df24e0afd285b98163b0b24ecde7ce87f2672e55bf70b12a7afec3c0" dependencies = [ "proc-macro2", "quote", - "syn 3.0.4", + "syn 3.0.6", ] [[package]] name = "darkbio-trust" -version = "0.5.1" +version = "0.6.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "f071d9b0c29b337c5342872ecf0b149f3b2b2ea9447c21cde1dffe53f810e2f1" +checksum = "68d3d06e8971c65286e956a3b8eadcde851e7f60ec4a854d04ce84b9db14725e" dependencies = [ "darkbio-crypto", "hex", @@ -467,10 +492,11 @@ dependencies = [ [[package]] name = "darkbio-wire" -version = "0.10.0" +version = "0.12.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "40f15a5cbf7a94fc3e71b048b1473049d427bce09b19e61daf27a616c176e53f" +checksum = "f85f05ce8ac70b644694edb9a8cbb66fd1e860e7544d57929d98c8557d9b5332" dependencies = [ + "darkbio-clock", "darkbio-cobs", "darkbio-crypto", "darkbio-trust", @@ -499,9 +525,9 @@ dependencies = [ [[package]] name = "der" -version = "0.8.1" +version = "0.8.2" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "a69dedd701da44b0536442edf09c81a64b0ab97a7a4a5e3d1971f00027cbc63d" +checksum = "a878c850e9e421b20262e9b41f9c860e4785fa07541c266b62ff9d1ef998a80a" dependencies = [ "const-oid 0.10.2", "der_derive", @@ -1147,7 +1173,7 @@ version = "0.11.0" source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "451913da69c775a56034ea8d9003d27ee8948e12443eae7c038ba100a4f21cb7" dependencies = [ - "der 0.8.1", + "der 0.8.2", "spki 0.8.0", ] @@ -1587,7 +1613,7 @@ source = "registry+https://github.com/rust-lang/crates.io-index" checksum = "1d9efca8738c78ee9484207732f728b1ef517bbb1833d6fc0879ca898a522f6f" dependencies = [ "base64ct", - "der 0.8.1", + "der 0.8.2", ] [[package]] @@ -1624,9 +1650,9 @@ dependencies = [ [[package]] name = "syn" -version = "3.0.4" +version = "3.0.6" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "e6275cddf4610d1775e6d1fe9469b2e77d0f39fd98fb7450901b821e0c53649f" +checksum = "8593e8e72159ed2257d083c7a454a85cbf854f37a0966d8d483aff8c8a3ebcee" dependencies = [ "proc-macro2", "quote", @@ -1635,22 +1661,22 @@ dependencies = [ [[package]] name = "thiserror" -version = "2.0.18" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "4288b5bcbc7920c07a1149a35cf9590a2aa808e0bc1eafaade0b80947865fbc4" +checksum = "09e52cb86a36cede5cb101bf8908837b3e4c6e5e59fe7fd85c23fb56200d189e" dependencies = [ "thiserror-impl", ] [[package]] name = "thiserror-impl" -version = "2.0.18" +version = "2.0.21" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "ebc4ee7f67670e9b64d05fa4253e753e016c6c95ff35b89b7941d6b856dec1d5" +checksum = "fe5197923287db20a58125f0bc85c062f7f2c892de97b18c356f9efb14b28524" dependencies = [ "proc-macro2", "quote", - "syn 2.0.117", + "syn 3.0.6", ] [[package]] @@ -2131,18 +2157,18 @@ dependencies = [ [[package]] name = "zeroize" -version = "1.8.2" +version = "1.9.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "b97154e67e32c85465826e8bcc1c59429aaaf107c1e4a9e53c8d8ccd5eff88d0" +checksum = "e13c156562582aa81c60cb29407084cdb54c4164760106ab78e6c5b0858cf64e" dependencies = [ "zeroize_derive", ] [[package]] name = "zeroize_derive" -version = "1.4.3" +version = "1.5.0" source = "registry+https://github.com/rust-lang/crates.io-index" -checksum = "85a5b4158499876c763cb03bc4e49185d3cccbabb15b33c627f7884f43db852e" +checksum = "3c50655cbb0fe3fc43170059e702f1ce5e19b84cec58dc87b037a09935c2f328" dependencies = [ "proc-macro2", "quote", diff --git a/Cargo.toml b/Cargo.toml index 3dd1307..0911cfe 100644 --- a/Cargo.toml +++ b/Cargo.toml @@ -29,9 +29,10 @@ test = true [dependencies] chrono = { version = "0.4.41", features = ["serde"] } clap = { version = "4.5.60", features = ["derive"] } -darkbio-crypto = { version = "0.18.2", features = ["cbor", "xdsa"] } -darkbio-trust = { version = "0.5.1", features = ["release", "staging", "develop"] } -darkbio-wire = "0.10.0" +darkbio-clock = { version = "0.3", features = ["crossbeam"] } +darkbio-crypto = { version = "0.19.1", features = ["cbor", "xdsa"] } +darkbio-trust = { version = "0.6.0", features = ["release", "staging", "develop"] } +darkbio-wire = "0.12.0" hex = "0.4.3" console = "0.15.11" base64 = "0.23.1" @@ -56,6 +57,9 @@ windows-sys = { version = "0.61", features = ["Win32_System_Console", "Win32_Sys [target.'cfg(unix)'.dependencies] signal-hook = "0.3" +[dev-dependencies] +darkbio-clock = { version = "0.3", features = ["crossbeam", "test-clock"] } + # Firmware uploads push megabytes through the crypto, which crawls unoptimized, # so dependencies build optimized under the dev profile too. The CLI and its # connection library stay unoptimized for debugging. diff --git a/connect/src/ark.rs b/connect/src/ark.rs index 5769778..95f7d7e 100644 --- a/connect/src/ark.rs +++ b/connect/src/ark.rs @@ -12,6 +12,7 @@ use crate::{ Dataset, Error, ExecutionProgress, Firmware, Identity, Registration, Request, Setup, Timing, UpdateProgress, UploadProgress, dataset, execution, }; +use darkbio_clock::Clock; use darkbio_wire::protocol::{self, Message, Promise, Requester, Responder, Session, schema}; use darkbio_wire::transport::{self, Verifier}; use std::io; @@ -44,6 +45,7 @@ impl Ark { /// Authenticates the peer under wire's handshake timeout, then selects cloud /// routing for the session. Returns the verifier's identity information. + /// Every operation of the connection reads time from the stream's clock. /// Failure closes the stream. pub(crate) fn attach( stream: transport::Stream, @@ -55,6 +57,7 @@ impl Ark { W: transport::Write + Send + 'static, V: Verifier, { + let clock = stream.clock(); let (session, info) = protocol::connect(stream, verifier).map_err(|err| { if let protocol::Error::Transport(cause) = &err && let transport::Error::RecvFailed(io) | transport::Error::SendFailed(io) = @@ -68,7 +71,7 @@ impl Ark { } Error::Handshake(err) })?; - let services = Arc::new(Services::new(&info, cloud(&info))); + let services = Arc::new(Services::new(&info, cloud(&info), &clock)); Ok((Self::start(session, services)?, info)) } @@ -157,6 +160,12 @@ pub struct Client { } impl Client { + /// Returns the clock of this client's connection. Every deadline passed to + /// the client is measured on it, so callers build their deadlines from it. + pub fn clock(&self) -> Clock { + self.services.clock().clone() + } + /// Sends a request and waits for its typed response under the chosen timing. /// An absolute deadline covers setup, queueing, sending and accepting the /// response; decoding is outside it. Reuse it to bound several calls. @@ -193,7 +202,12 @@ impl Client { request: R, timeout: Duration, ) -> Result { - let deadline = Instant::now().checked_add(timeout).ok_or(Error::Timeout)?; + let deadline = self + .services + .clock() + .now() + .checked_add(timeout) + .ok_or(Error::Timeout)?; self.call(request, deadline) } @@ -225,9 +239,11 @@ impl Client { Setup::Cloud => self.services.sync(&self.requester, timing)?, Setup::Relay => self.services.relay(&self.requester, timing)?, } - let deadline = R::WINDOW.map_or_else(|| timing.io(), |window| timing.window(window)); + let clock = self.services.clock(); + let deadline = + R::WINDOW.map_or_else(|| timing.io(clock), |window| timing.window(clock, window)); let setup = matches!(request, Message::DeviceInfoRequest(_)) - .then(|| (self.services.clone(), Instant::now())); + .then(|| (self.services.clone(), clock.now())); let promise = self .requester .request(request, deadline) @@ -246,7 +262,12 @@ impl Client { request: R, timeout: Duration, ) -> Result, Error> { - let deadline = Instant::now().checked_add(timeout).ok_or(Error::Timeout)?; + let deadline = self + .services + .clock() + .now() + .checked_add(timeout) + .ok_or(Error::Timeout)?; self.send(request, deadline) } @@ -314,7 +335,8 @@ impl Client { timing: impl Into, ) -> Result { let timing = timing.into(); - timing.check()?; + let clock = self.services.clock(); + timing.check(clock)?; let mut chunk = vec![0; size.min(1024 * 1024) as usize]; reader.read_exact(&mut chunk).map_err(|error| { if error.kind() == io::ErrorKind::TimedOut { @@ -323,7 +345,7 @@ impl Client { Error::DatasetRead(error) } })?; - timing.check()?; + timing.check(clock)?; self.call( schema::SlotIdentifyRequest { name: name.into(), @@ -404,7 +426,9 @@ impl Pending { /// /// Panics if a notification was already registered on this promise. pub fn notify(&mut self, sender: mpsc::Sender, event: E) { - self.promise.notify(sender, event); + self.promise.notify(move || { + let _ = sender.send(event); + }); } } @@ -430,7 +454,7 @@ mod tests { DeviceInfoRequest, OnboardingRequest, RelayAppToArkResponse, RelayArkToAppRequest, UnlockRequest, UnlockResponse, }; - use crate::testing::{Peer, answering, hangup, silent}; + use crate::testing::{Peer, answering, hangup, silent, test_clock, wait_deadline}; use std::thread; /// Budget for test I/O that is not exercising expiration. @@ -449,8 +473,9 @@ mod tests { fn verify( &self, attestation: &transport::Attestation, + now: std::time::SystemTime, ) -> Result<(crate::wire::crypto::xdsa::PublicKey, Identity), String> { - let (key, _) = crate::TrustMode::RootOrSelf.verify(attestation)?; + let (key, _) = crate::TrustMode::RootOrSelf.verify(attestation, now)?; let identity = Identity::Attested { env: self.0, device: crate::trust::device::Device { @@ -468,8 +493,9 @@ mod tests { } } + let clock = test_clock().clock(); for &env in crate::identity::ENVIRONMENTS { - let mut peer = Peer::spawn(Box::new(answering)); + let mut peer = Peer::spawn(&clock, Box::new(answering)); let mut selected = None; let (ark, identity) = Ark::attach(peer.stream(), &Attested(env), |identity| { let Identity::Attested { env, device } = identity else { @@ -484,7 +510,7 @@ mod tests { assert!(matches!(identity, Identity::Attested { env: actual, .. } if actual == env)); assert_eq!( ark.client() - .call(DeviceInfoRequest {}, Instant::now() + TIMEOUT) + .call(DeviceInfoRequest {}, clock.now() + TIMEOUT) .unwrap() .firmware_version, "1.0.0" @@ -495,7 +521,7 @@ mod tests { /// Failed authentication never invokes the cloud selector. #[test] fn test_cloud_selection_rejects_failed_handshake() { - let mut peer = Peer::spawn(Box::new(answering)); + let mut peer = Peer::spawn(&test_clock().clock(), Box::new(answering)); let wrong = crate::wire::crypto::xdsa::SecretKey::generate().public_key(); let result = Ark::attach( peer.stream(), @@ -536,41 +562,45 @@ mod tests { } } - let mut peer = Peer::spawn(Box::new(|session, request, responder| { - if !matches!(request, schema::host_to_ark::Content::Unlock(_)) { - return answering(session, request, responder); - } - let deadline = Instant::now() + TIMEOUT; - let error = session - .requester() - .request(RelayArkToAppRequest::default(), deadline) - .unwrap() - .wait::() - .unwrap_err(); - assert!(matches!( - error, protocol::Error::Remote(error) - if error.code == 0x100 && error.msg == "companion rejected authorization" - )); - responder - .fail( - schema::Error::reserved( - schema::ReservedErrors::Unavailable, - "authorization required", - ), - deadline, - ) - .unwrap() - .wait() - .unwrap(); - true - })); + let clock = test_clock().clock(); + let mut peer = Peer::spawn( + &clock, + Box::new(|session, request, responder| { + if !matches!(request, schema::host_to_ark::Content::Unlock(_)) { + return answering(session, request, responder); + } + let deadline = session.clock().now() + TIMEOUT; + let error = session + .requester() + .request(RelayArkToAppRequest::default(), deadline) + .unwrap() + .wait::() + .unwrap_err(); + assert!(matches!( + error, protocol::Error::Remote(error) + if error.code == 0x100 && error.msg == "companion rejected authorization" + )); + responder + .fail( + schema::Error::reserved( + schema::ReservedErrors::Unavailable, + "authorization required", + ), + deadline, + ) + .unwrap() + .wait() + .unwrap(); + true + }), + ); let (mut ark, _) = peer.attach().unwrap(); let client = ark.client(); - let pending = client.send(ManualUnlock, Instant::now() + TIMEOUT).unwrap(); + let pending = client.send(ManualUnlock, clock.now() + TIMEOUT).unwrap(); let (request, responder) = ark.recv().unwrap(); assert!(matches!(request, schema::ark_to_host::Content::RelayReq(_))); responder - .fail(Denied, Instant::now() + TIMEOUT) + .fail(Denied, clock.now() + TIMEOUT) .unwrap() .wait() .unwrap(); @@ -580,7 +610,7 @@ mod tests { )); assert_eq!( client - .call(DeviceInfoRequest {}, Instant::now() + TIMEOUT) + .call(DeviceInfoRequest {}, clock.now() + TIMEOUT) .unwrap() .firmware_version, "1.0.0" @@ -608,10 +638,11 @@ mod tests { content: Vec, } + let clock = test_clock().clock(); let signer = xdsa::SecretKey::generate(); let identity = signer.public_key(); - let attestation = self_attestation(&signer, identity.clone()); - let (host, remote) = memory::duplex(256 * 1024); + let attestation = self_attestation(&signer, identity.clone(), &clock); + let (host, remote) = memory::duplex(256 * 1024, &clock); let peer = thread::spawn(move || { let mut server = transport::Server::new(remote, signer, attestation); let transport::Event::Connected(sender) = server.recv().unwrap() else { @@ -663,7 +694,7 @@ mod tests { schema::ReservedErrors::Unsupported, "host does not serve device info", ), - Instant::now() + TIMEOUT, + clock.now() + TIMEOUT, ) .unwrap() .wait() @@ -686,23 +717,25 @@ mod tests { /// A reserved peer refusal is returned through the same request interface. #[test] fn test_requests() { - let mut peer = Peer::spawn(Box::new(answering)); + let clock = test_clock().clock(); + let mut peer = Peer::spawn(&clock, Box::new(answering)); let (ark, _) = peer.attach().unwrap(); let client = ark.client(); let (completed, events) = mpsc::channel(); let mut pending = client - .send(DeviceInfoRequest {}, Instant::now() + TIMEOUT) + .send(DeviceInfoRequest {}, clock.now() + TIMEOUT) .unwrap(); pending.notify(completed, 7); - assert_eq!(events.recv_timeout(TIMEOUT).unwrap(), 7); + assert_eq!(events.recv().unwrap(), 7); assert_eq!(pending.wait().unwrap().firmware_version, "1.0.0"); + let deadline = clock.now() + TIMEOUT; let callers: Vec<_> = (0..8) .map(|_| { let client = client.clone(); thread::spawn(move || { client - .call(DeviceInfoRequest {}, Instant::now() + TIMEOUT) + .call(DeviceInfoRequest {}, deadline) .unwrap() .firmware_version }) @@ -712,23 +745,24 @@ mod tests { assert_eq!(caller.join().unwrap(), "1.0.0"); } assert!( - matches!(client.call(OnboardingRequest::default(), Instant::now() + TIMEOUT), Err(Error::Remote(error)) if error.code == schema::ReservedErrors::Unsupported as u64) + matches!(client.call(OnboardingRequest::default(), clock.now() + TIMEOUT), Err(Error::Remote(error)) if error.code == schema::ReservedErrors::Unsupported as u64) ); } /// Dropping the owner closes pending requests and refuses surviving clients. #[test] fn test_owner_drop() { - let mut peer = Peer::spawn(silent()); + let clock = test_clock().clock(); + let mut peer = Peer::spawn(&clock, silent()); let (ark, _) = peer.attach().unwrap(); let client = ark.client(); let pending = client - .send(DeviceInfoRequest {}, Instant::now() + TIMEOUT) + .send(DeviceInfoRequest {}, clock.now() + TIMEOUT) .unwrap(); drop(ark); assert!(matches!(pending.wait(), Err(Error::Closed))); assert!(matches!( - client.call(DeviceInfoRequest {}, Instant::now() + TIMEOUT), + client.call(DeviceInfoRequest {}, clock.now() + TIMEOUT), Err(Error::Closed) )); } @@ -736,11 +770,12 @@ mod tests { /// A closer wakes both the receive loop and outstanding requests. #[test] fn test_close() { - let mut peer = Peer::spawn(silent()); + let clock = test_clock().clock(); + let mut peer = Peer::spawn(&clock, silent()); let (mut ark, _) = peer.attach().unwrap(); let pending = ark .client() - .send(DeviceInfoRequest {}, Instant::now() + TIMEOUT) + .send(DeviceInfoRequest {}, clock.now() + TIMEOUT) .unwrap(); let closer = ark.closer(); let receive = thread::spawn(move || ark.recv()); @@ -752,17 +787,18 @@ mod tests { /// A remote disconnect retains its reason for receives and later requests. #[test] fn test_disconnect() { - let mut peer = Peer::spawn(hangup()); + let clock = test_clock().clock(); + let mut peer = Peer::spawn(&clock, hangup()); let (mut ark, _) = peer.attach().unwrap(); assert!(matches!( ark.client() - .call(DeviceInfoRequest {}, Instant::now() + TIMEOUT), + .call(DeviceInfoRequest {}, clock.now() + TIMEOUT), Err(Error::Disconnected(_)) )); assert!(matches!(ark.recv(), Err(Error::Disconnected(_)))); assert!(matches!( ark.client() - .call(DeviceInfoRequest {}, Instant::now() + TIMEOUT), + .call(DeviceInfoRequest {}, clock.now() + TIMEOUT), Err(Error::Disconnected(_)) )); } @@ -770,16 +806,24 @@ mod tests { /// A short request timeout leaves a concurrent request's budget intact. #[test] fn test_timeouts() { - let mut peer = Peer::spawn(silent()); + let mut tester = test_clock(); + let clock = tester.clock(); + let mut peer = Peer::spawn(&clock, silent()); let (ark, _) = peer.attach().unwrap(); let client = ark.client(); let pending = client.send_timeout(DeviceInfoRequest {}, TIMEOUT).unwrap(); - assert!(matches!( - client - .clone() - .call_timeout(DeviceInfoRequest {}, Duration::from_millis(20)), - Err(Error::Timeout) - )); + + // The short call's deadline is the earliest one, and reaching it expires the call + let deadline = clock.now() + Duration::from_millis(20); + let short = thread::spawn({ + let client = client.clone(); + move || client.call_timeout(DeviceInfoRequest {}, Duration::from_millis(20)) + }); + wait_deadline(&tester, deadline); + tester.advance_to(deadline); + assert!(matches!(short.join().unwrap(), Err(Error::Timeout))); + + // The longer request is still pending until the owner closes ark.close(); assert!(matches!(pending.wait(), Err(Error::Closed))); } @@ -787,10 +831,12 @@ mod tests { /// Reusing a deadline across calls and cloned handles does not renew its budget. #[test] fn test_deadlines() { - let mut peer = Peer::spawn(Box::new(answering)); + let mut tester = test_clock(); + let clock = tester.clock(); + let mut peer = Peer::spawn(&clock, Box::new(answering)); let (ark, _) = peer.attach().unwrap(); let client = ark.client(); - let deadline = Instant::now() + Duration::from_secs(1); + let deadline = clock.now() + Duration::from_secs(1); assert_eq!( client .call(DeviceInfoRequest {}, deadline) @@ -799,7 +845,7 @@ mod tests { "1.0.0" ); // Spend the remaining operation budget before issuing the next request. - thread::sleep(deadline.saturating_duration_since(Instant::now())); + tester.advance_to(deadline); assert!(matches!( client.clone().call(DeviceInfoRequest {}, deadline), Err(Error::Timeout) @@ -816,34 +862,39 @@ mod tests { /// An unlock can wait for the host to return an opaque companion response. #[test] fn test_reverse_requests() { - let mut peer = Peer::spawn(Box::new(|session, _, responder| { - let deadline = Instant::now() + TIMEOUT; - // Unlock cannot complete until the application returns the opaque - // companion response through this reverse request. - let approval = session - .requester() - .request( - RelayArkToAppRequest { - id: 42, - req: vec![1, 2, 3], - }, - deadline, - ) - .unwrap() - .wait::() - .unwrap(); - assert_eq!(approval.id, 42); - assert_eq!(approval.res, [4, 5, 6]); - responder - .reply(UnlockResponse::default(), deadline) - .unwrap() - .wait() - .unwrap(); - true - })); + let clock = test_clock().clock(); + let mut peer = Peer::spawn( + &clock, + Box::new(|session, _, responder| { + let deadline = session.clock().now() + TIMEOUT; + // Unlock cannot complete until the application returns the opaque + // companion response through this reverse request. + let approval = session + .requester() + .request( + RelayArkToAppRequest { + id: 42, + req: vec![1, 2, 3], + }, + deadline, + ) + .unwrap() + .wait::() + .unwrap(); + assert_eq!(approval.id, 42); + assert_eq!(approval.res, [4, 5, 6]); + responder + .reply(UnlockResponse::default(), deadline) + .unwrap() + .wait() + .unwrap(); + true + }), + ); let (mut ark, _) = peer.attach().unwrap(); let client = ark.client(); - let operation = thread::spawn(move || client.call(ManualUnlock, Instant::now() + TIMEOUT)); + let deadline = clock.now() + TIMEOUT; + let operation = thread::spawn(move || client.call(ManualUnlock, deadline)); let (request, responder) = ark.recv().unwrap(); let schema::ark_to_host::Content::RelayReq(request) = request else { panic!("expected relay request") @@ -855,7 +906,7 @@ mod tests { id: request.id, res: vec![4, 5, 6], }, - Instant::now() + TIMEOUT, + clock.now() + TIMEOUT, ) .unwrap() .wait() diff --git a/connect/src/cloud/auth.rs b/connect/src/cloud/auth.rs index de1cef6..d112a33 100644 --- a/connect/src/cloud/auth.rs +++ b/connect/src/cloud/auth.rs @@ -8,13 +8,15 @@ use super::Failure; use crate::Timing; +use darkbio_clock::Clock; use std::sync::{Arc, RwLock}; use std::time::Instant; use ureq::http::{HeaderMap, StatusCode}; /// Caller-owned authentication for a cloud host. Connect supplies the HTTPS /// origin; credential storage, response recognition and login stay with the caller. -/// The same headers are used for HTTP requests and WebSocket upgrades. +/// The same headers are used for HTTP requests and WebSocket upgrades. Deadlines +/// are measured on the clock of the connection, which its clients return. pub trait CloudAuth: Send + Sync { /// Returns cached authentication headers without prompting, or an empty map. /// The deadline bounds lookup. Headers must not replace protocol headers. @@ -85,11 +87,12 @@ impl Authorization { .is_some_and(|provider| provider.rejected(origin, status, headers)) } - /// Refreshes credentials without extending an absolute operation deadline. - pub(super) fn login(&self, origin: &str, timing: Timing) -> Result<(), Failure> { + /// Refreshes credentials without extending an absolute operation deadline, + /// which is measured on the clock. + pub(super) fn login(&self, origin: &str, clock: &Clock, timing: Timing) -> Result<(), Failure> { let check = || { timing - .check() + .check(clock) .map_err(|_| Failure::Wire(darkbio_wire::protocol::Error::Timeout)) }; check()?; @@ -119,6 +122,9 @@ fn sensitive(mut headers: HeaderMap) -> HeaderMap { #[cfg(test)] pub(super) mod tests { use super::*; + use crate::testing::test_clock; + use darkbio_clock::TestClock; + use std::sync::Mutex; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; @@ -127,8 +133,10 @@ pub(super) mod tests { pub(in crate::cloud) struct Login { pub lookups: Arc, pub logins: Arc, - pub delay: Duration, pub fail: bool, + /// Test clock and the time each login spends on it, standing in for the + /// owner signing in through the browser. + pub browser: Option<(Arc>, Duration)>, } impl CloudAuth for Login { @@ -143,10 +151,15 @@ pub(super) mod tests { fn login(&self, _: &str, deadline: Option) -> Result { self.logins.fetch_add(1, Ordering::SeqCst); - std::thread::sleep(deadline.map_or(self.delay, |deadline| { - self.delay - .min(deadline.saturating_duration_since(Instant::now())) - })); + + // Spend the browser's time on the test clock, returning by the deadline as a provider must + if let Some((tester, delay)) = &self.browser { + let mut tester = tester.lock().unwrap(); + let now = tester.clock().now(); + tester.advance(deadline.map_or(*delay, |deadline| { + (*delay).min(deadline.saturating_duration_since(now)) + })); + } if self.fail { return Err("test login refused".into()); } @@ -165,18 +178,20 @@ pub(super) mod tests { #[test] fn credentials_are_cached_redacted_and_refreshed() { + let clock = test_clock().clock(); let auth = Authorization::default(); let login = Login::default(); auth.set(Arc::new(login.clone())); assert_eq!(login.lookups.load(Ordering::SeqCst), 0); - let deadline = Instant::now() + Duration::from_secs(1); + let deadline = clock.now() + Duration::from_secs(1); for _ in 0..2 { let headers = auth.headers("https://test.invalid", deadline); assert_eq!(headers["authorization"], "cached"); assert!(headers["authorization"].is_sensitive()); assert!(!format!("{headers:?}").contains("cached")); } - auth.login("https://test.invalid", deadline.into()).unwrap(); + auth.login("https://test.invalid", &clock, deadline.into()) + .unwrap(); let headers = auth.headers("https://test.invalid", deadline); assert_eq!(headers["authorization"], "refreshed"); assert!(headers["authorization"].is_sensitive()); @@ -187,23 +202,31 @@ pub(super) mod tests { #[test] fn login_retains_absolute_deadlines() { + // Let every browser login take 100 ms of the test clock + let tester = Arc::new(Mutex::new(test_clock())); + let clock = tester.lock().unwrap().clock(); let auth = Authorization::default(); let login = Login { - delay: Duration::from_millis(100), + browser: Some((tester, Duration::from_millis(100))), ..Default::default() }; auth.set(Arc::new(login.clone())); + + // An expired deadline refuses before the browser opens assert!(matches!( - auth.login("https://test.invalid", Timing::until(Instant::now())), + auth.login("https://test.invalid", &clock, Timing::until(clock.now())), Err(Failure::Wire(darkbio_wire::protocol::Error::Timeout)) )); assert_eq!(login.logins.load(Ordering::SeqCst), 0); + + // An inactivity allowance does not bound the login, an absolute deadline does let timing = Timing::inactivity(Duration::from_millis(1)); - auth.login("https://test.invalid", timing).unwrap(); + auth.login("https://test.invalid", &clock, timing).unwrap(); assert!(matches!( auth.login( "https://test.invalid", - timing.with_deadline(Instant::now() + Duration::from_millis(5)) + &clock, + timing.with_deadline(clock.now() + Duration::from_millis(5)) ), Err(Failure::Wire(darkbio_wire::protocol::Error::Timeout)) )); diff --git a/connect/src/cloud/dns.rs b/connect/src/cloud/dns.rs index 788f01a..da2f7cd 100644 --- a/connect/src/cloud/dns.rs +++ b/connect/src/cloud/dns.rs @@ -4,67 +4,81 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//! Shares unfinished DNS lookups across cloud socket attempts. +//! Shares unfinished DNS lookups across the cloud socket attempts of a connection. use super::{Failure, socket}; +use crate::timing::ClockExt; +use darkbio_clock::{Clock, sync}; use std::collections::HashMap; use std::net::{IpAddr, SocketAddr, ToSocketAddrs}; -use std::sync::{Arc, Condvar, LazyLock, Mutex}; +use std::sync::{Arc, Mutex}; use std::thread; use std::time::Instant; -/// Process-wide sharing of system lookups that cannot be cancelled on timeout. -static RESOLVER: LazyLock> = LazyLock::new(|| Arc::new(Resolver::default())); - /// Coalesces unfinished lookups by host and port, without caching completed DNS. -#[derive(Default)] -struct Resolver { +/// System lookups cannot be cancelled on timeout, so a retry joins the one +/// still running instead of starting another. +#[derive(Debug)] +pub(super) struct Resolver { + clock: Clock, // clock that the waiters' deadlines are measured on pending: Mutex>>, // Only unfinished system calls } /// One system lookup retained until every attached waiter releases it. -#[derive(Default)] +#[derive(Debug)] struct Lookup { /// Addresses or failure, published once by the resolver worker. - result: Mutex, Failure>>>, + result: sync::Mutex, Failure>>>, /// Wakes all callers when the shared system lookup returns. - ready: Condvar, + ready: sync::Condvar, } -/// A caller's deadline ends its wait, leaving the lookup available to retries. -pub(super) fn resolve( - host: &str, - port: u16, - deadline: Instant, -) -> Result, Failure> { - socket::remaining(deadline).map_err(socket::io_error)?; - if let Ok(ip) = host.parse::() { - return Ok(vec![SocketAddr::new(ip, port)]); +impl Resolver { + /// Creates a resolver whose waiters measure their deadlines on the clock. + pub(super) fn new(clock: &Clock) -> Arc { + Arc::new(Self { + clock: clock.clone(), + pending: Mutex::new(HashMap::new()), + }) + } + + /// A caller's deadline ends its wait, leaving the lookup available to retries. + pub(super) fn resolve( + self: &Arc, + host: &str, + port: u16, + deadline: Instant, + ) -> Result, Failure> { + self.clock.remaining(deadline).map_err(socket::io_error)?; + if let Ok(ip) = host.parse::() { + return Ok(vec![SocketAddr::new(ip, port)]); + } + let name = host.to_owned(); + self.lookup((name.clone(), port), deadline, move || { + (name.as_str(), port) + .to_socket_addrs() + .map(|addresses| addresses.collect()) + .map_err(socket::io_error) + }) } - let name = host.to_owned(); - RESOLVER.resolve((name.clone(), port), deadline, move || { - (name.as_str(), port) - .to_socket_addrs() - .map(|addresses| addresses.collect()) - .map_err(socket::io_error) - }) -} -impl Resolver { /// Completed lookups are removed so a later attachment refreshes DNS. An /// expired waiter neither cancels nor replaces a system call still running. - fn resolve( + fn lookup( self: &Arc, key: (String, u16), deadline: Instant, lookup: impl FnOnce() -> Result, Failure> + Send + 'static, ) -> Result, Failure> { - socket::remaining(deadline).map_err(socket::io_error)?; + self.clock.remaining(deadline).map_err(socket::io_error)?; let mut pending = self.pending.lock().expect("DNS lookups not poisoned"); let attempt = match pending.get(&key) { Some(attempt) => attempt.clone(), None => { - let attempt = Arc::new(Lookup::default()); + let attempt = Arc::new(Lookup { + result: sync::Mutex::new(None), + ready: sync::Condvar::new(&self.clock), + }); thread::Builder::new() .name("ark-relay-dns".into()) .spawn({ @@ -91,10 +105,10 @@ impl Resolver { if let Some(result) = &*result { return result.clone(); } - let left = socket::remaining(deadline).map_err(socket::io_error)?; + self.clock.remaining(deadline).map_err(socket::io_error)?; result = attempt .ready - .wait_timeout(result, left) + .wait_deadline(result, deadline) .expect("DNS result not poisoned") .0; } @@ -104,6 +118,7 @@ impl Resolver { #[cfg(test)] mod tests { use super::*; + use crate::testing::{test_clock, wait_deadline}; use darkbio_wire::protocol; use std::sync::mpsc; use std::time::Duration; @@ -111,45 +126,60 @@ mod tests { /// Retries join a stalled lookup, and the next completed attempt refreshes it. #[test] fn test_timeout_and_retry() { - let resolver = Arc::new(Resolver::default()); - let key = ("relay.invalid".into(), 443); - let (release, pause) = mpsc::channel(); + // Stall the first system lookup until released + let mut tester = test_clock(); + let clock = tester.clock(); + let resolver = Resolver::new(&clock); + let key = ("relay.invalid".to_string(), 443); + let (started, lookups) = mpsc::channel(); + let (release, pause) = mpsc::channel::<()>(); let address: SocketAddr = "127.0.0.1:443".parse().unwrap(); - assert!(matches!( - resolver.resolve( - key.clone(), - Instant::now() + Duration::from_millis(20), - move || { - pause.recv_timeout(Duration::from_secs(5)).unwrap(); - Ok(vec![address]) + let mut stalled = Some(move || { + started.send(()).unwrap(); + pause.recv().unwrap(); + Ok(vec![address]) + }); + + // The first waiter starts the lookup and three retries join it. Each one + // parks on its own deadline, which the clock then reaches. + for attempt in 0..4 { + let deadline = clock.now() + Duration::from_millis(20); + let waiter = thread::spawn({ + let resolver = resolver.clone(); + let key = key.clone(); + let stalled = stalled.take(); + move || match stalled { + Some(lookup) => resolver.lookup(key, deadline, lookup), + None => resolver.lookup(key, deadline, || panic!("duplicate DNS lookup")), } - ), - Err(Failure::Wire(protocol::Error::Timeout)) - )); - for _ in 0..3 { + }); + if attempt == 0 { + lookups.recv().unwrap(); + } + wait_deadline(&tester, deadline); + tester.advance_to(deadline); assert!(matches!( - resolver.resolve( - key.clone(), - Instant::now() + Duration::from_millis(20), - || panic!("duplicate DNS lookup") - ), + waiter.join().unwrap(), Err(Failure::Wire(protocol::Error::Timeout)) )); } + + // Releasing the lookup publishes its addresses and forgets it let attempt = resolver.pending.lock().unwrap().get(&key).unwrap().clone(); release.send(()).unwrap(); let result = attempt.result.lock().unwrap(); - let (result, timeout) = attempt + let result = attempt .ready - .wait_timeout_while(result, Duration::from_secs(5), |result| result.is_none()) + .wait_while(result, |result| result.is_none()) .unwrap(); - assert!(!timeout.timed_out()); assert_eq!(result.as_ref().unwrap().as_ref().unwrap(), &[address]); drop(result); assert!(resolver.pending.lock().unwrap().is_empty()); + + // A later attempt starts a fresh lookup instead of reusing the finished one assert!( resolver - .resolve(key, Instant::now() + Duration::from_secs(5), || Err( + .lookup(key, clock.now() + Duration::from_secs(5), || Err( Failure::Relay("fresh DNS result".into()) )) .is_err() diff --git a/connect/src/cloud/firmware.rs b/connect/src/cloud/firmware.rs index 999898b..f26ffa7 100644 --- a/connect/src/cloud/firmware.rs +++ b/connect/src/cloud/firmware.rs @@ -16,8 +16,6 @@ use darkbio_wire::protocol::Requester; use serde::Deserialize; use sha2::{Digest, Sha256}; use std::io::Read; -#[cfg(test)] -use std::time::Instant; /// Archive bytes per acknowledged transfer, amortizing USB and device write latency. const CHUNK_SIZE: usize = 1024 * 1024; @@ -87,10 +85,11 @@ impl Services { .try_lock() .map_err(|_| Error::Firmware("another firmware update is already running".into()))?; self.sync(requester, timing)?; + let clock = &self.clock; // Finish any browser login before asking the Ark to prepare an update. // A protected host can need login even when device sync is still fresh. if cloud.auth.configured() { - cloud.with_auth(timing, || cloud.identity(timing.io()))?; + cloud.with_auth(timing, || cloud.identity(timing.io(clock)))?; } progress(UpdateProgress::Preparing); let prepared = requester @@ -100,7 +99,7 @@ impl Services { sha256: firmware.sha256.to_vec(), bytes: firmware.size, }, - timing.approval(), + timing.approval(clock), )? .wait::()?; let access: Access = match http::get_authenticated( @@ -111,12 +110,12 @@ impl Services { .query("version", &firmware.version) .query("sha256", hex::encode(firmware.sha256)) .header("Dark-Auth", BASE64_URL_SAFE_NO_PAD.encode(prepared.auth)), - timing.io(), + timing.io(clock), ) { Err(Failure::AuthRequired) => { // Browser login can outlive the prepared proof. Leave a second // preparation and its possible approval to an explicit rerun. - cloud.auth.login(&cloud.origin, timing)?; + cloud.auth.login(&cloud.origin, clock, timing)?; return Err(Error::CloudAuth { origin: cloud.origin.clone(), message: "signed in to the cloud; rerun the firmware update".into(), @@ -134,7 +133,10 @@ impl Services { Error::Firmware(format!("invalid firmware access encoding: {error}")) })?; requester - .request(schema::FirmwareUpdateInitRequest { access }, timing.io())? + .request( + schema::FirmwareUpdateInitRequest { access }, + timing.io(clock), + )? .wait::()?; progress(UpdateProgress::Uploading { @@ -145,11 +147,11 @@ impl Services { progress(UpdateProgress::Verifying); requester - .request(schema::FirmwareUpdateVerifyRequest {}, timing.io())? + .request(schema::FirmwareUpdateVerifyRequest {}, timing.io(clock))? .wait::()?; progress(UpdateProgress::Installing); requester - .request(schema::FirmwareUpdateInstallRequest {}, timing.io())? + .request(schema::FirmwareUpdateInstallRequest {}, timing.io(clock))? .wait::()?; Ok(()) } @@ -164,16 +166,20 @@ fn upload( timing: Timing, progress: &mut impl FnMut(UpdateProgress), ) -> Result<(), Error> { + let clock = &requester.clock(); let mut uploaded = 0; let mut hash = Sha256::new(); while uploaded < firmware.size { - timing.check()?; + timing.check(clock)?; let size = (firmware.size - uploaded).min(CHUNK_SIZE as u64) as usize; let mut chunk = vec![0; size]; reader.read_exact(&mut chunk).map_err(Error::FirmwareRead)?; hash.update(&chunk); requester - .request(schema::FirmwareUpdateUploadRequest { chunk }, timing.io())? + .request( + schema::FirmwareUpdateUploadRequest { chunk }, + timing.io(clock), + )? .wait::()?; uploaded += size as u64; progress(UpdateProgress::Uploading { @@ -181,7 +187,7 @@ fn upload( total: firmware.size, }); } - timing.check()?; + timing.check(clock)?; if reader.read(&mut [0]).map_err(Error::FirmwareRead)? != 0 { return Err(Error::Integrity( "archive exceeds its advertised size".into(), @@ -200,11 +206,13 @@ mod tests { use super::*; use crate::cloud::tests::{TIMEOUT, attach}; use crate::schema::host_to_ark::Content; - use crate::testing::{Peer, answering}; + use crate::testing::{Peer, answering, test_clock}; use crate::trust::Realm; + use darkbio_clock::Clock; use std::io::Write; - use std::net::{TcpListener, TcpStream}; - use std::sync::{Arc, Mutex, mpsc}; + use std::net::{SocketAddr, TcpListener, TcpStream}; + use std::sync::atomic::{AtomicBool, Ordering}; + use std::sync::{Arc, Mutex}; use std::thread; use std::time::Duration; @@ -220,80 +228,85 @@ mod tests { /// request means later HTTP stages must never be contacted. struct Cloud { url: String, - stop: mpsc::Sender<()>, + address: SocketAddr, // listener that the stopping connection wakes + stopped: Arc, // marks the next connection as the signal to stop worker: Option>>, } impl Cloud { fn start(firmware: &Firmware, _bytes: Vec, access_status: u16) -> Self { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - listener.set_nonblocking(true).unwrap(); - let url = format!("http://{}/v1", listener.local_addr().unwrap()); + let address = listener.local_addr().unwrap(); + let url = format!("http://{address}/v1"); let key = format!( "/v1/firmware?version={}&sha256={}", firmware.version, hex::encode(firmware.sha256) ); - let (stop, stopped) = mpsc::channel(); - let worker = thread::spawn(move || { - let mut paths = Vec::new(); - let deadline = Instant::now() + TIMEOUT; - while stopped.try_recv().is_err() && Instant::now() < deadline { - let mut stream = match listener.accept() { - Ok((stream, _)) => stream, - Err(error) if error.kind() == std::io::ErrorKind::WouldBlock => { - thread::sleep(Duration::from_millis(1)); - continue; - } - Err(error) => panic!("test cloud: {error}"), - }; - stream.set_nonblocking(false).unwrap(); - stream.set_read_timeout(Some(TIMEOUT)).unwrap(); - stream.set_write_timeout(Some(TIMEOUT)).unwrap(); - let headers = headers(&mut stream); - let path = headers.split_whitespace().nth(1).unwrap(); - let (status, body) = match path { - "/v1/cloudsync/identity" => { - (200, br#"{"signer":"AQ==","crypto":"Ag=="}"#.as_slice()) + let stopped = Arc::new(AtomicBool::new(false)); + let worker = thread::spawn({ + let stopped = stopped.clone(); + move || { + let mut paths = Vec::new(); + loop { + let (mut stream, _) = listener.accept().unwrap(); + if stopped.load(Ordering::SeqCst) { + break paths; } - "/v1/cloudsync/time?challenge=03" => { - (200, br#"{"unixmilli":123,"signature":"BA=="}"#.as_slice()) - } - path if path == key => { - assert!(headers.to_lowercase().contains("dark-auth: -_8\r\n")); - (access_status, br#"{"access":"/w=="}"#.as_slice()) - } - other => panic!("unexpected HTTP request: {other}"), - }; - paths.push(path.to_owned()); - let header = format!( - "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", - body.len() - ); - // A failed wire upload may close the download mid-response. - let _ = stream - .write_all(header.as_bytes()) - .and_then(|()| stream.write_all(body)); + stream.set_read_timeout(Some(TIMEOUT)).unwrap(); + stream.set_write_timeout(Some(TIMEOUT)).unwrap(); + let headers = headers(&mut stream); + let path = headers.split_whitespace().nth(1).unwrap(); + let (status, body) = match path { + "/v1/cloudsync/identity" => { + (200, br#"{"signer":"AQ==","crypto":"Ag=="}"#.as_slice()) + } + "/v1/cloudsync/time?challenge=03" => { + (200, br#"{"unixmilli":123,"signature":"BA=="}"#.as_slice()) + } + path if path == key => { + assert!(headers.to_lowercase().contains("dark-auth: -_8\r\n")); + (access_status, br#"{"access":"/w=="}"#.as_slice()) + } + other => panic!("unexpected HTTP request: {other}"), + }; + paths.push(path.to_owned()); + let header = format!( + "HTTP/1.1 {status} Test\r\nContent-Length: {}\r\nConnection: close\r\n\r\n", + body.len() + ); + // A failed wire upload may close the download mid-response. + let _ = stream + .write_all(header.as_bytes()) + .and_then(|()| stream.write_all(body)); + } } - paths }); Self { url, - stop, + address, + stopped, worker: Some(worker), } } fn finish(mut self) -> Vec { - let _ = self.stop.send(()); + self.stop(); self.worker.take().unwrap().join().unwrap() } + + /// Wakes the accept loop with a connection of its own, which it takes as + /// the signal to stop. + fn stop(&self) { + self.stopped.store(true, Ordering::SeqCst); + let _ = TcpStream::connect(self.address); + } } impl Drop for Cloud { fn drop(&mut self) { - let _ = self.stop.send(()); if let Some(worker) = self.worker.take() { + self.stop(); let _ = worker.join(); } } @@ -312,6 +325,7 @@ mod tests { /// Records the update sequence and optionally refuses or disconnects during /// one stage. Only a fully uploaded archive is eligible for verification. fn peer( + clock: &Clock, firmware: &Firmware, fail: Option<&'static str>, ) -> (Peer, Arc>>) { @@ -319,79 +333,83 @@ mod tests { let stages = Arc::new(Mutex::new(Vec::new())); let observed = stages.clone(); let mut uploaded = Vec::new(); - let peer = Peer::spawn(Box::new(move |session, request, responder| { - let (stage, response): (_, darkbio_wire::protocol::Message) = match request { - Content::CloudSyncStart(request) => { - assert_eq!((request.signer, request.crypto), (vec![1], vec![2])); - ( - "sync-start", - schema::CloudSyncStartResponse { challenge: vec![3] }.into(), - ) - } - Content::CloudSyncFinish(request) => { - assert_eq!((request.unixmilli, request.signature), (123, vec![4])); - ( - "sync-finish", - schema::CloudSyncFinishResponse { accepted: 123 }.into(), - ) - } - Content::FirmwareUpdatePrep(request) => { - assert_eq!(request.version, firmware.version); - assert_eq!(request.sha256, firmware.sha256); - assert_eq!(request.bytes, firmware.size); - ( - "prepare", - schema::FirmwareUpdatePrepResponse { - auth: vec![0xfb, 0xff], - } - .into(), - ) - } - Content::FirmwareUpdateInit(request) => { - assert_eq!(request.access, [0xff]); - ("init", schema::FirmwareUpdateInitResponse {}.into()) - } - Content::FirmwareUpdateUpload(request) => { - assert!(!request.chunk.is_empty() && request.chunk.len() <= CHUNK_SIZE); - uploaded.extend(request.chunk); - ("upload", schema::FirmwareUpdateUploadResponse {}.into()) - } - Content::FirmwareUpdateVerify(_) => { - assert_eq!(uploaded.len() as u64, firmware.size); - assert_eq!(Sha256::digest(&uploaded).as_slice(), firmware.sha256); - ("verify", schema::FirmwareUpdateVerifyResponse {}.into()) + let peer = Peer::spawn( + clock, + Box::new(move |session, request, responder| { + let (stage, response): (_, darkbio_wire::protocol::Message) = match request { + Content::CloudSyncStart(request) => { + assert_eq!((request.signer, request.crypto), (vec![1], vec![2])); + ( + "sync-start", + schema::CloudSyncStartResponse { challenge: vec![3] }.into(), + ) + } + Content::CloudSyncFinish(request) => { + assert_eq!((request.unixmilli, request.signature), (123, vec![4])); + ( + "sync-finish", + schema::CloudSyncFinishResponse { accepted: 123 }.into(), + ) + } + Content::FirmwareUpdatePrep(request) => { + assert_eq!(request.version, firmware.version); + assert_eq!(request.sha256, firmware.sha256); + assert_eq!(request.bytes, firmware.size); + ( + "prepare", + schema::FirmwareUpdatePrepResponse { + auth: vec![0xfb, 0xff], + } + .into(), + ) + } + Content::FirmwareUpdateInit(request) => { + assert_eq!(request.access, [0xff]); + ("init", schema::FirmwareUpdateInitResponse {}.into()) + } + Content::FirmwareUpdateUpload(request) => { + assert!(!request.chunk.is_empty() && request.chunk.len() <= CHUNK_SIZE); + uploaded.extend(request.chunk); + ("upload", schema::FirmwareUpdateUploadResponse {}.into()) + } + Content::FirmwareUpdateVerify(_) => { + assert_eq!(uploaded.len() as u64, firmware.size); + assert_eq!(Sha256::digest(&uploaded).as_slice(), firmware.sha256); + ("verify", schema::FirmwareUpdateVerifyResponse {}.into()) + } + Content::FirmwareUpdateInstall(_) => { + ("install", schema::FirmwareUpdateInstallResponse {}.into()) + } + other => return answering(session, other, responder), + }; + observed.lock().unwrap().push(stage); + if fail == Some("disconnect") && stage == "install" { + return false; } - Content::FirmwareUpdateInstall(_) => { - ("install", schema::FirmwareUpdateInstallResponse {}.into()) + let deadline = session.clock().now() + TIMEOUT; + if fail == Some(stage) { + responder + .fail(schema::Error::new(0x777, "test update refusal"), deadline) + .unwrap(); + } else { + responder.reply(response, deadline).unwrap(); } - other => return answering(session, other, responder), - }; - observed.lock().unwrap().push(stage); - if fail == Some("disconnect") && stage == "install" { - return false; - } - let deadline = Instant::now() + TIMEOUT; - if fail == Some(stage) { - responder - .fail(schema::Error::new(0x777, "test update refusal"), deadline) - .unwrap(); - } else { - responder.reply(response, deadline).unwrap(); - } - true - })); + true + }), + ); (peer, stages) } #[test] fn test_update() { + let clock = test_clock().clock(); let bytes = vec![42; CHUNK_SIZE + 17]; let expected = firmware(&bytes); let cloud = Cloud::start(&expected, bytes.clone(), 200); - let (mut peer, stages) = peer(&expected, None); + let (mut peer, stages) = peer(&clock, &expected, None); let ark = attach(&mut peer, cloud.url.clone()); let client = ark.client(); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; let mut progress = Vec::new(); client .clone() @@ -445,6 +463,7 @@ mod tests { /// does not count as a successful reboot or trigger an installation retry. #[test] fn test_refusals() { + let clock = test_clock().clock(); for failure in [ "prepare", "init", @@ -456,14 +475,14 @@ mod tests { let bytes = vec![42; 17]; let firmware = firmware(&bytes); let cloud = Cloud::start(&firmware, bytes.clone(), 200); - let (mut peer, stages) = peer(&firmware, Some(failure)); + let (mut peer, stages) = peer(&clock, &firmware, Some(failure)); let ark = attach(&mut peer, cloud.url.clone()); let error = ark .client() .update_firmware( &firmware, &mut bytes.as_slice(), - Instant::now() + TIMEOUT, + clock.now() + TIMEOUT, |_| {}, ) .unwrap_err(); @@ -491,17 +510,18 @@ mod tests { /// installation, even if earlier upload chunks were accepted by the device. #[test] fn test_download_integrity() { + let clock = test_clock().clock(); for bytes in [vec![42; 16], vec![42; 18], vec![43; 17]] { let firmware = firmware(&[42; 17]); let cloud = Cloud::start(&firmware, bytes.clone(), 200); - let (mut peer, stages) = peer(&firmware, None); + let (mut peer, stages) = peer(&clock, &firmware, None); let ark = attach(&mut peer, cloud.url.clone()); assert!( ark.client() .update_firmware( &firmware, &mut bytes.as_slice(), - Instant::now() + TIMEOUT, + clock.now() + TIMEOUT, |_| {} ) .is_err() @@ -515,18 +535,20 @@ mod tests { /// an upload. Expiring the deadline after transfer also prevents verification. #[test] fn test_access_and_deadline() { + let mut tester = test_clock(); + let clock = tester.clock(); for expire in [false, true] { let bytes = vec![42; 17]; let firmware = firmware(&bytes); let cloud = Cloud::start(&firmware, bytes.clone(), if expire { 200 } else { 403 }); - let (mut peer, stages) = peer(&firmware, None); + let (mut peer, stages) = peer(&clock, &firmware, None); let ark = attach(&mut peer, cloud.url.clone()); - let deadline = Instant::now() + Duration::from_secs(1); + let deadline = clock.now() + Duration::from_secs(1); let error = ark .client() .update_firmware(&firmware, &mut bytes.as_slice(), deadline, |stage| { if expire && stage == UpdateProgress::Verifying { - thread::sleep(deadline.saturating_duration_since(Instant::now())); + tester.advance_to(deadline); } }) .unwrap_err(); @@ -564,25 +586,22 @@ mod tests { auth::tests::{Login, refused}, tests::{response, serve, sync_responses}, }; - use std::sync::atomic::Ordering; + let clock = test_clock().clock(); for expires_after_preparation in [false, true] { let bytes = vec![42; 17]; let firmware = firmware(&bytes); let mut responses = sync_responses(); if !expires_after_preparation { - responses.push((Duration::ZERO, refused(302))); + responses.push(refused(302)); } responses.push(sync_responses().remove(0)); - responses.push(( - Duration::ZERO, - if expires_after_preparation { - refused(403) - } else { - response(200, r#"{"access":"/w=="}"#) - }, - )); + responses.push(if expires_after_preparation { + refused(403) + } else { + response(200, r#"{"access":"/w=="}"#) + }); let (url, requests) = serve(responses); - let (mut peer, stages) = peer(&firmware, None); + let (mut peer, stages) = peer(&clock, &firmware, None); let mut ark = attach(&mut peer, url); let login = Login::default(); ark.set_cloud_auth(login.clone()); @@ -630,18 +649,23 @@ mod tests { /// The refusal stops the sequence before access keys or archives are fetched. #[test] fn test_emulator_refusal() { + let clock = test_clock().clock(); let bytes = vec![42]; let expected = firmware(&bytes); let cloud = Cloud::start(&expected, bytes.clone(), 200); - let (mut peer, stages) = peer(&expected, Some("prepare")); + let (mut peer, stages) = peer(&clock, &expected, Some("prepare")); let verifier = crate::TrustMode::Recover(Box::new(peer.identity.clone())); let (session, identity) = darkbio_wire::protocol::connect(peer.stream(), &verifier).unwrap(); - let mut services = Services::new(&identity, None); - services.cloud = Some(http::tests::api(cloud.url.clone(), Realm::Emulator)); + let mut services = Services::new(&identity, None, &session.clock()); + services.cloud = Some(http::tests::api( + cloud.url.clone(), + Realm::Emulator, + &session.clock(), + )); let ark = crate::Ark::start(session, Arc::new(services)).unwrap(); let client = ark.client(); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; let error = client .update_firmware(&expected, &mut bytes.as_slice(), deadline, |_| {}) .unwrap_err(); @@ -662,13 +686,17 @@ mod tests { /// Firmware discovery needs a selected environment and a live owner. #[test] fn test_identity_and_closure() { - let mut peer = Peer::spawn(Box::new(|_, _, _| panic!("unexpected device request"))); + let clock = test_clock().clock(); + let mut peer = Peer::spawn( + &clock, + Box::new(|_, _, _| panic!("unexpected device request")), + ); let verifier = crate::TrustMode::Recover(Box::new(peer.identity.clone())); let (session, identity) = darkbio_wire::protocol::connect(peer.stream(), &verifier).unwrap(); - let services = Services::new(&identity, None); + let services = Services::new(&identity, None, &session.clock()); let firmware = firmware(&[42]); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; assert!(matches!( services.update_firmware( &session.requester(), diff --git a/connect/src/cloud/http.rs b/connect/src/cloud/http.rs index 9f6ad27..8485abd 100644 --- a/connect/src/cloud/http.rs +++ b/connect/src/cloud/http.rs @@ -6,7 +6,7 @@ //! HTTP routes and payloads of the Ark cloud API. -use super::{Failure, auth}; +use super::{Failure, auth, dns}; use crate::schema::{CloudSyncFinishRequest, CloudSyncStartRequest}; use crate::trust::{Environment, Realm}; use crate::{Identity, Timing}; @@ -14,8 +14,10 @@ use base64::{ Engine, prelude::{BASE64_STANDARD, BASE64_URL_SAFE_NO_PAD}, }; +use darkbio_clock::Clock; use darkbio_wire::protocol; use serde::{Deserialize, de::DeserializeOwned}; +use std::sync::Arc; use std::time::Instant; /// Maximum JSON response, enough for cloud certificates, signed time or registry state. @@ -24,11 +26,14 @@ const MAX_RESPONSE: u64 = 64 * 1024; /// Cloud operations selected by the attestation or an explicit environment. #[derive(Debug)] pub(super) struct Api { + pub(super) clock: Clock, // clock that the deadlines are measured on + pub(super) auth: auth::Authorization, // Caller credentials, independent of the Ark proof pub(super) origin: String, // HTTPS origin shared by API and socket credentials pub(super) agent: ureq::Agent, // HTTP connections reused across the cloud exchange pub(super) url: String, // API of the selected environment pub(super) realm: Realm, // Realm selecting the device registry + pub(super) resolver: Arc, // lookups shared by this connection's cloud sockets serial: Option, // Attested serial, when available, checked against the registry } @@ -57,7 +62,11 @@ impl Api { /// Prepares cloud access without I/O. An explicit environment overrides the /// attested one. Its discovery realm is used only without an attested realm. - pub(super) fn new(identity: &Identity, cloud: Option<(Environment, Realm)>) -> Option { + pub(super) fn new( + identity: &Identity, + cloud: Option<(Environment, Realm)>, + clock: &Clock, + ) -> Option { let (env, realm, serial) = match identity { Identity::Attested { env, device } => ( cloud.as_ref().map_or(env, |(env, _)| env), @@ -70,11 +79,13 @@ impl Api { } }; Some(Self { + clock: clock.clone(), auth: auth::Authorization::default(), origin: api_url(*env).trim_end_matches("/v1").into(), agent: agent(), url: api_url(*env).into(), realm, + resolver: dns::Resolver::new(clock), serial, }) } @@ -120,7 +131,7 @@ impl Api { if !matches!(result, Err(Failure::AuthRequired)) { return result; } - self.auth.login(&self.origin, timing)?; + self.auth.login(&self.origin, &self.clock, timing)?; match attempt() { Err(Failure::AuthRequired) => Err(Failure::CloudAuth { origin: self.origin.clone(), @@ -294,7 +305,7 @@ fn send( request = request.header(name, value); } let remaining = deadline - .checked_duration_since(Instant::now()) + .checked_duration_since(api.clock.now()) .filter(|remaining| !remaining.is_zero()) .ok_or(protocol::Error::Timeout)?; let response = request @@ -330,19 +341,23 @@ pub(super) fn json( #[cfg(test)] pub(super) mod tests { use super::*; - use crate::cloud::tests::{TIMEOUT, http, response, serve}; + use crate::cloud::tests::{TIMEOUT, http, response, serve, serve_inner}; + use crate::testing::test_clock; use serde_json::json; - use std::thread; + use std::sync::mpsc; use std::time::Duration; - /// Redirects an attested connection to the loopback cloud. - pub(in crate::cloud) fn api(url: String, realm: Realm) -> Api { + /// Redirects an attested connection to the loopback cloud, measuring its + /// deadlines on the clock. + pub(in crate::cloud) fn api(url: String, realm: Realm, clock: &Clock) -> Api { Api { + clock: clock.clone(), auth: auth::Authorization::default(), origin: url.trim_end_matches("/v1").into(), agent: http(), url, realm, + resolver: dns::Resolver::new(clock), serial: Some("test-serial".into()), } } @@ -351,6 +366,7 @@ pub(super) mod tests { /// Without attestation, the supplied discovery realm selects the registry. #[test] fn test_cloud_routing() { + let clock = test_clock().clock(); let key = darkbio_crypto::xdsa::SecretKey::generate().public_key(); for &env in crate::identity::ENVIRONMENTS { let identity = Identity::Attested { @@ -366,11 +382,11 @@ pub(super) mod tests { expiry: Some(1), }, }; - let cloud = Api::new(&identity, None).unwrap(); + let cloud = Api::new(&identity, None, &clock).unwrap(); assert_eq!(cloud.url, api_url(env)); assert_eq!(cloud.realm, Realm::Emulator); for &selected in crate::identity::ENVIRONMENTS { - let cloud = Api::new(&identity, Some((selected, Realm::Hardware))).unwrap(); + let cloud = Api::new(&identity, Some((selected, Realm::Hardware)), &clock).unwrap(); assert_eq!(cloud.url, api_url(selected)); assert_eq!(cloud.realm, Realm::Emulator); assert!(cloud.relay_url().ends_with("/sandbox/relaying")); @@ -380,9 +396,9 @@ pub(super) mod tests { Identity::SelfSigned(key.clone()), Identity::Recovered(key.clone()), ] { - assert!(Api::new(&identity, None).is_none()); + assert!(Api::new(&identity, None, &clock).is_none()); for realm in [Realm::Hardware, Realm::Emulator] { - let cloud = Api::new(&identity, Some((env, realm))).unwrap(); + let cloud = Api::new(&identity, Some((env, realm)), &clock).unwrap(); assert_eq!(cloud.url, api_url(env)); assert_eq!(cloud.realm, realm); assert_eq!(cloud.serial, None); @@ -405,31 +421,26 @@ pub(super) mod tests { let signature = [0, 1, 0xfe, 0xff]; let unixmilli = (1u64 << 53) + 1; let (url, requests) = serve(vec![ - ( - Duration::ZERO, - response( - 200, - &json!({ - "signer": BASE64_STANDARD.encode(signer), - "crypto": BASE64_STANDARD.encode(crypto), - }) - .to_string(), - ), + response( + 200, + &json!({ + "signer": BASE64_STANDARD.encode(signer), + "crypto": BASE64_STANDARD.encode(crypto), + }) + .to_string(), ), - ( - Duration::ZERO, - response( - 200, - &json!({ - "unixmilli": unixmilli, - "signature": BASE64_STANDARD.encode(signature), - }) - .to_string(), - ), + response( + 200, + &json!({ + "unixmilli": unixmilli, + "signature": BASE64_STANDARD.encode(signature), + }) + .to_string(), ), ]); - let cloud = api(url, Realm::Hardware); - let deadline = Instant::now() + TIMEOUT; + let clock = test_clock().clock(); + let cloud = api(url, Realm::Hardware, &clock); + let deadline = clock.now() + TIMEOUT; let start = fetch_identity(&cloud, deadline).unwrap(); assert_eq!(start.signer, signer); assert_eq!(start.crypto, crypto); @@ -438,13 +449,13 @@ pub(super) mod tests { assert_eq!(finish.signature, signature); assert!( requests - .recv_timeout(TIMEOUT) + .recv() .unwrap() .starts_with("GET /v1/cloudsync/identity HTTP/1.1\r\n") ); assert!( requests - .recv_timeout(TIMEOUT) + .recv() .unwrap() .starts_with("GET /v1/cloudsync/time?challenge=00fbff HTTP/1.1\r\n") ); @@ -454,32 +465,33 @@ pub(super) mod tests { /// unpadded URL-safe authentication header. Inactive flags are retained. #[test] fn test_registry_routes() { + let clock = test_clock().clock(); for (realm, path) in [ (Realm::Hardware, "/v1/genuine"), (Realm::Emulator, "/v1/sandbox/genuine"), ] { - let (url, requests) = serve(vec![( - Duration::ZERO, - response( - 200, - &json!({ - "serial": "test-serial", - "enrolled": 123, - "disabled": true, - "expired": true, - "superseded": true, - }) - .to_string(), - ), + let (url, requests) = serve(vec![response( + 200, + &json!({ + "serial": "test-serial", + "enrolled": 123, + "disabled": true, + "expired": true, + "superseded": true, + }) + .to_string(), )]); - let registration = - fetch_registration(&api(url, realm), &[0xfb, 0xff], Instant::now() + TIMEOUT) - .unwrap(); + let registration = fetch_registration( + &api(url, realm, &clock), + &[0xfb, 0xff], + clock.now() + TIMEOUT, + ) + .unwrap(); assert_eq!(registration.serial, "test-serial"); assert_eq!(registration.enrolled, 123); assert!(registration.disabled && registration.expired && registration.superseded); assert!(!registration.active()); - let request = requests.recv_timeout(TIMEOUT).unwrap(); + let request = requests.recv().unwrap(); assert!(request.starts_with(&format!("GET {path} HTTP/1.1\r\n"))); assert!( request @@ -492,15 +504,13 @@ pub(super) mod tests { /// A registry reply for another serial cannot verify this connection. #[test] fn test_registry_identity() { - let (url, _requests) = serve(vec![( - Duration::ZERO, - response( - 200, - r#"{"serial":"another-ark","enrolled":123,"disabled":false,"expired":false,"superseded":false}"#, - ), + let clock = test_clock().clock(); + let (url, _requests) = serve(vec![response( + 200, + r#"{"serial":"another-ark","enrolled":123,"disabled":false,"expired":false,"superseded":false}"#, )]); assert!(matches!( - api(url, Realm::Hardware).genuine(&[1], Instant::now() + TIMEOUT), + api(url, Realm::Hardware, &clock).genuine(&[1], clock.now() + TIMEOUT), Err(Failure::Cloud(error)) if error.contains("serial does not match") )); } @@ -509,6 +519,7 @@ pub(super) mod tests { /// responses fail before a cloud payload can be forwarded to the Ark. #[test] fn test_bad_responses() { + let clock = test_clock().clock(); for (status, body) in [ (503, "unavailable".into()), (302, "redirect".into()), @@ -525,34 +536,38 @@ pub(super) mod tests { .to_string(), ), ] { - let (url, _requests) = serve(vec![(Duration::ZERO, response(status, &body))]); - assert!(fetch_identity(&api(url, Realm::Hardware), Instant::now() + TIMEOUT).is_err()); + let (url, _requests) = serve(vec![response(status, &body)]); + let cloud = api(url, Realm::Hardware, &clock); + assert!(fetch_identity(&cloud, clock.now() + TIMEOUT).is_err()); } - let (url, _requests) = serve(vec![( - Duration::ZERO, - response(200, r#"{"unixmilli":123,"signature":"!"}"#), - )]); - assert!(fetch_time(&api(url, Realm::Hardware), &[1], Instant::now() + TIMEOUT).is_err()); + let (url, _requests) = serve(vec![response(200, r#"{"unixmilli":123,"signature":"!"}"#)]); + let cloud = api(url, Realm::Hardware, &clock); + assert!(fetch_time(&cloud, &[1], clock.now() + TIMEOUT).is_err()); } /// A later HTTP request retains the original deadline. A stalled response /// also expires instead of leaving setup waiting indefinitely. #[test] fn test_deadlines() { + // A request after the clock reached the shared deadline fails without HTTP + let mut tester = test_clock(); + let clock = tester.clock(); let body = r#"{"signer":"AA==","crypto":"AA=="}"#; - let (url, _requests) = serve(vec![(Duration::ZERO, response(200, body))]); - let cloud = api(url, Realm::Hardware); - let deadline = Instant::now() + Duration::from_secs(1); + let (url, _requests) = serve(vec![response(200, body)]); + let cloud = api(url, Realm::Hardware, &clock); + let deadline = clock.now() + Duration::from_secs(1); fetch_identity(&cloud, deadline).unwrap(); - thread::sleep(deadline.saturating_duration_since(Instant::now())); + tester.advance_to(deadline); let error = fetch_time(&cloud, &[1], deadline).unwrap_err(); assert!(matches!(error, Failure::Wire(protocol::Error::Timeout))); - let (url, _requests) = serve(vec![(Duration::from_secs(1), response(200, body))]); + // A response the server never sends ends at the HTTP client's own timeout + let (_release, pause) = mpsc::channel(); + let (url, _requests) = serve_inner(vec![response(200, body)], Some(pause)); assert!(matches!( fetch_identity( - &api(url, Realm::Hardware), - Instant::now() + Duration::from_millis(50) + &api(url, Realm::Hardware, &clock), + clock.now() + Duration::from_millis(50) ), Err(Failure::Wire(protocol::Error::Timeout)) )); diff --git a/connect/src/cloud/mod.rs b/connect/src/cloud/mod.rs index ecff515..8637265 100644 --- a/connect/src/cloud/mod.rs +++ b/connect/src/cloud/mod.rs @@ -28,17 +28,19 @@ use crate::schema::{ GenuinityProofRequest, RelayArkToAppRequest, RelayJoinRequest, RelayJoinResponse, }; use crate::{Error, Identity, Timing}; +use darkbio_clock::{Clock, sync}; use darkbio_wire::protocol::{self, Requester, Responder}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Condvar, Mutex}; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::sync::{Arc, Mutex}; +use std::time::{Duration, Instant, UNIX_EPOCH}; /// Whether the reported cloud identity and clock can be reused for a request. -/// Sync is needed before the first use or at 15 seconds of clock drift, leaving -/// headroom for proof verification. Key rotation is detected by a refused proof; -/// the marker only records setup since boot. -pub fn cloud_synced(info: &crate::schema::DeviceInfoResponse) -> bool { - let now = SystemTime::now() +/// Sync is needed before the first use or at 15 seconds of drift from the +/// clock's wall time, leaving headroom for proof verification. Key rotation is +/// detected by a refused proof; the marker only records setup since boot. +pub fn cloud_synced(info: &crate::schema::DeviceInfoResponse, clock: &Clock) -> bool { + let now = clock + .system_time() .duration_since(UNIX_EPOCH) .unwrap_or_default() .as_secs(); @@ -54,6 +56,7 @@ fn synced_at(info: &crate::schema::DeviceInfoResponse, now: u64) -> bool { /// run outside its lock so independent requests and closure remain available. #[derive(Debug)] pub(crate) struct Services { + clock: Clock, // clock of the wire session, which every operation reads cloud: Option, // Absent without an attested or caller-supplied environment state: Mutex, // Current initialization attempt and connection lifecycle updating: Mutex<()>, // One firmware transfer at a time across client clones @@ -70,11 +73,11 @@ struct State { } /// One attempt's outcome, retained by its waiters even after a retry starts. -#[derive(Debug, Default)] +#[derive(Debug)] struct Attempt { - result: Mutex>>, // Shared success or the original failure - refreshed: AtomicBool, // This attempt exchanged keys and signed time - ready: Condvar, // Wakes waiters on completion or closure + result: sync::Mutex>>, // Shared success or the original failure + refreshed: AtomicBool, // This attempt exchanged keys and signed time + ready: sync::Condvar, // Wakes waiters on completion or closure } /// Failures shareable between callers joining the same initialization attempt. @@ -143,18 +146,26 @@ impl Services { error.into() } - /// Records cloud routing without starting network I/O. + /// Records cloud routing without starting network I/O. Every operation of the + /// connection reads its time from the clock of the wire session. pub(crate) fn new( identity: &Identity, cloud: Option<(crate::trust::Environment, crate::trust::Realm)>, + clock: &Clock, ) -> Self { Self { - cloud: http::Api::new(identity, cloud), + clock: clock.clone(), + cloud: http::Api::new(identity, cloud, clock), state: Mutex::new(State::default()), updating: Mutex::new(()), } } + /// Returns the clock of the wire session, which every operation reads. + pub(crate) fn clock(&self) -> &Clock { + &self.clock + } + /// Reuses fresh device state, caching that decision for one minute. Each /// caller bounds its own wait; only one exchange runs at a time. pub(crate) fn sync( @@ -189,7 +200,7 @@ impl Services { /// Serializes one prerequisite while allowing unrelated device traffic. /// Waiters retain the attempt they joined, even if a later caller retries it. fn ensure(&self, requester: &Requester, step: Step, timing: Timing) -> Result<(), Failure> { - let deadline = timing.io(); + let deadline = timing.io(&self.clock); let (attempt, leader) = { let mut state = self.state.lock().expect("cloud setup not poisoned"); if let Some(error) = &state.error { @@ -199,7 +210,7 @@ impl Services { _ if self.cloud.is_none() => return Err(Failure::MissingEnvironment), Step::Sync if state.synced.is_some_and(|(at, synced)| { - synced && at.elapsed() < Duration::from_secs(60) + synced && self.clock.elapsed(at) < Duration::from_secs(60) }) => { return Ok(()); @@ -209,7 +220,7 @@ impl Services { } _ => {} } - if Instant::now() >= deadline { + if self.clock.now() >= deadline { return Err(protocol::Error::Timeout.into()); } if matches!(step, Step::Refresh) { @@ -222,7 +233,7 @@ impl Services { match pending { Some(attempt) => (attempt.clone(), false), None => { - let attempt = Arc::new(Attempt::default()); + let attempt = Arc::new(Attempt::new(&self.clock)); *pending = Some(attempt.clone()); (attempt, true) } @@ -256,7 +267,7 @@ impl Services { result.and_then(|relay| { match step { Step::Sync | Step::Refresh => { - state.synced = Some((Instant::now(), true)); + state.synced = Some((self.clock.now(), true)); } Step::Relay => { let mut relay = relay.expect("relay setup returned an attachment"); @@ -281,14 +292,14 @@ impl Services { let cloud = self.cloud.as_ref().expect("cloud route available"); self.authenticate(requester, timing, || { let joined = requester - .request(RelayJoinRequest {}, timing.io())? + .request(RelayJoinRequest {}, timing.io(&self.clock))? .wait::()?; relay::Relay::connect( cloud, &cloud.relay_url(), &joined.auth, requester.clone(), - timing.io(), + timing.io(&self.clock), ) }) } @@ -323,7 +334,7 @@ impl Services { if self.cloud.is_none() { return Some((request, responder)); } - let deadline = Instant::now() + relay::EXCHANGE_TIMEOUT; + let deadline = self.clock.now() + relay::EXCHANGE_TIMEOUT; if let Err(error) = self.relay(requester, deadline) { relay::fail(responder, &error.to_string()); return None; @@ -348,9 +359,9 @@ impl Services { self.sync(requester, timing)?; self.authenticate(requester, timing, || { let proof = requester - .request(GenuinityProofRequest {}, timing.io())? + .request(GenuinityProofRequest {}, timing.io(&self.clock))? .wait::()?; - cloud.genuine(&proof.proof, timing.io()) + cloud.genuine(&proof.proof, timing.io(&self.clock)) }) .map_err(Into::into) } @@ -369,7 +380,7 @@ impl Services { .lock() .expect("cloud setup not poisoned") .synced - .filter(|(at, _)| at.elapsed() < Duration::from_secs(60)) + .filter(|&(at, _)| self.clock.elapsed(at) < Duration::from_secs(60)) .map(|(_, synced)| synced); let synced = match reported { Some(synced) => synced, @@ -381,13 +392,15 @@ impl Services { } } let cloud = self.cloud.as_ref().expect("cloud route available"); - let identity = cloud.with_auth(timing, || cloud.identity(timing.io()))?; + let identity = cloud.with_auth(timing, || cloud.identity(timing.io(&self.clock)))?; let started = requester - .request(identity, timing.io())? + .request(identity, timing.io(&self.clock))? .wait::()?; - let time = cloud.with_auth(timing, || cloud.time(&started.challenge, timing.io()))?; + let time = cloud.with_auth(timing, || { + cloud.time(&started.challenge, timing.io(&self.clock)) + })?; requester - .request(time, timing.io())? + .request(time, timing.io(&self.clock))? .wait::()?; tracing::info!(target: "darkbio_connect::setup", "cloud synchronized"); Ok(true) @@ -400,9 +413,9 @@ impl Services { timing: Timing, ) -> Result { let info = requester - .request(crate::schema::DeviceInfoRequest {}, timing.io())? + .request(crate::schema::DeviceInfoRequest {}, timing.io(&self.clock))? .wait::()?; - Ok(cloud_synced(&info)) + Ok(cloud_synced(&info, &self.clock)) } /// Reuses device info already requested by the caller. A delayed response @@ -413,7 +426,7 @@ impl Services { && state.syncing.is_none() && state.synced.is_none_or(|(at, _)| at < requested) { - state.synced = Some((requested, cloud_synced(info))); + state.synced = Some((requested, cloud_synced(info, &self.clock))); } } @@ -453,6 +466,15 @@ enum Step { } impl Attempt { + /// Starts an attempt whose waiters measure their deadlines on the clock. + fn new(clock: &Clock) -> Self { + Self { + result: sync::Mutex::new(None), + refreshed: AtomicBool::new(false), + ready: sync::Condvar::new(clock), + } + } + /// Publishes the first outcome, preserving closure if it won the race. fn finish(&self, result: Result<(), Failure>) { let mut outcome = self.result.lock().expect("cloud attempt not poisoned"); @@ -469,14 +491,14 @@ impl Attempt { if let Some(result) = &*outcome { return result.clone(); } - let remaining = deadline - .checked_duration_since(Instant::now()) - .ok_or(protocol::Error::Timeout)?; - outcome = self + let (next, wait) = self .ready - .wait_timeout(outcome, remaining) - .expect("cloud attempt not poisoned") - .0; + .wait_deadline(outcome, deadline) + .expect("cloud attempt not poisoned"); + outcome = next; + if wait.timed_out() && outcome.is_none() { + return Err(protocol::Error::Timeout.into()); + } } } } @@ -489,7 +511,7 @@ pub(crate) mod tests { use crate::schema::{ CloudSyncFinishResponse, CloudSyncStartResponse, DeviceInfoRequest, GenuinityProofResponse, }; - use crate::testing::{Peer, answering}; + use crate::testing::{Peer, answering, test_clock, wait_deadline}; use crate::trust::Realm; use crate::{Ark, TrustMode}; use std::io::{Read, Write}; @@ -503,36 +525,25 @@ pub(crate) mod tests { pub(super) const TIMEOUT: Duration = Duration::from_secs(5); /// Serves scripted responses and captures requests, closing each connection - /// after its response. Accepts are bounded so a failed test leaves no waiter. - pub(crate) fn serve(responses: Vec<(Duration, String)>) -> (String, mpsc::Receiver) { + /// after its response. A server left waiting by a failed test blocks only + /// its own thread. + pub(crate) fn serve(responses: Vec) -> (String, mpsc::Receiver) { serve_inner(responses, None) } /// Holds the first response until released, making setup races deterministic. - fn serve_inner( - responses: Vec<(Duration, String)>, + pub(super) fn serve_inner( + responses: Vec, mut pause: Option>, ) -> (String, mpsc::Receiver) { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - listener.set_nonblocking(true).unwrap(); let url = format!("http://{}/v1", listener.local_addr().unwrap()); let (sender, receiver) = mpsc::channel(); thread::spawn(move || { - for (delay, response) in responses { - let deadline = Instant::now() + TIMEOUT; - let mut stream = loop { - match listener.accept() { - Ok((stream, _)) => break stream, - Err(err) if err.kind() == std::io::ErrorKind::WouldBlock => { - if Instant::now() >= deadline { - return; - } - thread::sleep(Duration::from_millis(1)); - } - Err(_) => return, - } + for response in responses { + let Ok((mut stream, _)) = listener.accept() else { + return; }; - stream.set_nonblocking(false).unwrap(); stream.set_read_timeout(Some(TIMEOUT)).unwrap(); stream.set_write_timeout(Some(TIMEOUT)).unwrap(); let mut request = Vec::new(); @@ -545,11 +556,10 @@ pub(crate) mod tests { } sender.send(String::from_utf8(request).unwrap()).unwrap(); if let Some(pause) = pause.take() - && pause.recv_timeout(TIMEOUT).is_err() + && pause.recv().is_err() { return; } - thread::sleep(delay); let _ = stream.write_all(response.as_bytes()); } }); @@ -575,11 +585,14 @@ pub(crate) mod tests { } /// Attaches a real wire session with cloud routes redirected to the test server. + /// The connection runs on the clock of the peer's stream. pub(crate) fn attach(peer: &mut Peer, url: String) -> Ark { let verifier = TrustMode::Recover(Box::new(peer.identity.clone())); let (session, _) = protocol::connect(peer.stream(), &verifier).unwrap(); + let clock = session.clock(); let services = Arc::new(Services { - cloud: Some(super::http::tests::api(url, Realm::Hardware)), + clock: clock.clone(), + cloud: Some(super::http::tests::api(url, Realm::Hardware, &clock)), state: Mutex::new(State::default()), updating: Mutex::new(()), }); @@ -588,87 +601,84 @@ pub(crate) mod tests { /// Peer that only issues a proof after sync, retaining counts for duplicate /// initialization checks. Optionally refuses its first start request. - fn peer(refuse_first: bool) -> (Peer, Arc, Arc) { + fn peer(clock: &Clock, refuse_first: bool) -> (Peer, Arc, Arc) { let starts = Arc::new(AtomicUsize::new(0)); let proofs = Arc::new(AtomicUsize::new(0)); - let peer = Peer::spawn(Box::new({ - let starts = starts.clone(); - let proofs = proofs.clone(); - let mut synced = false; - move |session, request, responder| { - let deadline = Instant::now() + TIMEOUT; - match request { - Content::CloudSyncStart(request) => { - assert_eq!(request.signer, [1]); - assert_eq!(request.crypto, [2]); - if starts.fetch_add(1, Ordering::SeqCst) == 0 && refuse_first { + let peer = Peer::spawn( + clock, + Box::new({ + let starts = starts.clone(); + let proofs = proofs.clone(); + let mut synced = false; + move |session, request, responder| { + let deadline = session.clock().now() + TIMEOUT; + match request { + Content::CloudSyncStart(request) => { + assert_eq!(request.signer, [1]); + assert_eq!(request.crypto, [2]); + if starts.fetch_add(1, Ordering::SeqCst) == 0 && refuse_first { + responder + .fail( + crate::schema::Error::new(0x111, "identity rejected"), + deadline, + ) + .unwrap() + .wait() + .unwrap(); + } else { + responder + .reply(CloudSyncStartResponse { challenge: vec![3] }, deadline) + .unwrap() + .wait() + .unwrap(); + } + } + Content::CloudSyncFinish(request) => { + assert_eq!(request.unixmilli, 123); + assert_eq!(request.signature, [4]); + synced = true; + responder + .reply(CloudSyncFinishResponse { accepted: 123 }, deadline) + .unwrap() + .wait() + .unwrap(); + } + Content::GenuinityProof(_) => { + assert!(synced, "proof requested before cloud sync"); + proofs.fetch_add(1, Ordering::SeqCst); responder - .fail( - crate::schema::Error::new(0x111, "identity rejected"), + .reply( + GenuinityProofResponse { + proof: vec![0xfb, 0xff], + }, deadline, ) .unwrap() .wait() .unwrap(); - } else { + } + Content::SlotList(_) => { + assert!(synced, "slots requested before cloud sync"); responder - .reply(CloudSyncStartResponse { challenge: vec![3] }, deadline) + .reply(crate::schema::SlotListResponse::default(), deadline) .unwrap() .wait() .unwrap(); } + request => return answering(session, request, responder), } - Content::CloudSyncFinish(request) => { - assert_eq!(request.unixmilli, 123); - assert_eq!(request.signature, [4]); - synced = true; - responder - .reply(CloudSyncFinishResponse { accepted: 123 }, deadline) - .unwrap() - .wait() - .unwrap(); - } - Content::GenuinityProof(_) => { - assert!(synced, "proof requested before cloud sync"); - proofs.fetch_add(1, Ordering::SeqCst); - responder - .reply( - GenuinityProofResponse { - proof: vec![0xfb, 0xff], - }, - deadline, - ) - .unwrap() - .wait() - .unwrap(); - } - Content::SlotList(_) => { - assert!(synced, "slots requested before cloud sync"); - responder - .reply(crate::schema::SlotListResponse::default(), deadline) - .unwrap() - .wait() - .unwrap(); - } - request => return answering(session, request, responder), + true } - true - } - })); + }), + ); (peer, starts, proofs) } /// JSON replies used by the real wire peer's cloud synchronization exchange. - pub(super) fn sync_responses() -> Vec<(Duration, String)> { + pub(super) fn sync_responses() -> Vec { vec![ - ( - Duration::ZERO, - response(200, r#"{"signer":"AQ==","crypto":"Ag=="}"#), - ), - ( - Duration::ZERO, - response(200, r#"{"unixmilli":123,"signature":"BA=="}"#), - ), + response(200, r#"{"signer":"AQ==","crypto":"Ag=="}"#), + response(200, r#"{"unixmilli":123,"signature":"BA=="}"#), ] } @@ -676,19 +686,24 @@ pub(crate) mod tests { /// independently and subsequent operations reuse the successful exchange. #[test] fn test_shared_setup() { + // Hold the leader's sync at its first HTTP request + let mut tester = test_clock(); + let clock = tester.clock(); let mut responses = sync_responses(); - responses.push((Duration::ZERO, response(200, r#"{"serial":"test-serial","enrolled":123,"disabled":false,"expired":false,"superseded":false}"#))); + responses.push(response(200, r#"{"serial":"test-serial","enrolled":123,"disabled":false,"expired":false,"superseded":false}"#)); let (release, pause) = mpsc::channel(); let (url, requests) = serve_inner(responses, Some(pause)); - let (mut peer, starts, proofs) = peer(false); + let (mut peer, starts, proofs) = peer(&clock, false); let ark = attach(&mut peer, url); let client = ark.client(); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; let leader = thread::spawn({ let client = client.clone(); move || client.call(GenuinityProofRequest {}, deadline) }); - requests.recv_timeout(TIMEOUT).unwrap(); + requests.recv().unwrap(); + + // Status bypasses the pending setup assert_eq!( client .call(DeviceInfoRequest {}, deadline) @@ -697,12 +712,18 @@ pub(crate) mod tests { "1.0.0" ); assert_eq!(starts.load(Ordering::SeqCst), 0); - assert!(matches!( - client - .clone() - .call_timeout(GenuinityProofRequest {}, Duration::from_millis(20)), - Err(Error::Timeout) - )); + + // A short waiter on the pending setup expires alone, at the earliest deadline on the clock + let short = clock.now() + Duration::from_millis(20); + let waiter = thread::spawn({ + let client = client.clone(); + move || client.call_timeout(GenuinityProofRequest {}, Duration::from_millis(20)) + }); + wait_deadline(&tester, short); + tester.advance_to(short); + assert!(matches!(waiter.join().unwrap(), Err(Error::Timeout))); + + // A later waiter and the leader both finish on the one exchange let follower = thread::spawn({ let client = client.clone(); move || client.call(GenuinityProofRequest {}, deadline) @@ -715,28 +736,24 @@ pub(crate) mod tests { assert_eq!(proofs.load(Ordering::SeqCst), 3); assert!( requests - .recv_timeout(TIMEOUT) + .recv() .unwrap() .starts_with("GET /v1/cloudsync/time?challenge=03 ") ); - assert!( - requests - .recv_timeout(TIMEOUT) - .unwrap() - .starts_with("GET /v1/genuine ") - ); + assert!(requests.recv().unwrap().starts_with("GET /v1/genuine ")); } /// A refused attempt preserves its device error and does not poison a retry. #[test] fn test_setup_retry() { + let clock = test_clock().clock(); let mut responses = sync_responses(); responses.insert(0, responses[0].clone()); let (url, _requests) = serve(responses); - let (mut peer, starts, proofs) = peer(true); + let (mut peer, starts, proofs) = peer(&clock, true); let ark = attach(&mut peer, url); let client = ark.client(); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; assert!(matches!(client.call(GenuinityProofRequest {}, deadline), Err(Error::Remote(error)) if error.code == 0x111)); client.call(GenuinityProofRequest {}, deadline).unwrap(); @@ -748,17 +765,18 @@ pub(crate) mod tests { /// and prevents that response from starting any request on the closed Ark. #[test] fn test_close_during_setup() { + let clock = test_clock().clock(); let (release, pause) = mpsc::channel(); let (url, requests) = serve_inner(vec![sync_responses().remove(0)], Some(pause)); - let (mut peer, starts, _) = peer(false); + let (mut peer, starts, _) = peer(&clock, false); let ark = attach(&mut peer, url); let client = ark.client(); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; let leader = thread::spawn({ let client = client.clone(); move || client.call(GenuinityProofRequest {}, deadline) }); - requests.recv_timeout(TIMEOUT).unwrap(); + requests.recv().unwrap(); let waiter = thread::spawn({ let client = client.clone(); move || client.call(GenuinityProofRequest {}, deadline) @@ -774,8 +792,9 @@ pub(crate) mod tests { /// requests fail clearly when the caller did not supply an environment. #[test] fn test_unattested_setup() { + let clock = test_clock().clock(); for recover in [false, true] { - let mut peer = Peer::spawn(Box::new(answering)); + let mut peer = Peer::spawn(&clock, Box::new(answering)); let policy = if recover { TrustMode::Recover(Box::new(peer.identity.clone())) } else { @@ -784,19 +803,19 @@ pub(crate) mod tests { let (ark, _) = Ark::attach(peer.stream(), &policy, |_| None).unwrap(); let client = ark.client(); assert!(matches!( - client.call(GenuinityProofRequest {}, Instant::now() + TIMEOUT), + client.call(GenuinityProofRequest {}, clock.now() + TIMEOUT), Err(Error::MissingEnvironment) )); assert!(matches!( - client.genuine(Instant::now() + TIMEOUT), + client.genuine(clock.now() + TIMEOUT), Err(Error::MissingEnvironment) )); assert!(matches!( - client.call(crate::schema::UnlockRequest {}, Instant::now() + TIMEOUT), + client.call(crate::schema::UnlockRequest {}, clock.now() + TIMEOUT), Err(Error::MissingEnvironment) )); client - .call(DeviceInfoRequest {}, Instant::now() + TIMEOUT) + .call(DeviceInfoRequest {}, clock.now() + TIMEOUT) .unwrap(); } } @@ -805,15 +824,16 @@ pub(crate) mod tests { /// Registry authentication uses their opaque proofs without an attested serial. #[test] fn test_unattested_cloud() { + let clock = test_clock().clock(); for recover in [false, true] { for realm in [Realm::Hardware, Realm::Emulator] { let mut responses = sync_responses(); - responses.push((Duration::ZERO, response(200, r#"{"serial":"registry-serial","enrolled":123,"disabled":false,"expired":false,"superseded":false}"#))); - responses.push((Duration::ZERO, response(403, "registry refused proof"))); + responses.push(response(200, r#"{"serial":"registry-serial","enrolled":123,"disabled":false,"expired":false,"superseded":false}"#)); + responses.push(response(403, "registry refused proof")); responses.extend(sync_responses()); - responses.push((Duration::ZERO, response(403, "registry refused proof"))); + responses.push(response(403, "registry refused proof")); let (url, requests) = serve(responses); - let (mut peer, starts, proofs) = peer(false); + let (mut peer, starts, proofs) = peer(&clock, false); let policy = if recover { TrustMode::Recover(Box::new(peer.identity.clone())) } else { @@ -824,13 +844,13 @@ pub(crate) mod tests { assert_eq!(matches!(identity, Identity::Recovered(_)), recover); let env = crate::identity::ENVIRONMENTS[0]; - let mut services = Services::new(&identity, Some((env, realm))); + let mut services = Services::new(&identity, Some((env, realm)), &session.clock()); let cloud = services.cloud.as_mut().unwrap(); cloud.url = url; cloud.agent = http(); let ark = Ark::start(session, Arc::new(services)).unwrap(); let client = ark.client(); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; client.call(DeviceInfoRequest {}, deadline).unwrap(); assert_eq!(starts.load(Ordering::SeqCst), 0); @@ -862,7 +882,7 @@ pub(crate) mod tests { "/v1/cloudsync/time?challenge=03", registry, ] { - let request = requests.recv_timeout(TIMEOUT).unwrap(); + let request = requests.recv().unwrap(); assert!(request.starts_with(&format!("GET {path} HTTP/1.1\r\n"))); } } @@ -902,68 +922,74 @@ pub(crate) mod tests { #[test] fn test_authentication_refresh() { use crate::schema; + let clock = test_clock().clock(); for operation in ["genuine", "relaying", "pairing"] { for (status, retried) in [(400, 400), (403, 403), (403, 200), (503, 503)] { if retried == 200 && operation != "genuine" { continue; } - let mut responses = vec![(Duration::ZERO, response(status, "refused"))]; + let mut responses = vec![response(status, "refused")]; if status == 403 { responses.extend(sync_responses()); - responses.push((Duration::ZERO, response(retried, if retried == 200 { + responses.push(response(retried, if retried == 200 { r#"{"serial":"test-serial","enrolled":123,"disabled":false,"expired":false,"superseded":false}"# - } else { "still refused" }))); + } else { "still refused" })); } let (url, requests) = serve(responses); let proofs = Arc::new(AtomicUsize::new(0)); - let mut peer = Peer::spawn(Box::new({ - let proofs = proofs.clone(); - let mut refreshed = false; - move |_, request, responder| { - let deadline = Instant::now() + TIMEOUT; - let proof = vec![u8::from(refreshed)]; - let response: protocol::Message = match request { - Content::DeviceInfo(_) => schema::DeviceInfoResponse { - cloud_synced: true, - cloud_clock: SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs(), - ..Default::default() - } - .into(), - Content::CloudSyncStart(_) => { - schema::CloudSyncStartResponse { challenge: vec![3] }.into() - } - Content::CloudSyncFinish(_) => { - refreshed = true; - schema::CloudSyncFinishResponse { accepted: 123 }.into() - } - Content::GenuinityProof(_) => { - proofs.fetch_add(1, Ordering::SeqCst); - schema::GenuinityProofResponse { proof }.into() - } - Content::RelayJoin(_) => { - proofs.fetch_add(1, Ordering::SeqCst); - schema::RelayJoinResponse { auth: proof }.into() - } - Content::PairingAuth(_) => { - proofs.fetch_add(1, Ordering::SeqCst); - schema::PairingAuthResponse { - auth: proof, - fprint: vec![8; 32], + let mut peer = Peer::spawn( + &clock, + Box::new({ + let proofs = proofs.clone(); + let mut refreshed = false; + move |session, request, responder| { + let deadline = session.clock().now() + TIMEOUT; + let proof = vec![u8::from(refreshed)]; + let response: protocol::Message = match request { + Content::DeviceInfo(_) => schema::DeviceInfoResponse { + cloud_synced: true, + cloud_clock: session + .clock() + .system_time() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(), + ..Default::default() } - .into() - } - other => panic!("unexpected request: {other:?}"), - }; - responder.reply(response, deadline).unwrap(); - true - } - })); + .into(), + Content::CloudSyncStart(_) => { + schema::CloudSyncStartResponse { challenge: vec![3] }.into() + } + Content::CloudSyncFinish(_) => { + refreshed = true; + schema::CloudSyncFinishResponse { accepted: 123 }.into() + } + Content::GenuinityProof(_) => { + proofs.fetch_add(1, Ordering::SeqCst); + schema::GenuinityProofResponse { proof }.into() + } + Content::RelayJoin(_) => { + proofs.fetch_add(1, Ordering::SeqCst); + schema::RelayJoinResponse { auth: proof }.into() + } + Content::PairingAuth(_) => { + proofs.fetch_add(1, Ordering::SeqCst); + schema::PairingAuthResponse { + auth: proof, + fprint: vec![8; 32], + } + .into() + } + other => panic!("unexpected request: {other:?}"), + }; + responder.reply(response, deadline).unwrap(); + true + } + }), + ); let ark = attach(&mut peer, url); let client = ark.client(); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; let result = match operation { "genuine" => client.genuine(deadline).map(drop), "relaying" => client.attach_relay(deadline), @@ -1015,48 +1041,58 @@ pub(crate) mod tests { #[test] fn caller_authentication_retries_fresh_proofs_without_cloud_sync() { use auth::tests::{Login, refused}; + let clock = test_clock().clock(); for operation in ["genuine", "relaying", "pairing"] { for fail_login in [false, true] { - let mut responses = vec![(Duration::ZERO, refused(403))]; + let mut responses = vec![refused(403)]; if !fail_login { - responses.push((Duration::ZERO, if operation == "genuine" { + responses.push(if operation == "genuine" { response(200, r#"{"serial":"test-serial","enrolled":123,"disabled":false,"expired":false,"superseded":false}"#) - } else { refused(403) })); + } else { refused(403) }); } let (url, requests) = serve(responses); let proofs = Arc::new(AtomicUsize::new(0)); - let mut peer = Peer::spawn(Box::new({ - let proofs = proofs.clone(); - move |_, request, responder| { - let response: protocol::Message = match request { - Content::DeviceInfo(_) => crate::schema::DeviceInfoResponse { - cloud_synced: true, - cloud_clock: SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs(), - ..Default::default() - } - .into(), - Content::GenuinityProof(_) => crate::schema::GenuinityProofResponse { - proof: vec![proofs.fetch_add(1, Ordering::SeqCst) as u8], - } - .into(), - Content::RelayJoin(_) => crate::schema::RelayJoinResponse { - auth: vec![proofs.fetch_add(1, Ordering::SeqCst) as u8], - } - .into(), - Content::PairingAuth(_) => crate::schema::PairingAuthResponse { - auth: vec![proofs.fetch_add(1, Ordering::SeqCst) as u8], - fprint: vec![8; 32], - } - .into(), - other => panic!("unexpected request: {other:?}"), - }; - responder.reply(response, Instant::now() + TIMEOUT).unwrap(); - true - } - })); + let mut peer = Peer::spawn( + &clock, + Box::new({ + let proofs = proofs.clone(); + move |session, request, responder| { + let response: protocol::Message = match request { + Content::DeviceInfo(_) => crate::schema::DeviceInfoResponse { + cloud_synced: true, + cloud_clock: session + .clock() + .system_time() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(), + ..Default::default() + } + .into(), + Content::GenuinityProof(_) => { + crate::schema::GenuinityProofResponse { + proof: vec![proofs.fetch_add(1, Ordering::SeqCst) as u8], + } + .into() + } + Content::RelayJoin(_) => crate::schema::RelayJoinResponse { + auth: vec![proofs.fetch_add(1, Ordering::SeqCst) as u8], + } + .into(), + Content::PairingAuth(_) => crate::schema::PairingAuthResponse { + auth: vec![proofs.fetch_add(1, Ordering::SeqCst) as u8], + fprint: vec![8; 32], + } + .into(), + other => panic!("unexpected request: {other:?}"), + }; + responder + .reply(response, session.clock().now() + TIMEOUT) + .unwrap(); + true + } + }), + ); let mut ark = attach(&mut peer, url); let login = Login { fail: fail_login, @@ -1109,10 +1145,10 @@ pub(crate) mod tests { #[test] fn cloud_sync_logs_in_before_sending_certificates_to_the_ark() { use auth::tests::{Login, refused}; - let mut responses = vec![(Duration::ZERO, refused(302))]; + let mut responses = vec![refused(302)]; responses.extend(sync_responses()); let (url, requests) = serve(responses); - let (mut peer, starts, _) = peer(false); + let (mut peer, starts, _) = peer(&test_clock().clock(), false); let mut ark = attach(&mut peer, url); let login = Login::default(); ark.set_cloud_auth(login.clone()); @@ -1142,6 +1178,7 @@ pub(crate) mod tests { examples: vec!["first".into(), "second".into()], }], }; + let clock = test_clock().clock(); for initially_synced in [false, true] { let mut responses = sync_responses(); if !initially_synced { @@ -1149,46 +1186,51 @@ pub(crate) mod tests { } let (url, requests) = serve(responses); let infos = Arc::new(AtomicUsize::new(0)); - let mut peer = Peer::spawn(Box::new({ - let infos = infos.clone(); - let paths = expected.clone(); - let mut synced = initially_synced; - move |session, request, responder| { - let deadline = Instant::now() + TIMEOUT; - let response: protocol::Message = match request { - Content::DeviceInfo(_) => { - infos.fetch_add(1, Ordering::SeqCst); - let clock = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs(); - schema::DeviceInfoResponse { - cloud_clock: clock, - cloud_synced: synced, - ..Default::default() + let mut peer = Peer::spawn( + &clock, + Box::new({ + let infos = infos.clone(); + let paths = expected.clone(); + let mut synced = initially_synced; + move |session, request, responder| { + let deadline = session.clock().now() + TIMEOUT; + let response: protocol::Message = match request { + Content::DeviceInfo(_) => { + infos.fetch_add(1, Ordering::SeqCst); + let clock = session + .clock() + .system_time() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + schema::DeviceInfoResponse { + cloud_clock: clock, + cloud_synced: synced, + ..Default::default() + } + .into() } - .into() - } - Content::DatasetPaths(_) => { - assert!(synced, "request served before sync"); - paths.clone().into() - } - Content::CloudSyncStart(_) => { - schema::CloudSyncStartResponse { challenge: vec![3] }.into() - } - Content::CloudSyncFinish(_) => { - synced = true; - schema::CloudSyncFinishResponse { accepted: 123 }.into() - } - other => return answering(session, other, responder), - }; - responder.reply(response, deadline).unwrap(); - true - } - })); + Content::DatasetPaths(_) => { + assert!(synced, "request served before sync"); + paths.clone().into() + } + Content::CloudSyncStart(_) => { + schema::CloudSyncStartResponse { challenge: vec![3] }.into() + } + Content::CloudSyncFinish(_) => { + synced = true; + schema::CloudSyncFinishResponse { accepted: 123 }.into() + } + other => return answering(session, other, responder), + }; + responder.reply(response, deadline).unwrap(); + true + } + }), + ); let ark = attach(&mut peer, url); let client = ark.client(); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; let info = client.call(schema::DeviceInfoRequest {}, deadline).unwrap(); assert_eq!(info.cloud_synced, initially_synced); for _ in 0..2 { @@ -1203,18 +1245,8 @@ pub(crate) mod tests { if initially_synced { 0 } else { 2 } ); client.sync(deadline).unwrap(); - assert!( - requests - .recv_timeout(TIMEOUT) - .unwrap() - .contains("/cloudsync/identity") - ); - assert!( - requests - .recv_timeout(TIMEOUT) - .unwrap() - .contains("/cloudsync/time") - ); + assert!(requests.recv().unwrap().contains("/cloudsync/identity")); + assert!(requests.recv().unwrap().contains("/cloudsync/time")); assert_eq!(infos.load(Ordering::SeqCst), 1); } } @@ -1222,11 +1254,12 @@ pub(crate) mod tests { /// Waiting on an older status response must not undo a completed refresh. #[test] fn test_delayed_device_info_retains_newer_sync() { + let clock = test_clock().clock(); let (url, requests) = serve(sync_responses()); - let (mut peer, starts, _) = peer(false); + let (mut peer, starts, _) = peer(&clock, false); let ark = attach(&mut peer, url); let client = ark.client(); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; let pending = client.send(DeviceInfoRequest {}, deadline).unwrap(); client.sync(deadline).unwrap(); assert!(!pending.wait().unwrap().cloud_synced); @@ -1242,6 +1275,7 @@ pub(crate) mod tests { #[test] fn test_unavailable_retry_requires_lost_sync() { use crate::schema::{self, ReservedErrors}; + let clock = test_clock().clock(); for (code, reset, repeat, retries) in [ (ReservedErrors::Unavailable as u64, true, false, 1), (ReservedErrors::Unavailable as u64, true, true, 1), @@ -1255,54 +1289,62 @@ pub(crate) mod tests { vec![] }); let count = Arc::new(AtomicUsize::new(0)); - let mut peer = Peer::spawn(Box::new({ - let count = count.clone(); - let mut synced = true; - move |session, request, responder| { - let deadline = Instant::now() + TIMEOUT; - let response: protocol::Message = match request { - Content::DeviceInfo(_) => { - let clock = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs(); - schema::DeviceInfoResponse { - cloud_clock: clock, - cloud_synced: synced, - ..Default::default() + let mut peer = Peer::spawn( + &clock, + Box::new({ + let count = count.clone(); + let mut synced = true; + move |session, request, responder| { + let deadline = session.clock().now() + TIMEOUT; + let response: protocol::Message = match request { + Content::DeviceInfo(_) => { + let clock = session + .clock() + .system_time() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + schema::DeviceInfoResponse { + cloud_clock: clock, + cloud_synced: synced, + ..Default::default() + } + .into() } - .into() - } - Content::SlotList(_) => { - let attempt = count.fetch_add(1, Ordering::SeqCst); - if attempt == 0 || repeat { - if reset { - synced = false; + Content::SlotList(_) => { + let attempt = count.fetch_add(1, Ordering::SeqCst); + if attempt == 0 || repeat { + if reset { + synced = false; + } + responder + .fail( + schema::Error::new(code, "original refusal"), + deadline, + ) + .unwrap(); + return true; } - responder - .fail(schema::Error::new(code, "original refusal"), deadline) - .unwrap(); - return true; + schema::SlotListResponse::default().into() } - schema::SlotListResponse::default().into() - } - Content::CloudSyncStart(_) => { - schema::CloudSyncStartResponse { challenge: vec![3] }.into() - } - Content::CloudSyncFinish(_) => { - synced = true; - schema::CloudSyncFinishResponse { accepted: 123 }.into() - } - other => return answering(session, other, responder), - }; - responder.reply(response, deadline).unwrap(); - true - } - })); + Content::CloudSyncStart(_) => { + schema::CloudSyncStartResponse { challenge: vec![3] }.into() + } + Content::CloudSyncFinish(_) => { + synced = true; + schema::CloudSyncFinishResponse { accepted: 123 }.into() + } + other => return answering(session, other, responder), + }; + responder.reply(response, deadline).unwrap(); + true + } + }), + ); let ark = attach(&mut peer, url); let result = ark .client() - .call(schema::SlotListRequest {}, Instant::now() + TIMEOUT); + .call(schema::SlotListRequest {}, clock.now() + TIMEOUT); assert_eq!(count.load(Ordering::SeqCst), 1 + retries); if retries == 1 && !repeat { assert!(result.is_ok()); diff --git a/connect/src/cloud/pairing.rs b/connect/src/cloud/pairing.rs index ddb1d81..9b6627f 100644 --- a/connect/src/cloud/pairing.rs +++ b/connect/src/cloud/pairing.rs @@ -11,9 +11,10 @@ use super::{ socket::{self, Connection, Socket, socket_mut}, }; use crate::{Error, Timing, schema}; +use darkbio_clock::Clock; use darkbio_crypto::{cbor::Cbor, cose}; use darkbio_wire::protocol::Requester; -use std::time::{Duration, Instant, SystemTime, UNIX_EPOCH}; +use std::time::{Duration, Instant, UNIX_EPOCH}; use tungstenite::Message; /// Pairing stages. The caller renders the rendezvous or turns it into a QR code; @@ -26,8 +27,9 @@ pub enum PairingProgress { colo: String, /// Rendezvous secret shared with the companion through the pairing code. secret: [u8; 32], - /// End of the scan window in Unix seconds. - deadline: u64, + /// End of the scan window on the connection's clock, when pairing + /// stops waiting for the companion. + deadline: Instant, /// Pairing encryption key fingerprint conveyed to the companion out of band. fingerprint: Vec, }, @@ -61,84 +63,118 @@ impl Services { &self, requester: &Requester, timing: Timing, - mut progress: impl FnMut(PairingProgress), + progress: impl FnMut(PairingProgress), ) -> Result<(), Error> { self.sync(requester, timing)?; + let clock = &self.clock; let cloud = self.cloud.as_ref().ok_or(Error::MissingEnvironment)?; let (mut socket, fingerprint) = self.authenticate(requester, timing, || { let auth = requester - .request(schema::PairingAuthRequest {}, timing.io())? + .request(schema::PairingAuthRequest {}, timing.io(clock))? .wait::()?; let socket = socket::connect( cloud, &cloud.pairing_url(), &auth.auth, "Pairing", - timing.io(), + timing.io(clock), )?; Ok((socket, auth.fprint)) })?; - // These claims locate the rendezvous and bound scanning. The Ark verifies - // the companion identity and storage messages forwarded below. - let rendezvous: Rendezvous = cose::peek(&receive(&mut socket, timing.io())?) - .map_err(|err| Error::Pairing(err.to_string()))?; - let expires = scan_deadline(rendezvous.deadline)?; - progress(PairingProgress::Rendezvous { - colo: rendezvous.colo, - secret: rendezvous.secret, - deadline: rendezvous.deadline, - fingerprint, - }); - let wait = timing.limit(expires); - let identity = receive(&mut socket, wait).map_err(|err| { - if matches!(err, Error::Timeout) && wait == expires { - Error::PairingExpired - } else { - err - } - })?; - progress(PairingProgress::Identity); - requester - .request( - schema::PairingSetAppIdentityRequest { identity }, - timing.io(), - )? - .wait::()?; - let app_key = receive(&mut socket, timing.approval())?; - progress(PairingProgress::Storage); - let storage = requester - .request(schema::PairingSetAppStorageRequest { app_key }, timing.io())? - .wait::()?; - send(&mut socket, storage.ark_keys, timing.io())?; - let app_ack = receive(&mut socket, timing.approval())?; - requester - .request(schema::PairingAckArkStorageRequest { app_ack }, timing.io())? - .wait::()?; - progress(PairingProgress::Approval); - let accepted = requester - .request( - schema::PairingAcceptanceRequest {}, - timing.window(crate::timing::PAIRING_WINDOW), - )? - .wait::()?; - send(&mut socket, accepted.confirm, timing.io())?; - progress(PairingProgress::Formatting); - let completed = requester - .request( - schema::PairingCompletionRequest {}, - timing.window(crate::timing::PAIRING_WINDOW), - )? - .wait::()?; - send(&mut socket, completed.confirm, timing.io())?; + exchange(requester, timing, &mut socket, fingerprint, progress)?; let _ = socket.close(None); Ok(()) } } -/// Converts the cloud's Unix deadline once, then waits on the monotonic clock. -fn scan_deadline(deadline: u64) -> Result { - let start = Instant::now(); - let now = SystemTime::now() +/// Presents the rendezvous, then forwards each opaque exchange between the +/// companion and the Ark. The owner's scan waits under the cloud's deadline, +/// which only a caller's earlier absolute deadline cuts short. +fn exchange( + requester: &Requester, + timing: Timing, + channel: &mut impl Channel, + fingerprint: Vec, + mut progress: impl FnMut(PairingProgress), +) -> Result<(), Error> { + let clock = &requester.clock(); + + // These claims locate the rendezvous and bound scanning. The Ark verifies + // the companion identity and storage messages forwarded below. + let rendezvous: Rendezvous = cose::peek(&channel.receive(timing.io(clock))?) + .map_err(|err| Error::Pairing(err.to_string()))?; + let expires = scan_deadline(clock, rendezvous.deadline)?; + let wait = timing.limit(expires); + progress(PairingProgress::Rendezvous { + colo: rendezvous.colo, + secret: rendezvous.secret, + deadline: wait, + fingerprint, + }); + let identity = channel.receive(wait).map_err(|err| { + if matches!(err, Error::Timeout) && wait == expires { + Error::PairingExpired + } else { + err + } + })?; + progress(PairingProgress::Identity); + requester + .request( + schema::PairingSetAppIdentityRequest { identity }, + timing.io(clock), + )? + .wait::()?; + let app_key = channel.receive(timing.approval(clock))?; + progress(PairingProgress::Storage); + let storage = requester + .request( + schema::PairingSetAppStorageRequest { app_key }, + timing.io(clock), + )? + .wait::()?; + channel.send(storage.ark_keys, timing.io(clock))?; + let app_ack = channel.receive(timing.approval(clock))?; + requester + .request( + schema::PairingAckArkStorageRequest { app_ack }, + timing.io(clock), + )? + .wait::()?; + progress(PairingProgress::Approval); + let accepted = requester + .request( + schema::PairingAcceptanceRequest {}, + timing.window(clock, crate::timing::PAIRING_WINDOW), + )? + .wait::()?; + channel.send(accepted.confirm, timing.io(clock))?; + progress(PairingProgress::Formatting); + let completed = requester + .request( + schema::PairingCompletionRequest {}, + timing.window(clock, crate::timing::PAIRING_WINDOW), + )? + .wait::()?; + channel.send(completed.confirm, timing.io(clock)) +} + +/// Rendezvous with the companion, carrying opaque payloads both ways, each +/// exchange under its own deadline. +trait Channel { + /// Waits for one binary payload until the deadline. + fn receive(&mut self, deadline: Instant) -> Result, Error>; + + /// Sends one payload within the deadline. + fn send(&mut self, bytes: Vec, deadline: Instant) -> Result<(), Error>; +} + +/// Converts the cloud's Unix deadline once, against the clock's wall time, then +/// waits on the clock's monotonic time. +fn scan_deadline(clock: &Clock, deadline: u64) -> Result { + let start = clock.now(); + let now = clock + .system_time() .duration_since(UNIX_EPOCH) .map_err(|err| Error::Pairing(err.to_string()))?; let remaining = Duration::from_secs(deadline) @@ -192,35 +228,56 @@ fn send(socket: &mut Connection, bytes: Vec, deadline: Instant) -> Result<() Ok(()) } +impl Channel for Connection { + /// Waits on the cloud's pairing socket, answering its control frames. + fn receive(&mut self, deadline: Instant) -> Result, Error> { + receive(self, deadline) + } + + /// Sends through the cloud's pairing socket. + fn send(&mut self, bytes: Vec, deadline: Instant) -> Result<(), Error> { + send(self, bytes, deadline) + } +} + #[cfg(test)] mod tests { use super::*; use crate::{ Ark, TrustMode, cloud::{State, http}, - testing::Peer, + testing::{Peer, test_clock, wait_deadline}, trust::Realm, }; + use darkbio_clock::crossbeam_channel; use darkbio_crypto::xdsa; use darkbio_wire::protocol::{self, schema::host_to_ark::Content}; use std::{ net::TcpListener, - sync::{Arc, Mutex}, + sync::{Arc, Mutex, mpsc}, thread, }; const TIMEOUT: Duration = Duration::from_secs(5); #[test] fn scan_uses_cloud_deadline_and_retains_caller_bound() { - assert!(matches!(scan_deadline(0), Err(Error::PairingExpired))); - let now = Instant::now(); - let epoch = SystemTime::now() - .duration_since(UNIX_EPOCH) - .unwrap() - .as_secs(); - let expiry = scan_deadline(epoch + 60).unwrap(); - assert!(expiry > now + Duration::from_secs(58)); - assert!(expiry <= now + Duration::from_secs(60)); + // Pin wall time to a whole second, so the cloud's deadline maps exactly + let mut tester = test_clock(); + tester.set_system_time(UNIX_EPOCH + Duration::from_secs(1_789_000_000)); + let clock = tester.clock(); + assert!(matches!( + scan_deadline(&clock, 0), + Err(Error::PairingExpired) + )); + assert!(matches!( + scan_deadline(&clock, 1_789_000_000), + Err(Error::PairingExpired) + )); + let now = clock.now(); + let expiry = scan_deadline(&clock, 1_789_000_060).unwrap(); + assert_eq!(expiry, now + Duration::from_secs(60)); + + // The scan keeps its own deadline, cut only by the caller's absolute one let timing = Timing::inactivity(Duration::from_millis(1)); assert_eq!(timing.limit(expiry), expiry); let caller = now + Duration::from_secs(1); @@ -231,6 +288,7 @@ mod tests { #[test] #[allow(clippy::result_large_err)] // Tungstenite's HTTP callback owns its rejection. fn close_retains_timeout_and_other_reasons() { + let clock = test_clock().clock(); for reason in ["pairing timed out", "companion disconnected"] { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("ws://{}/pairing", listener.local_addr().unwrap()); @@ -255,9 +313,9 @@ mod tests { })) .unwrap(); }); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; let mut socket = socket::connect( - &http::tests::api(url.clone(), Realm::Hardware), + &http::tests::api(url.clone(), Realm::Hardware, &clock), &url, &[], "Pairing", @@ -274,19 +332,183 @@ mod tests { } } + /// Signs a rendezvous at `signed` that expires at `deadline`, both in Unix + /// seconds. + fn rendezvous(signed: u64, deadline: u64) -> Vec { + let rendezvous = Rendezvous { + colo: "OTP".into(), + secret: [9; 32], + deadline, + }; + cose::sign_at( + rendezvous, + (), + &xdsa::SecretKey::generate(), + b"pairing-v1", + signed as i64, + ) + .unwrap() + } + + /// Ark answering each pairing request with fixed opaque payloads, checking + /// the ones it receives. It refuses the owner's acceptance when asked to. + fn pairing_peer(clock: &Clock, refuse: bool) -> Peer { + Peer::spawn( + clock, + Box::new(move |_, request, responder| { + let deadline = responder.clock().now() + TIMEOUT; + match request { + Content::PairingAuth(_) => responder.reply( + schema::PairingAuthResponse { + auth: vec![251, 255], + fprint: vec![8; 32], + }, + deadline, + ), + Content::PairingSetAppId(request) => { + assert_eq!(request.identity, [1, 0, 255]); + responder.reply(schema::PairingSetAppIdentityResponse {}, deadline) + } + Content::PairingSetAppStorage(request) => { + assert_eq!(request.app_key, [2, 0, 254]); + responder.reply( + schema::PairingSetAppStorageResponse { + ark_keys: vec![3, 0, 253], + }, + deadline, + ) + } + Content::PairingAckArkStorage(request) => { + assert_eq!(request.app_ack, [4, 0, 252]); + responder.reply(schema::PairingAckArkStorageResponse {}, deadline) + } + Content::PairingAccept(_) if refuse => { + responder.fail(schema::Error::new(0x1234, "owner refused"), deadline) + } + Content::PairingAccept(_) => responder.reply( + schema::PairingAcceptanceResponse { + confirm: vec![5, 0, 251], + }, + deadline, + ), + Content::PairingComplete(_) => responder.reply( + schema::PairingCompletionResponse { + confirm: vec![6, 0, 250], + }, + deadline, + ), + _ => panic!("unexpected pairing request {request:?}"), + } + .unwrap(); + true + }), + ) + } + + /// The owner's scan outlives the machine allowance under the cloud's + /// deadline, while a caller's earlier absolute deadline still ends it. The + /// presented rendezvous counts down to the deadline the scan waits on. + #[test] + fn test_scan_waits_under_the_cloud_deadline() { + /// Companion side of the rendezvous, which the test hands each payload + /// to. Every receive reports its deadline before it waits. + struct Companion { + clock: Clock, // clock the receives wait on + payloads: crossbeam_channel::Receiver>, // payloads the test hands over + waits: mpsc::Sender, // deadline of every receive + } + impl Channel for Companion { + fn receive(&mut self, deadline: Instant) -> Result, Error> { + self.waits.send(deadline).unwrap(); + self.clock + .recv_deadline(&self.payloads, deadline) + .map_err(|_| Error::Timeout) + } + fn send(&mut self, _: Vec, _: Instant) -> Result<(), Error> { + Ok(()) + } + } + + for caller in [None, Some(Duration::from_secs(10))] { + // Pin wall time to a whole second, so the cloud's deadline maps exactly + let mut tester = test_clock(); + tester.set_system_time(UNIX_EPOCH + Duration::from_secs(1_789_000_000)); + let clock = tester.clock(); + let start = clock.now(); + + // Start pairing with the rendezvous at hand but the companion silent + let mut peer = pairing_peer(&clock, false); + let verifier = TrustMode::Recover(Box::new(peer.identity.clone())); + let (session, _) = protocol::connect(peer.stream(), &verifier).unwrap(); + let (hand, payloads) = crossbeam_channel::unbounded(); + let (waits, receives) = mpsc::channel(); + let (progress, stages) = mpsc::channel(); + hand.send(rendezvous(1_789_000_000, 1_789_000_060)).unwrap(); + let allowance = Timing::inactivity(Duration::from_secs(1)); + let timing = caller.map_or(allowance, |caller| allowance.with_deadline(start + caller)); + let pairing = thread::spawn({ + let requester = session.requester(); + let mut companion = Companion { + clock: clock.clone(), + payloads, + waits, + }; + move || { + exchange(&requester, timing, &mut companion, vec![8; 32], |stage| { + progress.send(stage).unwrap() + }) + } + }); + + // The rendezvous comes within the allowance, then the scan waits on + // the cloud's deadline or the caller's earlier one, which is also + // the deadline the rendezvous is presented with + assert_eq!(receives.recv().unwrap(), start + Duration::from_secs(1)); + let scan = start + caller.unwrap_or(Duration::from_secs(60)); + assert_eq!(receives.recv().unwrap(), scan); + assert!(matches!( + stages.recv().unwrap(), + PairingProgress::Rendezvous { deadline, .. } if deadline == scan + )); + wait_deadline(&tester, scan); + + // Passing the machine allowance leaves the scan waiting + tester.advance(Duration::from_secs(2)); + assert_eq!(tester.next_deadline(), Some(scan)); + + // A late companion completes the exchange, a caller's deadline ends it + if caller.is_none() { + for payload in [vec![1, 0, 255], vec![2, 0, 254], vec![4, 0, 252]] { + hand.send(payload).unwrap(); + } + pairing.join().unwrap().unwrap(); + } else { + tester.advance_to(scan); + assert!(matches!(pairing.join().unwrap(), Err(Error::Timeout))); + } + } + } + /// The host only relays sealed payloads. Both completion messages and a - /// refusal retain their protocol ordering; a scan can outlive an I/O wait. + /// refusal retain their protocol ordering. #[test] #[allow(clippy::result_large_err)] // Tungstenite requires a full HTTP rejection response. fn pairing_exchange_and_refusal() { + // Pin wall time to a whole second, so the cloud's deadline maps exactly + let mut tester = test_clock(); + tester.set_system_time(UNIX_EPOCH + Duration::from_secs(1_789_000_000)); + let clock = tester.clock(); + let start = clock.now(); + for refuse in [false, true] { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("http://{}/v1", listener.local_addr().unwrap()); - let expiry = SystemTime::now() + let signed = clock + .system_time() .duration_since(UNIX_EPOCH) .unwrap() - .as_secs() - + 60; + .as_secs(); + let expiry = signed + 60; let cloud = thread::spawn(move || { let (stream, _) = listener.accept().unwrap(); stream.set_read_timeout(Some(TIMEOUT)).unwrap(); @@ -307,16 +529,9 @@ mod tests { }, ) .unwrap(); - let rendezvous = Rendezvous { - colo: "OTP".into(), - secret: [9; 32], - deadline: expiry, - }; - let signed = - cose::sign(rendezvous, (), &xdsa::SecretKey::generate(), b"pairing-v1") - .unwrap(); - socket.send(Message::Binary(signed.into())).unwrap(); - thread::sleep(Duration::from_millis(1200)); + socket + .send(Message::Binary(rendezvous(signed, expiry).into())) + .unwrap(); socket.send(Message::Ping(vec![9].into())).unwrap(); socket .send(Message::Binary(vec![1, 0, 255].into())) @@ -341,59 +556,14 @@ mod tests { assert!(matches!(socket.read().unwrap(), Message::Close(_))); } }); - let mut peer = Peer::spawn(Box::new(move |_, request, responder| { - let deadline = Instant::now() + TIMEOUT; - match request { - Content::PairingAuth(_) => responder.reply( - schema::PairingAuthResponse { - auth: vec![251, 255], - fprint: vec![8; 32], - }, - deadline, - ), - Content::PairingSetAppId(request) => { - assert_eq!(request.identity, [1, 0, 255]); - responder.reply(schema::PairingSetAppIdentityResponse {}, deadline) - } - Content::PairingSetAppStorage(request) => { - assert_eq!(request.app_key, [2, 0, 254]); - responder.reply( - schema::PairingSetAppStorageResponse { - ark_keys: vec![3, 0, 253], - }, - deadline, - ) - } - Content::PairingAckArkStorage(request) => { - assert_eq!(request.app_ack, [4, 0, 252]); - responder.reply(schema::PairingAckArkStorageResponse {}, deadline) - } - Content::PairingAccept(_) if refuse => { - responder.fail(schema::Error::new(0x1234, "owner refused"), deadline) - } - Content::PairingAccept(_) => responder.reply( - schema::PairingAcceptanceResponse { - confirm: vec![5, 0, 251], - }, - deadline, - ), - Content::PairingComplete(_) => responder.reply( - schema::PairingCompletionResponse { - confirm: vec![6, 0, 250], - }, - deadline, - ), - _ => panic!("unexpected pairing request {request:?}"), - } - .unwrap(); - true - })); + let mut peer = pairing_peer(&clock, refuse); let verifier = TrustMode::Recover(Box::new(peer.identity.clone())); let (session, _) = protocol::connect(peer.stream(), &verifier).unwrap(); let services = Arc::new(Services { - cloud: Some(http::tests::api(url, Realm::Hardware)), + clock: clock.clone(), + cloud: Some(http::tests::api(url, Realm::Hardware, &clock)), state: Mutex::new(State { - synced: Some((Instant::now(), true)), + synced: Some((clock.now(), true)), ..Default::default() }), updating: Mutex::new(()), @@ -420,7 +590,7 @@ mod tests { } assert!( matches!(&stages[0], PairingProgress::Rendezvous { colo, secret, deadline, fingerprint } - if *deadline == expiry && colo == "OTP" && secret == &[9;32] && fingerprint == &vec![8;32]) + if *deadline == start + Duration::from_secs(60) && colo == "OTP" && secret == &[9;32] && fingerprint == &vec![8;32]) ); cloud.join().unwrap(); } diff --git a/connect/src/cloud/relay.rs b/connect/src/cloud/relay.rs index cbb6e58..ed7ce15 100644 --- a/connect/src/cloud/relay.rs +++ b/connect/src/cloud/relay.rs @@ -10,11 +10,16 @@ //! The wire dispatcher only admits reverse requests to a bounded queue. Relay //! failure refuses retained Ark requests; a later operation may attach again, //! but no request is replayed and the underlying wire session stays independent. +//! +//! The worker waits on the socket through mio, so its heartbeat and write +//! timers run on real time. Exchange deadlines belong to the wire session and +//! are measured on its clock. use super::{ Failure, - socket::{self, Socket, io_error, remaining, socket_error, socket_mut}, + socket::{self, Socket, io_error, socket_error, socket_mut}, }; +use crate::timing::ClockExt; use darkbio_crypto::cbor::{self, Cbor}; use darkbio_wire::protocol::{self, Promise, Requester, Responder, schema}; use mio::{Events, Interest, Poll, Token, Waker}; @@ -72,8 +77,11 @@ struct Worker { #[derive(Debug)] struct Shared { state: Mutex, // Admission and closure are atomic with respect to each other - wake: Waker, // Signals queued Ark traffic or local closure + wake: Arc, // signals queued Ark traffic, wire completions or local closure socket: TcpStream, // Interrupts reads even inside WebSocket message assembly + /// Notifies a test once the last handle to this relay is gone. + #[cfg(test)] + dropped: Mutex>>, } /// Reverse requests awaiting admission and the first attachment failure. @@ -95,7 +103,7 @@ impl Relay { deadline: Instant, ) -> Result { let mut socket = socket::connect(api, url, auth, "Relaying", deadline)?; - remaining(deadline).map_err(io_error)?; + api.clock.remaining(deadline).map_err(io_error)?; let Socket::Blocking { stream, .. } = socket_mut(&mut socket) else { unreachable!() }; @@ -115,8 +123,10 @@ impl Relay { *socket_mut(&mut socket) = Socket::Connected(connected); let shared = Arc::new(Shared { state: Mutex::new(State::default()), - wake: Waker::new(poll.registry(), WAKE).map_err(io_error)?, + wake: Arc::new(Waker::new(poll.registry(), WAKE).map_err(io_error)?), socket: shutdown, + #[cfg(test)] + dropped: Mutex::new(None), }); Ok(Self { shared, @@ -124,7 +134,7 @@ impl Relay { socket, poll, requester, - heartbeat: Heartbeat::new(PING_INTERVAL, PONG_TIMEOUT), + heartbeat: Heartbeat::attach(), }), }) } @@ -211,6 +221,16 @@ impl Shared { } } +#[cfg(test)] +impl Drop for Shared { + /// Notifies the test that the relay and its worker released their handles. + fn drop(&mut self) { + if let Some(sender) = self.dropped.get_mut().unwrap().take() { + let _ = sender.send(()); + } + } +} + /// Recognizes an incomplete nonblocking operation that the poll loop can resume. fn would_block(error: &tungstenite::Error) -> bool { matches!(error, tungstenite::Error::Io(error) if error.kind() == io::ErrorKind::WouldBlock) @@ -219,10 +239,9 @@ fn would_block(error: &tungstenite::Error) -> bool { /// An unavailable relay fails the Ark's reverse request, allowing its original /// operation to finish with an error. Enqueueing the reply does not wait on I/O. pub(super) fn fail(responder: Responder, reason: &str) { - let _ = responder.fail( - schema::Error::reserved(schema::ReservedErrors::Unavailable, reason), - Instant::now() + protocol::DEFAULT_AUTOREPLY_TIMEOUT, - ); + let error = schema::Error::reserved(schema::ReservedErrors::Unavailable, reason); + let deadline = responder.clock().now() + protocol::DEFAULT_AUTOREPLY_TIMEOUT; + let _ = responder.fail(error, deadline); } /// Current cloud envelope. Only the envelope is interpreted; all bodies stay sealed. @@ -316,12 +335,23 @@ struct Heartbeat { } impl Heartbeat { - /// Schedules the first probe without sending traffic during attachment. - fn new(interval: Duration, timeout: Duration) -> Self { + /// Schedules the first probe of a relay attaching now. The worker's socket + /// timers run on real time, so the schedule starts on it too. + #[expect( + clippy::disallowed_methods, + reason = "the heartbeat is one of the worker's socket timers, which run on real time from attachment" + )] + fn attach() -> Self { + Self::new(PING_INTERVAL, PONG_TIMEOUT, Instant::now()) + } + + /// Schedules the first probe an interval after `now`, sending no traffic + /// during attachment. + fn new(interval: Duration, timeout: Duration, now: Instant) -> Self { Self { interval, timeout, - next: Instant::now() + interval, + next: now + interval, sequence: 0, pending: None, } @@ -358,8 +388,51 @@ impl Heartbeat { } } +/// Bounds output the cloud socket has not taken yet. The first write or +/// deferred flush starts the bound, later ones keep its deadline, and only a +/// completed flush ends it. +#[derive(Debug, Default)] +struct Backlog { + /// Time the pending output must be flushed by, while there is some. + deadline: Option, +} + +impl Backlog { + /// Starts the bound at `now` for output left to flush, keeping the + /// deadline of a bound already running. + fn start(&mut self, now: Instant) { + self.deadline.get_or_insert(now + WRITE_TIMEOUT); + } + + /// Ends the bound once a flush took all output. + fn flushed(&mut self) { + self.deadline = None; + } + + /// Whether output is left to flush, which holds back the next frame. + fn active(&self) -> bool { + self.deadline.is_some() + } + + /// Whether the output left to flush missed its deadline by `now`. + fn expired(&self, now: Instant) -> bool { + self.deadline.is_some_and(|deadline| now >= deadline) + } + + /// Time left after `now` before the deadline, bounding the next poll. + fn remaining(&self, now: Instant) -> Option { + self.deadline + .map(|deadline| deadline.saturating_duration_since(now)) + } +} + /// Writes one frame at a time while receiving concurrently. Wire completions -/// are polled only while app requests are in flight; idle wakeups check liveness. +/// and queued Ark requests wake the poll; idle wakeups check liveness. The +/// socket's own timers read real time and exchange deadlines the session's clock. +#[expect( + clippy::disallowed_methods, + reason = "the worker waits on the cloud socket through mio, so its heartbeat and write timers run on real time" +)] fn pump( mut socket: WebSocket>, mut poll: Poll, @@ -367,6 +440,7 @@ fn pump( shared: Arc, mut heartbeat: Heartbeat, ) { + let clock = requester.clock(); let mut events = Events::with_capacity(8); // The two directions have independent ID spaces. Only the worker changes // these maps, so completions never need the shared admission lock. @@ -375,7 +449,7 @@ fn pump( let (answered, answers) = mpsc::channel(); let mut output = VecDeque::new(); let mut bytes = 0usize; - let mut writing = None; + let mut writing = Backlog::default(); let result = (|| -> Result<(), Failure> { loop { { @@ -386,12 +460,12 @@ fn pump( // Admission stops while the socket is backlogged; reading and // completion handling continue independently of that backlog. if output.is_empty() - && writing.is_none() + && !writing.active() && pending.len() < MAX_INFLIGHT && let Some((request, responder, deadline)) = state.queue.pop_front() { state.bytes -= request.req.len(); - if deadline <= Instant::now() { + if deadline <= clock.now() { fail(responder, "relay request timed out in queue"); } else if let std::collections::hash_map::Entry::Vacant(entry) = pending.entry(request.id) @@ -409,9 +483,10 @@ fn pump( } // Expired exchanges release their responder even if no further // traffic arrives. Responses without a pending responder are discarded. + let now = clock.now(); let expired: Vec<_> = pending .iter() - .filter(|(_, (_, deadline))| Instant::now() >= *deadline) + .filter(|(_, (_, deadline))| now >= *deadline) .map(|(id, _)| *id) .collect(); for id in expired { @@ -438,11 +513,11 @@ fn pump( enqueue(&mut output, &mut bytes, Frame::Response(id, response.res))?; } } - if writing.is_none() + if !writing.active() && let Some(frame) = output.pop_front() { bytes -= frame.len(); - writing = Some(Instant::now() + WRITE_TIMEOUT); + writing.start(Instant::now()); match socket.write(Message::Binary(frame.into())) { Ok(()) => {} Err(error) if would_block(&error) => {} @@ -463,9 +538,14 @@ fn pump( } let mut promise = requester.request( schema::RelayAppToArkRequest { id, req }, - Instant::now() + EXCHANGE_TIMEOUT, + clock.now() + EXCHANGE_TIMEOUT, )?; - promise.notify(answered.clone(), id); + let answered = answered.clone(); + let wake = shared.wake.clone(); + promise.notify(move || { + let _ = answered.send(id); + let _ = wake.wake(); + }); inbound.insert(id, promise); } Frame::Response(id, res) => { @@ -488,7 +568,7 @@ fn pump( batch_full = index == 31; } if let Some(ping) = heartbeat.ping(Instant::now())? { - writing.get_or_insert_with(|| Instant::now() + WRITE_TIMEOUT); + writing.start(Instant::now()); match socket.write(Message::Ping(ping.to_vec().into())) { Ok(()) => {} Err(error) if would_block(&error) => {} @@ -496,14 +576,12 @@ fn pump( } } match socket.flush() { - Ok(()) => writing = None, - Err(error) if would_block(&error) => { - writing.get_or_insert_with(|| Instant::now() + WRITE_TIMEOUT); - } + Ok(()) => writing.flushed(), + Err(error) if would_block(&error) => writing.start(Instant::now()), Err(error) => return Err(socket_error(error)), } - if let Some(deadline) = writing { - remaining(deadline).map_err(io_error)?; + if writing.expired(Instant::now()) { + return Err(Failure::Wire(protocol::Error::Timeout)); } let queued = !shared .state @@ -512,22 +590,20 @@ fn pump( .queue .is_empty(); if batch_full - || (writing.is_none() + || (!writing.active() && (!output.is_empty() || (queued && pending.len() < MAX_INFLIGHT))) { continue; } - // Wire completions arrive on a channel without a mio wakeup. Poll - // briefly only while those requests exist; otherwise wait for I/O. - let deadline = pending + // Wait for readiness, a wakeup or the earliest deadline. Exchanges end + // on the session's clock, the socket's own timers on real time. + let (now, session) = (Instant::now(), clock.now()); + let timeout = pending .values() - .map(|(_, deadline)| *deadline) - .chain(writing) - .chain(Some(heartbeat.deadline())) - .chain((!inbound.is_empty()).then(|| Instant::now() + Duration::from_millis(10))) + .map(|(_, deadline)| deadline.saturating_duration_since(session)) + .chain(writing.remaining(now)) + .chain(Some(heartbeat.deadline().saturating_duration_since(now))) .min(); - let timeout = - deadline.map(|deadline| deadline.saturating_duration_since(Instant::now())); match poll.poll(&mut events, timeout) { Ok(()) => {} Err(error) if error.kind() == io::ErrorKind::Interrupted => {} @@ -566,9 +642,10 @@ fn enqueue(output: &mut VecDeque>, bytes: &mut usize, frame: Frame) -> R mod tests { use super::*; use crate::cloud::tests::{TIMEOUT, attach, response}; - use crate::testing::{Peer, answering}; + use crate::testing::{Peer, answering, test_clock, wait_deadline}; use crate::{Error, schema::host_to_ark::Content}; use crate::{cloud::http, trust::Realm}; + use darkbio_clock::Clock; use std::io::{Read, Write}; use std::net::TcpListener; use std::sync::atomic::{AtomicUsize, Ordering}; @@ -577,23 +654,12 @@ mod tests { /// IDs retain all sixty-four bits, including numbers beyond JSON precision. const ID: u64 = (1 << 63) + 7; + /// Accepts the next cloud connection, bounding its I/O in case a test fails. fn accept(listener: &TcpListener) -> TcpStream { - let deadline = Instant::now() + TIMEOUT; - loop { - match listener.accept() { - Ok((stream, _)) => { - stream.set_nonblocking(false).unwrap(); - stream.set_read_timeout(Some(TIMEOUT)).unwrap(); - stream.set_write_timeout(Some(TIMEOUT)).unwrap(); - return stream; - } - Err(error) if error.kind() == io::ErrorKind::WouldBlock => { - assert!(Instant::now() < deadline, "test cloud was never contacted"); - thread::sleep(Duration::from_millis(1)); - } - Err(error) => panic!("test accept: {error}"), - } - } + let (stream, _) = listener.accept().unwrap(); + stream.set_read_timeout(Some(TIMEOUT)).unwrap(); + stream.set_write_timeout(Some(TIMEOUT)).unwrap(); + stream } /// Reads headers without consuming any bytes of the first WebSocket frame. @@ -613,7 +679,6 @@ mod tests { mut serve: impl FnMut(usize, TcpStream) + Send + 'static, ) -> (String, thread::JoinHandle<()>) { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - listener.set_nonblocking(true).unwrap(); let url = format!("http://{}/v1", listener.local_addr().unwrap()); let worker = thread::spawn(move || { for (path, body) in [ @@ -664,199 +729,214 @@ mod tests { } } + /// Makes the relay probe at attachment and again right after every pong, + /// giving each pong `timeout`. + fn probe_at_once(relay: &mut Relay, timeout: Duration) { + let heartbeat = &mut relay.worker.as_mut().unwrap().heartbeat; + let attached = heartbeat.deadline() - PING_INTERVAL; + *heartbeat = Heartbeat::new(Duration::ZERO, timeout, attached); + } + /// Keeps serving app requests while an unlock waits for its reverse response. - fn peer() -> (Peer, Arc, Arc) { + fn peer(clock: &Clock) -> (Peer, Arc, Arc) { let joins = Arc::new(AtomicUsize::new(0)); let syncs = Arc::new(AtomicUsize::new(0)); - let peer = Peer::spawn(Box::new({ - let joins = joins.clone(); - let syncs = syncs.clone(); - let mut synced = false; - let mut next_id = ID; - move |session, request, responder| { - let deadline = Instant::now() + TIMEOUT; - match request { - Content::DeviceInfo(_) => { - let clock = std::time::SystemTime::now() - .duration_since(std::time::UNIX_EPOCH) - .unwrap() - .as_secs(); - responder - .reply( - schema::DeviceInfoResponse { - cloud_clock: clock, - cloud_synced: syncs.load(Ordering::SeqCst) > 0, - ..Default::default() - }, - deadline, - ) - .unwrap(); - } - - Content::CloudSyncStart(request) => { - assert_eq!(request.signer, [1]); - assert_eq!(request.crypto, [2]); - syncs.fetch_add(1, Ordering::SeqCst); - responder - .reply( - schema::CloudSyncStartResponse { challenge: vec![3] }, - deadline, - ) - .unwrap(); - } - Content::CloudSyncFinish(request) => { - assert_eq!(request.unixmilli, 123); - assert_eq!(request.signature, [4]); - synced = true; - responder - .reply(schema::CloudSyncFinishResponse { accepted: 123 }, deadline) - .unwrap(); - } - Content::RelayJoin(_) => { - assert!(synced); - joins.fetch_add(1, Ordering::SeqCst); - responder - .reply( - schema::RelayJoinResponse { - auth: vec![0xfb, 0xff], - }, - deadline, - ) - .unwrap(); - } - Content::ExecUploadStart(request) => { - assert!(synced); - assert_eq!(request.bytes, 4); - responder - .reply( - schema::ExecutionUploadStartResponse { taskid: 42 }, - deadline, - ) - .unwrap(); - } - Content::ExecUploadChunk(request) => { - assert_eq!(request.taskid, 42); - assert_eq!(request.chunk, [0, 97, 115, 109]); - responder - .reply(schema::ExecutionUploadChunkResponse {}, deadline) - .unwrap(); - } - Content::ExecStatus(request) => { - assert_eq!(request.taskid, 42); - responder - .reply( - schema::ExecutionStatusResponse { - pending: false, - result: Some(schema::ExecutionResultResponse { - success: true, + let peer = Peer::spawn( + clock, + Box::new({ + let joins = joins.clone(); + let syncs = syncs.clone(); + let mut synced = false; + let mut next_id = ID; + move |session, request, responder| { + let deadline = session.clock().now() + TIMEOUT; + match request { + Content::DeviceInfo(_) => { + let clock = session + .clock() + .system_time() + .duration_since(std::time::UNIX_EPOCH) + .unwrap() + .as_secs(); + responder + .reply( + schema::DeviceInfoResponse { + cloud_clock: clock, + cloud_synced: syncs.load(Ordering::SeqCst) > 0, ..Default::default() - }), - }, - deadline, - ) - .unwrap(); - } - request @ (Content::Unlock(_) - | Content::ExecSched(_) - | Content::SlotRepair(_) - | Content::SlotDelete(_) - | Content::FirmwareUpdatePrep(_) - | Content::SlotUploadStart(_)) => { - let (reply, preflight, authorize): (protocol::Message, _, _) = match request - { - Content::Unlock(_) => (schema::UnlockResponse {}.into(), true, true), - Content::ExecSched(_) => { - (schema::ExecutionScheduleResponse {}.into(), true, true) - } - Content::SlotRepair(_) => { - (schema::SlotRepairResponse {}.into(), true, true) - } - Content::SlotDelete(_) => { - (schema::SlotDeleteResponse {}.into(), true, true) + }, + deadline, + ) + .unwrap(); + } + + Content::CloudSyncStart(request) => { + assert_eq!(request.signer, [1]); + assert_eq!(request.crypto, [2]); + syncs.fetch_add(1, Ordering::SeqCst); + responder + .reply( + schema::CloudSyncStartResponse { challenge: vec![3] }, + deadline, + ) + .unwrap(); + } + Content::CloudSyncFinish(request) => { + assert_eq!(request.unixmilli, 123); + assert_eq!(request.signature, [4]); + synced = true; + responder + .reply(schema::CloudSyncFinishResponse { accepted: 123 }, deadline) + .unwrap(); + } + Content::RelayJoin(_) => { + assert!(synced); + joins.fetch_add(1, Ordering::SeqCst); + responder + .reply( + schema::RelayJoinResponse { + auth: vec![0xfb, 0xff], + }, + deadline, + ) + .unwrap(); + } + Content::ExecUploadStart(request) => { + assert!(synced); + assert_eq!(request.bytes, 4); + responder + .reply( + schema::ExecutionUploadStartResponse { taskid: 42 }, + deadline, + ) + .unwrap(); + } + Content::ExecUploadChunk(request) => { + assert_eq!(request.taskid, 42); + assert_eq!(request.chunk, [0, 97, 115, 109]); + responder + .reply(schema::ExecutionUploadChunkResponse {}, deadline) + .unwrap(); + } + Content::ExecStatus(request) => { + assert_eq!(request.taskid, 42); + responder + .reply( + schema::ExecutionStatusResponse { + pending: false, + result: Some(schema::ExecutionResultResponse { + success: true, + ..Default::default() + }), + }, + deadline, + ) + .unwrap(); + } + request @ (Content::Unlock(_) + | Content::ExecSched(_) + | Content::SlotRepair(_) + | Content::SlotDelete(_) + | Content::FirmwareUpdatePrep(_) + | Content::SlotUploadStart(_)) => { + let (reply, preflight, authorize): (protocol::Message, _, _) = + match request { + Content::Unlock(_) => { + (schema::UnlockResponse {}.into(), true, true) + } + Content::ExecSched(_) => { + (schema::ExecutionScheduleResponse {}.into(), true, true) + } + Content::SlotRepair(_) => { + (schema::SlotRepairResponse {}.into(), true, true) + } + Content::SlotDelete(_) => { + (schema::SlotDeleteResponse {}.into(), true, true) + } + Content::FirmwareUpdatePrep(request) => ( + schema::FirmwareUpdatePrepResponse::default().into(), + false, + request.version != "unpaired", + ), + Content::SlotUploadStart(request) => ( + schema::SlotUploadStartResponse { session: 42 }.into(), + false, + request.kind() != schema::SlotKind::SlotReferenceGenome, + ), + _ => unreachable!(), + }; + if !authorize { + responder.reply(reply, deadline).unwrap(); + return true; } - Content::FirmwareUpdatePrep(request) => ( - schema::FirmwareUpdatePrepResponse::default().into(), - false, - request.version != "unpaired", - ), - Content::SlotUploadStart(request) => ( - schema::SlotUploadStartResponse { session: 42 }.into(), - false, - request.kind() != schema::SlotKind::SlotReferenceGenome, - ), - _ => unreachable!(), - }; - if !authorize { - responder.reply(reply, deadline).unwrap(); - return true; + let id = next_id; + next_id += 1; + assert!( + !preflight || joins.load(Ordering::SeqCst) > 0, + "request preceded relay attachment" + ); + let promise = session + .requester() + .request( + schema::RelayArkToAppRequest { + id, + req: vec![1, 2, 3], + }, + deadline, + ) + .unwrap(); + thread::spawn(move || { + let result = match promise.wait::() { + Ok(answer) => { + assert_eq!(answer.id, id); + match answer.res.as_slice() { + [4, 5, 6] => responder.reply(reply, deadline), + [0] => responder.fail( + schema::Error::new(0x506, "authorization denied"), + deadline, + ), + _ => panic!("companion response was altered"), + } + } + Err(protocol::Error::Remote(error)) => { + responder.fail(error, deadline) + } + Err(_) => return, + }; + let _ = result; + }); } - let id = next_id; - next_id += 1; - assert!( - !preflight || joins.load(Ordering::SeqCst) > 0, - "request preceded relay attachment" - ); - let promise = session - .requester() - .request( - schema::RelayArkToAppRequest { - id, - req: vec![1, 2, 3], - }, - deadline, - ) - .unwrap(); - thread::spawn(move || { - let result = match promise.wait::() { - Ok(answer) => { - assert_eq!(answer.id, id); - match answer.res.as_slice() { - [4, 5, 6] => responder.reply(reply, deadline), - [0] => responder.fail( - schema::Error::new(0x506, "authorization denied"), - deadline, + Content::RelayReq(request) => { + if request.req == [0] { + responder + .fail( + schema::Error::reserved( + schema::ReservedErrors::Unsupported, + "test refusal", ), - _ => panic!("companion response was altered"), - } - } - Err(protocol::Error::Remote(error)) => { - responder.fail(error, deadline) - } - Err(_) => return, - }; - let _ = result; - }); - } - Content::RelayReq(request) => { - if request.req == [0] { + deadline, + ) + .unwrap(); + return true; + } + assert_eq!(request.id, ID); + assert_eq!(request.req, [9, 8, 7]); responder - .fail( - schema::Error::reserved( - schema::ReservedErrors::Unsupported, - "test refusal", - ), + .reply( + schema::RelayArkToAppResponse { + id: ID, + res: vec![6, 5, 4], + }, deadline, ) .unwrap(); - return true; } - assert_eq!(request.id, ID); - assert_eq!(request.req, [9, 8, 7]); - responder - .reply( - schema::RelayArkToAppResponse { - id: ID, - res: vec![6, 5, 4], - }, - deadline, - ) - .unwrap(); + other => return answering(session, other, responder), } - other => return answering(session, other, responder), + true } - true - } - })); + }), + ); (peer, joins, syncs) } @@ -915,17 +995,18 @@ mod tests { Frame::Response(id, vec![0]).encode().into(), )) .unwrap(); - pause.recv_timeout(TIMEOUT).unwrap(); + pause.recv().unwrap(); }); - let (mut peer, joins, syncs) = peer(); + let clock = test_clock().clock(); + let (mut peer, joins, syncs) = peer(&clock); let ark = attach(&mut peer, url); let client = ark.client(); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; client.call(schema::DeviceInfoRequest {}, deadline).unwrap(); assert_eq!(joins.load(Ordering::SeqCst), 0); assert_eq!(syncs.load(Ordering::SeqCst), 0); client.call(schema::UnlockRequest {}, deadline).unwrap(); - stages.recv_timeout(TIMEOUT).unwrap(); + stages.recv().unwrap(); assert!( matches!(client.clone().call(schema::UnlockRequest {}, deadline), Err(Error::Remote(error)) if error.code == 0x506) ); @@ -959,9 +1040,9 @@ mod tests { Frame::Response(id, vec![4, 5, 6]).encode().into(), )) .unwrap(); - pause.recv_timeout(TIMEOUT).unwrap(); + pause.recv().unwrap(); }); - let (mut peer, joins, _) = peer(); + let (mut peer, joins, _) = peer(&test_clock().clock()); let ark = attach(&mut peer, url); ark.client() .call_timeout(schema::UnlockRequest {}, TIMEOUT) @@ -996,13 +1077,14 @@ mod tests { let result = client.execute( 4, &mut [0, 97, 115, 109].as_slice(), - Instant::now() + TIMEOUT, + client.clock().now() + TIMEOUT, |_| {}, )?; assert!(result.success); Ok(()) }, ]; + let clock = test_clock().clock(); for call in calls { let (release, pause) = mpsc::channel(); let (url, server) = cloud(1, move |_, stream| { @@ -1015,9 +1097,9 @@ mod tests { Frame::Response(id, vec![4, 5, 6]).encode().into(), )) .unwrap(); - pause.recv_timeout(TIMEOUT).unwrap(); + pause.recv().unwrap(); }); - let (mut peer, joins, _) = peer(); + let (mut peer, joins, _) = peer(&clock); let ark = attach(&mut peer, url); call(&ark.client()).unwrap(); assert_eq!(joins.load(Ordering::SeqCst), 1); @@ -1030,6 +1112,7 @@ mod tests { /// counterparts attach when the Ark first requests authorization. #[test] fn test_conditional_authorization() { + let clock = test_clock().clock(); for firmware in [false, true] { let (release, pause) = mpsc::channel(); let (url, server) = cloud(1, move |_, stream| { @@ -1042,9 +1125,9 @@ mod tests { Frame::Response(id, vec![4, 5, 6]).encode().into(), )) .unwrap(); - pause.recv_timeout(TIMEOUT).unwrap(); + pause.recv().unwrap(); }); - let (mut peer, joins, _) = peer(); + let (mut peer, joins, _) = peer(&clock); let ark = attach(&mut peer, url); let client = ark.client(); for authorize in [false, true] { @@ -1084,12 +1167,13 @@ mod tests { /// while local requests and two concurrent authorizations remain available. #[test] fn test_shared_attachment() { + // Hold the leader's relay attachment at its upgrade let (seen, attempts) = mpsc::channel(); let (release, pause) = mpsc::channel(); let (finish, done) = mpsc::channel(); let (url, server) = cloud(1, move |_, stream| { seen.send(()).unwrap(); - pause.recv_timeout(TIMEOUT).unwrap(); + pause.recv().unwrap(); let mut socket = upgrade(stream); for _ in 0..2 { let Frame::Request(id, req) = frame(&mut socket) else { @@ -1102,24 +1186,33 @@ mod tests { )) .unwrap(); } - done.recv_timeout(TIMEOUT).unwrap(); + done.recv().unwrap(); }); - let (mut peer, joins, syncs) = peer(); + let mut tester = test_clock(); + let clock = tester.clock(); + let (mut peer, joins, syncs) = peer(&clock); let ark = attach(&mut peer, url); let client = ark.client(); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; let leader = thread::spawn({ let client = client.clone(); move || client.call(schema::UnlockRequest {}, deadline) }); - attempts.recv_timeout(TIMEOUT).unwrap(); + attempts.recv().unwrap(); client.call(schema::DeviceInfoRequest {}, deadline).unwrap(); - assert!(matches!( - client - .clone() - .call_timeout(schema::UnlockRequest {}, Duration::from_millis(20)), - Err(Error::Timeout) - )); + + // A short caller joining the attachment expires alone, at the earliest + // deadline on the clock + let short = clock.now() + Duration::from_millis(20); + let waiter = thread::spawn({ + let client = client.clone(); + move || client.call_timeout(schema::UnlockRequest {}, Duration::from_millis(20)) + }); + wait_deadline(&tester, short); + tester.advance_to(short); + assert!(matches!(waiter.join().unwrap(), Err(Error::Timeout))); + + // The leader and a later caller authorize over the one attachment let follower = thread::spawn({ let client = client.clone(); move || client.call(schema::UnlockRequest {}, deadline) @@ -1136,19 +1229,32 @@ mod tests { /// A call deadline still bounds authorization after successful cloud setup. #[test] fn test_authorization_deadline() { + // The companion receives the authorization request but never answers + let (reached, authorizing) = mpsc::channel(); let (release, pause) = mpsc::channel(); let (url, server) = cloud(1, move |_, stream| { let mut socket = upgrade(stream); assert!(matches!(frame(&mut socket), Frame::Request(_, _))); - pause.recv_timeout(TIMEOUT).unwrap(); + reached.send(()).unwrap(); + pause.recv().unwrap(); }); - let (mut peer, _, _) = peer(); + let mut tester = test_clock(); + let clock = tester.clock(); + let (mut peer, _, _) = peer(&clock); let ark = attach(&mut peer, url); let client = ark.client(); - assert!(matches!( - client.call_timeout(schema::UnlockRequest {}, Duration::from_millis(500)), - Err(Error::Timeout) - )); + + // The unlock waits for approval until the clock reaches its deadline + let deadline = clock.now() + Duration::from_millis(500); + let unlock = thread::spawn({ + let client = client.clone(); + move || client.call(schema::UnlockRequest {}, deadline) + }); + authorizing.recv().unwrap(); + tester.advance_to(deadline); + assert!(matches!(unlock.join().unwrap(), Err(Error::Timeout))); + + // Local requests keep working on the same connection client .call_timeout(schema::DeviceInfoRequest {}, TIMEOUT) .unwrap(); @@ -1160,6 +1266,7 @@ mod tests { /// replaying the unlock or disrupting local device requests. #[test] fn test_reconnect() { + let clock = test_clock().clock(); for refused in [false, true] { let (release, pause) = mpsc::channel(); let (url, server) = cloud(2, move |attempt, mut stream| { @@ -1182,13 +1289,13 @@ mod tests { Frame::Response(id, vec![4, 5, 6]).encode().into(), )) .unwrap(); - pause.recv_timeout(TIMEOUT).unwrap(); + pause.recv().unwrap(); } }); - let (mut peer, joins, syncs) = peer(); + let (mut peer, joins, syncs) = peer(&clock); let ark = attach(&mut peer, url); let client = ark.client(); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; let error = client.call(schema::UnlockRequest {}, deadline).unwrap_err(); if refused { assert!(matches!(error, Error::Cloud(message) if message.contains("503"))); @@ -1218,17 +1325,18 @@ mod tests { seen.send(()).unwrap(); assert!(matches!(socket.read(), Err(_) | Ok(Message::Close(_)))); }); - let (mut peer, _, _) = peer(); + let clock = test_clock().clock(); + let (mut peer, _, _) = peer(&clock); let ark = attach(&mut peer, url); let client = ark.client(); let pending = client - .send(schema::UnlockRequest {}, Instant::now() + TIMEOUT) + .send(schema::UnlockRequest {}, clock.now() + TIMEOUT) .unwrap(); - requests.recv_timeout(TIMEOUT).unwrap(); + requests.recv().unwrap(); drop(ark); assert!(matches!(pending.wait(), Err(Error::Closed))); assert!(matches!( - client.call(schema::UnlockRequest {}, Instant::now() + TIMEOUT), + client.call(schema::UnlockRequest {}, clock.now() + TIMEOUT), Err(Error::Closed) )); server.join().unwrap(); @@ -1238,65 +1346,152 @@ mod tests { /// its deadline. A valid pong schedules a fresh probe with a different ID. #[test] fn test_heartbeat() { - let mut heartbeat = Heartbeat::new(PING_INTERVAL, PONG_TIMEOUT); - let now = heartbeat.deadline(); + // Probe first once an interval has passed since the start + let mut tester = test_clock(); + let clock = tester.clock(); + let mut heartbeat = Heartbeat::new(PING_INTERVAL, PONG_TIMEOUT, clock.now()); + assert!(heartbeat.ping(clock.now()).unwrap().is_none()); + tester.advance(PING_INTERVAL); + let now = clock.now(); let first = heartbeat.ping(now).unwrap().unwrap(); let deadline = heartbeat.deadline(); assert!(heartbeat.ping(now).unwrap().is_none()); + + // Only the pong echoing the probe acknowledges it and schedules the next heartbeat.pong(&[0; 8], now); assert_eq!(heartbeat.deadline(), deadline); heartbeat.pong(&first, now); assert_eq!(heartbeat.deadline(), now + PING_INTERVAL); - let second = heartbeat.ping(heartbeat.deadline()).unwrap().unwrap(); + + // An old probe's pong or one arriving at the deadline leaves the probe to expire + tester.advance(PING_INTERVAL); + let second = heartbeat.ping(clock.now()).unwrap().unwrap(); assert_ne!(first, second); let deadline = heartbeat.deadline(); - heartbeat.pong(&first, deadline - Duration::from_millis(1)); - heartbeat.pong(&second, deadline); - assert!(heartbeat.ping(deadline).is_err()); + tester.advance_to(deadline - Duration::from_millis(1)); + heartbeat.pong(&first, clock.now()); + tester.advance_to(deadline); + heartbeat.pong(&second, clock.now()); + assert!(heartbeat.ping(clock.now()).is_err()); + } + + /// Output left to flush starts the backlog bound once. Later blockage keeps + /// the original deadline, where the bound expires, and a completed flush + /// clears it for the next output. + #[test] + fn test_backlog() { + // The first blockage starts the bound + let mut tester = test_clock(); + let clock = tester.clock(); + let mut backlog = Backlog::default(); + assert!(!backlog.active()); + let start = clock.now(); + backlog.start(start); + assert!(backlog.active()); + assert_eq!(backlog.remaining(start), Some(WRITE_TIMEOUT)); + + // Blocking again keeps the original deadline, which ends the bound on time + tester.advance(Duration::from_secs(3)); + backlog.start(clock.now()); + assert_eq!( + backlog.remaining(clock.now()), + Some(WRITE_TIMEOUT - Duration::from_secs(3)) + ); + tester.advance_to(start + WRITE_TIMEOUT - Duration::from_millis(1)); + assert!(!backlog.expired(clock.now())); + tester.advance_to(start + WRITE_TIMEOUT); + assert!(backlog.expired(clock.now())); + + // A completed flush clears the bound, and the next output starts afresh + backlog.flushed(); + assert!(!backlog.active()); + assert!(!backlog.expired(clock.now())); + backlog.start(clock.now()); + assert_eq!(backlog.remaining(clock.now()), Some(WRITE_TIMEOUT)); + } + + /// The first probe is due an interval after attachment, so a worker that + /// starts that late probes at once. + #[test] + fn test_heartbeat_starts_at_attachment() { + let listener = TcpListener::bind("127.0.0.1:0").unwrap(); + let url = format!("ws://{}/v1/relaying", listener.local_addr().unwrap()); + let (probed, probes) = mpsc::channel(); + let server = thread::spawn(move || { + let mut socket = upgrade(accept(&listener)); + assert!(matches!(socket.read().unwrap(), Message::Ping(_))); + probed.send(()).unwrap(); + let _ = socket.get_mut().read_to_end(&mut Vec::new()); + }); + let clock = test_clock().clock(); + let mut peer = Peer::spawn(&clock, Box::new(answering)); + let verifier = crate::TrustMode::Recover(Box::new(peer.identity.clone())); + let (session, _) = protocol::connect(peer.stream(), &verifier).unwrap(); + let api = http::tests::api(url.clone(), Realm::Hardware, &clock); + let deadline = clock.now() + TIMEOUT; + let mut relay = + Relay::connect(&api, &url, &[0xfb, 0xff], session.requester(), deadline).unwrap(); + + // Start the worker as if a whole interval passed since attachment + relay.worker.as_mut().unwrap().heartbeat.next -= PING_INTERVAL; + relay.start().unwrap(); + probes.recv().unwrap(); + relay.close(); + server.join().unwrap(); } /// The socket pump accepts matching pongs, then ends an unresponsive relay /// without disrupting local wire calls. A replacement can attach afterward. + /// Both relays probe at once. The first gives every pong an hour, and the + /// test waits for the server to answer three probes. The second gives its + /// pong no time, and the test waits for the server to see its socket end. #[test] fn test_heartbeat_disconnect() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - listener.set_nonblocking(true).unwrap(); let url = format!("ws://{}/v1/relaying", listener.local_addr().unwrap()); + let (answered, pongs) = mpsc::channel(); + let (gone, ended) = mpsc::channel(); let (release, pause) = mpsc::channel(); - let (seen, pings) = mpsc::channel(); let server = thread::spawn(move || { + // Answer three probes, each one sent only once the previous pong matched let mut socket = upgrade(accept(&listener)); for _ in 0..3 { assert!(matches!(socket.read().unwrap(), Message::Ping(_))); socket.flush().unwrap(); } - assert!(matches!(socket.read().unwrap(), Message::Ping(_))); - // Replace tungstenite's automatic pong with one that does not match. - socket.send(Message::Pong(vec![0].into())).unwrap(); - seen.send(()).unwrap(); - assert!(matches!(socket.read(), Err(_) | Ok(Message::Close(_)))); + answered.send(()).unwrap(); + let _ = socket.get_mut().read_to_end(&mut Vec::new()); + + // Leave the next relay's probe unanswered until the relay goes away + let mut socket = upgrade(accept(&listener)); + let _ = socket.get_mut().read_to_end(&mut Vec::new()); + gone.send(()).unwrap(); + + // Hold a replacement attachment open until the test ends let _replacement = upgrade(accept(&listener)); - pause.recv_timeout(TIMEOUT).unwrap(); + pause.recv().unwrap(); }); - let mut peer = Peer::spawn(Box::new(answering)); + let clock = test_clock().clock(); + let mut peer = Peer::spawn(&clock, Box::new(answering)); let verifier = crate::TrustMode::Recover(Box::new(peer.identity.clone())); let (session, _) = protocol::connect(peer.stream(), &verifier).unwrap(); - let deadline = Instant::now() + TIMEOUT; - let mut relay = Relay::connect( - &http::tests::api(url.clone(), Realm::Hardware), - &url, - &[0xfb, 0xff], - session.requester(), - deadline, - ) - .unwrap(); - relay.worker.as_mut().unwrap().heartbeat = - Heartbeat::new(Duration::from_millis(20), Duration::from_millis(200)); + let api = http::tests::api(url.clone(), Realm::Hardware, &clock); + let deadline = clock.now() + TIMEOUT; + + // A relay whose probes are answered keeps probing + let mut relay = + Relay::connect(&api, &url, &[0xfb, 0xff], session.requester(), deadline).unwrap(); + probe_at_once(&mut relay, Duration::from_secs(3600)); relay.start().unwrap(); - pings.recv_timeout(TIMEOUT).unwrap(); - while relay.connected() && Instant::now() < deadline { - thread::sleep(Duration::from_millis(1)); - } + pongs.recv().unwrap(); + relay.close(); + + // A relay whose probe expires at once ends, leaving the wire session usable + let mut relay = + Relay::connect(&api, &url, &[0xfb, 0xff], session.requester(), deadline).unwrap(); + probe_at_once(&mut relay, Duration::ZERO); + relay.start().unwrap(); + ended.recv().unwrap(); let error = relay.shared.state.lock().unwrap().error.clone(); assert_eq!( error.as_deref(), @@ -1308,14 +1503,10 @@ mod tests { .unwrap() .wait::() .unwrap(); - let mut replacement = Relay::connect( - &http::tests::api(url.clone(), Realm::Hardware), - &url, - &[0xfb, 0xff], - session.requester(), - deadline, - ) - .unwrap(); + + // A replacement attaches afterward + let mut replacement = + Relay::connect(&api, &url, &[0xfb, 0xff], session.requester(), deadline).unwrap(); replacement.start().unwrap(); assert!(replacement.connected()); release.send(()).unwrap(); @@ -1326,9 +1517,9 @@ mod tests { /// while it is assembling an unfinished fragmented message. #[test] fn test_close_fragmented_message() { + let clock = test_clock().clock(); for started in [false, true] { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - listener.set_nonblocking(true).unwrap(); let url = format!("ws://{}/v1/relaying", listener.local_addr().unwrap()); let (sent, fragments) = mpsc::channel(); let server = thread::spawn(move || { @@ -1339,56 +1530,56 @@ mod tests { sent.send(()).unwrap(); assert!(matches!(socket.get_mut().read(&mut [0]), Ok(0) | Err(_))); }); - let mut peer = Peer::spawn(Box::new(answering)); + let mut peer = Peer::spawn(&clock, Box::new(answering)); let verifier = crate::TrustMode::Recover(Box::new(peer.identity.clone())); let (session, _) = protocol::connect(peer.stream(), &verifier).unwrap(); - let deadline = Instant::now() + TIMEOUT; + let deadline = clock.now() + TIMEOUT; let mut relay = Relay::connect( - &http::tests::api(url.clone(), Realm::Hardware), + &http::tests::api(url.clone(), Realm::Hardware, &clock), &url, &[0xfb, 0xff], session.requester(), deadline, ) .unwrap(); + let (dropped, released) = mpsc::channel(); + *relay.shared.dropped.lock().unwrap() = Some(dropped); if started { relay.start().unwrap(); } - fragments.recv_timeout(TIMEOUT).unwrap(); + fragments.recv().unwrap(); relay.close(); server.join().unwrap(); - while Arc::strong_count(&relay.shared) != 1 && Instant::now() < deadline { - thread::sleep(Duration::from_millis(1)); - } - assert_eq!( - Arc::strong_count(&relay.shared), - 1, - "relay worker must exit" - ); + + // The worker exits, so dropping this attachment releases the last handle + drop(relay); + released.recv().unwrap(); } } /// Every read of a stalled upgrade retains the caller's original deadline. + /// The socket's own timeout is real, so the upgrade stalls for the whole + /// 100 ms budget that the clock leaves it. #[test] fn test_handshake_deadline() { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - listener.set_nonblocking(true).unwrap(); let url = format!("ws://{}/v1/relaying", listener.local_addr().unwrap()); let (release, pause) = mpsc::channel(); let server = thread::spawn(move || { let mut stream = accept(&listener); headers(&mut stream); - pause.recv_timeout(TIMEOUT).unwrap(); + pause.recv().unwrap(); }); - let mut peer = Peer::spawn(Box::new(answering)); + let clock = test_clock().clock(); + let mut peer = Peer::spawn(&clock, Box::new(answering)); let verifier = crate::TrustMode::Recover(Box::new(peer.identity.clone())); let (session, _) = protocol::connect(peer.stream(), &verifier).unwrap(); let result = Relay::connect( - &http::tests::api(url.clone(), Realm::Hardware), + &http::tests::api(url.clone(), Realm::Hardware, &clock), &url, &[1], session.requester(), - Instant::now() + Duration::from_millis(100), + clock.now() + Duration::from_millis(100), ); assert!( matches!(result, Err(Failure::Wire(protocol::Error::Timeout))), diff --git a/connect/src/cloud/socket.rs b/connect/src/cloud/socket.rs index dbf1ae9..78cad83 100644 --- a/connect/src/cloud/socket.rs +++ b/connect/src/cloud/socket.rs @@ -6,12 +6,14 @@ //! Authenticated cloud sockets with blocking deadlines and relay readiness. -use super::{Failure, dns, http::Api}; +use super::{Failure, http::Api}; +use crate::timing::ClockExt; use base64::{Engine, prelude::BASE64_URL_SAFE_NO_PAD}; +use darkbio_clock::Clock; use darkbio_wire::protocol; use std::io::{self, Read, Write}; use std::net::TcpStream; -use std::time::{Duration, Instant}; +use std::time::Instant; use tungstenite::{ WebSocket, client::IntoClientRequest, handshake::HandshakeError, protocol::WebSocketConfig, stream::MaybeTlsStream, @@ -62,14 +64,15 @@ pub(super) fn connect( 80 }); - let addresses = dns::resolve(&host, port, deadline)?; + let addresses = api.resolver.resolve(&host, port, deadline)?; let mut failure = io::Error::new( io::ErrorKind::AddrNotAvailable, "cloud socket has no address", ); let mut connected = None; for address in addresses { - match TcpStream::connect_timeout(&address, remaining(deadline).map_err(io_error)?) { + let left = api.clock.remaining(deadline).map_err(io_error)?; + match TcpStream::connect_timeout(&address, left) { Ok(stream) => { connected = Some(stream); break; @@ -86,7 +89,11 @@ pub(super) fn connect( .max_frame_size(Some(MAX_MESSAGE)); let (socket, response) = tungstenite::client_tls_with_config( request, - Socket::Blocking { stream, deadline }, + Socket::Blocking { + clock: api.clock.clone(), + stream, + deadline, + }, Some(config), None, ) @@ -119,6 +126,8 @@ pub(super) fn connect( pub(super) enum Socket { /// Handshake or pairing stream with a shared read and write bound. Blocking { + /// Clock of the connection, which the deadline is measured on. + clock: Clock, /// Connected TCP socket, optionally wrapped by TLS above this adapter. stream: TcpStream, /// Absolute bound checked again before each blocking I/O. @@ -132,8 +141,12 @@ impl Read for Socket { /// Applies the remaining blocking deadline or returns readiness-based I/O. fn read(&mut self, bytes: &mut [u8]) -> io::Result { match self { - Self::Blocking { stream, deadline } => { - stream.set_read_timeout(Some(remaining(*deadline)?))?; + Self::Blocking { + clock, + stream, + deadline, + } => { + stream.set_read_timeout(Some(clock.remaining(*deadline)?))?; stream.read(bytes) } Self::Connected(stream) => stream.read(bytes), @@ -145,8 +158,12 @@ impl Write for Socket { /// Applies the remaining blocking deadline or writes through the relay socket. fn write(&mut self, bytes: &[u8]) -> io::Result { match self { - Self::Blocking { stream, deadline } => { - stream.set_write_timeout(Some(remaining(*deadline)?))?; + Self::Blocking { + clock, + stream, + deadline, + } => { + stream.set_write_timeout(Some(clock.remaining(*deadline)?))?; stream.write(bytes) } Self::Connected(stream) => stream.write(bytes), @@ -155,8 +172,12 @@ impl Write for Socket { /// Flushes the underlying stream under the same bound as a write. fn flush(&mut self) -> io::Result<()> { match self { - Self::Blocking { stream, deadline } => { - stream.set_write_timeout(Some(remaining(*deadline)?))?; + Self::Blocking { + clock, + stream, + deadline, + } => { + stream.set_write_timeout(Some(clock.remaining(*deadline)?))?; stream.flush() } Self::Connected(stream) => stream.flush(), @@ -173,14 +194,6 @@ pub(super) fn socket_mut(socket: &mut WebSocket>) -> &mut } } -/// Returns a positive OS timeout; zero would mean an unbounded wait on some APIs. -pub(super) fn remaining(deadline: Instant) -> io::Result { - deadline - .checked_duration_since(Instant::now()) - .filter(|left| !left.is_zero()) - .ok_or_else(|| io::Error::from(io::ErrorKind::TimedOut)) -} - /// Preserves timeout classification across blocking socket error conventions. pub(super) fn io_error(error: io::Error) -> Failure { match error.kind() { @@ -208,6 +221,7 @@ mod tests { http::tests::api, tests::TIMEOUT, }; + use crate::testing::test_clock; use crate::{Timing, trust::Realm}; use std::net::TcpListener; use std::sync::{Arc, atomic::Ordering}; @@ -216,6 +230,7 @@ mod tests { #[test] #[allow(clippy::result_large_err)] // Tungstenite's server callback owns its HTTP response. fn socket_upgrades_and_reconnects_use_refreshed_credentials() { + let clock = test_clock().clock(); for subprotocol in ["Pairing", "Relaying"] { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("ws://{}/v1/{subprotocol}", listener.local_addr().unwrap()); @@ -251,14 +266,14 @@ mod tests { drop(socket); } }); - let cloud = api(url.clone(), Realm::Hardware); + let cloud = api(url.clone(), Realm::Hardware, &clock); let login = Login::default(); cloud.auth.set(Arc::new(login.clone())); let timing = Timing::inactivity(TIMEOUT); for _ in 0..2 { let socket = cloud .with_auth(timing, || { - connect(&cloud, &url, &[0xfb, 0xff], subprotocol, timing.io()) + connect(&cloud, &url, &[0xfb, 0xff], subprotocol, timing.io(&clock)) }) .unwrap(); drop(socket); diff --git a/connect/src/dataset.rs b/connect/src/dataset.rs index a96cb6f..5498f69 100644 --- a/connect/src/dataset.rs +++ b/connect/src/dataset.rs @@ -7,10 +7,11 @@ //! Dataset identification, streaming and processing on the Ark. use crate::{Error, Timing, schema}; +use darkbio_clock::Clock; use darkbio_wire::protocol::{Message, Promise, Requester}; use sha2::{Digest, Sha256}; use std::io::{self, Read}; -use std::time::{Duration, Instant}; +use std::time::Duration; /// Prefix supplied to the Ark for file identification and upload preparation. const IDENTIFY_SIZE: usize = 1024 * 1024; @@ -62,7 +63,8 @@ pub struct Dataset { /// Identifies once, resends that same head in the authorized start request and /// streams the rest. A failed session is cancelled within the remaining deadline; -/// cleanup never replaces the original error or retries an upload. +/// cleanup never replaces the original error or retries an upload. Deadlines +/// are measured on the clock of the requester's session. pub(crate) fn upload( requester: &Requester, dataset: &Dataset, @@ -70,6 +72,7 @@ pub(crate) fn upload( timing: impl Into, mut progress: impl FnMut(UploadProgress), ) -> Result<(), Error> { + let clock = &requester.clock(); let timing = timing.into(); if dataset.size == 0 { return Err(Error::Dataset("dataset is empty".into())); @@ -77,6 +80,7 @@ pub(crate) fn upload( let head = read_chunk( reader, dataset.size.min(IDENTIFY_SIZE as u64) as usize, + clock, timing, None, )?; @@ -85,7 +89,7 @@ pub(crate) fn upload( hash.update(&head); } if head.len() as u64 == dataset.size { - finish_read(reader, hash.take(), dataset, timing)?; + finish_read(reader, hash.take(), dataset, clock, timing)?; } let kind = match dataset.slot { Some(kind) => kind, @@ -99,7 +103,7 @@ pub(crate) fn upload( chunk: head.clone(), kinds: Vec::new(), }, - timing.io(), + timing.io(clock), )? .wait::()?; if !identified.rejection.is_empty() { @@ -120,7 +124,7 @@ pub(crate) fn upload( size: dataset.size, chunk: head, }, - timing.approval(), + timing.approval(clock), )? .wait::()? .session; @@ -134,20 +138,20 @@ pub(crate) fn upload( let mut pending: Option<(Promise, u64)> = None; while sent < dataset.size { let size = (dataset.size - sent).min(CHUNK_SIZE as u64) as usize; - let chunk = read_chunk(reader, size, timing, Some(CHUNK_INTERVAL))?; + let chunk = read_chunk(reader, size, clock, timing, Some(CHUNK_INTERVAL))?; let size = chunk.len() as u64; if let Some(hash) = &mut hash { hash.update(&chunk); } sent += size; if sent == dataset.size { - finish_read(reader, hash.take(), dataset, timing)?; + finish_read(reader, hash.take(), dataset, clock, timing)?; } // Keep at most two chunks outstanding so device writes can overlap // the next transfer. The last acknowledgement is awaited too. let next = requester.request( schema::SlotUploadChunkRequest { session, chunk }, - timing.io(), + timing.io(clock), )?; if let Some((previous, bytes)) = pending.take() { previous.wait::()?; @@ -169,7 +173,10 @@ pub(crate) fn upload( } loop { let status = requester - .request(schema::SlotUploadProcessRequest { session }, timing.io())? + .request( + schema::SlotUploadProcessRequest { session }, + timing.io(clock), + )? .wait::()?; if !status.failure.is_empty() { return Err(Error::Dataset(status.failure)); @@ -188,11 +195,11 @@ pub(crate) fn upload( if done { return Ok(()); } - timing.pause(POLL_INTERVAL)?; + timing.pause(clock, POLL_INTERVAL)?; } })(); if result.is_err() { - let cleanup = timing.io().min(Instant::now() + Duration::from_secs(1)); + let cleanup = timing.io(clock).min(clock.now() + Duration::from_secs(1)); let _ = requester .request(schema::SlotUploadCancelRequest { session }, cleanup) .and_then(|pending| pending.wait::()); @@ -206,25 +213,26 @@ pub(crate) fn upload( fn read_chunk( reader: &mut impl Read, size: usize, + clock: &Clock, timing: Timing, interval: Option, ) -> Result, Error> { - let start = Instant::now(); + let start = clock.now(); let mut chunk = vec![0; size]; let mut filled = 0; while filled < size { - timing.check()?; + timing.check(clock)?; match reader.read(&mut chunk[filled..]) { Ok(0) => return Err(read_error(io::ErrorKind::UnexpectedEof.into())), Ok(count) => filled += count, Err(err) if err.kind() == io::ErrorKind::Interrupted => continue, Err(err) => return Err(read_error(err)), } - if interval.is_some_and(|interval| start.elapsed() >= interval) { + if interval.is_some_and(|interval| clock.elapsed(start) >= interval) { break; } } - timing.check()?; + timing.check(clock)?; chunk.truncate(filled); Ok(chunk) } @@ -235,10 +243,11 @@ fn finish_read( reader: &mut impl Read, hash: Option, dataset: &Dataset, + clock: &Clock, timing: Timing, ) -> Result<(), Error> { loop { - timing.check()?; + timing.check(clock)?; match reader.read(&mut [0]) { Ok(0) => break, Ok(_) => { @@ -250,7 +259,7 @@ fn finish_read( Err(error) => return Err(read_error(error)), } } - timing.check()?; + timing.check(clock)?; if let (Some(hash), Some(expected)) = (hash, dataset.sha256) && hash.finalize().as_slice() != expected { @@ -274,11 +283,13 @@ fn read_error(error: io::Error) -> Error { mod tests { use super::*; use crate::TrustMode; - use crate::testing::Peer; + use crate::testing::{Peer, test_clock, wait_deadline}; + use darkbio_clock::TestClock; use darkbio_wire::protocol::{self, Session}; use schema::host_to_ark::Content; use std::collections::VecDeque; - use std::sync::{Arc, Mutex}; + use std::sync::{Arc, Mutex, mpsc}; + use std::thread; const TIMEOUT: Duration = Duration::from_secs(10); @@ -288,6 +299,8 @@ mod tests { head: Vec, bytes: Vec, kind: Option, + /// Notifies a test each time an upload chunk reaches the Ark. + chunks: Option>, } fn status(phase: u64, progress: u64) -> schema::SlotUploadProcessResponse { @@ -321,96 +334,103 @@ mod tests { /// Records the actual wire exchange, optionally refusing a stage. A cancel /// refusal must never replace the failure that caused cleanup. fn peer( + clock: &Clock, fail: Option<&'static str>, reports: Vec, ) -> (Peer, Arc>) { let observed = Arc::new(Mutex::new(Observed::default())); let shared = observed.clone(); let mut reports = VecDeque::from(reports); - let peer = Peer::spawn(Box::new(move |session, request, responder| { - if matches!(request, Content::DeviceInfo(_)) { - return crate::testing::answering(session, request, responder); - } - let deadline = Instant::now() + TIMEOUT; - let mut observed = shared.lock().unwrap(); - let (stage, response): (_, Message) = match request { - Content::CloudSyncStart(_) => ( - "sync-start", - schema::CloudSyncStartResponse { challenge: vec![3] }.into(), - ), - Content::CloudSyncFinish(_) => ( - "sync-finish", - schema::CloudSyncFinishResponse { accepted: 123 }.into(), - ), - Content::SlotIdentify(request) => { - assert_eq!(request.name, "sample.vcf.gz"); - assert!(request.kinds.is_empty()); - observed.head = request.chunk; - ( - "peek", - schema::SlotIdentifyResponse { - kind: 2, - summary: "Variant calls".into(), - rejection: if fail == Some("identify") { - "unrecognized dataset".into() - } else { - String::new() - }, - ..Default::default() - } - .into(), - ) - } - Content::SlotUploadStart(request) => { - assert_eq!(request.name, "sample.vcf.gz"); - if !observed.head.is_empty() { - assert_eq!(request.chunk, observed.head); - } - observed.bytes.extend(request.chunk); - observed.kind = Some(request.kind); - ( - "start", - schema::SlotUploadStartResponse { session: 7 }.into(), - ) - } - Content::SlotUploadChunk(request) => { - assert_eq!(request.session, 7); - assert!(request.chunk.len() <= CHUNK_SIZE); - observed.bytes.extend(request.chunk); - ("chunk", schema::SlotUploadChunkResponse {}.into()) + let peer = Peer::spawn( + clock, + Box::new(move |session, request, responder| { + if matches!(request, Content::DeviceInfo(_)) { + return crate::testing::answering(session, request, responder); } - Content::SlotUploadProcess(request) => { - assert_eq!(request.session, 7); - ( - "process", - reports - .pop_front() - .unwrap_or_else(|| status(2, 10_000)) + let deadline = session.clock().now() + TIMEOUT; + let mut observed = shared.lock().unwrap(); + let (stage, response): (_, Message) = match request { + Content::CloudSyncStart(_) => ( + "sync-start", + schema::CloudSyncStartResponse { challenge: vec![3] }.into(), + ), + Content::CloudSyncFinish(_) => ( + "sync-finish", + schema::CloudSyncFinishResponse { accepted: 123 }.into(), + ), + Content::SlotIdentify(request) => { + assert_eq!(request.name, "sample.vcf.gz"); + assert!(request.kinds.is_empty()); + observed.head = request.chunk; + ( + "peek", + schema::SlotIdentifyResponse { + kind: 2, + summary: "Variant calls".into(), + rejection: if fail == Some("identify") { + "unrecognized dataset".into() + } else { + String::new() + }, + ..Default::default() + } .into(), - ) - } - Content::SlotUploadCancel(request) => { - assert_eq!(request.session, 7); - ("cancel", schema::SlotUploadCancelResponse {}.into()) + ) + } + Content::SlotUploadStart(request) => { + assert_eq!(request.name, "sample.vcf.gz"); + if !observed.head.is_empty() { + assert_eq!(request.chunk, observed.head); + } + observed.bytes.extend(request.chunk); + observed.kind = Some(request.kind); + ( + "start", + schema::SlotUploadStartResponse { session: 7 }.into(), + ) + } + Content::SlotUploadChunk(request) => { + assert_eq!(request.session, 7); + assert!(request.chunk.len() <= CHUNK_SIZE); + observed.bytes.extend(request.chunk); + if let Some(chunks) = &observed.chunks { + let _ = chunks.send(()); + } + ("chunk", schema::SlotUploadChunkResponse {}.into()) + } + Content::SlotUploadProcess(request) => { + assert_eq!(request.session, 7); + ( + "process", + reports + .pop_front() + .unwrap_or_else(|| status(2, 10_000)) + .into(), + ) + } + Content::SlotUploadCancel(request) => { + assert_eq!(request.session, 7); + ("cancel", schema::SlotUploadCancelResponse {}.into()) + } + _ => panic!("unexpected request"), + }; + observed.stages.push(stage); + if fail == Some(stage) || stage == "cancel" && fail.is_some() { + responder + .fail( + schema::Error { + code: 0x778, + msg: format!("refused {stage}"), + }, + deadline, + ) + .unwrap(); + } else { + responder.reply(response, deadline).unwrap(); } - _ => panic!("unexpected request"), - }; - observed.stages.push(stage); - if fail == Some(stage) || stage == "cancel" && fail.is_some() { - responder - .fail( - schema::Error { - code: 0x778, - msg: format!("refused {stage}"), - }, - deadline, - ) - .unwrap(); - } else { - responder.reply(response, deadline).unwrap(); - } - true - })); + true + }), + ); (peer, observed) } @@ -418,19 +438,36 @@ mod tests { /// arrive exactly once, and an early phase reaching 100% is not completion. #[test] fn test_upload() { + let mut tester = test_clock(); + let clock = tester.clock(); for size in [17, IDENTIFY_SIZE, IDENTIFY_SIZE + 2 * CHUNK_SIZE + 29] { + // Upload and process the source, the first report asking for another poll let bytes: Vec<_> = (0..size).map(|i| (i % 251) as u8).collect(); - let (mut peer, observed) = peer(None, vec![status(1, 10_000), status(2, 10_000)]); + let (mut peer, observed) = + peer(&clock, None, vec![status(1, 10_000), status(2, 10_000)]); let session = attach(&mut peer); - let mut progress = Vec::new(); - upload( - &session.requester(), - &source(size, None), - &mut bytes.as_slice(), - Instant::now() + TIMEOUT, - |stage| progress.push(stage), - ) - .unwrap(); + let deadline = clock.now() + TIMEOUT; + let uploading = thread::spawn({ + let requester = session.requester(); + let bytes = bytes.clone(); + move || { + let mut progress = Vec::new(); + let result = upload( + &requester, + &source(size, None), + &mut bytes.as_slice(), + deadline, + |stage| progress.push(stage), + ); + result.map(|()| progress) + } + }); + + // End the pause between the two reports once the uploader sleeps in it + let poll = clock.now() + POLL_INTERVAL; + wait_deadline(&tester, poll); + tester.advance_to(poll); + let progress = uploading.join().unwrap().unwrap(); let observed = observed.lock().unwrap(); assert_eq!(observed.bytes, bytes); assert_eq!(observed.kind, Some(2)); @@ -457,24 +494,21 @@ mod tests { /// reader models a source that only continues once the Ark receives it. #[test] fn test_slow_source_flushes_partial_chunks() { + /// Source whose second read takes a whole flush interval of the test + /// clock, and whose third waits until the Ark received a chunk. struct Slow<'a> { + tester: &'a mut TestClock, bytes: &'a [u8], reads: usize, - observed: Arc>, + chunks: mpsc::Receiver<()>, } impl Read for Slow<'_> { fn read(&mut self, buffer: &mut [u8]) -> io::Result { self.reads += 1; if self.reads == 2 { - std::thread::sleep(CHUNK_INTERVAL); + self.tester.advance(CHUNK_INTERVAL); } else if self.reads == 3 { - let deadline = Instant::now() + Duration::from_secs(1); - while self.observed.lock().unwrap().bytes.len() <= IDENTIFY_SIZE { - if Instant::now() >= deadline { - return Err(io::ErrorKind::TimedOut.into()); - } - std::thread::yield_now(); - } + self.chunks.recv().unwrap(); } let size = if self.reads == 1 { IDENTIFY_SIZE @@ -485,19 +519,24 @@ mod tests { self.bytes.read(&mut buffer[..size]) } } + let mut tester = test_clock(); + let clock = tester.clock(); let bytes = vec![42; IDENTIFY_SIZE + 128 * 1024]; - let (mut peer, observed) = peer(None, vec![]); + let (mut peer, observed) = peer(&clock, None, vec![]); + let (arrived, chunks) = mpsc::channel(); + observed.lock().unwrap().chunks = Some(arrived); let session = attach(&mut peer); let mut reader = Slow { + tester: &mut tester, bytes: &bytes, reads: 0, - observed: observed.clone(), + chunks, }; upload( &session.requester(), &source(bytes.len(), None), &mut reader, - Instant::now() + TIMEOUT, + clock.now() + TIMEOUT, |_| {}, ) .unwrap(); @@ -517,15 +556,16 @@ mod tests { /// malformed progress report can be mistaken for completed processing. #[test] fn test_failures() { + let clock = test_clock().clock(); for fail in ["identify", "peek", "start", "chunk", "process"] { let bytes = vec![42; IDENTIFY_SIZE + 2 * CHUNK_SIZE + 1]; - let (mut peer, observed) = peer(Some(fail), vec![]); + let (mut peer, observed) = peer(&clock, Some(fail), vec![]); let session = attach(&mut peer); let result = upload( &session.requester(), &source(bytes.len(), None), &mut bytes.as_slice(), - Instant::now() + TIMEOUT, + clock.now() + TIMEOUT, |_| {}, ); if fail == "identify" { @@ -564,13 +604,13 @@ mod tests { status(3, 10_000), status(2, 10_001), ] { - let (mut peer, observed) = peer(None, vec![report]); + let (mut peer, observed) = peer(&clock, None, vec![report]); let session = attach(&mut peer); let result = upload( &session.requester(), &source(1, None), &mut [42].as_slice(), - Instant::now() + TIMEOUT, + clock.now() + TIMEOUT, |_| {}, ); assert!(matches!(result, Err(Error::Dataset(_)))); @@ -582,6 +622,7 @@ mod tests { /// bad length or hash prevents processing, for both small and large files. #[test] fn test_integrity() { + let clock = test_clock().clock(); for size in [17, IDENTIFY_SIZE + CHUNK_SIZE + 17] { let original = vec![42; size]; let hash = Sha256::digest(&original).into(); @@ -595,13 +636,13 @@ mod tests { "corrupt" => bytes[size - 1] ^= 1, _ => {} } - let (mut peer, observed) = peer(None, vec![]); + let (mut peer, observed) = peer(&clock, None, vec![]); let session = attach(&mut peer); let result = upload( &session.requester(), &source(size, Some(hash)), &mut bytes.as_slice(), - Instant::now() + TIMEOUT, + clock.now() + TIMEOUT, |_| {}, ); let observed = observed.lock().unwrap(); @@ -622,43 +663,47 @@ mod tests { /// Responses arriving in reverse order must not advance the source early. #[test] fn test_transfer_window() { - let (notice, notices) = std::sync::mpsc::channel(); - let (release, released) = std::sync::mpsc::channel(); + let clock = test_clock().clock(); + let (notice, notices) = mpsc::channel(); + let (release, released) = mpsc::channel(); let mut held = None; let mut chunks = 0; - let mut peer = Peer::spawn(Box::new(move |_, request, responder| { - let deadline = Instant::now() + TIMEOUT; - match request { - Content::SlotUploadStart(_) => { - responder - .reply(schema::SlotUploadStartResponse { session: 7 }, deadline) - .unwrap(); - } - Content::SlotUploadChunk(_) => { - chunks += 1; - if chunks == 1 { - held = Some(responder); - } else { + let mut peer = Peer::spawn( + &clock, + Box::new(move |session, request, responder| { + let deadline = session.clock().now() + TIMEOUT; + match request { + Content::SlotUploadStart(_) => { responder - .reply(schema::SlotUploadChunkResponse {}, deadline) + .reply(schema::SlotUploadStartResponse { session: 7 }, deadline) .unwrap(); - if chunks == 2 { - notice.send(()).unwrap(); - released.recv_timeout(TIMEOUT).unwrap(); - held.take() - .unwrap() + } + Content::SlotUploadChunk(_) => { + chunks += 1; + if chunks == 1 { + held = Some(responder); + } else { + responder .reply(schema::SlotUploadChunkResponse {}, deadline) .unwrap(); + if chunks == 2 { + notice.send(()).unwrap(); + released.recv().unwrap(); + held.take() + .unwrap() + .reply(schema::SlotUploadChunkResponse {}, deadline) + .unwrap(); + } } } + Content::SlotUploadProcess(_) => { + responder.reply(status(2, 10_000), deadline).unwrap(); + } + _ => panic!("unexpected request"), } - Content::SlotUploadProcess(_) => { - responder.reply(status(2, 10_000), deadline).unwrap(); - } - _ => panic!("unexpected request"), - } - true - })); + true + }), + ); let session = attach(&mut peer); let bytes = vec![42; IDENTIFY_SIZE + 3 * CHUNK_SIZE]; let source = source(bytes.len(), Some(Sha256::digest(&bytes).into())); @@ -677,7 +722,8 @@ mod tests { } let counted = read.clone(); let requester = session.requester(); - let worker = std::thread::spawn(move || { + let deadline = clock.now() + TIMEOUT; + let worker = thread::spawn(move || { upload( &requester, &source, @@ -685,11 +731,11 @@ mod tests { bytes: &bytes, count: counted, }, - Instant::now() + TIMEOUT, + deadline, |_| {}, ) }); - notices.recv_timeout(TIMEOUT).unwrap(); + notices.recv().unwrap(); assert_eq!( read.load(std::sync::atomic::Ordering::SeqCst), IDENTIFY_SIZE + 2 * CHUNK_SIZE @@ -715,7 +761,8 @@ mod tests { self.bytes.read(&mut buf[..size]) } } - let (mut peer, _) = peer(None, vec![]); + let clock = test_clock().clock(); + let (mut peer, _) = peer(&clock, None, vec![]); let session = attach(&mut peer); let mut reader = Fragmented { calls: 0, @@ -725,7 +772,7 @@ mod tests { &session.requester(), &source(5, None), &mut reader, - Instant::now() + TIMEOUT, + clock.now() + TIMEOUT, |_| {}, ) .unwrap(); @@ -740,24 +787,19 @@ mod tests { #[test] fn test_client_setup() { use crate::cloud::tests::{response, serve}; + let clock = test_clock().clock(); let (url, requests) = serve(vec![ - ( - Duration::ZERO, - response(200, r#"{"signer":"AQ==","crypto":"Ag=="}"#), - ), - ( - Duration::ZERO, - response(200, r#"{"unixmilli":123,"signature":"BA=="}"#), - ), + response(200, r#"{"signer":"AQ==","crypto":"Ag=="}"#), + response(200, r#"{"unixmilli":123,"signature":"BA=="}"#), ]); - let (mut peer, observed) = peer(None, vec![]); + let (mut peer, observed) = peer(&clock, None, vec![]); let ark = crate::cloud::tests::attach(&mut peer, url); for _ in 0..2 { ark.client() .upload_dataset( &source(1, None), &mut [42].as_slice(), - Instant::now() + TIMEOUT, + clock.now() + TIMEOUT, |_| {}, ) .unwrap(); @@ -775,15 +817,10 @@ mod tests { &observed.stages[..3], &["sync-start", "sync-finish", "peek"] ); + assert!(requests.recv().unwrap().contains("/cloudsync/identity")); assert!( requests - .recv_timeout(TIMEOUT) - .unwrap() - .contains("/cloudsync/identity") - ); - assert!( - requests - .recv_timeout(TIMEOUT) + .recv() .unwrap() .contains("/cloudsync/time?challenge=03") ); @@ -792,21 +829,35 @@ mod tests { /// successive replies report the same progress percentage. #[test] fn test_processing_renews_waits() { + // Upload with a machine allowance shorter than the pause between polls + let mut tester = test_clock(); + let clock = tester.clock(); let (mut peer, observed) = peer( + &clock, None, vec![status(1, 100), status(1, 100), status(2, 10_000)], ); let session = attach(&mut peer); - let started = Instant::now(); - upload( - &session.requester(), - &source(1, None), - &mut [42].as_slice(), - Timing::inactivity(Duration::from_millis(250)), - |_| {}, - ) - .unwrap(); - assert!(started.elapsed() >= 2 * POLL_INTERVAL); + let uploading = thread::spawn({ + let requester = session.requester(); + move || { + upload( + &requester, + &source(1, None), + &mut [42].as_slice(), + Timing::inactivity(Duration::from_millis(250)), + |_| {}, + ) + } + }); + + // Each pause lasts a whole poll interval, each report renewing the allowance + for _ in 0..2 { + let poll = clock.now() + POLL_INTERVAL; + wait_deadline(&tester, poll); + tester.advance_to(poll); + } + uploading.join().unwrap().unwrap(); assert!(!observed.lock().unwrap().stages.contains(&"cancel")); } } diff --git a/connect/src/device.rs b/connect/src/device.rs index 9a820a8..437a309 100644 --- a/connect/src/device.rs +++ b/connect/src/device.rs @@ -10,6 +10,7 @@ use crate::emulator::Instance; use crate::trust::{Environment, Realm}; use crate::{Ark, Error, Identity, TrustMode, emulator, hardware}; +use darkbio_clock::Clock; use std::fmt; /// Kind of Ark reported by discovery. Authentication establishes its identity @@ -172,7 +173,8 @@ impl Device { self.open(verifier, |identity| Some(env(identity))) } - /// Opens the retained transport with any caller-supplied cloud route. + /// Opens the retained transport with any caller-supplied cloud route. The + /// connection runs on the real clock, which its clients hand to callers. fn open( &self, verifier: &TrustMode, @@ -183,9 +185,12 @@ impl Device { DeviceKind::Emulator => Realm::Emulator, }; let cloud = |identity: &Identity| env(identity).map(|env| (env, realm)); + let clock = Clock::real(); match &self.source { - Source::Usb(info) => hardware::connect(info, verifier, cloud), - Source::Registry(instance) => emulator::connect(&instance.url(), verifier, cloud), + Source::Usb(info) => hardware::connect(info, verifier, cloud, &clock), + Source::Registry(instance) => { + emulator::connect(&instance.url(), verifier, cloud, &clock) + } } } } diff --git a/connect/src/emulator/ws.rs b/connect/src/emulator/ws.rs index cf21742..1800655 100644 --- a/connect/src/emulator/ws.rs +++ b/connect/src/emulator/ws.rs @@ -9,14 +9,19 @@ //! One worker owns the WebSocket and handles binary data, Ping/Pong and Close. //! Socket readiness and explicit wakeups drive its nonblocking I/O. The adapters //! bound input buffering and wait for output under wire's write deadline. +//! +//! Wire's deadlines are measured on the connection's clock. The worker waits on +//! the socket through mio, so the bound on its own control replies runs on real time. +use crate::timing::ClockExt; use crate::{Ark, Error, wire}; +use darkbio_clock::{Clock, crossbeam_channel, sync}; use mio::{Events, Interest, Poll, Token, Waker}; use std::collections::VecDeque; use std::io::{self, Read, Write}; use std::net::{Shutdown, TcpStream, ToSocketAddrs}; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Condvar, Mutex, mpsc}; +use std::sync::{Arc, mpsc}; use std::thread; use std::time::{Duration, Instant}; use tungstenite::client::IntoClientRequest; @@ -39,10 +44,12 @@ const INBOUND_LIMIT: usize = 2 * MAX_MESSAGE; /// Opens a plain WebSocket endpoint and authenticates its wire session. /// After address resolution, TCP establishment and HTTP upgrade share the /// handshake timeout. The encrypted wire handshake starts its own timeout. +/// The connection measures its deadlines on the clock. pub(crate) fn connect>( url: &str, verifier: &V, cloud: impl FnOnce(&crate::Identity) -> Option<(crate::trust::Environment, crate::trust::Realm)>, + clock: &Clock, ) -> Result<(Ark, V::Info), Error> { // Resolve the endpoint before starting the TCP and HTTP handshake budget. let request = url.into_client_request().map_err(Error::Upgrade)?; @@ -61,15 +68,15 @@ pub(crate) fn connect>( .to_socket_addrs() .map_err(Error::Unreachable)?; // Failed address attempts consume the same budget as the eventual upgrade. - let deadline = Instant::now() + transport::DEFAULT_HANDSHAKE_TIMEOUT; + let deadline = clock.now() + transport::DEFAULT_HANDSHAKE_TIMEOUT; let mut last_error = io::Error::new( io::ErrorKind::AddrNotAvailable, "host resolves to no address", ); let mut connected = None; for address in addresses { - match TcpStream::connect_timeout(&address, remaining(deadline).map_err(Error::Unreachable)?) - { + let left = clock.remaining(deadline).map_err(Error::Unreachable)?; + match TcpStream::connect_timeout(&address, left) { Ok(stream) => { connected = Some(stream); break; @@ -88,6 +95,7 @@ pub(crate) fn connect>( let (socket, _) = tungstenite::client::client_with_config( request, Socket::Handshake { + clock: clock.clone(), stream: tcp, deadline, }, @@ -103,7 +111,7 @@ pub(crate) fn connect>( HandshakeError::Failure(err) => Error::Upgrade(err), })?; // Transfer the upgraded socket to its worker and give wire blocking adapters. - let (reader, writer, shutdown) = adapters(socket).map_err(Error::Unreachable)?; + let (reader, writer, shutdown) = adapters(socket, clock).map_err(Error::Unreachable)?; Ark::attach( transport::Stream::new(reader, writer, shutdown), verifier, @@ -116,6 +124,8 @@ pub(crate) fn connect>( enum Socket { /// TCP stream whose individual I/O calls share the upgrade deadline. Handshake { + /// Clock of the connection, which the deadline is measured on. + clock: Clock, /// TCP connection being upgraded, before readiness registration. stream: TcpStream, /// Shared absolute bound for every HTTP upgrade read and write. @@ -129,8 +139,12 @@ impl Read for Socket { /// Applies the upgrade deadline or delegates to the registered socket. fn read(&mut self, buf: &mut [u8]) -> io::Result { match self { - Self::Handshake { stream, deadline } => { - stream.set_read_timeout(Some(remaining(*deadline)?))?; + Self::Handshake { + clock, + stream, + deadline, + } => { + stream.set_read_timeout(Some(clock.remaining(*deadline)?))?; stream.read(buf) } Self::Connected(stream) => stream.read(buf), @@ -142,8 +156,12 @@ impl Write for Socket { /// Applies the upgrade deadline or writes through readiness-aware I/O. fn write(&mut self, buf: &[u8]) -> io::Result { match self { - Self::Handshake { stream, deadline } => { - stream.set_write_timeout(Some(remaining(*deadline)?))?; + Self::Handshake { + clock, + stream, + deadline, + } => { + stream.set_write_timeout(Some(clock.remaining(*deadline)?))?; stream.write(buf) } Self::Connected(stream) => stream.write(buf), @@ -159,14 +177,6 @@ impl Write for Socket { } } -/// Returns the time left before a deadline, refusing an already expired budget. -fn remaining(deadline: Instant) -> io::Result { - deadline - .checked_duration_since(Instant::now()) - .filter(|left| !left.is_zero()) - .ok_or_else(|| io::Error::from(io::ErrorKind::TimedOut)) -} - /// Reports output refused after the connection or its worker has closed. fn closed() -> io::Error { io::Error::new(io::ErrorKind::NotConnected, "WebSocket connection closed") @@ -196,10 +206,14 @@ struct Incoming { /// Input and closure state shared by the worker and its blocking adapters. struct Shared { - closed: AtomicBool, // Local shutdown signal checked by every participant - incoming: Mutex, // Buffered input and the worker's ending result - available: Condvar, // Wakes the reader for input, failure or closure - wake: Waker, // Wakes the worker when an adapter changes its work + clock: Clock, // clock that wire's deadlines are measured on + closed: AtomicBool, // Local shutdown signal checked by every participant + incoming: sync::Mutex, // Buffered input and the worker's ending result + available: sync::Condvar, // Wakes the reader for input, failure or closure + wake: Waker, // Wakes the worker when an adapter changes its work + /// Notifies a test each time the flush of a writer's frame waits for the socket. + #[cfg(test)] + blocked: std::sync::Mutex>>, } impl Shared { @@ -237,6 +251,14 @@ impl Shared { incoming.chunks.push_back(bytes); self.available.notify_one(); } + + /// Notifies a waiting test that the flush of a writer's frame waits for the socket. + #[cfg(test)] + fn flush_blocked(&self) { + if let Some(sender) = self.blocked.lock().unwrap().as_ref() { + let _ = sender.send(()); + } + } } /// One flush submitted by the writer, acknowledged when the worker finishes it. @@ -244,12 +266,16 @@ struct Outgoing { bytes: Vec, // Complete frame accumulated since the previous flush deadline: Instant, // Original write deadline, including the queue wait /// Receives the local write result without blocking the socket worker. - done: mpsc::SyncSender>, + done: crossbeam_channel::Sender>, } /// Starts the socket worker and returns adapters with their shutdown operation. /// The returned shutdown wakes blocking I/O and closes the underlying socket. -fn adapters(mut socket: WebSocket) -> io::Result<(Reader, Writer, impl FnOnce() + Send)> { +/// The adapters measure wire's deadlines on the clock. +fn adapters( + mut socket: WebSocket, + clock: &Clock, +) -> io::Result<(Reader, Writer, impl FnOnce() + Send + use<>)> { let Socket::Handshake { stream, .. } = socket.get_ref() else { unreachable!("socket upgraded once") }; @@ -267,10 +293,13 @@ fn adapters(mut socket: WebSocket) -> io::Result<(Reader, Writer, impl F // Reads and writes must use mio's socket, which rearms readiness on Windows. *socket.get_mut() = Socket::Connected(connected); let shared = Arc::new(Shared { + clock: clock.clone(), closed: AtomicBool::new(false), - incoming: Mutex::new(Incoming::default()), - available: Condvar::new(), + incoming: sync::Mutex::new(Incoming::default()), + available: sync::Condvar::new(clock), wake: Waker::new(poll.registry(), WAKE)?, + #[cfg(test)] + blocked: std::sync::Mutex::new(None), }); let (outgoing, outbox) = mpsc::channel(); thread::Builder::new().name("ark-websocket".into()).spawn({ @@ -295,17 +324,58 @@ fn adapters(mut socket: WebSocket) -> io::Result<(Reader, Writer, impl F )) } +/// Bounds output the socket has not taken yet, the worker's own control +/// replies included. A deferred flush starts the bound, later ones keep its +/// deadline, and only a completed flush ends it. +#[derive(Debug, Default)] +struct Backlog { + /// Time the pending output must be flushed by, while there is some. + deadline: Option, +} + +impl Backlog { + /// Starts the bound at `now` for output left to flush, keeping the + /// deadline of a bound already running. + fn start(&mut self, now: Instant) { + self.deadline + .get_or_insert(now + transport::DEFAULT_WRITE_TIMEOUT); + } + + /// Ends the bound once a flush took all output. + fn flushed(&mut self) { + self.deadline = None; + } + + /// Whether the output left to flush missed its deadline by `now`. + fn expired(&self, now: Instant) -> bool { + self.deadline.is_some_and(|deadline| now >= deadline) + } + + /// Time left after `now` before the deadline, bounding the next poll. + fn remaining(&self, now: Instant) -> Option { + self.deadline + .map(|deadline| deadline.saturating_duration_since(now)) + } +} + /// Drives the WebSocket while preserving input progress during blocked output. /// One adapter writer submits flushes serially and waits for each acknowledgement. +/// Frame deadlines are wire's and measured on the connection's clock; control +/// replies are bounded on real time. +#[expect( + clippy::disallowed_methods, + reason = "the worker waits on the socket through mio, so the bound on its own control replies runs on real time" +)] fn pump( mut socket: WebSocket, mut poll: Poll, shared: Arc, outbox: mpsc::Receiver, ) { + let clock = &shared.clock; let mut events = Events::with_capacity(8); let mut active: Option = None; - let mut control_deadline = None; + let mut backlog = Backlog::default(); let mut peer_closed = false; let result = (|| -> io::Result<()> { loop { @@ -313,17 +383,17 @@ fn pump( return Ok(()); } if let Some(frame) = &active { - remaining(frame.deadline)?; + clock.remaining(frame.deadline)?; } - if let Some(deadline) = control_deadline { - remaining(deadline)?; + if backlog.expired(Instant::now()) { + return Err(io::Error::from(io::ErrorKind::TimedOut)); } // Admit at most one flush. Its bytes stay in the WebSocket until // output completes, while the original deadline continues to run. if active.is_none() && !peer_closed { match outbox.try_recv() { Ok(mut frame) => { - if remaining(frame.deadline).is_err() { + if clock.remaining(frame.deadline).is_err() { let _ = frame .done .send(Err(io::Error::from(io::ErrorKind::TimedOut))); @@ -365,7 +435,7 @@ fn pump( // An acknowledgement covers everything queued before this flush. match socket.flush() { Ok(()) => { - control_deadline = None; + backlog.flushed(); if let Some(frame) = active.take() { let _ = frame.done.send(Ok(())); } @@ -374,8 +444,11 @@ fn pump( } } Err(err) if would_block(&err) => { - control_deadline - .get_or_insert_with(|| Instant::now() + transport::DEFAULT_WRITE_TIMEOUT); + #[cfg(test)] + if active.is_some() { + shared.flush_blocked(); + } + backlog.start(Instant::now()); } Err(tungstenite::Error::ConnectionClosed) => return Ok(()), Err(err) => return Err(socket_error(err)), @@ -385,14 +458,12 @@ fn pump( } // Either socket readiness or an adapter wake may enable more work. // Deadlines must also wake an otherwise idle or blocked connection. - let deadline = active + let timeout = active .as_ref() - .map(|frame| frame.deadline) + .map(|frame| frame.deadline.saturating_duration_since(clock.now())) .into_iter() - .chain(control_deadline) + .chain(backlog.remaining(Instant::now())) .min(); - let timeout = - deadline.map(|deadline| deadline.saturating_duration_since(Instant::now())); match poll.poll(&mut events, timeout) { Ok(()) => {} Err(err) if err.kind() == io::ErrorKind::Interrupted => {} @@ -457,9 +528,10 @@ impl Read for Reader { .wait(incoming) .expect("WebSocket input not poisoned"), Some(deadline) => { + self.shared.clock.remaining(deadline)?; self.shared .available - .wait_timeout(incoming, remaining(deadline)?) + .wait_deadline(incoming, deadline) .expect("WebSocket input not poisoned") .0 } @@ -469,6 +541,11 @@ impl Read for Reader { } impl transport::Read for Reader { + /// Returns the connection's clock, which the read deadlines are measured on. + fn clock(&self) -> Clock { + self.shared.clock.clone() + } + /// Bounds future input waits, leaving already buffered bytes available. fn set_read_deadline(&mut self, deadline: Option) -> io::Result<()> { self.deadline = deadline; @@ -491,7 +568,7 @@ impl Write for Writer { return Err(closed()); } if let Some(deadline) = self.deadline { - remaining(deadline)?; + self.shared.clock.remaining(deadline)?; } self.pending.extend_from_slice(bytes); Ok(bytes.len()) @@ -503,14 +580,15 @@ impl Write for Writer { if self.shared.closed.load(Ordering::Acquire) { return Err(closed()); } + let clock = &self.shared.clock; let deadline = self .deadline - .unwrap_or_else(|| Instant::now() + transport::DEFAULT_WRITE_TIMEOUT); - remaining(deadline)?; + .unwrap_or_else(|| clock.now() + transport::DEFAULT_WRITE_TIMEOUT); + clock.remaining(deadline)?; if self.pending.is_empty() { return Ok(()); } - let (done, result) = mpsc::sync_channel(1); + let (done, result) = crossbeam_channel::bounded(1); self.outgoing .send(Outgoing { bytes: std::mem::take(&mut self.pending), @@ -519,15 +597,23 @@ impl Write for Writer { }) .map_err(|_| closed())?; self.shared.wake.wake()?; - match result.recv_timeout(remaining(deadline)?) { + clock.remaining(deadline)?; + match clock.recv_deadline(&result, deadline) { Ok(result) => result, - Err(mpsc::RecvTimeoutError::Timeout) => Err(io::Error::from(io::ErrorKind::TimedOut)), - Err(mpsc::RecvTimeoutError::Disconnected) => Err(closed()), + Err(crossbeam_channel::RecvTimeoutError::Timeout) => { + Err(io::Error::from(io::ErrorKind::TimedOut)) + } + Err(crossbeam_channel::RecvTimeoutError::Disconnected) => Err(closed()), } } } impl transport::Write for Writer { + /// Returns the connection's clock, which the write deadline is measured on. + fn clock(&self) -> Clock { + self.shared.clock.clone() + } + /// Installs the shared bound for accumulating and flushing the next frame. fn set_write_deadline(&mut self, deadline: Instant) -> io::Result<()> { self.deadline = Some(deadline); @@ -539,13 +625,16 @@ impl transport::Write for Writer { #[cfg(test)] mod tests { use super::*; - use crate::testing::{Peer, answering, hangup}; + use crate::testing::{Peer, answering, hangup, test_clock, wait_deadline}; use darkbio_wire::memory::Duplex; use std::net::TcpListener; use std::thread; + use tungstenite::protocol::Role; /// Exercises the actual adapters without a wire session consuming their bytes. - fn socket_pair() -> (WebSocket, WebSocket) { + /// The upgrade's socket timeouts are measured from the clock, which the + /// tests never advance through them. + fn socket_pair(clock: &Clock) -> (WebSocket, WebSocket) { let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let addr = listener.local_addr().unwrap(); let server = thread::spawn(move || { @@ -558,8 +647,9 @@ mod tests { let (client, _) = tungstenite::client( format!("ws://{addr}/v1/usb"), Socket::Handshake { + clock: clock.clone(), stream: tcp, - deadline: Instant::now() + Duration::from_secs(2), + deadline: clock.now() + Duration::from_secs(2), }, ) .unwrap(); @@ -569,13 +659,9 @@ mod tests { /// Ping and Close receive protocol replies while the application is idle. #[test] fn test_control_frames() { - let (socket, mut server) = socket_pair(); - let (mut reader, _writer, shutdown) = adapters(socket).unwrap(); - transport::Read::set_read_deadline( - &mut reader, - Some(Instant::now() + Duration::from_secs(2)), - ) - .unwrap(); + let clock = test_clock().clock(); + let (socket, mut server) = socket_pair(&clock); + let (mut reader, _writer, shutdown) = adapters(socket, &clock).unwrap(); server .send(Message::Ping(Bytes::from_static(b"probe"))) .unwrap(); @@ -592,28 +678,45 @@ mod tests { /// A read respects its deadline, and shutdown also wakes a read without one. #[test] fn test_read_deadline_and_close() { - let (socket, _server) = socket_pair(); - let (mut reader, _writer, shutdown) = adapters(socket).unwrap(); - transport::Read::set_read_deadline( - &mut reader, - Some(Instant::now() + Duration::from_millis(20)), - ) - .unwrap(); - assert_eq!( - reader.read(&mut [0]).unwrap_err().kind(), - io::ErrorKind::TimedOut - ); + let mut tester = test_clock(); + let clock = tester.clock(); + let (socket, _server) = socket_pair(&clock); + let (mut reader, _writer, shutdown) = adapters(socket, &clock).unwrap(); + + // The read waits on its deadline and ends once the clock reaches it + let deadline = clock.now() + Duration::from_millis(20); + transport::Read::set_read_deadline(&mut reader, Some(deadline)).unwrap(); + let reading = thread::spawn(move || { + let result = reader.read(&mut [0]); + (reader, result) + }); + wait_deadline(&tester, deadline); + tester.advance_to(deadline); + let (mut reader, result) = reading.join().unwrap(); + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::TimedOut); + + // Shutdown wakes a read that waits without a deadline. The reader is the + // only thread that waits on this clock, so the park after its start is + // the read's own. No deadline is listed for that park. transport::Read::set_read_deadline(&mut reader, None).unwrap(); - let reading = thread::spawn(move || reader.read(&mut [0])); + let (started, reading) = mpsc::channel(); + let read = thread::spawn(move || { + started.send(()).unwrap(); + reader.read(&mut [0]) + }); + reading.recv().unwrap(); + tester.wait_blocked(1); + assert_eq!(tester.next_deadline(), None); shutdown(); - assert_eq!(reading.join().unwrap().unwrap(), 0); + assert_eq!(read.join().unwrap().unwrap(), 0); } /// Consuming buffered input releases capacity for the remaining messages. #[test] fn test_input_buffering() { - let (socket, mut server) = socket_pair(); - let (mut reader, _writer, shutdown) = adapters(socket).unwrap(); + let clock = test_clock().clock(); + let (socket, mut server) = socket_pair(&clock); + let (mut reader, _writer, shutdown) = adapters(socket, &clock).unwrap(); let sending = thread::spawn(move || { let message = Bytes::from(vec![7; MAX_MESSAGE]); for _ in 0..4 { @@ -621,11 +724,6 @@ mod tests { } server }); - transport::Read::set_read_deadline( - &mut reader, - Some(Instant::now() + Duration::from_secs(3)), - ) - .unwrap(); let mut message = vec![0; MAX_MESSAGE]; for _ in 0..4 { reader.read_exact(&mut message).unwrap(); @@ -639,35 +737,41 @@ mod tests { /// Blocked output retains its deadline while incoming traffic still reaches the reader. #[test] fn test_output_backpressure() { - let (socket, mut server) = socket_pair(); - let (_reader, mut writer, shutdown) = adapters(socket).unwrap(); + // Flood the peer with output it never drains, under one write deadline, + // until the worker's flush of a frame waits for the socket + let mut tester = test_clock(); + let clock = tester.clock(); + let (socket, mut server) = socket_pair(&clock); + let (_reader, mut writer, shutdown) = adapters(socket, &clock).unwrap(); let shared = writer.shared.clone(); - let (started, writing) = mpsc::channel(); + let (blocked, flushes) = mpsc::channel(); + *shared.blocked.lock().unwrap() = Some(blocked); + let deadline = clock.now() + Duration::from_millis(300); + transport::Write::set_write_deadline(&mut writer, deadline).unwrap(); let sending = thread::spawn(move || -> io::Result<()> { - let deadline = Instant::now() + Duration::from_millis(300); - transport::Write::set_write_deadline(&mut writer, deadline).unwrap(); - started.send(()).unwrap(); let message = vec![9; MAX_MESSAGE]; loop { writer.write_all(&message)?; writer.flush()?; } }); - writing.recv().unwrap(); - // The peer sends input but deliberately never drains the client's output. + flushes.recv().unwrap(); + + // The peer sends input, which reaches the reader past the stuck output server .send(Message::Binary(Bytes::from_static(b"incoming"))) .unwrap(); - let mut incoming = shared.incoming.lock().unwrap(); - while incoming.bytes == 0 && !incoming.ended { - incoming = shared - .available - .wait_timeout(incoming, Duration::from_secs(1)) - .unwrap() - .0; - } + let incoming = shared.incoming.lock().unwrap(); + let incoming = shared + .available + .wait_while(incoming, |incoming| incoming.bytes == 0 && !incoming.ended) + .unwrap(); assert_eq!(incoming.chunks.front().unwrap().as_ref(), b"incoming"); drop(incoming); + + // The writer waits for the stuck flush and gives up at its deadline + wait_deadline(&tester, deadline); + tester.advance_to(deadline); assert_eq!( sending.join().unwrap().unwrap_err().kind(), io::ErrorKind::TimedOut @@ -675,22 +779,70 @@ mod tests { shutdown(); } - /// How often the bridge turns from one direction to the other. - const ROUND: Duration = Duration::from_millis(10); + /// A blocked flush starts the output bound once. Later blocked flushes keep + /// the original deadline, where the bound expires, and a completed flush + /// clears it for the next blockage. + #[test] + fn test_backlog() { + // The first blocked flush starts the bound + let mut tester = test_clock(); + let clock = tester.clock(); + let limit = transport::DEFAULT_WRITE_TIMEOUT; + let mut backlog = Backlog::default(); + assert_eq!(backlog.remaining(clock.now()), None); + let start = clock.now(); + backlog.start(start); + assert_eq!(backlog.remaining(start), Some(limit)); + + // Blocking again keeps the original deadline, which ends the bound on time + tester.advance(Duration::from_secs(3)); + backlog.start(clock.now()); + assert_eq!( + backlog.remaining(clock.now()), + Some(limit - Duration::from_secs(3)) + ); + tester.advance_to(start + limit - Duration::from_millis(1)); + assert!(!backlog.expired(clock.now())); + tester.advance_to(start + limit); + assert!(backlog.expired(clock.now())); + + // A completed flush clears the bound, and the next blockage starts afresh + backlog.flushed(); + assert!(!backlog.expired(clock.now())); + assert_eq!(backlog.remaining(clock.now()), None); + backlog.start(clock.now()); + assert_eq!(backlog.remaining(clock.now()), Some(limit)); + } // Serves one WebSocket client on the listener the way an emulator does, // carrying its binary messages into the peer's stream and the stream's - // bytes back out as messages, until either side ends. + // bytes back out as messages, until either side ends. Each direction has + // its own thread and socket handle and blocks on its own input. fn bridge(listener: TcpListener, stream: Duplex) { let (mut reader, mut writer) = stream.into_halves(); let (tcp, _) = listener.accept().unwrap(); let mut socket = tungstenite::accept(tcp).unwrap(); - socket + + // Probe the client with a ping, then carry the peer's output to it, + // ending the connection once the peer is gone + let mut outgoing = + WebSocket::from_raw_socket(socket.get_ref().try_clone().unwrap(), Role::Server, None); + outgoing .send(Message::Ping(Bytes::from_static(b"keepalive"))) .unwrap(); + let sending = thread::spawn(move || { + let mut buf = vec![0u8; 64 * 1024]; + while let Ok(count @ 1..) = reader.read(&mut buf) { + let message = Message::Binary(Bytes::copy_from_slice(&buf[..count])); + if outgoing.send(message).is_err() { + break; + } + } + let _ = outgoing.get_ref().shutdown(Shutdown::Both); + }); + + // Carry the client's messages into the peer, until the client goes away let mut pong_received = false; - socket.get_ref().set_read_timeout(Some(ROUND)).unwrap(); - let mut buf = vec![0u8; 64 * 1024]; loop { match socket.read() { Ok(Message::Binary(data)) => { @@ -706,30 +858,12 @@ mod tests { assert_eq!(bytes.as_ref(), b"keepalive"); pong_received = true; } - Ok(Message::Close(_)) | Err(tungstenite::Error::ConnectionClosed) => break, + Ok(Message::Close(_)) | Err(_) => break, Ok(_) => {} - Err(tungstenite::Error::Io(err)) - if matches!( - err.kind(), - io::ErrorKind::WouldBlock | io::ErrorKind::TimedOut - ) => {} - Err(_) => break, - } - transport::Read::set_read_deadline(&mut reader, Some(Instant::now() + ROUND)).unwrap(); - match reader.read(&mut buf) { - Ok(0) => break, - Ok(n) => { - if socket - .send(Message::Binary(Bytes::copy_from_slice(&buf[..n]))) - .is_err() - { - break; - } - } - Err(err) if err.kind() == io::ErrorKind::TimedOut => {} - Err(_) => break, } } + drop(writer); + sending.join().unwrap(); assert!( pong_received, "client must answer Ping while carrying wire traffic" @@ -740,7 +874,8 @@ mod tests { // it, the requests answered and the close ending the socket. #[test] fn test_socket_session() { - let mut peer = Peer::spawn(Box::new(answering)); + let clock = test_clock().clock(); + let mut peer = Peer::spawn(&clock, Box::new(answering)); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("ws://{}/v1/usb", listener.local_addr().unwrap()); let stream = peer.stream(); @@ -750,6 +885,7 @@ mod tests { &url, &crate::TrustMode::Recover(Box::new(peer.identity.clone())), |_| None, + &clock, ) .unwrap(); assert_eq!( @@ -768,7 +904,8 @@ mod tests { // behind it is gone. #[test] fn test_socket_lost() { - let mut peer = Peer::spawn(hangup()); + let clock = test_clock().clock(); + let mut peer = Peer::spawn(&clock, hangup()); let listener = TcpListener::bind("127.0.0.1:0").unwrap(); let url = format!("ws://{}/v1/usb", listener.local_addr().unwrap()); let stream = peer.stream(); @@ -778,6 +915,7 @@ mod tests { &url, &crate::TrustMode::Recover(Box::new(peer.identity.clone())), |_| None, + &clock, ) .unwrap(); let err = ark diff --git a/connect/src/execution.rs b/connect/src/execution.rs index b9096db..763f747 100644 --- a/connect/src/execution.rs +++ b/connect/src/execution.rs @@ -7,9 +7,10 @@ //! App uploads, companion authorization and execution results. use crate::{Error, Timing, schema}; +use darkbio_clock::Clock; use darkbio_wire::protocol::{Message, Promise, Requester}; use std::io::{self, Read}; -use std::time::{Duration, Instant}; +use std::time::Duration; const CHUNK_SIZE: usize = 2 * 1024 * 1024 - 32 * 1024; // Leave room for sealing and framing /// Delay between status requests while the Ark retains a pending task. @@ -44,6 +45,7 @@ pub enum ExecutionProgress { /// Streams at most two outstanding chunks, waits for authorization and retrieves /// the result once. Scheduling establishes the relay through the caller's client. +/// Deadlines and the running time are measured on the clock of the requester's session. pub(crate) fn execute( requester: &Requester, size: u64, @@ -52,6 +54,7 @@ pub(crate) fn execute( mut progress: impl FnMut(ExecutionProgress), schedule: impl FnOnce(u64) -> Result<(), Error>, ) -> Result { + let clock = &requester.clock(); let timing = timing.into(); if size == 0 { return Err(Error::Execution("app is empty".into())); @@ -60,7 +63,7 @@ pub(crate) fn execute( let taskid = requester .request( schema::ExecutionUploadStartRequest { bytes: size }, - timing.io(), + timing.io(clock), )? .wait::()? .taskid; @@ -74,20 +77,20 @@ pub(crate) fn execute( let mut uploaded = 0; let mut pending: Option<(Promise, u64)> = None; while sent < size { - timing.check()?; + timing.check(clock)?; let bytes = (size - sent).min(CHUNK_SIZE as u64) as usize; let mut chunk = vec![0; bytes]; reader.read_exact(&mut chunk).map_err(read_error)?; - timing.check()?; + timing.check(clock)?; sent += bytes as u64; if sent == size { - finish_read(reader, timing)?; + finish_read(reader, clock, timing)?; } // Submit the next chunk before waiting for the previous one, keeping // at most two outstanding while device writes overlap transport I/O. let next = requester.request( schema::ExecutionUploadChunkRequest { taskid, chunk }, - timing.io(), + timing.io(clock), )?; if let Some((previous, bytes)) = pending.take() { previous.wait::()?; @@ -111,24 +114,24 @@ pub(crate) fn execute( schedule(taskid)?; // Retrieving a completed status consumes the result on the Ark. This // workflow is the sole poller and never retries a completed retrieval. - let started = Instant::now(); + let started = clock.now(); loop { progress(ExecutionProgress::Running { - elapsed: started.elapsed(), + elapsed: clock.elapsed(started), }); let status = requester - .request(schema::ExecutionStatusRequest { taskid }, timing.io())? + .request(schema::ExecutionStatusRequest { taskid }, timing.io(clock))? .wait::()?; match (status.pending, status.result) { (false, Some(result)) => return Ok(result), (true, None) => {} _ => return Err(Error::Execution("invalid execution status".into())), } - timing.pause(POLL_INTERVAL)?; + timing.pause(clock, POLL_INTERVAL)?; } })(); if result.is_err() { - let cleanup = timing.io().min(Instant::now() + Duration::from_secs(1)); + let cleanup = timing.io(clock).min(clock.now() + Duration::from_secs(1)); let _ = requester .request(schema::ExecutionCancelRequest { taskid }, cleanup) .and_then(|pending| pending.wait::()); @@ -138,9 +141,9 @@ pub(crate) fn execute( /// Check EOF before the final chunk so a growing or misdeclared source never /// reaches scheduling. Interrupted reads do not indicate the end of a file. -fn finish_read(reader: &mut impl Read, timing: Timing) -> Result<(), Error> { +fn finish_read(reader: &mut impl Read, clock: &Clock, timing: Timing) -> Result<(), Error> { loop { - timing.check()?; + timing.check(clock)?; match reader.read(&mut [0]) { Ok(0) => break, Ok(_) => return Err(Error::Execution("app exceeds its advertised size".into())), @@ -148,7 +151,7 @@ fn finish_read(reader: &mut impl Read, timing: Timing) -> Result<(), Error> { Err(error) => return Err(read_error(error)), } } - timing.check()?; + timing.check(clock)?; Ok(()) } @@ -165,11 +168,12 @@ fn read_error(error: io::Error) -> Error { mod tests { use super::*; use crate::TrustMode; - use crate::testing::Peer; + use crate::testing::{Peer, test_clock, wait_deadline}; use darkbio_wire::protocol::{self, Session}; use schema::host_to_ark::Content; use std::collections::VecDeque; use std::sync::{Arc, Mutex, mpsc}; + use std::thread; const TIMEOUT: Duration = Duration::from_secs(10); @@ -191,60 +195,64 @@ mod tests { } fn peer( + clock: &Clock, fail: Option<&'static str>, reports: Vec, ) -> (Peer, Arc>) { let observed = Arc::new(Mutex::new(Observed::default())); let shared = observed.clone(); let mut reports = VecDeque::from(reports); - let peer = Peer::spawn(Box::new(move |_, request, responder| { - let mut observed = shared.lock().unwrap(); - let (stage, response): (_, Message) = match request { - Content::ExecUploadStart(request) => { - observed.size = request.bytes; - ( - "start", - schema::ExecutionUploadStartResponse { taskid: 7 }.into(), - ) - } - Content::ExecUploadChunk(request) => { - assert_eq!(request.taskid, 7); - assert!(request.chunk.len() <= CHUNK_SIZE); - observed.bytes.extend(request.chunk); - ("chunk", schema::ExecutionUploadChunkResponse {}.into()) - } - Content::ExecSched(request) => { - assert_eq!(request.taskid, 7); - assert_eq!(observed.bytes.len() as u64, observed.size); - ("schedule", schema::ExecutionScheduleResponse {}.into()) - } - Content::ExecStatus(request) => { - assert_eq!(request.taskid, 7); - ( - "status", - reports.pop_front().expect("unexpected status poll").into(), - ) - } - Content::ExecCancel(request) => { - assert_eq!(request.taskid, 7); - ("cancel", schema::ExecutionCancelResponse {}.into()) + let peer = Peer::spawn( + clock, + Box::new(move |session, request, responder| { + let mut observed = shared.lock().unwrap(); + let (stage, response): (_, Message) = match request { + Content::ExecUploadStart(request) => { + observed.size = request.bytes; + ( + "start", + schema::ExecutionUploadStartResponse { taskid: 7 }.into(), + ) + } + Content::ExecUploadChunk(request) => { + assert_eq!(request.taskid, 7); + assert!(request.chunk.len() <= CHUNK_SIZE); + observed.bytes.extend(request.chunk); + ("chunk", schema::ExecutionUploadChunkResponse {}.into()) + } + Content::ExecSched(request) => { + assert_eq!(request.taskid, 7); + assert_eq!(observed.bytes.len() as u64, observed.size); + ("schedule", schema::ExecutionScheduleResponse {}.into()) + } + Content::ExecStatus(request) => { + assert_eq!(request.taskid, 7); + ( + "status", + reports.pop_front().expect("unexpected status poll").into(), + ) + } + Content::ExecCancel(request) => { + assert_eq!(request.taskid, 7); + ("cancel", schema::ExecutionCancelResponse {}.into()) + } + _ => panic!("unexpected request"), + }; + observed.stages.push(stage); + let deadline = session.clock().now() + TIMEOUT; + if fail == Some(stage) || stage == "cancel" && fail.is_some() { + responder + .fail( + schema::Error::new(0x778, format!("refused {stage}")), + deadline, + ) + .unwrap(); + } else { + responder.reply(response, deadline).unwrap(); } - _ => panic!("unexpected request"), - }; - observed.stages.push(stage); - let deadline = Instant::now() + TIMEOUT; - if fail == Some(stage) || stage == "cancel" && fail.is_some() { - responder - .fail( - schema::Error::new(0x778, format!("refused {stage}")), - deadline, - ) - .unwrap(); - } else { - responder.reply(response, deadline).unwrap(); - } - true - })); + true + }), + ); (peer, observed) } @@ -259,7 +267,7 @@ mod tests { reader: &mut impl Read, progress: impl FnMut(ExecutionProgress), ) -> Result { - let deadline = Instant::now() + TIMEOUT; + let deadline = requester.clock().now() + TIMEOUT; execute(requester, size, reader, deadline, progress, |taskid| { requester .request(schema::ExecutionScheduleRequest { taskid }, deadline)? @@ -272,9 +280,13 @@ mod tests { /// the result is retrieved, retaining binary output even when the app failed. #[test] fn test_execution() { + let mut tester = test_clock(); + let clock = tester.clock(); for (size, success) in [(17, false), (3 * CHUNK_SIZE + 29, true)] { + // Run the app, the first status reporting it still pending let expected = result(success); let (mut peer, observed) = peer( + &clock, None, vec![ schema::ExecutionStatusResponse { @@ -289,14 +301,23 @@ mod tests { ); let session = attach(&mut peer); let bytes: Vec<_> = (0..size).map(|i| (i % 251) as u8).collect(); - let mut progress = Vec::new(); - let actual = run( - &session.requester(), - size as u64, - &mut bytes.as_slice(), - |stage| progress.push(stage), - ) - .unwrap(); + let running = thread::spawn({ + let requester = session.requester(); + let bytes = bytes.clone(); + move || { + let mut progress = Vec::new(); + let result = run(&requester, size as u64, &mut bytes.as_slice(), |stage| { + progress.push(stage) + }); + result.map(|result| (result, progress)) + } + }); + + // End the pause between the two status polls once the runner sleeps in it + let poll = clock.now() + POLL_INTERVAL; + wait_deadline(&tester, poll); + tester.advance_to(poll); + let (actual, progress) = running.join().unwrap().unwrap(); assert_eq!(actual, expected); let observed = observed.lock().unwrap(); assert_eq!(observed.bytes, bytes); @@ -328,8 +349,10 @@ mod tests { /// A refused start has no task ID and must not attempt cancellation. #[test] fn test_refusals() { + let clock = test_clock().clock(); for fail in ["start", "chunk", "schedule", "status"] { let (mut peer, observed) = peer( + &clock, Some(fail), vec![schema::ExecutionStatusResponse { pending: false, @@ -360,6 +383,7 @@ mod tests { /// A missing result or contradictory pending flag must not become success. #[test] fn test_invalid_status() { + let clock = test_clock().clock(); for report in [ schema::ExecutionStatusResponse::default(), schema::ExecutionStatusResponse { @@ -367,7 +391,7 @@ mod tests { result: Some(result(true)), }, ] { - let (mut peer, observed) = peer(None, vec![report]); + let (mut peer, observed) = peer(&clock, None, vec![report]); let session = attach(&mut peer); assert!(matches!( run(&session.requester(), 1, &mut [42].as_slice(), |_| {}), @@ -381,9 +405,10 @@ mod tests { /// change after their first chunk. Read failures cancel the allocated task. #[test] fn test_source_length() { + let clock = test_clock().clock(); for size in [17, CHUNK_SIZE + 17] { for extra in [-1_i64, 1] { - let (mut peer, observed) = peer(None, vec![]); + let (mut peer, observed) = peer(&clock, None, vec![]); let session = attach(&mut peer); let bytes = vec![42; (size as i64 + extra) as usize]; assert!( @@ -406,63 +431,67 @@ mod tests { /// The last chunk remains outstanding until explicitly released by the test. #[test] fn test_upload_window() { + let clock = test_clock().clock(); let (notice, notices) = mpsc::channel(); let (release, released) = mpsc::channel(); let mut first = None; let mut chunks = 0; - let mut peer = Peer::spawn(Box::new(move |_, request, responder| { - let deadline = Instant::now() + TIMEOUT; - match request { - Content::ExecUploadStart(_) => { - responder - .reply(schema::ExecutionUploadStartResponse { taskid: 7 }, deadline) - .unwrap(); - } - Content::ExecUploadChunk(_) => { - chunks += 1; - if chunks == 1 { - first = Some(responder); - } else { - if chunks == 3 { - notice.send(()).unwrap(); - released.recv_timeout(TIMEOUT).unwrap(); - } + let mut peer = Peer::spawn( + &clock, + Box::new(move |session, request, responder| { + let deadline = session.clock().now() + TIMEOUT; + match request { + Content::ExecUploadStart(_) => { responder - .reply(schema::ExecutionUploadChunkResponse {}, deadline) + .reply(schema::ExecutionUploadStartResponse { taskid: 7 }, deadline) .unwrap(); - if chunks == 2 { - first - .take() - .unwrap() + } + Content::ExecUploadChunk(_) => { + chunks += 1; + if chunks == 1 { + first = Some(responder); + } else { + if chunks == 3 { + notice.send(()).unwrap(); + released.recv().unwrap(); + } + responder .reply(schema::ExecutionUploadChunkResponse {}, deadline) .unwrap(); + if chunks == 2 { + first + .take() + .unwrap() + .reply(schema::ExecutionUploadChunkResponse {}, deadline) + .unwrap(); + } } } + Content::ExecSched(_) => { + responder + .reply(schema::ExecutionScheduleResponse {}, deadline) + .unwrap(); + } + Content::ExecStatus(_) => { + responder + .reply( + schema::ExecutionStatusResponse { + pending: false, + result: Some(result(true)), + }, + deadline, + ) + .unwrap(); + } + _ => panic!("unexpected request"), } - Content::ExecSched(_) => { - responder - .reply(schema::ExecutionScheduleResponse {}, deadline) - .unwrap(); - } - Content::ExecStatus(_) => { - responder - .reply( - schema::ExecutionStatusResponse { - pending: false, - result: Some(result(true)), - }, - deadline, - ) - .unwrap(); - } - _ => panic!("unexpected request"), - } - true - })); + true + }), + ); let session = attach(&mut peer); let requester = session.requester(); let (updates, progress) = mpsc::channel(); - let worker = std::thread::spawn(move || { + let worker = thread::spawn(move || { let bytes = vec![42; CHUNK_SIZE * 3]; run( &requester, @@ -473,7 +502,7 @@ mod tests { }, ) }); - notices.recv_timeout(TIMEOUT).unwrap(); + notices.recv().unwrap(); assert!( !progress .try_iter() @@ -487,59 +516,63 @@ mod tests { /// requires an application receive loop or a second connection. #[test] fn test_cancellation() { + let clock = test_clock().clock(); let (notice, notices) = mpsc::channel(); let mut held = None; - let mut peer = Peer::spawn(Box::new(move |_, request, responder| { - let deadline = Instant::now() + TIMEOUT; - match request { - Content::ExecUploadStart(_) => { - responder - .reply(schema::ExecutionUploadStartResponse { taskid: 7 }, deadline) - .unwrap(); - } - Content::ExecUploadChunk(_) => { - responder - .reply(schema::ExecutionUploadChunkResponse {}, deadline) - .unwrap(); - } - Content::ExecSched(_) => { - responder - .reply(schema::ExecutionScheduleResponse {}, deadline) - .unwrap(); - } - Content::ExecStatus(_) => { - held = Some(responder); - notice.send(()).unwrap(); - } - Content::ExecCancel(request) => { - assert_eq!(request.taskid, 7); - responder - .reply(schema::ExecutionCancelResponse {}, deadline) - .unwrap(); - held.take() - .unwrap() - .reply( - schema::ExecutionStatusResponse { - pending: false, - result: Some(result(false)), - }, - deadline, - ) - .unwrap(); + let mut peer = Peer::spawn( + &clock, + Box::new(move |session, request, responder| { + let deadline = session.clock().now() + TIMEOUT; + match request { + Content::ExecUploadStart(_) => { + responder + .reply(schema::ExecutionUploadStartResponse { taskid: 7 }, deadline) + .unwrap(); + } + Content::ExecUploadChunk(_) => { + responder + .reply(schema::ExecutionUploadChunkResponse {}, deadline) + .unwrap(); + } + Content::ExecSched(_) => { + responder + .reply(schema::ExecutionScheduleResponse {}, deadline) + .unwrap(); + } + Content::ExecStatus(_) => { + held = Some(responder); + notice.send(()).unwrap(); + } + Content::ExecCancel(request) => { + assert_eq!(request.taskid, 7); + responder + .reply(schema::ExecutionCancelResponse {}, deadline) + .unwrap(); + held.take() + .unwrap() + .reply( + schema::ExecutionStatusResponse { + pending: false, + result: Some(result(false)), + }, + deadline, + ) + .unwrap(); + } + _ => panic!("unexpected request"), } - _ => panic!("unexpected request"), - } - true - })); + true + }), + ); let session = attach(&mut peer); let requester = session.requester(); - let worker = std::thread::spawn(move || run(&requester, 1, &mut [42].as_slice(), |_| {})); - notices.recv_timeout(TIMEOUT).unwrap(); + let worker = thread::spawn(move || run(&requester, 1, &mut [42].as_slice(), |_| {})); + notices.recv().unwrap(); session .requester() .request( schema::ExecutionCancelRequest { taskid: 7 }, - Instant::now() + TIMEOUT, + clock.now() + TIMEOUT, ) .unwrap() .wait::() @@ -564,7 +597,9 @@ mod tests { self.bytes.read(&mut buffer[..1]) } } + let clock = test_clock().clock(); let (mut peer, _) = peer( + &clock, None, vec![schema::ExecutionStatusResponse { pending: false, @@ -587,7 +622,7 @@ mod tests { &session.requester(), 1, &mut [42].as_slice(), - Instant::now(), + clock.now(), |_| {}, |_| panic!("expired execution scheduled"), ); diff --git a/connect/src/hardware/usb.rs b/connect/src/hardware/usb.rs index 1703641..9a332b0 100644 --- a/connect/src/hardware/usb.rs +++ b/connect/src/hardware/usb.rs @@ -13,26 +13,27 @@ //! A zero length packet closes a frame that ended on a packet boundary. Flushes //! reap finished transfers without draining the ring, so consecutive frames //! can overlap on the bus. -//! Every wait is bounded by the deadline the wire installed and ends early -//! once the connection is closed. +//! Every wait is bounded by the deadline the wire installed, measured on the +//! connection's clock, and ends early once the connection is closed. use crate::ark::Ark; use crate::{Error, wire}; +use darkbio_clock::{Clock, sync}; use nusb::descriptors::TransferType; use nusb::transfer::{ Buffer, Bulk, Completion, Direction, EndpointDirection, In, Out, TransferError, }; use nusb::{ErrorKind, MaybeFuture}; use std::io::{self, Read, Write}; +use std::sync::Arc; use std::sync::atomic::{AtomicBool, Ordering}; -use std::sync::{Arc, Condvar, Mutex}; use std::task::{Context, Poll, Wake, Waker}; use std::time::Instant; use wire::transport::{self, Verifier}; /// Class, subclass and protocol of the vendor interface carrying the wire. It -/// tells the interface from the mass storage a development Ark exposes too, -/// which is bulk in both directions as well. +/// distinguishes the interface from the mass storage a development Ark exposes +/// too, which is bulk in both directions as well. const VENDOR_INTERFACE: (u8, u8, u8) = (0xff, 1, 2); /// Separator between the parts of the product string an Ark enumerates @@ -57,11 +58,13 @@ pub(crate) fn name(product: &str) -> Option<&str> { } /// Opens the Ark and runs the wire handshake over it, the verifier deciding -/// whether to trust the attestation it presents. +/// whether to trust the attestation it presents. The connection measures its +/// deadlines on the clock. pub(crate) fn connect>( info: &nusb::DeviceInfo, verifier: &V, cloud: impl FnOnce(&crate::Identity) -> Option<(crate::trust::Environment, crate::trust::Realm)>, + clock: &Clock, ) -> Result<(Ark, V::Info), Error> { let device = info.open().wait().map_err(Error::Usb)?; let config = device @@ -121,8 +124,8 @@ pub(crate) fn connect>( // Wrap the endpoints into the wire's reader and writer, each woken by // its own transfers finishing and by the close let closed = Arc::new(AtomicBool::new(false)); - let reads = Arc::new(Notifier::default()); - let writes = Arc::new(Notifier::default()); + let reads = Arc::new(Notifier::new(clock)); + let writes = Arc::new(Notifier::new(clock)); let reader = Reader::new(ep_in, reads.clone(), closed.clone()); let writer = Writer::new(ep_out, writes.clone(), closed.clone()); @@ -179,18 +182,32 @@ impl Transfers for nusb::Endpoint { /// Wakes a direction waiting on its endpoint, a transfer finishing or the /// connection closing being what there is to wake for. -#[derive(Default)] struct Notifier { - woken: Mutex, // Whether a wake arrived since the wait last looked - wake: Condvar, // Signalled on every wake + clock: Clock, // clock that the wire's deadlines are measured on + woken: sync::Mutex, // Whether a wake arrived since the wait last looked + wake: sync::Condvar, // Signalled on every wake } impl Notifier { + /// Creates a notifier whose waits end at deadlines on the clock. + fn new(clock: &Clock) -> Self { + Self { + clock: clock.clone(), + woken: sync::Mutex::new(false), + wake: sync::Condvar::new(clock), + } + } + /// Wakes the waiting direction, or its next wait if none is on. fn notify(&self) { *self.woken.lock().expect("USB wake state not poisoned") = true; self.wake.notify_all(); } + + /// Returns whether an optional deadline has passed on the clock. + fn expired(&self, deadline: Option) -> bool { + deadline.is_some_and(|deadline| self.clock.now() >= deadline) + } } impl Wake for Notifier { @@ -205,11 +222,6 @@ impl Wake for Notifier { } } -/// Returns whether an optional deadline has expired. -fn expired(deadline: Option) -> bool { - deadline.is_some_and(|deadline| Instant::now() >= deadline) -} - /// Waits for the next transfer of the queue to finish, giving up without one /// once the deadline passes or the connection is closed. Without a deadline /// only a finished transfer or the close end the wait. A deadline already @@ -228,7 +240,7 @@ fn finished( } let mut woken = notifier.woken.lock().expect("USB wake state not poisoned"); while !*woken { - if closed.load(Ordering::Acquire) || expired(deadline) { + if closed.load(Ordering::Acquire) || notifier.expired(deadline) { return None; } woken = match deadline { @@ -237,10 +249,9 @@ fn finished( .wait(woken) .expect("USB wake state not poisoned"), Some(deadline) => { - let left = deadline.saturating_duration_since(Instant::now()); notifier .wake - .wait_timeout(woken, left) + .wait_deadline(woken, deadline) .expect("USB wake state not poisoned") .0 } @@ -323,7 +334,7 @@ impl Read for Reader { if self.closed.load(Ordering::Acquire) { return Ok(0); } - if expired(self.deadline) { + if self.notifier.expired(self.deadline) { return Err(io::Error::from(io::ErrorKind::TimedOut)); } let Some(completion) = @@ -349,6 +360,11 @@ impl Read for Reader { } impl transport::Read for Reader { + /// Returns the connection's clock, which the read deadlines are measured on. + fn clock(&self) -> Clock { + self.notifier.clock.clone() + } + /// Bounds future waits without discarding bytes from a completed transfer. fn set_read_deadline(&mut self, deadline: Option) -> io::Result<()> { self.deadline = deadline; @@ -426,7 +442,7 @@ impl Writer { /// without waiting for the rest. fn reap(&mut self) -> io::Result<()> { while self.queue.in_flight() > 0 { - let now = Some(Instant::now()); + let now = Some(self.notifier.clock.now()); let Some(completion) = finished(&mut self.queue, &self.notifier, &self.closed, now) else { return Ok(()); @@ -444,7 +460,7 @@ impl Write for Writer { if self.closed.load(Ordering::Acquire) { return Err(closed()); } - if expired(self.deadline) { + if self.notifier.expired(self.deadline) { return Err(io::Error::from(io::ErrorKind::TimedOut)); } // Chunks already queued stay queued in order, so a wait for room @@ -467,7 +483,7 @@ impl Write for Writer { if self.closed.load(Ordering::Acquire) { return Err(closed()); } - if expired(self.deadline) { + if self.notifier.expired(self.deadline) { return Err(io::Error::from(io::ErrorKind::TimedOut)); } // A frame ending on a packet boundary leaves the device's read open, @@ -484,6 +500,11 @@ impl Write for Writer { } impl transport::Write for Writer { + /// Returns the connection's clock, which the write deadline is measured on. + fn clock(&self) -> Clock { + self.notifier.clock.clone() + } + /// Installs one bound for subsequent writes, queue waits and frame flushes. fn set_write_deadline(&mut self, deadline: Instant) -> io::Result<()> { self.deadline = Some(deadline); @@ -494,7 +515,10 @@ impl transport::Write for Writer { #[cfg(test)] mod tests { use super::*; + use crate::testing::{test_clock, wait_deadline}; + use darkbio_clock::TestClock; use std::collections::VecDeque; + use std::sync::Mutex; use std::thread; use std::time::Duration; @@ -560,6 +584,22 @@ mod tests { } } + // Waits until a direction looked at the ring and then parked on the + // deadline, or without one. The direction is the only thread that waits on + // the clock, and the test clears the ring's waker before it starts. + fn parked(fake: &Fake, tester: &TestClock, deadline: Option) { + while fake.lock().unwrap().waker.is_none() { + thread::yield_now(); + } + match deadline { + Some(deadline) => wait_deadline(tester, deadline), + None => { + tester.wait_blocked(1); + assert_eq!(tester.next_deadline(), None); + } + } + } + // Lengths of the transfers queued, oldest first. fn queued(fake: &Fake) -> Vec { fake.lock() @@ -570,17 +610,17 @@ mod tests { .collect() } - fn reader() -> (Reader, Fake, Arc) { + fn reader(clock: &Clock) -> (Reader, Fake, Arc) { let fake = Fake::default(); let closed = Arc::new(AtomicBool::new(false)); - let reader = Reader::new(fake.clone(), Arc::new(Notifier::default()), closed.clone()); + let reader = Reader::new(fake.clone(), Arc::new(Notifier::new(clock)), closed.clone()); (reader, fake, closed) } - fn writer() -> (Writer, Fake, Arc) { + fn writer(clock: &Clock) -> (Writer, Fake, Arc) { let fake = Fake::default(); let closed = Arc::new(AtomicBool::new(false)); - let writer = Writer::new(fake.clone(), Arc::new(Notifier::default()), closed.clone()); + let writer = Writer::new(fake.clone(), Arc::new(Notifier::new(clock)), closed.clone()); (writer, fake, closed) } @@ -589,7 +629,7 @@ mod tests { // empty one is skipped rather than ending the stream. #[test] fn test_read_serves_transfers() { - let (mut reader, fake, _closed) = reader(); + let (mut reader, fake, _closed) = reader(&test_clock().clock()); assert_eq!(fake.in_flight(), TRANSFERS); finish(&fake, 3, Ok(())); @@ -610,47 +650,51 @@ mod tests { // ends it with the stream. #[test] fn test_read_waits() { - let (mut reader, fake, closed) = reader(); - let mut buf = [0u8; 8]; - - let started = Instant::now(); - transport::Read::set_read_deadline(&mut reader, Some(started + Duration::from_millis(50))) - .unwrap(); - assert_eq!( - reader.read(&mut buf).unwrap_err().kind(), - io::ErrorKind::TimedOut - ); - assert!(started.elapsed() >= Duration::from_millis(50)); + let mut tester = test_clock(); + let clock = tester.clock(); + let (mut reader, fake, closed) = reader(&clock); + let notifier = reader.notifier.clone(); + + // The read waits on its deadline and ends once the clock reaches it + let deadline = clock.now() + Duration::from_millis(50); + transport::Read::set_read_deadline(&mut reader, Some(deadline)).unwrap(); + fake.lock().unwrap().waker = None; + let reading = thread::spawn(move || { + let result = reader.read(&mut [0u8; 8]); + assert!(clock.now() >= deadline); + (reader, result) + }); + parked(&fake, &tester, Some(deadline)); + tester.advance_to(deadline); + let (mut reader, result) = reading.join().unwrap(); + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::TimedOut); transport::Read::set_read_deadline(&mut reader, None).unwrap(); - let arriving = { - let fake = fake.clone(); - thread::spawn(move || { - thread::sleep(Duration::from_millis(20)); - finish(&fake, 5, Ok(())); - }) - }; - assert_eq!(reader.read(&mut buf).unwrap(), 5); - arriving.join().unwrap(); - - let closing = { - let closed = closed.clone(); - let notifier = reader.notifier.clone(); - thread::spawn(move || { - thread::sleep(Duration::from_millis(20)); - closed.store(true, Ordering::Release); - notifier.notify(); - }) - }; - assert_eq!(reader.read(&mut buf).unwrap(), 0); - closing.join().unwrap(); + // A transfer finishing during the wait ends it with its data + fake.lock().unwrap().waker = None; + let reading = thread::spawn(move || { + let result = reader.read(&mut [0u8; 8]); + (reader, result) + }); + parked(&fake, &tester, None); + finish(&fake, 5, Ok(())); + let (mut reader, result) = reading.join().unwrap(); + assert_eq!(result.unwrap(), 5); + + // Closing the connection ends the wait with the stream + fake.lock().unwrap().waker = None; + let reading = thread::spawn(move || reader.read(&mut [0u8; 8])); + parked(&fake, &tester, None); + closed.store(true, Ordering::Release); + notifier.notify(); + assert_eq!(reading.join().unwrap().unwrap(), 0); } // Tests that a failed transfer fails the read, the device going away // reported as the connection lost. #[test] fn test_read_failure() { - let (mut reader, fake, _closed) = reader(); + let (mut reader, fake, _closed) = reader(&test_clock().clock()); finish(&fake, 0, Err(TransferError::Disconnected)); assert_eq!( reader.read(&mut [0u8; 8]).unwrap_err().kind(), @@ -664,7 +708,9 @@ mod tests { // close refuses output. #[test] fn test_write_chunks() { - let (mut writer, fake, closed) = writer(); + let mut tester = test_clock(); + let clock = tester.clock(); + let (mut writer, fake, closed) = writer(&clock); let data = vec![7u8; 100_000]; assert_eq!(writer.write(&data).unwrap(), 100_000); assert_eq!(queued(&fake), [TRANSFER_SIZE, 100_000 - TRANSFER_SIZE]); @@ -675,25 +721,39 @@ mod tests { } assert_eq!(fake.in_flight(), TRANSFERS); - let started = Instant::now(); - transport::Write::set_write_deadline(&mut writer, started + Duration::from_millis(50)) - .unwrap(); - assert_eq!( - writer.write(&chunk).unwrap_err().kind(), - io::ErrorKind::TimedOut - ); - assert!(started.elapsed() >= Duration::from_millis(50)); + // A full ring waits for room until the clock reaches the deadline + let deadline = clock.now() + Duration::from_millis(50); + transport::Write::set_write_deadline(&mut writer, deadline).unwrap(); + fake.lock().unwrap().waker = None; + let writing = thread::spawn({ + let clock = clock.clone(); + let chunk = chunk.clone(); + move || { + let result = writer.write(&chunk); + assert!(clock.now() >= deadline); + (writer, result) + } + }); + parked(&fake, &tester, Some(deadline)); + tester.advance_to(deadline); + let (mut writer, result) = writing.join().unwrap(); + assert_eq!(result.unwrap_err().kind(), io::ErrorKind::TimedOut); // Make one completion available so only the second chunk waits // for its deadline. let two = vec![7u8; 2 * TRANSFER_SIZE]; finish(&fake, 0, Ok(())); - transport::Write::set_write_deadline( - &mut writer, - Instant::now() + Duration::from_millis(100), - ) - .unwrap(); - assert_eq!(writer.write(&two).unwrap(), TRANSFER_SIZE); + let deadline = clock.now() + Duration::from_millis(100); + transport::Write::set_write_deadline(&mut writer, deadline).unwrap(); + fake.lock().unwrap().waker = None; + let writing = thread::spawn(move || { + let result = writer.write(&two); + (writer, result) + }); + parked(&fake, &tester, Some(deadline)); + tester.advance_to(deadline); + let (mut writer, result) = writing.join().unwrap(); + assert_eq!(result.unwrap(), TRANSFER_SIZE); closed.store(true, Ordering::Release); assert_eq!( @@ -712,7 +772,7 @@ mod tests { // flush without the flush draining the ring. #[test] fn test_flush() { - let (mut writer, fake, _closed) = writer(); + let (mut writer, fake, _closed) = writer(&test_clock().clock()); assert_eq!(writer.write(&[1u8; 2 * PACKET]).unwrap(), 2 * PACKET); writer.flush().unwrap(); diff --git a/connect/src/identity.rs b/connect/src/identity.rs index 86d2017..843e77e 100644 --- a/connect/src/identity.rs +++ b/connect/src/identity.rs @@ -74,8 +74,12 @@ impl Verifier for TrustMode { /// Trust outcome returned alongside the authenticated handshake key. type Info = Identity; - /// Verifies the attestation or returns the pinned key selected for recovery. - fn verify(&self, attestation: &Attestation) -> Result<(xdsa::PublicKey, Identity), String> { + /// Verifies the attestation at `now` or returns the pinned key selected for recovery. + fn verify( + &self, + attestation: &Attestation, + now: SystemTime, + ) -> Result<(xdsa::PublicKey, Identity), String> { // Recovery authenticates key possession without consulting the attestation. if let TrustMode::Recover(key) = self { return Ok((*key.clone(), Identity::Recovered(*key.clone()))); @@ -84,7 +88,7 @@ impl Verifier for TrustMode { // from a known root that fails to verify is a hard error, only unknown // signers fall through to the self-signed check. Retain their diagnostic // so an unknown signer is not obscured by the self-signed fallback. - let now = SystemTime::now() + let now = now .duration_since(UNIX_EPOCH) .map_err(|err| err.to_string())? .as_secs(); @@ -118,7 +122,7 @@ impl Verifier for TrustMode { #[cfg(test)] mod tests { use super::*; - use crate::testing::self_attestation; + use crate::testing::{self_attestation, test_clock}; use darkbio_crypto::{cbor, cose}; // Tests that the root trust mode accepts a self-signed attestation with the @@ -126,19 +130,25 @@ mod tests { // recovery mode pins the given identity regardless of the attestation. #[test] fn test_trust_modes() { + let clock = test_clock().clock(); let identity = xdsa::SecretKey::generate(); let foreign = xdsa::SecretKey::generate(); // Self-signed attestation proves possession of its key only. - let attestation = self_attestation(&identity, identity.public_key()); - let (key, info) = TrustMode::RootOrSelf.verify(&attestation).unwrap(); + let attestation = self_attestation(&identity, identity.public_key(), &clock); + let (key, info) = TrustMode::RootOrSelf + .verify(&attestation, clock.system_time()) + .unwrap(); assert_eq!(key.fingerprint(), identity.public_key().fingerprint()); assert!(matches!(info, Identity::SelfSigned(_))); assert_eq!(info.realm(), None); // Attestation by an unknown key, refused - let attestation = self_attestation(&foreign, identity.public_key()); - let error = TrustMode::RootOrSelf.verify(&attestation).err().unwrap(); + let attestation = self_attestation(&foreign, identity.public_key(), &clock); + let error = TrustMode::RootOrSelf + .verify(&attestation, clock.system_time()) + .err() + .unwrap(); assert!(error.contains(&hex::encode(foreign.fingerprint().to_bytes()))); assert!(error.contains("unknown key")); assert!(!error.contains("--features")); @@ -146,7 +156,7 @@ mod tests { // Recovery mode, the pinned key is used regardless of the attestation let pinned = xdsa::SecretKey::generate().public_key(); let (key, info) = TrustMode::Recover(Box::new(pinned.clone())) - .verify(&attestation) + .verify(&attestation, clock.system_time()) .unwrap(); assert_eq!(key.fingerprint(), pinned.fingerprint()); assert!(matches!(info, Identity::Recovered(_))); @@ -156,8 +166,9 @@ mod tests { /// Known device roots verify signatures instead of accepting claimed fingerprints. #[test] fn test_known_signer() { + let clock = test_clock().clock(); let key = xdsa::SecretKey::generate(); - let attestation = self_attestation(&key, key.public_key()); + let attestation = self_attestation(&key, key.public_key(), &clock); for fingerprint in [ "8b842c20bb8083a1635140e58675f3b95a100ac0e39ab82fa6cb2ef23eb532fb", // Release hardware "7d725c5cb3f80ef4e17bb98ea1f14683714a7eaffaa78cded17ff0e2c0c96ffa", // Staging hardware @@ -173,7 +184,10 @@ mod tests { header.kid = claimed; envelope.protected = cbor::encode(&header).unwrap(); let forged = Attestation::new(cbor::encode(&envelope).unwrap()).unwrap(); - let error = TrustMode::RootOrSelf.verify(&forged).err().unwrap(); + let error = TrustMode::RootOrSelf + .verify(&forged, clock.system_time()) + .err() + .unwrap(); if root.role == trust::roots::Role::CloudAttester { assert!(error.contains(fingerprint), "{error}"); assert!(error.contains(&root.to_string()), "{error}"); @@ -187,12 +201,16 @@ mod tests { /// Invalid self-signatures retain their cryptographic error. #[test] fn test_invalid_attestation() { + let clock = test_clock().clock(); let key = xdsa::SecretKey::generate(); - let attestation = self_attestation(&key, key.public_key()); + let attestation = self_attestation(&key, key.public_key(), &clock); let mut bytes = attestation.as_bytes().to_vec(); *bytes.last_mut().unwrap() ^= 1; let attestation = Attestation::new(bytes).unwrap(); - let error = TrustMode::RootOrSelf.verify(&attestation).err().unwrap(); + let error = TrustMode::RootOrSelf + .verify(&attestation, clock.system_time()) + .err() + .unwrap(); assert!(error.starts_with("cwt:"), "{error}"); assert!(!error.contains("--features")); assert!(!error.contains("not among the trusted roots")); diff --git a/connect/src/incoming.rs b/connect/src/incoming.rs index 792fb96..7942f5e 100644 --- a/connect/src/incoming.rs +++ b/connect/src/incoming.rs @@ -10,7 +10,6 @@ use crate::cloud::Services; use darkbio_wire::protocol::{self, Responder, Session, schema}; use std::collections::VecDeque; use std::sync::{Arc, Condvar, Mutex}; -use std::time::Instant; /// Bounded application queue. Closure discards requests and retains the first /// ending reason for every subsequent receive. @@ -73,13 +72,12 @@ impl Incoming { || bytes > protocol::DEFAULT_MAX_INBOUND_BYTES.saturating_sub(state.bytes) { drop(state); - let _ = responder.fail( - schema::Error::reserved( - schema::ReservedErrors::Unavailable, - "host request queue full", - ), - Instant::now() + protocol::DEFAULT_AUTOREPLY_TIMEOUT, + let error = schema::Error::reserved( + schema::ReservedErrors::Unavailable, + "host request queue full", ); + let deadline = responder.clock().now() + protocol::DEFAULT_AUTOREPLY_TIMEOUT; + let _ = responder.fail(error, deadline); return; } state.bytes += bytes; diff --git a/connect/src/lib.rs b/connect/src/lib.rs index 2d85e90..cd15384 100644 --- a/connect/src/lib.rs +++ b/connect/src/lib.rs @@ -21,14 +21,15 @@ //! //! ```no_run //! use darkbio_connect::{Error, TrustMode, schema}; -//! use std::time::{Duration, Instant}; +//! use std::time::Duration; //! //! # fn main() -> Result<(), Error> { //! let found = darkbio_connect::list(); //! let device = found.select(None)?; //! let (ark, identity) = device.connect(&TrustMode::RootOrSelf)?; -//! let info = ark.client().call( -//! schema::DeviceInfoRequest {}, Instant::now() + Duration::from_secs(10), +//! let client = ark.client(); +//! let info = client.call( +//! schema::DeviceInfoRequest {}, client.clock().now() + Duration::from_secs(10), //! )?; //! # Ok(()) //! # } @@ -47,10 +48,11 @@ //! //! Calls and workflows accept an [`Instant`](std::time::Instant) for one fixed //! deadline or [`Timing`] for an inactivity allowance, optionally combined with -//! that deadline. No timeout state lives on the shared client. Approval requests -//! use their protocol window when an inactivity allowance is supplied. Arbitrary -//! readers and progress callbacks run on the caller's thread and must bound their -//! own blocking work. +//! that deadline. No timeout state lives on the shared client. Deadlines are +//! measured on the connection's [`clock::Clock`], which [`Client::clock`] returns. +//! Approval requests use their protocol window when an inactivity allowance is +//! supplied. Arbitrary readers and progress callbacks run on the caller's thread +//! and must bound their own blocking work. //! //! [`Client::identify_dataset`] identifies a file without opening an upload session. //! [`Client::upload_dataset`] streams a [`Dataset`] and waits for processing. @@ -102,6 +104,8 @@ mod testing; pub use ark::{Ark, Client, Closer, Pending}; pub use cloud::{CloudAuth, Firmware, PairingProgress, Registration, UpdateProgress, cloud_synced}; +/// Clocks that connections measure their deadlines on, as [`Client::clock`] returns. +pub use darkbio_clock as clock; pub use darkbio_wire as wire; pub use darkbio_wire::protocol::schema; pub use darkbio_wire::protocol::{CodedError, Promise, Responder}; diff --git a/connect/src/testing.rs b/connect/src/testing.rs index 25fc29f..090dcba 100644 --- a/connect/src/testing.rs +++ b/connect/src/testing.rs @@ -4,9 +4,10 @@ // Use of this source code is governed by a BSD-style // license that can be found in the LICENSE file. -//! Scripted wire peers and attestations for connection and trust tests. +//! Scripted wire peers, attestations and test clocks for connection and trust tests. use crate::{Ark, Error, Identity, TrustMode}; +use darkbio_clock::{Clock, TestClock}; use darkbio_crypto::cwt::claims::{self, eat}; use darkbio_crypto::{cwt, xdsa}; use darkbio_trust::CRYPTO_DOMAIN_DEVICE_ATTESTATION; @@ -17,15 +18,37 @@ use darkbio_wire::protocol::schema::{self, DeviceInfoResponse, UnlockResponse}; use darkbio_wire::protocol::{Responder, Server, Session}; use darkbio_wire::transport::Attestation; use std::thread::{self, JoinHandle}; -use std::time::{Duration, Instant}; +use std::time::{Duration, Instant, UNIX_EPOCH}; /// Bytes buffered per direction of a peer's stream, enough for the handshake /// and a few messages to flow without the other side reading. const CAPACITY: usize = 256 * 1024; -/// Hardware attestation of an identity, signed by the given key. Signed by the -/// identity itself, it is the placeholder of an Ark that was never onboarded. -pub fn self_attestation(signer: &xdsa::SecretKey, identity: xdsa::PublicKey) -> Attestation { +/// Creates a stopped clock a day ahead of real time, so a stray read of the +/// real clock stands out from the test's time. +pub fn test_clock() -> TestClock { + let mut tester = TestClock::new(); + tester.advance(Duration::from_secs(86400)); + tester +} + +/// Blocks until the earliest wait or timer on the clock is due at `deadline`. +/// The advance that reaches the deadline then wakes it, whenever the test makes +/// that advance. +pub fn wait_deadline(tester: &TestClock, deadline: Instant) { + while tester.next_deadline() != Some(deadline) { + thread::yield_now(); + } +} + +/// Hardware attestation of an identity, signed by the given key at the clock's +/// wall time. Signed by the identity itself, it is the placeholder of an Ark +/// that was never onboarded. +pub fn self_attestation( + signer: &xdsa::SecretKey, + identity: xdsa::PublicKey, + clock: &Clock, +) -> Attestation { let claims = HardwareClaims { sub: claims::Subject { sub: "test-device".into(), @@ -37,7 +60,18 @@ pub fn self_attestation(signer: &xdsa::SecretKey, identity: xdsa::PublicKey) -> hwm: eat::HwModel { hw_model: vec![] }, hwv: eat::HwVersion::new("test-version".into()), }; - let cwt = cwt::issue(&claims, signer, CRYPTO_DOMAIN_DEVICE_ATTESTATION).unwrap(); + let timestamp = clock + .system_time() + .duration_since(UNIX_EPOCH) + .unwrap() + .as_secs(); + let cwt = cwt::issue_at( + &claims, + signer, + CRYPTO_DOMAIN_DEVICE_ATTESTATION, + timestamp as i64, + ) + .unwrap(); Attestation::new(cwt).unwrap() } @@ -49,7 +83,7 @@ pub type Script = Box bool + Send>; /// requests with a request of the peer's own ahead of the reply, and anything /// else with the protocol's UNSUPPORTED error. pub fn answering(session: &Session, request: Request, responder: Responder) -> bool { - let deadline = Instant::now() + Duration::from_secs(10); + let deadline = session.clock().now() + Duration::from_secs(10); let queued = match request { Request::DeviceInfo(_) => responder.reply( DeviceInfoResponse { @@ -104,13 +138,14 @@ pub struct Peer { } impl Peer { - /// Starts a peer serving the client per the script. - pub fn spawn(mut script: Script) -> Self { + /// Starts a peer serving the client per the script. Both ends of its stream + /// measure their deadlines on the clock. + pub fn spawn(clock: &Clock, mut script: Script) -> Self { let signer = xdsa::SecretKey::generate(); let identity = signer.public_key(); - let attestation = self_attestation(&signer, identity.clone()); + let attestation = self_attestation(&signer, identity.clone(), clock); - let (host, ark) = memory::duplex(CAPACITY); + let (host, ark) = memory::duplex(CAPACITY, clock); let thread = thread::spawn(move || { let mut server = Server::new(ark, signer, attestation); let Ok(mut session) = server.accept() else { diff --git a/connect/src/timing.rs b/connect/src/timing.rs index e4e8f71..217fb20 100644 --- a/connect/src/timing.rs +++ b/connect/src/timing.rs @@ -6,6 +6,8 @@ //! Bounds carried by one operation, never stored on a shared client. +use darkbio_clock::Clock; +use std::io; use std::time::{Duration, Instant}; /// Device approval window with time for relay forwarding and the final reply. @@ -53,21 +55,21 @@ impl Timing { self } - /// Deadline for the next machine response. - pub(crate) fn io(self) -> Instant { - self.bound(self.inactivity) + /// Deadline for the next machine response, measured on the clock. + pub(crate) fn io(self, clock: &Clock) -> Instant { + self.bound(clock, self.inactivity) } /// Approval windows include a small allowance for forwarding and replies. /// Callers using only an absolute deadline retain that exact bound. - pub(crate) fn approval(self) -> Instant { - self.window(APPROVAL_WINDOW) + pub(crate) fn approval(self, clock: &Clock) -> Instant { + self.window(clock, APPROVAL_WINDOW) } /// Replaces an inactivity allowance with a protocol-specific wait window. /// An absolute-only timing retains its original deadline. - pub(crate) fn window(self, window: Duration) -> Instant { - self.bound(self.inactivity.map(|_| window)) + pub(crate) fn window(self, clock: &Clock, window: Duration) -> Instant { + self.bound(clock, self.inactivity.map(|_| window)) } /// Clips a protocol deadline without applying the machine wait allowance. @@ -77,10 +79,10 @@ impl Timing { /// Caller-supplied readers own their per-read timeout. Only a workflow's /// absolute deadline can expire while an otherwise active reader runs. - pub(crate) fn check(self) -> Result<(), crate::Error> { + pub(crate) fn check(self, clock: &Clock) -> Result<(), crate::Error> { if self .deadline - .is_some_and(|deadline| Instant::now() >= deadline) + .is_some_and(|deadline| clock.now() >= deadline) { Err(crate::Error::Timeout) } else { @@ -88,25 +90,26 @@ impl Timing { } } - /// Poll cadence is independent of the response allowance. - pub(crate) fn pause(self, interval: Duration) -> Result<(), crate::Error> { + /// Poll cadence is independent of the response allowance. The pause sleeps + /// on the clock, never past the absolute deadline. + pub(crate) fn pause(self, clock: &Clock, interval: Duration) -> Result<(), crate::Error> { let wait = match self.deadline { Some(deadline) => interval.min( deadline - .checked_duration_since(Instant::now()) + .checked_duration_since(clock.now()) .filter(|wait| !wait.is_zero()) .ok_or(crate::Error::Timeout)?, ), None => interval, }; - std::thread::sleep(wait); + clock.sleep(wait); Ok(()) } /// Chooses the earlier bound, treating duration overflow as immediate expiry. - fn bound(self, timeout: Option) -> Instant { + fn bound(self, clock: &Clock, timeout: Option) -> Instant { let wait = timeout.map(|timeout| { - let now = Instant::now(); + let now = clock.now(); now.checked_add(timeout).unwrap_or(now) }); match (self.deadline, wait) { @@ -125,27 +128,47 @@ impl From for Timing { } } +/// Time left on a clock before a deadline, as blocking OS calls take it. +pub(crate) trait ClockExt { + /// Returns the time left before the deadline as a positive OS timeout. A + /// passed deadline fails with `TimedOut`, since a zero timeout means an + /// unbounded wait on some APIs. + fn remaining(&self, deadline: Instant) -> io::Result; +} + +impl ClockExt for Clock { + fn remaining(&self, deadline: Instant) -> io::Result { + deadline + .checked_duration_since(self.now()) + .filter(|left| !left.is_zero()) + .ok_or_else(|| io::Error::from(io::ErrorKind::TimedOut)) + } +} + /// Deadline clipping and protocol window regressions. #[cfg(test)] mod tests { use super::*; + use crate::testing::test_clock; /// A renewed machine allowance and a longer approval window cannot extend /// a caller's absolute workflow deadline. #[test] fn test_absolute_bound() { - let deadline = Instant::now() + Duration::from_secs(1); + let clock = test_clock().clock(); + let deadline = clock.now() + Duration::from_secs(1); let timing = Timing::inactivity(Duration::from_secs(60)).with_deadline(deadline); - assert_eq!(timing.io(), deadline); - assert_eq!(timing.approval(), deadline); - assert_eq!(Timing::until(deadline).io(), deadline); + assert_eq!(timing.io(&clock), deadline); + assert_eq!(timing.approval(&clock), deadline); + assert_eq!(Timing::until(deadline).io(&clock), deadline); } /// Short inactivity limits do not shorten the device's approval window. #[test] fn test_approval_window() { + let clock = test_clock().clock(); let timing = Timing::inactivity(Duration::from_millis(10)); - assert!(timing.io() < Instant::now() + Duration::from_secs(1)); - assert!(timing.approval() > Instant::now() + Duration::from_secs(39)); + assert_eq!(timing.io(&clock), clock.now() + Duration::from_millis(10)); + assert!(timing.approval(&clock) > clock.now() + Duration::from_secs(39)); } } diff --git a/src/access.rs b/src/access.rs index 820fff3..40c5636 100644 --- a/src/access.rs +++ b/src/access.rs @@ -7,24 +7,30 @@ //! Cloudflare Access login for internal API and package hosts. use crate::{context::Context, error::Error}; +use darkbio_clock::{Clock, crossbeam_channel}; use darkbio_connect::Error as ConnectError; use std::io::{self, Read}; -use std::process::{Command, Stdio}; -use std::sync::mpsc; +use std::process::{Command, ExitStatus, Stdio}; use std::thread; use std::time::{Duration, Instant}; use ureq::http::{HeaderMap, HeaderValue, StatusCode}; +/// Pace of the polls that observe a running helper's exit. +const POLL: Duration = Duration::from_millis(10); + /// Prompt policy copied into a connection without retaining its session owner. pub(crate) struct Login { + clock: Clock, // connection's clock, which its deadlines are measured on output: crate::output::Output, // Invocation's shared diagnostic stream interactive: bool, // Whether browser login is permitted } impl Login { /// Captures CLI policy without looking up credentials or contacting a host. - pub fn new(context: &Context) -> Self { + /// The connection's clock measures the deadlines it passes in. + pub fn new(context: &Context, clock: Clock) -> Self { Self { + clock, output: context.output.clone(), interactive: context.interactive(), } @@ -42,6 +48,7 @@ impl darkbio_connect::CloudAuth for Login { } token( Command::new("cloudflared").args(["access", "token", "--app", origin]), + &self.clock, deadline, ) .ok() @@ -62,7 +69,7 @@ impl darkbio_connect::CloudAuth for Login { ) { return Err("cloud access refused for an unsupported host".into()); } - let window = Instant::now() + Duration::from_secs(600); + let window = self.clock.now() + Duration::from_secs(600); let token = self.authenticate(origin, deadline.map_or(window, |bound| bound.min(window)))?; headers(&token) @@ -91,26 +98,34 @@ impl Login { ); token( Command::new("cloudflared").args(["access", "login", "--app", origin]), + &self.clock, deadline, ) .map_err(|error| error.to_string()) } } -/// Asks cloudflared for a cached application token, treating failure as no credentials. -pub(crate) fn cached(context: &Context, origin: &str) -> Option { +/// Asks cloudflared for a cached application token, treating failure as no +/// credentials. The lookup gets one command budget on the connection's clock. +pub(crate) fn cached(context: &Context, clock: &Clock, origin: &str) -> Option { token( Command::new("cloudflared").args(["access", "token", "--app", origin]), - context.deadline(), + clock, + context.deadline(clock), ) .ok() } /// Starts browser login only when stdin prompts are permitted, under a separate -/// human login window. Noninteractive callers receive the manual login command. -pub(crate) fn authenticate(context: &Context, origin: &str) -> Result { - Login::new(context) - .authenticate(origin, Instant::now() + Duration::from_secs(600)) +/// human login window on the connection's clock. Noninteractive callers receive +/// the manual login command. +pub(crate) fn authenticate( + context: &Context, + clock: &Clock, + origin: &str, +) -> Result { + Login::new(context, clock.clone()) + .authenticate(origin, clock.now() + Duration::from_secs(600)) .map_err(|err| { Error::new(4, "login-required", err) .hint(format!("run `cloudflared access login --app {origin}`")) @@ -164,8 +179,10 @@ fn challenge(origin: &str, redirect: &str) -> bool { /// Captures credentials without forwarding helper output to the terminal. /// Expiration kills and reaps the helper, including while its output is blocked. -fn token(command: &mut Command, deadline: Instant) -> Result { - remaining(deadline)?; +/// The helper's exit and output arrive as events, which the deadline bounds on +/// the clock. +fn token(command: &mut Command, clock: &Clock, deadline: Instant) -> Result { + remaining(clock, deadline)?; let mut child = Child( command .stdin(Stdio::null()) @@ -184,7 +201,7 @@ fn token(command: &mut Command, deadline: Instant) -> Result Result ConnectError::Timeout, - mpsc::RecvTimeoutError::Disconnected => { - ConnectError::Cloud("cloudflared output reader stopped".into()) - } - })? - .map_err(|error| { - ConnectError::Cloud(format!("could not read cloudflared token: {error}")) - })?; + + // Watch the helper on a thread of its own, which reports its exit + let (report, exited) = crossbeam_channel::bounded(1); + let (abandon, abandoned) = crossbeam_channel::bounded(0); + let watcher = thread::Builder::new() + .name("access-helper".into()) + .spawn(move || watch(child, report, abandoned)) + .map_err(ConnectError::Worker)?; + + // Wait for the exit and the output, then stop the watcher, which kills and + // reaps a helper that still runs + let result = settle(clock, deadline, &exited, &output); + drop(abandon); + let _ = watcher.join(); + let output = result?; let token = String::from_utf8_lossy(&output); let token = token.trim(); if output.len() > 64 * 1024 @@ -239,10 +246,78 @@ fn token(command: &mut Command, deadline: Instant) -> Result Result { +/// Waits until the deadline for the helper's exit and then for its output. A +/// failed helper is reported by its exit status, never by its output. +fn settle( + clock: &Clock, + deadline: Instant, + exited: &crossbeam_channel::Receiver>, + output: &crossbeam_channel::Receiver>>, +) -> Result, ConnectError> { + // The exit comes first, since a failed helper's output is never read + remaining(clock, deadline)?; + let status = clock + .recv_deadline(exited, deadline) + .map_err(|error| match error { + crossbeam_channel::RecvTimeoutError::Timeout => ConnectError::Timeout, + crossbeam_channel::RecvTimeoutError::Disconnected => { + ConnectError::Cloud("cloudflared watcher stopped".into()) + } + })? + .map_err(|error| ConnectError::Cloud(format!("could not wait for cloudflared: {error}")))?; + if !status.success() { + return Err(ConnectError::Cloud(format!( + "cloudflared login failed ({status})" + ))); + } + // The output follows within the same deadline + remaining(clock, deadline)?; + clock + .recv_deadline(output, deadline) + .map_err(|error| match error { + crossbeam_channel::RecvTimeoutError::Timeout => ConnectError::Timeout, + crossbeam_channel::RecvTimeoutError::Disconnected => { + ConnectError::Cloud("cloudflared output reader stopped".into()) + } + })? + .map_err(|error| ConnectError::Cloud(format!("could not read cloudflared token: {error}"))) +} + +/// Polls the helper until it exits and reports the exit. A waiter that gives +/// up disconnects `abandoned`, which ends the polls at once. Dropping the +/// helper kills and reaps it either way. +#[expect( + clippy::disallowed_methods, + reason = "try_wait observes the helper's exit, and real time only paces its polls" +)] +fn watch( + mut child: Child, + report: crossbeam_channel::Sender>, + abandoned: crossbeam_channel::Receiver<()>, +) { + loop { + match child.0.try_wait() { + Ok(Some(status)) => { + let _ = report.send(Ok(status)); + return; + } + Ok(None) => {} + Err(error) => { + let _ = report.send(Err(error)); + return; + } + } + if abandoned.recv_timeout(POLL) != Err(crossbeam_channel::RecvTimeoutError::Timeout) { + return; + } + } +} + +/// Requires a positive remaining helper budget on the clock before spawning or +/// receiving. +fn remaining(clock: &Clock, deadline: Instant) -> Result { deadline - .checked_duration_since(Instant::now()) + .checked_duration_since(clock.now()) .filter(|duration| !duration.is_zero()) .ok_or(ConnectError::Timeout) } @@ -261,6 +336,8 @@ impl Drop for Child { #[cfg(test)] mod tests { use super::*; + use crate::testing::wait_deadline; + use darkbio_clock::TestClock; #[test] fn access_refusals_do_not_include_device_proof_errors() { @@ -310,7 +387,9 @@ mod tests { use clap::Parser; use darkbio_connect::CloudAuth; let options = crate::args::Cli::parse_from(["ark", "--no-input"]).options; + let clock = TestClock::new().clock(); let login = Login { + clock: clock.clone(), output: crate::output::Output::new(&options), interactive: false, }; @@ -326,7 +405,7 @@ mod tests { ); assert!( login - .headers("https://api.dark.bio", Instant::now()) + .headers("https://api.dark.bio", clock.now()) .is_empty() ); } @@ -351,10 +430,12 @@ mod tests { #[cfg(unix)] #[test] fn test_token() { - let deadline = Instant::now() + Duration::from_secs(5); + let clock = TestClock::new().clock(); + let deadline = clock.now() + Duration::from_secs(5); assert_eq!( token( Command::new("sh").args(["-c", "printf 'e30.e30.signature\\n'"]), + &clock, deadline ) .unwrap(), @@ -366,11 +447,13 @@ mod tests { "printf 'e30.e30.private-token'; exit 1", "head -c 70000 /dev/zero", ] { - let error = token(Command::new("sh").args(["-c", script]), deadline).unwrap_err(); + let error = + token(Command::new("sh").args(["-c", script]), &clock, deadline).unwrap_err(); assert!(!error.to_string().contains("private-token")); } let error = token( &mut Command::new("/nonexistent/ark-test-cloudflared"), + &clock, deadline, ) .unwrap_err(); @@ -382,21 +465,115 @@ mod tests { #[cfg(unix)] #[test] fn test_deadline() { + // An expired deadline refuses before starting anything + let mut tester = TestClock::new(); + let clock = tester.clock(); assert!(matches!( token( &mut Command::new("/nonexistent/ark-test-cloudflared"), - Instant::now() + &clock, + clock.now() ), Err(ConnectError::Timeout) )); - let start = Instant::now(); + + // A helper announces its start through a named pipe, then would run for an hour + let pipe = std::env::temp_dir().join(format!("ark-access-test-{}", std::process::id())); + assert!( + Command::new("mkfifo") + .arg(&pipe) + .status() + .unwrap() + .success() + ); + let script = format!("echo started > '{}'; exec sleep 3600", pipe.display()); + let deadline = clock.now() + Duration::from_millis(50); + let waiting = thread::spawn(move || { + token(Command::new("sh").args(["-c", &script]), &clock, deadline) + }); + std::fs::read(&pipe).unwrap(); + std::fs::remove_file(&pipe).unwrap(); + + // Reaching the deadline during the wait for the exit ends it with a + // timeout. The helper is killed and reaped before the call returns. + wait_deadline(&tester, deadline); + tester.advance_to(deadline); assert!(matches!( - token( - Command::new("sh").args(["-c", "exec sleep 5"]), - start + Duration::from_millis(50) - ), + waiting.join().unwrap(), Err(ConnectError::Timeout) )); - assert!(start.elapsed() < Duration::from_secs(1)); + } + + /// Exit status of a helper that ended with `code`. + #[cfg(unix)] + fn exited(code: i32) -> ExitStatus { + std::os::unix::process::ExitStatusExt::from_raw(code << 8) + } + + /// Exit status of a helper that ended with `code`. + #[cfg(windows)] + fn exited(code: i32) -> ExitStatus { + std::os::windows::process::ExitStatusExt::from_raw(code as u32) + } + + /// The helper's exit and then its output settle the wait. A failed or + /// unobservable exit fails it without the output, and the deadline ends + /// the wait for either event. + #[test] + fn test_settle() { + // Settles one helper's events on a thread, under a deadline a second + // away. An exit given here is queued before the wait starts. + let mut tester = TestClock::new(); + let settling = |tester: &TestClock, exit: Option>| { + let clock = tester.clock(); + let deadline = clock.now() + Duration::from_secs(1); + let (exits, exited) = crossbeam_channel::unbounded(); + let (output, outputs) = crossbeam_channel::unbounded(); + if let Some(exit) = exit { + exits.send(exit).unwrap(); + } + let result = thread::spawn(move || settle(&clock, deadline, &exited, &outputs)); + (exits, output, deadline, result) + }; + + // The exit and the output arrive while the wait is on + let (exit, output, deadline, result) = settling(&tester, None); + wait_deadline(&tester, deadline); + exit.send(Ok(exited(0))).unwrap(); + output.send(Ok(b"e30.e30.signature".to_vec())).unwrap(); + assert_eq!(result.join().unwrap().unwrap(), b"e30.e30.signature"); + + // A failed exit fails the wait without the output queued ahead of it + let (exit, output, _, result) = settling(&tester, None); + output.send(Ok(b"e30.e30.private-token".to_vec())).unwrap(); + exit.send(Ok(exited(1))).unwrap(); + let error = result.join().unwrap().unwrap_err().to_string(); + assert!(error.contains("login failed") && !error.contains("private-token")); + + // So does an exit the watcher could not observe + let (exit, _output, _, result) = settling(&tester, None); + exit.send(Err(io::Error::other("no such process"))).unwrap(); + assert!( + result + .join() + .unwrap() + .unwrap_err() + .to_string() + .contains("could not wait") + ); + + // The deadline ends a wait for the exit, the only wait with a timer + let (_exit, _output, deadline, result) = settling(&tester, None); + wait_deadline(&tester, deadline); + tester.advance_to(deadline); + assert!(matches!(result.join().unwrap(), Err(ConnectError::Timeout))); + + // It also ends a wait for the output. The exit is queued first, so its + // receive arms no timer and the one armed belongs to the output wait. + let (_exit, _output, deadline, result) = settling(&tester, Some(Ok(exited(0)))); + tester.wait_timers(1); + assert_eq!(tester.next_deadline(), Some(deadline)); + tester.advance_to(deadline); + assert!(matches!(result.join().unwrap(), Err(ConnectError::Timeout))); } } diff --git a/src/args.rs b/src/args.rs index a0bcb31..b915c50 100644 --- a/src/args.rs +++ b/src/args.rs @@ -312,6 +312,10 @@ pub(crate) fn slot_name(id: i32) -> String { } /// Reject durations that cannot be represented as monotonic deadlines. +#[expect( + clippy::disallowed_methods, + reason = "flags are parsed before any connection clock exists, and the timeout must fit a deadline on the real monotonic clock that connections run on" +)] fn parse_timeout(value: &str) -> Result { let seconds = value .parse::() diff --git a/src/context.rs b/src/context.rs index 673ac3a..3517733 100644 --- a/src/context.rs +++ b/src/context.rs @@ -11,6 +11,7 @@ use crate::{ error::Error, output::Output, }; +use darkbio_clock::Clock; use darkbio_connect::{ Ark, Client, Device, Identity, Timing, TrustMode, schema, trust::Environment, }; @@ -62,9 +63,10 @@ impl Context { pub fn timing(&self) -> Timing { Timing::inactivity(Duration::from_secs(self.options.timeout)) } - /// Starts one fixed wait budget, for operations that need a single bound. - pub fn deadline(&self) -> Instant { - Instant::now() + Duration::from_secs(self.options.timeout) + /// Starts one fixed wait budget on a connection's clock, for operations + /// that need a single bound. + pub fn deadline(&self, clock: &Clock) -> Instant { + clock.now() + Duration::from_secs(self.options.timeout) } /// Permits stdin prompts only for a terminal outside JSON and no-input modes. pub fn interactive(&self) -> bool { @@ -104,7 +106,8 @@ impl Context { /// Opens an endpoint with CLI routing precedence and records it for interruption. /// An optional reboot deadline bounds device-info I/O; transport establishment - /// and the wire handshake retain their own connection timeouts. + /// and the wire handshake retain their own connection timeouts. The output + /// and the login helper time their waits on the new connection's clock. pub fn open_until( &self, device: Device, @@ -118,6 +121,8 @@ impl Context { let (mut ark, identity) = device.connect_with_env(&trust, |identity| { environment(self.options.env, identity, device.env()) })?; + let client = ark.client(); + self.output.connection(client.clock()); let env = environment(self.options.env, &identity, device.env()); if let Identity::Attested { env: attested, .. } = &identity && self.options.env.is_some_and(|env| env != *attested) @@ -129,9 +134,8 @@ impl Context { } self.output.environment(env); if env != Environment::Release { - ark.set_cloud_auth(crate::access::Login::new(self)); + ark.set_cloud_auth(crate::access::Login::new(self, client.clock())); } - let client = ark.client(); self.interrupt.connection(client.clone(), ark.closer()); let info = client.call(schema::DeviceInfoRequest {}, timing)?; self.output.event("step", "connected and authenticated"); diff --git a/src/data/mod.rs b/src/data/mod.rs index 33787c9..671d92d 100644 --- a/src/data/mod.rs +++ b/src/data/mod.rs @@ -19,13 +19,13 @@ use crate::{ error::Error, progress::{Processing, Transfer, Update}, }; +use darkbio_clock::Clock; use darkbio_connect::{ Dataset, UploadProgress, schema::{self, SlotState, SlotStatus}, }; use serde_json::{Value, json}; use std::io::Seek; -use std::time::Instant; /// Dispatches dataset commands after applying CLI unlock and dry-run policy. /// Path entries, slot metadata and refusal messages come from the Ark; the CLI @@ -251,8 +251,9 @@ fn upload( .document_with(&value, |theme| crate::output::human::document(theme, &view)); } file.rewind()?; - let started = Instant::now(); - let mut progress = Progress::new(context, identified.kind); + let clock = connection.client.clock(); + let started = clock.now(); + let mut progress = Progress::new(context, clock.clone(), identified.kind); context.interrupt.partial(value.clone()); let result = connection.client.upload_dataset( &Dataset { @@ -267,14 +268,14 @@ fn upload( progress.update(stage); value["uploaded_bytes"] = json!(progress.uploaded); value["phases"] = json!(progress.phases); - value["duration_seconds"] = json!(started.elapsed().as_secs()); + value["duration_seconds"] = json!(clock.elapsed(started).as_secs()); context.interrupt.partial(value.clone()); }, ); context.interrupt.clear(); value["uploaded_bytes"] = json!(progress.uploaded); value["phases"] = json!(progress.phases); - value["duration_seconds"] = json!(started.elapsed().as_secs()); + value["duration_seconds"] = json!(clock.elapsed(started).as_secs()); context.output.document(&value)?; result.map_err(Into::into) } @@ -298,13 +299,13 @@ pub(crate) struct Progress<'a> { } impl<'a> Progress<'a> { /// Starts a new dataset's progress without inheriting a previous transfer's estimates. - pub fn new(context: &'a Context, slot: i32) -> Self { + pub fn new(context: &'a Context, clock: Clock, slot: i32) -> Self { Self { context, slot, - transfer: Transfer::new(context.output.terminal()), + transfer: Transfer::new(context.output.terminal(), clock.clone()), last_upload: None, - processing: Processing::new(context.output.terminal()), + processing: Processing::new(context.output.terminal(), clock), uploaded: 0, phases: Vec::new(), } diff --git a/src/data/reference.rs b/src/data/reference.rs index 5de4f0c..a31d40c 100644 --- a/src/data/reference.rs +++ b/src/data/reference.rs @@ -18,6 +18,7 @@ use crate::{ http, output::Output, }; +use darkbio_clock::Clock; use darkbio_connect::{Dataset, schema::SlotStatus}; use serde_json::{Value, json}; use sha2::{Digest, Sha256}; @@ -230,10 +231,15 @@ fn install( directory: Option<&Path>, ) -> Result<(), Error> { let mut directory = directory; + let clock = connection.client.clock(); if let Some(directory) = directory { let path = directory.join(&source.hash); if let Ok(mut file) = File::open(&path) { - let mut progress = Progress::new(context, source.dataset.slot.expect("reference slot")); + let mut progress = Progress::new( + context, + clock.clone(), + source.dataset.slot.expect("reference slot"), + ); let result = connection.client.upload_dataset( &source.dataset, &mut file, @@ -280,9 +286,13 @@ fn install( } None => None, }; - let mut reader = Reader::new(&agent, source, &context.output, entry)?; + let mut reader = Reader::new(&agent, source, &context.output, clock.clone(), entry)?; let last_upload = reader.last_upload.clone(); - let mut progress = Progress::new(context, source.dataset.slot.expect("reference slot")); + let mut progress = Progress::new( + context, + clock.clone(), + source.dataset.slot.expect("reference slot"), + ); let result = connection.client.upload_dataset( &source.dataset, &mut reader, @@ -293,7 +303,7 @@ fn install( darkbio_connect::UploadProgress::Started { .. } | darkbio_connect::UploadProgress::Uploading { .. } ) { - last_upload.set(Some(Instant::now())); + last_upload.set(Some(clock.now())); } progress.update(stage); }, @@ -392,6 +402,8 @@ fn valid_range(response: &ureq::http::Response, offset: u64, size: u /// Hashes both sources together and marks network failures for download retry policy. /// The cache lock stays held across replay, response validation and queued writes. struct Reader<'a> { + /// Connection clock that measures the upload window. + clock: Clock, /// HTTP client whose first request waits until the prefix has been read. agent: &'a ureq::Agent, /// Advertised URL, length and digest for this attempt. @@ -426,6 +438,7 @@ impl<'a> Reader<'a> { agent: &'a ureq::Agent, source: &'a Source, output: &'a Output, + clock: Clock, entry: Option, ) -> io::Result { let offset = entry @@ -439,6 +452,7 @@ impl<'a> Reader<'a> { .map(|entry| entry.prefix().map(|file| file.take(offset))) .transpose()?; Ok(Self { + clock, agent, source, output, @@ -570,7 +584,7 @@ impl Read for Reader<'_> { if self .last_upload .get() - .is_some_and(|last| last.elapsed() >= Duration::from_secs(30)) + .is_some_and(|last| self.clock.elapsed(last) >= Duration::from_secs(30)) { self.failed = true; return Err(io::Error::new( @@ -586,6 +600,7 @@ impl Read for Reader<'_> { mod tests { use super::*; use clap::Parser; + use darkbio_clock::TestClock; use darkbio_connect::schema::{SlotDownload, SlotOrigin, SlotState}; use std::io::Write; use std::net::TcpListener; @@ -767,7 +782,8 @@ mod tests { let agent = http::agent(Duration::from_secs(1), 0); let output = output(); let source = source("https://example.com/reference.gz".into(), &[1, 2, 3]); - let mut reader = Reader::new(&agent, &source, &output, None).unwrap(); + let clock = TestClock::new().clock(); + let mut reader = Reader::new(&agent, &source, &output, clock, None).unwrap(); reader.network = Some(ureq::Body::builder().data([1, 2]).into_reader()); let error = reader.read_to_end(&mut Vec::new()).unwrap_err(); assert_eq!(error.kind(), io::ErrorKind::UnexpectedEof); @@ -801,12 +817,19 @@ mod tests { fn download_stall_tracks_the_upload_window() { let agent = http::agent(Duration::from_secs(1), 0); let output = output(); - let source = source("https://example.com/reference.gz".into(), &[42]); - let mut reader = Reader::new(&agent, &source, &output, None).unwrap(); - reader - .last_upload - .set(Some(Instant::now() - Duration::from_secs(31))); - reader.network = Some(ureq::Body::builder().data([42]).into_reader()); + let source = source("https://example.com/reference.gz".into(), &[42, 43]); + let mut tester = TestClock::new(); + let mut reader = Reader::new(&agent, &source, &output, tester.clock(), None).unwrap(); + reader.network = Some(ureq::Body::builder().data([42, 43]).into_reader()); + + // A read just inside the upload window passes + reader.last_upload.set(Some(tester.clock().now())); + tester.advance(Duration::from_secs(29)); + assert_eq!(reader.read(&mut [0]).unwrap(), 1); + assert!(!reader.failed); + + // A read at the end of the window fails the download + tester.advance(Duration::from_secs(1)); assert_eq!( reader.read(&mut [0]).unwrap_err().kind(), io::ErrorKind::TimedOut @@ -832,7 +855,8 @@ mod tests { .unwrap(); let agent = agent(); let output = output(); - let mut reader = Reader::new(&agent, &source, &output, Some(entry)).unwrap(); + let clock = TestClock::new().clock(); + let mut reader = Reader::new(&agent, &source, &output, clock, Some(entry)).unwrap(); let mut received = vec![0; cache::CHUNK]; reader.read_exact(&mut received).unwrap(); assert_eq!(received, bytes[..cache::CHUNK]); @@ -897,6 +921,7 @@ mod tests { ); let agent = agent(); let output = output(); + let clock = TestClock::new().clock(); { let entry = cache::Entry::open( &directory.0, @@ -905,7 +930,8 @@ mod tests { source.dataset.size, ) .unwrap(); - let mut reader = Reader::new(&agent, &source, &output, Some(entry)).unwrap(); + let mut reader = + Reader::new(&agent, &source, &output, clock.clone(), Some(entry)).unwrap(); let mut received = Vec::new(); assert!(reader.read_to_end(&mut received).is_err()); assert_eq!(received, bytes[..cache::CHUNK]); @@ -916,7 +942,7 @@ mod tests { cache::Entry::open(&directory.0, &source.hash, &source.url, source.dataset.size) .unwrap(); assert_eq!(entry.len().unwrap(), 0); - let mut reader = Reader::new(&agent, &source, &output, Some(entry)).unwrap(); + let mut reader = Reader::new(&agent, &source, &output, clock, Some(entry)).unwrap(); let mut received = Vec::new(); reader.read_to_end(&mut received).unwrap(); assert_eq!(received, bytes); @@ -948,7 +974,8 @@ mod tests { .unwrap(); let agent = agent(); let output = output(); - let mut reader = Reader::new(&agent, &source, &output, Some(entry)).unwrap(); + let clock = TestClock::new().clock(); + let mut reader = Reader::new(&agent, &source, &output, clock, Some(entry)).unwrap(); let mut received = Vec::new(); reader.read_to_end(&mut received).unwrap(); assert_eq!(received, bytes); diff --git a/src/data/scenarios.rs b/src/data/scenarios.rs index 9fee0f6..0d71d74 100644 --- a/src/data/scenarios.rs +++ b/src/data/scenarios.rs @@ -102,7 +102,7 @@ fn slots() -> Vec { } /// Runs one scenario in a child test process, capturing its real stdout and -/// stderr so results and hints can be told apart. +/// stderr so results and hints stay distinguishable. fn capture(scenario: &str, json: bool) -> (String, String) { let output = std::process::Command::new(std::env::current_exe().unwrap()) .args([ diff --git a/src/device.rs b/src/device.rs index 7a487e2..71de482 100644 --- a/src/device.rs +++ b/src/device.rs @@ -70,6 +70,7 @@ pub(crate) fn status(context: &Context, recovery: args::Recovery) -> Result<(), fn status_value(connection: &Connection) -> Value { let current = connection.require_current().is_ok(); let info = &connection.info; + let clock = connection.client.clock(); let reported = format!("{} - {}", info.version_str, info.revision_str); let (trust, serial, realm, model, mismatch, env) = match &connection.identity { Identity::Attested { env, device } => ( @@ -99,7 +100,7 @@ fn status_value(connection: &Connection) -> Value { json!({"name":connection.device.name(),"serial":serial, "hardware":{"version":info.version_str,"revision":info.revision_str,"model":model}, "firmware":{"version":info.firmware_version,"published":timestamp(info.firmware_publish)}, - "trust":trust,"environment":env,"realm":realm,"synced":current.then(|| darkbio_connect::cloud_synced(info)),"paired":current.then_some(info.paired),"unlocked":current.then_some(info.unlocked), + "trust":trust,"environment":env,"realm":realm,"synced":current.then(|| darkbio_connect::cloud_synced(info, &clock)),"paired":current.then_some(info.paired),"unlocked":current.then_some(info.unlocked), "identity":hex::encode(connection.identity.key().fingerprint().to_bytes()), "pubkey":hex::encode(connection.identity.key().to_bytes()),"mismatch":mismatch}) } diff --git a/src/doctor.rs b/src/doctor.rs index 34dac8a..0ebf498 100644 --- a/src/doctor.rs +++ b/src/doctor.rs @@ -7,6 +7,7 @@ //! Independent diagnostics composed from connection primitives. use crate::{context::Context, error::Error, firmware::Packages, update}; +use darkbio_clock::Clock; use darkbio_connect::schema; use serde_json::{Value, json}; @@ -91,7 +92,7 @@ pub(crate) fn run(context: &Context) -> Result<(), Error> { if connection.env.is_none() { checks.skip("firmware", "cloud environment unknown"); } else { - match Packages::new(context, connection.env) + match Packages::new(context, connection.client.clock(), connection.env) .and_then(|mut packages| packages.list(context)) { Ok(firmwares) => checks.ok( @@ -210,7 +211,7 @@ impl Checks<'_> { // Look up within --timeout, keeping the answer for later commands let running = update::running(); let channel = update::Channel::for_version(&running); - let asked = chrono::Utc::now(); + let asked = chrono::DateTime::from(Clock::real().system_time()); let result = update::refresh( &crate::data::cache::directory(), channel, diff --git a/src/execution.rs b/src/execution.rs index 2db8f83..dbd9364 100644 --- a/src/execution.rs +++ b/src/execution.rs @@ -16,7 +16,6 @@ use crate::{ use base64::{Engine, prelude::BASE64_STANDARD}; use darkbio_connect::{ExecutionProgress, schema}; use serde_json::{Value, json}; -use std::time::Instant; /// Cancels an explicit task or uploads and runs a local app after unlock. /// The connector owns protocol sequencing; the CLI owns progress, partial results @@ -39,8 +38,9 @@ pub(crate) fn run(context: &Context, command: args::App) -> Result<(), Error> { let connection = context.connect(None)?; context.require_unlocked(&connection, false)?; let mut value = json!({"task":null,"app":{"name":null,"version":null},"success":null,"stdout":null,"stderr":null,"duration_seconds":null}); + let clock = connection.client.clock(); let mut started = None; - let mut transfer = Transfer::new(context.output.terminal()); + let mut transfer = Transfer::new(context.output.terminal(), clock.clone()); let report_interval = if context.output.terminal() { 1 } else { 5 }; let mut reported = None; let result = @@ -66,7 +66,7 @@ pub(crate) fn run(context: &Context, command: args::App) -> Result<(), Error> { format!("run {} (Ark Companion on your phone)", path.display()), ), ExecutionProgress::Running { elapsed } => { - started.get_or_insert_with(|| Instant::now() - elapsed); + started.get_or_insert_with(|| clock.now() - elapsed); let seconds = elapsed.as_secs(); if reported.is_none_or(|last| seconds >= last + report_interval) { context @@ -88,9 +88,7 @@ pub(crate) fn run(context: &Context, command: args::App) -> Result<(), Error> { }; value["app"] = json!({"name":result.app_name,"version":result.app_version}); value["success"] = json!(result.success); - let duration = started - .expect("successful execution reported running") - .elapsed(); + let duration = clock.elapsed(started.expect("successful execution reported running")); value["duration_seconds"] = json!(duration.as_secs()); bytes(&mut value, "stdout", &result.stdout); bytes(&mut value, "stderr", &result.stderr); diff --git a/src/firmware/mod.rs b/src/firmware/mod.rs index 7aa8fd1..4b7c391 100644 --- a/src/firmware/mod.rs +++ b/src/firmware/mod.rs @@ -15,10 +15,11 @@ use crate::{ http, progress::Transfer, }; +use darkbio_clock::Clock; use darkbio_connect::{UpdateProgress, schema, trust::Environment}; use package::Package; use serde_json::{Value, json}; -use std::time::{Duration, Instant}; +use std::time::Duration; /// Verification window covering old-session closure and discovery after installation. pub(crate) const REBOOT_WAIT: Duration = Duration::from_secs(120); @@ -62,7 +63,8 @@ pub(crate) fn check_compatibility(info: &schema::DeviceInfoResponse) -> Result<( /// authorized update sequence. Partial results distinguish installation from return. pub(crate) fn run(context: &Context, command: args::Firmware) -> Result<(), Error> { let connection = context.connect_recovery(None)?; - let mut packages = Packages::new(context, connection.env)?; + let clock = connection.client.clock(); + let mut packages = Packages::new(context, clock.clone(), connection.env)?; let firmwares = packages.list(context)?; if let args::Firmware::List = command { return listing(context, &connection, &firmwares); @@ -138,7 +140,7 @@ pub(crate) fn run(context: &Context, command: args::Firmware) -> Result<(), Erro path: package::path(target), response: None, }; - let mut transfer = Transfer::new(context.output.terminal()); + let mut transfer = Transfer::new(context.output.terminal(), clock.clone()); let result = connection.client.update_firmware( &target.firmware(), &mut reader, @@ -191,7 +193,7 @@ pub(crate) fn run(context: &Context, command: args::Firmware) -> Result<(), Erro .output .event("progress", "waiting for the Ark to reboot"); context.output.wait("waiting for the Ark to return", None); - let started = Instant::now(); + let started = clock.now(); let result = verify_reboot(context, &connection, &target.version, &mut value); context.output.finish(); if result.is_ok() { @@ -199,7 +201,7 @@ pub(crate) fn run(context: &Context, command: args::Firmware) -> Result<(), Erro "progress", format!( "returned after {} s; verified {}", - started.elapsed().as_secs(), + clock.elapsed(started).as_secs(), target.version ), ); @@ -211,31 +213,33 @@ pub(crate) fn run(context: &Context, command: args::Firmware) -> Result<(), Erro } /// Installation acknowledges before scheduling reboot. Observe the old session -/// ending first, including when reinstalling the same build. +/// ending first, including when reinstalling the same build. The reboot window +/// and the pauses between attempts run on the old connection's clock. fn verify_reboot( context: &Context, connection: &Connection, target: &str, value: &mut Value, ) -> Result<(), Error> { - let deadline = Instant::now() + REBOOT_WAIT; + let clock = connection.client.clock(); + let deadline = clock.now() + REBOOT_WAIT; loop { match connection.client.call( schema::DeviceInfoRequest {}, context.timing().with_deadline(deadline), ) { - Ok(_) => std::thread::sleep(Duration::from_millis(250)), + Ok(_) => clock.sleep(Duration::from_millis(250)), Err(darkbio_connect::Error::Closed | darkbio_connect::Error::Disconnected(_)) => break, Err(error) => return Err(error.into()), } - if Instant::now() >= deadline { + if clock.now() >= deadline { return Err(reboot_timeout()); } } connection.ark.close(); let device = &connection.device; let key = connection.identity.key(); - while deadline.saturating_duration_since(Instant::now()) + while deadline.saturating_duration_since(clock.now()) > darkbio_connect::wire::transport::DEFAULT_HANDSHAKE_TIMEOUT { let found = darkbio_connect::list(); @@ -270,9 +274,9 @@ fn verify_reboot( }; } } - std::thread::sleep(Duration::from_millis(500)); + clock.sleep(Duration::from_millis(500)); } - std::thread::sleep(deadline.saturating_duration_since(Instant::now())); + clock.sleep_until(deadline); Err(reboot_timeout()) } /// Reports that installation was not verified within the reboot window. @@ -399,6 +403,8 @@ fn listing(context: &Context, connection: &Connection, firmwares: &[Package]) -> /// Environment-specific package client and optional Access credentials. pub(crate) struct Packages { + /// Connection's clock, which bounds the Access login helpers. + clock: Clock, /// HTTPS downloader retaining connections and refusing automatic redirects. agent: ureq::Agent, /// Selected package origin; credentials are sent only to this host. @@ -408,7 +414,7 @@ pub(crate) struct Packages { } impl Packages { /// Selects the package host and tries cached credentials without prompting for login. - pub fn new(context: &Context, env: Option) -> Result { + pub fn new(context: &Context, clock: Clock, env: Option) -> Result { let origin = match env.ok_or_else(|| { Error::new(4, "environment-unknown", "cloud environment unknown") .hint("select one with --env") @@ -418,11 +424,12 @@ impl Packages { Environment::Develop => "https://pkg.darkbio.dev", }; let token = if origin != "https://pkg.dark.bio" { - access::cached(context, origin) + access::cached(context, &clock, origin) } else { None }; let result = Self { + clock, agent: http::agent(Duration::from_secs(context.options.timeout), 0), origin, token, @@ -467,7 +474,7 @@ impl Packages { }; let response = fetch(self.token.as_deref())?; let response = if access::required(self.origin, response.status(), response.headers()) { - self.token = Some(access::authenticate(context, self.origin)?); + self.token = Some(access::authenticate(context, &self.clock, self.origin)?); fetch(self.token.as_deref())? } else { response diff --git a/src/help.rs b/src/help.rs index 8a1b275..d6f7321 100644 --- a/src/help.rs +++ b/src/help.rs @@ -529,8 +529,8 @@ mod tests { fn shared_options_are_listed_on_the_root_page_only() { let theme = Theme::test(80, Color::Off, false); let mut root = command(&theme); - // Examples mention the flags too, so the listing is told by the text - // clap prints beside each option. + // Examples mention the flags too, so the test identifies the listing by + // the text clap prints beside each option. let listed = [ "--timeout ", "Print the complete result as JSON", diff --git a/src/help/datasets.md b/src/help/datasets.md index dc735ab..67bc0b0 100644 --- a/src/help/datasets.md +++ b/src/help/datasets.md @@ -16,7 +16,7 @@ damage, requires, size_bytes, build, version and download. show adds required_by and cached. The reading list keeps to short columns, while show also prints the full description and format. Both show dependency state alongside each name. -- description explains the slot's data to its owner. format tells whoever fills +- description explains the slot's data to its owner. format shows whoever fills the slot which file it accepts, the shape that file needs, what the Ark refuses and whether the owner approves the upload. - size_bytes is the bytes on the Ark's disk for this slot, zero when empty. diff --git a/src/http.rs b/src/http.rs index 8387e70..0233823 100644 --- a/src/http.rs +++ b/src/http.rs @@ -16,6 +16,12 @@ use ureq::unversioned::{ /// Creates an HTTPS download client with bounded network waits and caller-selected /// redirect allowance. Active bodies can outlive many inactivity windows. pub(crate) fn agent(timeout: Duration, redirects: u32) -> ureq::Agent { + agent_over(DefaultConnector::default(), timeout, redirects) +} + +/// Builds the download client over the connector that opens its transports, +/// wrapping each one in the inactivity bound. +fn agent_over(connector: impl Connector, timeout: Duration, redirects: u32) -> ureq::Agent { let config = ureq::Agent::config_builder() .https_only(true) .max_redirects(redirects) @@ -27,7 +33,7 @@ pub(crate) fn agent(timeout: Duration, redirects: u32) -> ureq::Agent { .build(); ureq::Agent::with_parts( config, - DefaultConnector::default().chain(Inactivity(timeout)), + connector.chain(Inactivity(timeout)), DefaultResolver::default(), ) } @@ -131,67 +137,168 @@ impl Transport for Idle { #[cfg(test)] mod tests { use super::*; - use std::{ - io::{Read, Write}, - net::TcpListener, - thread, - time::Instant, - }; - - /// Active downloads can take many inactivity windows. A silent body still - /// expires, including ureq's wrapped io::Error timeout representation. + use crate::testing::wait_deadline; + use darkbio_clock::{Clock, TestClock, crossbeam_channel}; + use std::io::{self, Read}; + use std::sync::{Mutex, mpsc}; + use std::thread; + use std::time::Instant; + use ureq::unversioned::transport::LazyBuffers; + + /// Transport whose input the test hands over, waited for on the test clock. + /// Every wait reports its deadline before it starts. + #[derive(Debug)] + struct Scripted { + /// Clock the input waits run on. + clock: Clock, + /// Buffers the client reads and writes through. + buffers: LazyBuffers, + /// Input the test hands over. + input: crossbeam_channel::Receiver>, + /// Deadline of every input wait, reported before it starts. + waits: mpsc::Sender>, + } + + impl Transport for Scripted { + fn buffers(&mut self) -> &mut dyn Buffers { + &mut self.buffers + } + + fn transmit_output(&mut self, _: usize, _: NextTimeout) -> Result<(), ureq::Error> { + Ok(()) + } + + fn await_input(&mut self, timeout: NextTimeout) -> Result { + let deadline = + (!timeout.after.is_not_happening()).then(|| self.clock.now() + *timeout.after); + self.waits.send(deadline).unwrap(); + let input = match deadline { + Some(deadline) => self + .clock + .recv_deadline(&self.input, deadline) + .map_err(|_| ureq::Error::Timeout(timeout.reason))?, + None => self.input.recv().unwrap(), + }; + self.buffers.input_append_buf()[..input.len()].copy_from_slice(&input); + self.buffers.input_appended(input.len()); + Ok(true) + } + + fn is_open(&mut self) -> bool { + true + } + + fn is_tls(&self) -> bool { + true + } + } + + /// Opens the scripted transport for the one connection a test makes. + #[derive(Debug)] + struct Script(Mutex>); + + impl Connector for Script { + type Out = Scripted; + + fn connect( + &self, + _: &ConnectionDetails, + _: Option<()>, + ) -> Result, ureq::Error> { + Ok(self.0.lock().unwrap().take()) + } + } + + /// Builds a download client with a 300 ms allowance over a scripted + /// transport, returning where to hand it input and where its waits report. + fn scripted( + clock: &Clock, + ) -> ( + ureq::Agent, + crossbeam_channel::Sender>, + mpsc::Receiver>, + ) { + let (hand, input) = crossbeam_channel::unbounded(); + let (waits, reported) = mpsc::channel(); + let transport = Scripted { + clock: clock.clone(), + buffers: LazyBuffers::new(16 * 1024, 16 * 1024), + input, + waits, + }; + let connector = Script(Mutex::new(Some(transport))); + let agent = agent_over(connector, Duration::from_millis(300), 0); + (agent, hand, reported) + } + + /// Active downloads can take many inactivity windows, since each wait for + /// body input gets the whole allowance. A silent body still expires at the + /// end of its window, and an earlier HTTP deadline wins over the allowance. #[test] fn body_timeout_measures_each_wait() { - for stalled in [false, true] { - let listener = TcpListener::bind("127.0.0.1:0").unwrap(); - let url = format!("http://{}/reference", listener.local_addr().unwrap()); - let server = thread::spawn(move || { - let (mut stream, _) = listener.accept().unwrap(); - stream - .set_read_timeout(Some(Duration::from_secs(5))) - .unwrap(); - let mut head = Vec::new(); - while !head.ends_with(b"\r\n\r\n") { - let mut byte = [0]; - stream.read_exact(&mut byte).unwrap(); - head.push(byte[0]); - } - stream - .write_all(b"HTTP/1.1 200 OK\r\nContent-Length: 8\r\nConnection: close\r\n\r\n") - .unwrap(); - for _ in 0..8 { - thread::sleep(if stalled { - Duration::from_millis(600) - } else { - Duration::from_millis(75) - }); - if stream.write_all(&[42]).is_err() { - break; - } - } - }); - let agent = agent(Duration::from_millis(300), 5); - let mut response = agent - .get(&url) - .config() - .https_only(false) - .proxy(None) - .build() - .call() - .unwrap(); - let start = Instant::now(); - let mut bytes = Vec::new(); - let result = response.body_mut().as_reader().read_to_end(&mut bytes); - if stalled { - assert_eq!(read_error(result.unwrap_err()).class, 7); - } else { - result.unwrap(); - assert_eq!(bytes, [42; 8]); - assert!(start.elapsed() > Duration::from_millis(300)); - } - drop(response); - server.join().unwrap(); + // A request's earlier deadline bounds the wait for its response + let mut tester = TestClock::new(); + let clock = tester.clock(); + let (agent, hand, reported) = scripted(&clock); + hand.send(b"HTTP/1.1 200 OK\r\nContent-Length: 1\r\n\r\n*".to_vec()) + .unwrap(); + let body = agent + .get("https://127.0.0.1/reference") + .config() + .timeout_global(Some(Duration::from_millis(100))) + .build() + .call() + .unwrap() + .into_body() + .read_to_vec() + .unwrap(); + assert_eq!(body, b"*"); + let response = reported.recv().unwrap().unwrap(); + assert!(response <= clock.now() + Duration::from_millis(100)); + + // A body arriving a byte at a time outlives one window, each wait + // getting the whole allowance + let (agent, hand, reported) = scripted(&clock); + hand.send(b"HTTP/1.1 200 OK\r\nContent-Length: 4\r\n\r\n".to_vec()) + .unwrap(); + let reading = thread::spawn(move || { + let mut response = agent.get("https://127.0.0.1/reference").call().unwrap(); + let mut body = Vec::new(); + let result = response.body_mut().as_reader().read_to_end(&mut body); + (body, result) + }); + let response = reported.recv().unwrap().unwrap(); + assert!(response <= clock.now() + Duration::from_millis(300)); + for _ in 0..3 { + assert_eq!( + reported.recv().unwrap(), + Some(clock.now() + Duration::from_millis(300)) + ); + tester.advance(Duration::from_millis(200)); + hand.send(vec![42]).unwrap(); } + + // The last byte never comes, so its wait expires at the end of its window + let deadline = reported.recv().unwrap().unwrap(); + assert_eq!(deadline, clock.now() + Duration::from_millis(300)); + wait_deadline(&tester, deadline); + tester.advance_to(deadline); + let (body, result) = reading.join().unwrap(); + assert_eq!(body, [42; 3]); + assert_eq!(read_error(result.unwrap_err()).class, 7); + } + + /// Body readers wrap ureq's timeout in an io::Error, which still classifies + /// as a timeout. + #[test] + fn test_wrapped_body_timeouts_are_timeouts() { + let wrapped = ureq::Error::Timeout(ureq::Timeout::RecvBody).into_io(); + assert_eq!(wrapped.kind(), io::ErrorKind::Other); + assert_eq!(read_error(wrapped).class, 7); + assert_eq!( + read_error(io::Error::from(io::ErrorKind::ConnectionReset)).class, + 4 + ); } #[test] diff --git a/src/main.rs b/src/main.rs index 3d25892..22e7b87 100644 --- a/src/main.rs +++ b/src/main.rs @@ -25,9 +25,13 @@ mod progress; mod style; mod update; +#[cfg(test)] +mod testing; + use args::{Cli, Command}; use clap::{FromArgMatches, Parser}; use context::Context; +use darkbio_clock::Clock; use error::Error; use serde_json::{Value, json}; use std::process::ExitCode; @@ -94,7 +98,8 @@ fn main() -> ExitCode { output, interrupt, }; - // Valid commands print the release note, except help, completions, --version, a bare run and doctor + // Valid commands print the release note, except help, completions, --version, a bare run and doctor. + // The note comes before any connection, so it reads the real clock. if validation.is_ok() && !cli.help && !cli.version @@ -103,7 +108,10 @@ fn main() -> ExitCode { None | Some(Command::Help { .. } | Command::Completions { .. } | Command::Doctor) ) { - update::start(&context.output, chrono::Utc::now()); + update::start( + &context.output, + chrono::DateTime::from(Clock::real().system_time()), + ); } let result = if let Err(error) = validation { Err(error) diff --git a/src/output.rs b/src/output.rs index 958ce22..7175ec7 100644 --- a/src/output.rs +++ b/src/output.rs @@ -11,14 +11,16 @@ pub(crate) mod human; use crate::args::Options; use crate::error::Error; use crate::style::{self, Role, Theme}; +use darkbio_clock::{Clock, crossbeam_channel}; use darkbio_connect::trust::Environment; use serde_json::{Value, json}; use std::io::{self, Write}; use std::sync::{ - Arc, Mutex, + Arc, Mutex, MutexGuard, atomic::{AtomicBool, Ordering}, }; -use std::time::{Duration, Instant, SystemTime}; +use std::thread::JoinHandle; +use std::time::{Duration, Instant}; /// Clonable output handle for one invocation. Result emission is claimed once; /// events and live stderr lines share terminal state across clones. @@ -42,8 +44,59 @@ struct State { environment_noted: AtomicBool, /// Serializes result claims and writes; acquired before the terminal lock. result: Mutex<()>, + /// Clock of the latest connection, which times every wait display. + clock: Mutex>, + /// Worker redrawing the active wait. Its lock orders every change of the + /// wait display with the worker's replacement, and comes before the + /// terminal lock. + ticker: Mutex>, /// Serializes stderr line changes and spacing around human result blocks. - terminal: Mutex, + /// The redraw worker shares it, but never the rest of the output. + terminal: Arc>, +} + +/// Owner of a worker that redraws a wait display once a second on a clock. +/// Dropping it stops the worker and waits for it to exit. The worker holds +/// only what it draws with, never its owner, so the owner is never dropped on +/// the worker's own thread. +struct Ticker { + /// Disconnects when dropped, which wakes the worker to exit. + stop: Option>, + /// Joined on drop, so no redraw outlives the display. + worker: Option>, +} + +impl Ticker { + /// Starts a worker that draws a second after its start and then a second + /// after each draw, until the owner drops it. + fn start(clock: Clock, mut draw: impl FnMut(Instant) + Send + 'static) -> Self { + let (stop, stopped) = crossbeam_channel::bounded::<()>(0); + let worker = std::thread::spawn(move || { + loop { + let next = clock.now() + Duration::from_secs(1); + if clock.recv_deadline(&stopped, next) + != Err(crossbeam_channel::RecvTimeoutError::Timeout) + { + break; + } + draw(clock.now()); + } + }); + Self { + stop: Some(stop), + worker: Some(worker), + } + } +} + +impl Drop for Ticker { + /// Wakes the worker and waits for it to exit. + fn drop(&mut self) { + drop(self.stop.take()); + if let Some(worker) = self.worker.take() { + let _ = worker.join(); + } + } } /// Current terminal layout, guarded independently of the result claim. @@ -53,8 +106,6 @@ struct Terminal { live: Option<(String, bool)>, /// Active elapsed-time or countdown display, if any. waiting: Option, - /// Monotonic timer generation used to retire previous wait workers. - generation: u64, /// Whether stderr has printed content that needs spacing before a result. err_printed: bool, /// Whether a human stdout block needs separation from the next stderr event. @@ -63,11 +114,9 @@ struct Terminal { /// Presentation-only wait state; it never controls an operation's deadline. struct Waiting { - /// Identifies the worker allowed to redraw this wait after each timer tick. - generation: u64, /// Short activity name displayed beside the elapsed or remaining time. label: String, - /// Host time when this wait display began. + /// Clock time when this wait display began. started: Instant, /// Fixed countdown bound, absent for an elapsed-time display. until: Option, @@ -85,9 +134,15 @@ impl Output { printed: AtomicBool::new(false), environment_noted: AtomicBool::new(false), result: Mutex::new(()), - terminal: Mutex::new(Terminal::default()), + clock: Mutex::new(None), + ticker: Mutex::new(None), + terminal: Arc::new(Mutex::new(Terminal::default())), })) } + /// Times later wait displays on the clock of a newly opened connection. + pub fn connection(&self, clock: Clock) { + *self.0.clock.lock().expect("output not poisoned") = Some(clock); + } /// Whether the caller requested JSON for both output streams. pub fn json(&self) -> bool { self.0.json @@ -172,8 +227,7 @@ impl Output { /// Ends live stderr activity and writes a complete stdout result block. /// The caller holds the result lock; terminal state is acquired second. fn write_result(&self, text: &str) -> Result<(), Error> { - let mut terminal = self.0.terminal.lock().expect("output not poisoned"); - terminal.waiting = None; + let mut terminal = self.end_wait(); close_line(&mut terminal, &mut io::stderr().lock()); let mut stdout = io::stdout().lock(); if self.0.out.interactive && self.0.err.interactive && terminal.err_printed { @@ -230,10 +284,11 @@ impl Output { return; } { - let mut terminal = self.0.terminal.lock().expect("output not poisoned"); - if !matches!(kind, "note" | "warning" | "step" | "log") { - terminal.waiting = None; - } + let mut terminal = if matches!(kind, "note" | "warning" | "step" | "log") { + self.0.terminal.lock().expect("output not poisoned") + } else { + self.end_wait() + }; let mut stderr = io::stderr().lock(); close_line(&mut terminal, &mut stderr); if self.json() { @@ -267,8 +322,7 @@ impl Output { if self.0.quiet { return; } - let mut terminal = self.0.terminal.lock().expect("output not poisoned"); - terminal.waiting = None; + let mut terminal = self.end_wait(); let mut stderr = io::stderr().lock(); if terminal .live @@ -350,7 +404,7 @@ impl Output { /// Presents a scan URL and a QR code when terminal width and Unicode permit. /// The caller uses this renderer only outside JSON, where the URL is structured. - pub fn pairing(&self, url: &str, deadline: SystemTime) { + pub fn pairing(&self, url: &str, deadline: Instant) { self.event("approve", "scan in Ark Companion"); self.finish(); { @@ -392,54 +446,49 @@ impl Output { } /// The timer only draws while a wait is active. It never bounds the call. - pub fn wait(&self, label: &str, until: Option) { + /// It runs on the latest connection's clock and stays hidden before one. + pub fn wait(&self, label: &str, until: Option) { if !self.0.err.interactive || self.0.quiet { return; } - let generation; + self.show_wait(label, until, || io::stderr().lock()); + } + + /// Replaces any earlier wait display with one drawn to `screen`, which the + /// worker then redraws every second. The replacement happens under the + /// ticker lock, so a concurrent change of the display lands before or after + /// it as a whole. + fn show_wait( + &self, + label: &str, + until: Option, + screen: impl Fn() -> W + Send + 'static, + ) { + let Some(clock) = self.0.clock.lock().expect("output not poisoned").clone() else { + return; + }; + // Stop the earlier worker before touching the terminal, which it draws on + let mut ticker = self.0.ticker.lock().expect("output not poisoned"); + drop(ticker.take()); + + // Publish the new display and draw its first frame at once { let mut terminal = self.0.terminal.lock().expect("output not poisoned"); - terminal.generation += 1; - generation = terminal.generation; + let now = clock.now(); terminal.waiting = Some(Waiting { - generation, label: label.into(), - started: Instant::now(), - until: until.and_then(|end| { - Instant::now() - .checked_add(end.duration_since(SystemTime::now()).unwrap_or_default()) - }), + started: now, + until, }); - tick( - &self.0.err, - &mut terminal, - &mut io::stderr().lock(), - Instant::now(), - ); + tick(&self.0.err, &mut terminal, &mut screen(), now); } - // A replacement wait invalidates this generation. The weak reference - // also lets the worker exit when the invocation releases its output. - let state = Arc::downgrade(&self.0); - std::thread::spawn(move || { - loop { - std::thread::sleep(Duration::from_secs(1)); - let Some(state) = state.upgrade() else { break }; - let mut terminal = state.terminal.lock().expect("output not poisoned"); - if !terminal - .waiting - .as_ref() - .is_some_and(|wait| wait.generation == generation) - { - break; - } - tick( - &state.err, - &mut terminal, - &mut io::stderr().lock(), - Instant::now(), - ); - } - }); + // Install the worker before another change can take the ticker lock + let theme = self.0.err.clone(); + let terminal = self.0.terminal.clone(); + *ticker = Some(Ticker::start(clock, move |now| { + let mut terminal = terminal.lock().expect("output not poisoned"); + tick(&theme, &mut terminal, &mut screen(), now); + })); } /// Prints and flushes a prompt without reading stdin or deciding whether to ask. @@ -509,12 +558,22 @@ impl Output { /// Stops the active timer and closes its live line; safe to call repeatedly. pub fn finish(&self) { - let mut terminal = self.0.terminal.lock().expect("output not poisoned"); - terminal.waiting = None; + let mut terminal = self.end_wait(); let mut stderr = io::stderr().lock(); close_line(&mut terminal, &mut stderr); let _ = stderr.flush(); } + + /// Ends the wait display and returns the terminal for the caller's next + /// write. The worker is stopped under the ticker lock and joined before + /// the terminal lock is taken, since each redraw takes that lock too. + fn end_wait(&self) -> MutexGuard<'_, Terminal> { + let mut ticker = self.0.ticker.lock().expect("output not poisoned"); + drop(ticker.take()); + let mut terminal = self.0.terminal.lock().expect("output not poisoned"); + terminal.waiting = None; + terminal + } } /// Styles and wraps one human event while preserving its recognizable prefix. @@ -603,7 +662,162 @@ pub(crate) fn scalar(value: &Value) -> String { mod tests { use super::*; use crate::style::Color; + use crate::testing::wait_deadline; use clap::Parser; + use darkbio_clock::TestClock; + use std::sync::{TryLockError, mpsc}; + use std::thread; + + /// Captures the frames that wait displays draw in place of stderr. + #[derive(Clone, Default)] + struct Screen(Arc>>); + + impl Screen { + /// Returns the frames drawn so far, oldest first. + fn frames(&self) -> Vec { + String::from_utf8(self.0.lock().unwrap().clone()) + .unwrap() + .split(style::CLEAR_LINE) + .filter(|frame| !frame.is_empty()) + .map(String::from) + .collect() + } + } + + impl Write for Screen { + fn write(&mut self, bytes: &[u8]) -> io::Result { + self.0.lock().unwrap().extend_from_slice(bytes); + Ok(bytes.len()) + } + + fn flush(&mut self) -> io::Result<()> { + Ok(()) + } + } + + /// Builds an output whose waits run on the clock. + fn output(clock: Clock) -> Output { + let options = crate::args::Cli::try_parse_from(["ark"]).unwrap().options; + let output = Output::new(&options); + output.connection(clock); + output + } + + /// Shows a wait on a thread of its own, drawing it on the screen. + fn spawn_wait(output: &Output, label: &'static str, screen: &Screen) -> thread::JoinHandle<()> { + let (output, screen) = (output.clone(), screen.clone()); + thread::spawn(move || output.show_wait(label, None, move || screen.clone())) + } + + /// Waits until another thread holds the output's ticker lock. + fn ticker_held(output: &Output) { + loop { + match output.0.ticker.try_lock() { + Err(TryLockError::WouldBlock) => return, + free => drop(free), + } + thread::yield_now(); + } + } + + /// A wait arriving while another replaces the display goes after it, so its + /// display is the one that its sole worker redraws. Dropping the output then + /// stops that worker while it waits for the next redraw. + #[test] + fn test_concurrent_waits_keep_one_worker() { + // Hold the terminal, so the first wait stops inside its replacement + let mut tester = TestClock::new(); + let start = tester.clock().now(); + let output = output(tester.clock()); + let screen = Screen::default(); + let terminal = output.0.terminal.lock().unwrap(); + let first = spawn_wait(&output, "first", &screen); + ticker_held(&output); + + // The second wait queues behind the first one's replacement + let second = spawn_wait(&output, "second", &screen); + drop(terminal); + first.join().unwrap(); + second.join().unwrap(); + let frames = screen.frames(); + assert_eq!(frames.len(), 2); + assert!(frames[0].contains("first") && frames[0].contains("0 s elapsed")); + assert!(frames[1].contains("second") && frames[1].contains("0 s elapsed")); + + // Its worker alone redraws it a second later, then waits another second + wait_deadline(&tester, start + Duration::from_secs(1)); + tester.advance_to(start + Duration::from_secs(1)); + wait_deadline(&tester, start + Duration::from_secs(2)); + let frames = screen.frames(); + assert_eq!(frames.len(), 3); + assert!(frames[2].contains("second") && frames[2].contains("1 s elapsed")); + + // Dropping the output stops the waiting worker + drop(output); + assert_eq!(tester.next_deadline(), None); + assert_eq!(screen.frames().len(), 3); + } + + /// Ending the display while a wait replaces it lands after the replacement, + /// so no worker is left once the end returns. + #[test] + fn test_end_stops_a_pending_worker() { + // Hold the terminal, so the wait stops inside its replacement + let tester = TestClock::new(); + let output = output(tester.clock()); + let screen = Screen::default(); + let terminal = output.0.terminal.lock().unwrap(); + let waiting = spawn_wait(&output, "waiting", &screen); + ticker_held(&output); + + // The end queues behind the replacement and stops its worker + let ending = thread::spawn({ + let output = output.clone(); + move || drop(output.end_wait()) + }); + drop(terminal); + waiting.join().unwrap(); + ending.join().unwrap(); + assert!(output.0.ticker.lock().unwrap().is_none()); + assert!(output.0.terminal.lock().unwrap().waiting.is_none()); + assert_eq!(tester.next_deadline(), None); + } + + /// Dropping a ticker during a redraw returns only after the redraw finished + /// and the worker exited, without another redraw. + #[test] + fn test_ticker_drop_waits_for_a_redraw() { + // The worker redraws a second after its start and holds the redraw open + let mut tester = TestClock::new(); + let clock = tester.clock(); + let (entered, redraws) = mpsc::channel(); + let (release, gate) = mpsc::channel::<()>(); + let mut ticker = Ticker::start(clock.clone(), move |now| { + entered.send(now).unwrap(); + gate.recv().unwrap(); + }); + let first = clock.now() + Duration::from_secs(1); + wait_deadline(&tester, first); + tester.advance_to(first); + assert_eq!(redraws.recv().unwrap(), first); + + // Pass the stop signal through a helper, which lets the redraw finish + // only once the drop has started cancelling the worker + let (tap, tapped) = crossbeam_channel::bounded::<()>(0); + let stop = ticker.stop.replace(tap); + let releasing = thread::spawn(move || { + assert_eq!(tapped.recv(), Err(crossbeam_channel::RecvError)); + drop(stop); + release.send(()).unwrap(); + }); + + // The drop itself waits for the worker, so its closure is gone the + // moment the drop returns + drop(ticker); + assert_eq!(redraws.try_recv(), Err(mpsc::TryRecvError::Disconnected)); + releasing.join().unwrap(); + assert_eq!(tester.next_deadline(), None); + } #[test] fn events_keep_prefixes_and_style_inline_commands() { @@ -625,10 +839,9 @@ mod tests { #[test] fn waiting_line_is_erased_but_completed_progress_stays() { let theme = Theme::test(80, Color::Off, true); - let started = Instant::now(); + let started = TestClock::new().clock().now(); let mut terminal = Terminal { waiting: Some(Waiting { - generation: 1, label: "waiting".into(), started, until: None, diff --git a/src/pairing.rs b/src/pairing.rs index db1d6fe..70f8639 100644 --- a/src/pairing.rs +++ b/src/pairing.rs @@ -12,7 +12,6 @@ use darkbio_connect::{ trust::{Environment, Realm}, }; use serde_json::json; -use std::time::{Duration, UNIX_EPOCH}; /// Pairs an unpaired Ark, translating connector stages into the owner's scan and /// approval instructions. Link construction and terminal presentation stay in the CLI. @@ -71,9 +70,7 @@ pub(crate) fn run(context: &Context) -> Result<(), Error> { } ); if context.output.terminal() { - context - .output - .pairing(&url, UNIX_EPOCH + Duration::from_secs(deadline)); + context.output.pairing(&url, deadline); previous = Some("rendezvous"); } else { context diff --git a/src/progress.rs b/src/progress.rs index e66df51..f42fe5e 100644 --- a/src/progress.rs +++ b/src/progress.rs @@ -7,6 +7,7 @@ //! Transfer rates and per-step estimates for terminal progress. use crate::style::{Role, Theme}; +use darkbio_clock::Clock; use darkbio_connect::schema::SlotUploadProcessResponse; use std::collections::VecDeque; use std::time::{Duration, Instant}; @@ -119,6 +120,8 @@ impl Update { /// Samples only acknowledged bytes, starting with the first upload report so /// cloud setup and approval do not enter the rate estimate. pub(super) struct Transfer { + /// Connection's clock, which times the samples. + clock: Clock, /// Rolling counter samples, in bytes for uploads and basis points for processing. rate: Rate, /// Emission cadence, independent of the sampling cadence. @@ -127,8 +130,9 @@ pub(super) struct Transfer { impl Transfer { /// Starts without rate history so setup and approval cannot skew the first estimate. - pub(super) fn new(human: bool) -> Self { + pub(super) fn new(human: bool, clock: Clock) -> Self { Self { + clock, rate: Rate::default(), report: Report::new(human), } @@ -136,10 +140,10 @@ impl Transfer { /// Samples acknowledged bytes and emits an observation only when reporting is due. pub(super) fn update(&mut self, uploaded: u64, total: u64) -> Option { - self.update_at(uploaded, total, Instant::now()) + self.update_at(uploaded, total, self.clock.now()) } - /// Updates byte-rate history at the supplied host time, even if output is throttled. + /// Updates byte-rate history at the supplied clock time, even if output is throttled. fn update_at(&mut self, uploaded: u64, total: u64, now: Instant) -> Option { let rate = self.rate.sample(uploaded, now); let percent = percent(uploaded, total); @@ -187,8 +191,10 @@ impl Transfer { } /// Progress percentages belong to individual steps. Device timestamps identify -/// a restarted step; elapsed time is measured by the host's monotonic clock. +/// a restarted step; elapsed time is measured on the connection's monotonic clock. pub(super) struct Processing { + /// Connection's clock, which times the samples. + clock: Clock, /// Last report, retained to finish a phase when the next report advances past it. previous: Option, /// Basis-point progress samples for the current processing step only. @@ -199,8 +205,9 @@ pub(super) struct Processing { impl Processing { /// Starts without a step identity or estimate; the first report establishes both. - pub(super) fn new(human: bool) -> Self { + pub(super) fn new(human: bool, clock: Clock) -> Self { Self { + clock, previous: None, rate: Rate::default(), report: Report::new(human), @@ -227,12 +234,11 @@ impl Processing { completed.phase_progress = 10_000; Self::observation(&completed, None) }); - completed - .into_iter() - .chain(self.update_at(status, Instant::now())) + let now = self.clock.now(); + completed.into_iter().chain(self.update_at(status, now)) } - /// Builds a step-specific estimate from basis points and monotonic host time. + /// Builds a step-specific estimate from basis points and monotonic clock time. fn update_at(&mut self, status: &SlotUploadProcessResponse, now: Instant) -> Option { let phase = (status.proc_start, status.phase_in, status.phase_start); if self.previous.as_ref().is_none_or(|previous| { @@ -416,6 +422,7 @@ fn human_eta(remaining: u64, rate: Option) -> String { #[cfg(test)] mod tests { use super::*; + use darkbio_clock::TestClock; #[test] fn bar_keeps_speed_and_eta_in_the_available_width() { @@ -457,8 +464,9 @@ mod tests { /// newly transferred would inflate speed and shorten the ETA. #[test] fn test_transfer_estimate() { - let start = Instant::now(); - let mut transfer = Transfer::new(false); + let clock = TestClock::new().clock(); + let start = clock.now(); + let mut transfer = Transfer::new(false, clock); let mib = 1024 * 1024; let first = transfer.update_at(25 * mib, 100 * mib, start).unwrap().text; assert!(first.contains("speed estimating... | ETA estimating...")); @@ -528,9 +536,10 @@ mod tests { use crate::style::Color; for width in [60, 80, 100, 140] { let theme = Theme::test(width, Color::True, true); - let start = Instant::now(); + let clock = TestClock::new().clock(); + let start = clock.now(); let total = 290 * 1024 * 1024; - let mut transfer = Transfer::new(true); + let mut transfer = Transfer::new(true, clock.clone()); transfer.update_at(0, total, start); let mut upload = transfer .update_at(total, total, start + Duration::from_secs(8)) @@ -549,7 +558,7 @@ mod tests { if width >= 140 { assert!(rendered.contains("290.0/290.0 MiB")); } - let mut processing = Processing::new(true); + let mut processing = Processing::new(true, clock); for (phase, progress) in [(1, 8200), (2, 9200), (2, 10_000)] { report.phase_in = phase; report.phase_progress = progress; @@ -579,7 +588,7 @@ mod tests { #[test] fn advancing_completes_the_previous_phase_before_rendering_the_next() { - let mut processing = Processing::new(true); + let mut processing = Processing::new(true, TestClock::new().clock()); let first: Vec<_> = processing.update(&status(1, 8200)).collect(); assert_eq!(first.len(), 1); assert_eq!(first[0].percent, 82); @@ -613,7 +622,7 @@ mod tests { ..status(2, 2000) }, ] { - let mut processing = Processing::new(true); + let mut processing = Processing::new(true, TestClock::new().clock()); assert_eq!(processing.update(&status(1, 8200)).count(), 1); assert!(processing.update(&next).all(|update| update.percent != 100)); } @@ -623,8 +632,9 @@ mod tests { /// partway through it. A restarted step must not inherit its previous rate. #[test] fn test_step_estimate() { - let start = Instant::now(); - let mut processing = Processing::new(false); + let clock = TestClock::new().clock(); + let start = clock.now(); + let mut processing = Processing::new(false, clock); assert!( processing .update_at(&status(1, 1000), start) @@ -681,7 +691,7 @@ mod tests { /// over instead of underflowing or producing an estimate from another run. #[test] fn test_rate_stall_and_reset() { - let start = Instant::now(); + let start = TestClock::new().clock().now(); let mut rate = Rate::default(); assert_eq!(rate.sample(0, start), None); assert_eq!( @@ -702,7 +712,7 @@ mod tests { /// stay quiet. Completion is printed even inside the normal interval. #[test] fn test_report_cadence() { - let start = Instant::now(); + let start = TestClock::new().clock().now(); let mut report = Report::new(false); assert!(report.due(91, start)); assert!(!report.due(92, start + Duration::from_secs(1))); @@ -713,8 +723,9 @@ mod tests { #[test] fn test_human_transfer_cadence() { - let start = Instant::now(); - let mut transfer = Transfer::new(true); + let clock = TestClock::new().clock(); + let start = clock.now(); + let mut transfer = Transfer::new(true, clock); assert!(transfer.update_at(0, 100, start).is_some()); assert!( transfer @@ -737,8 +748,9 @@ mod tests { #[test] fn test_human_step_cadence() { - let start = Instant::now(); - let mut processing = Processing::new(true); + let clock = TestClock::new().clock(); + let start = clock.now(); + let mut processing = Processing::new(true, clock); assert!(processing.update_at(&status(1, 1000), start).is_some()); let next = start + Duration::from_millis(500); assert!(processing.update_at(&status(2, 1000), next).is_some()); diff --git a/src/testing.rs b/src/testing.rs new file mode 100644 index 0000000..12f7bac --- /dev/null +++ b/src/testing.rs @@ -0,0 +1,20 @@ +// ark: command line interface to Ark enclaves +// Copyright 2026 Dark Bio AG. All rights reserved. +// +// Use of this source code is governed by a BSD-style +// license that can be found in the LICENSE file. + +//! Test clock gates shared by the command modules' tests. + +use darkbio_clock::TestClock; +use std::thread; +use std::time::Instant; + +/// Blocks until the earliest wait or timer on the clock is due at `deadline`. +/// The advance that reaches the deadline then wakes it, whenever the test makes +/// that advance. +pub(crate) fn wait_deadline(tester: &TestClock, deadline: Instant) { + while tester.next_deadline() != Some(deadline) { + thread::yield_now(); + } +} diff --git a/src/update.rs b/src/update.rs index df11f1a..c031ea1 100644 --- a/src/update.rs +++ b/src/update.rs @@ -8,6 +8,7 @@ use crate::output::Output; use chrono::{DateTime, Utc}; +use darkbio_clock::Clock; use semver::Version; use serde::{Deserialize, Serialize}; use std::fs::{self, File}; @@ -25,7 +26,7 @@ const CACHE_LIMIT: u64 = 4 * 1024; /// Largest development release response accepted from GitHub, in bytes. const RESPONSE_LIMIT: u64 = 1024 * 1024; -/// Release channel of a build, told apart by its version's prerelease field. +/// Release channel of a build, distinguished by its version's prerelease field. #[derive(Clone, Copy, Debug, Deserialize, Eq, PartialEq, Serialize)] #[serde(rename_all = "lowercase")] pub(crate) enum Channel { @@ -187,23 +188,18 @@ pub(crate) fn start(output: &Output, now: DateTime) { } /// Runs the lookup in the detached copy, which ends within 30 s whatever happens. +/// The copy opens no connection, so its clock is the real one. pub(crate) fn run() { // Under CI the copy does nothing if disabled() { return; } - let started = Instant::now(); - let asked = Utc::now(); + let clock = Clock::real(); + let started = clock.now(); + let asked = DateTime::::from(clock.system_time()); // End the process at 30 s, and skip the lookup when nothing can enforce that - if thread::Builder::new() - .name("ark-update-watchdog".into()) - .spawn(move || { - thread::sleep(Duration::from_secs(30).saturating_sub(started.elapsed())); - std::process::exit(0); - }) - .is_err() - { + if watchdog(started, Duration::from_secs(30)).is_err() { return; } @@ -217,6 +213,22 @@ pub(crate) fn run() { ); } +/// Ends the process once `limit` has passed since `started` on the real clock, +/// whatever the lookup is doing. +#[expect( + clippy::disallowed_methods, + reason = "the watchdog bounds the real lifetime of the detached lookup process" +)] +fn watchdog(started: Instant, limit: Duration) -> io::Result<()> { + thread::Builder::new() + .name("ark-update-watchdog".into()) + .spawn(move || { + thread::sleep(limit.saturating_sub(started.elapsed())); + std::process::exit(0); + }) + .map(drop) +} + /// Stamps a new attempt, keeping the version last found on the same channel. fn claim(directory: &Path, channel: Channel, now: DateTime) -> io::Result<()> { Answer { @@ -353,7 +365,7 @@ fn develop(bytes: &[u8]) -> Result { /// Words the upgrade advice for the way this executable was installed. pub(crate) fn hint(channel: Channel) -> String { - // Gather the paths that tell the install methods apart + // Gather the paths that distinguish the install methods let executable = std::env::current_exe().ok(); let home = directories::BaseDirs::new(); let cargo_home = std::env::var_os("CARGO_HOME"); diff --git a/tests/palette.rs b/tests/palette.rs index 5f34e14..815c201 100644 --- a/tests/palette.rs +++ b/tests/palette.rs @@ -31,6 +31,17 @@ fn test_update_entry_point_is_silent_under_ci() { assert!(output.stderr.is_empty()); } +/// Stamps a kept update answer as asked now. The spawned ark judges the +/// answer's age against the real wall time, so the stamp reads it too. +#[cfg(unix)] +#[expect( + clippy::disallowed_methods, + reason = "the spawned binary compares the answer's stamp with the real wall time" +)] +fn asked_now() -> String { + chrono::Utc::now().to_rfc3339() +} + /// A fresh isolated answer produces one stderr note while help and invalid invocations stay quiet. #[cfg(unix)] #[test] @@ -64,7 +75,7 @@ fn test_update_note_preserves_command_output_and_excludes_noncommands() { let newest = format!("{}.0.0", version.major + 1); let answer = serde_json::to_vec(&serde_json::json!({ "channel": if version.pre.is_empty() { "release" } else { "develop" }, - "asked": chrono::Utc::now().to_rfc3339(), + "asked": asked_now(), "newest": newest, })) .unwrap(); @@ -226,7 +237,8 @@ fn command_tree_output_conforms() { for (path, page) in commands() { let args: Vec<_> = path.iter().map(String::as_str).collect(); // The shared options are listed once, on the root page. Examples - // mention the flags too, so the listing is told by its description. + // mention the flags too, so the test identifies the listing by its + // description. for option in ["--timeout ", "Print the complete result as JSON"] { assert_eq!(page.contains(option), path.is_empty(), "{path:?}: {option}"); }