feat(core): streaming AEAD for large files (STREAM over XChaCha20-Poly1305)

crates/encryptoglyph/src/lib.rs

@@ -22,6 +22,7 @@ pub mod audit;
 pub mod wasm;
 pub mod kek;
 pub mod index;
+pub mod stream;
 
 pub const MAGIC: &[u8; 2] = b"EG";
 pub const VERSION_LEGACY: u8 = 1;

crates/encryptoglyph/src/stream.rs

@@ -0,0 +1,136 @@
+//! Streaming AEAD for large files and backups (backlog #10).
+//!
+//! An independent STREAM-style construction (the Hoang, Reyhanitabar, Rogaway and Vizár online-AE
+//! design that age, Tink and libsodium secretstream also use), built on XChaCha20-Poly1305.
+//!
+//! Header (43 bytes): `b"EGS" | ver=1 | chunk_size u32 BE | salt[16] | nonce_prefix[19]`.
+//! Per-stream key = HKDF-SHA256(ikm = key, salt, info = "encryptoglyph/stream/v1").
+//! Chunk i nonce = `nonce_prefix | i u32 BE | last u8` and AAD = `header | aad`.
+//! The final chunk (possibly empty) is flagged, so truncation, reordering, dropping chunks and
+//! appending data all fail to decrypt.
+use crate::{Error, KEY_LEN};
+use chacha20poly1305::aead::{Aead, KeyInit, Payload};
+use chacha20poly1305::{XChaCha20Poly1305, XNonce};
+use hkdf::Hkdf;
+use sha2::Sha256;
+use std::io::{Read, Write};
+use zeroize::Zeroizing;
+
+pub const STREAM_MAGIC: &[u8; 3] = b"EGS";
+pub const STREAM_VERSION: u8 = 1;
+pub const HEADER_LEN: usize = 3 + 1 + 4 + 16 + 19;
+pub const DEFAULT_CHUNK: u32 = 64 * 1024;
+const TAG: usize = 16;
+const MAX_CHUNK: u32 = 16 * 1024 * 1024;
+
+fn cipher(key: &[u8], salt: &[u8]) -> Result<XChaCha20Poly1305, Error> {
+    if key.len() != KEY_LEN {
+        return Err(Error::InvalidKey);
+    }
+    let mut k = Zeroizing::new([0u8; 32]);
+    Hkdf::<Sha256>::new(Some(salt), key)
+        .expand(b"encryptoglyph/stream/v1", &mut *k)
+        .map_err(|_| Error::InvalidKey)?;
+    XChaCha20Poly1305::new_from_slice(&*k).map_err(|_| Error::InvalidKey)
+}
+
+fn nonce(prefix: &[u8], i: u32, last: bool) -> XNonce {
+    let mut n = [0u8; 24];
+    n[..19].copy_from_slice(prefix);
+    n[19..23].copy_from_slice(&i.to_be_bytes());
+    n[23] = last as u8;
+    XNonce::from(n)
+}
+
+fn read_full<R: Read>(r: &mut R, buf: &mut [u8]) -> Result<usize, Error> {
+    let mut n = 0;
+    while n < buf.len() {
+        match r.read(&mut buf[n..]) {
+            Ok(0) => break,
+            Ok(k) => n += k,
+            Err(e) if e.kind() == std::io::ErrorKind::Interrupted => {}
+            Err(_) => return Err(Error::InvalidArgument),
+        }
+    }
+    Ok(n)
+}
+
+/// Encrypt everything from `input` to `output`. Returns the number of plaintext bytes.
+pub fn encrypt_stream<R: Read, W: Write>(key: &[u8], aad: &[u8], chunk: u32, input: &mut R, output: &mut W) -> Result<u64, Error> {
+    if chunk == 0 || chunk > MAX_CHUNK {
+        return Err(Error::InvalidArgument);
+    }
+    let mut header = Vec::with_capacity(HEADER_LEN);
+    header.extend_from_slice(STREAM_MAGIC);
+    header.push(STREAM_VERSION);
+    header.extend_from_slice(&chunk.to_be_bytes());
+    let mut rnd = [0u8; 35];
+    getrandom::getrandom(&mut rnd).map_err(|_| Error::InvalidArgument)?;
+    header.extend_from_slice(&rnd);
+    let c = cipher(key, &header[8..24])?;
+    let mut ad = header.clone();
+    ad.extend_from_slice(aad);
+    output.write_all(&header).map_err(|_| Error::InvalidArgument)?;
+
+    let cs = chunk as usize;
+    let mut cur = Zeroizing::new(vec![0u8; cs]);
+    let mut next = Zeroizing::new(vec![0u8; cs]);
+    let mut n = read_full(input, &mut cur)?;
+    let (mut i, mut total) = (0u32, 0u64);
+    loop {
+        // Look ahead one chunk to know whether this one is last.
+        let m = if n == cs { read_full(input, &mut next)? } else { 0 };
+        let last = m == 0;
+        let ct = c
+            .encrypt(&nonce(&header[24..43], i, last), Payload { msg: &cur[..n], aad: &ad })
+            .map_err(|_| Error::InvalidArgument)?;
+        output.write_all(&ct).map_err(|_| Error::InvalidArgument)?;
+        total += n as u64;
+        if last {
+            return Ok(total);
+        }
+        i = i.checked_add(1).ok_or(Error::InvalidArgument)?;
+        std::mem::swap(&mut cur, &mut next);
+        n = m;
+    }
+}
+
+/// Decrypt a stream. Plaintext is written only after each chunk authenticates. If this returns an
+/// error, the caller must discard any output already written.
+pub fn decrypt_stream<R: Read, W: Write>(key: &[u8], aad: &[u8], input: &mut R, output: &mut W) -> Result<u64, Error> {
+    let mut header = [0u8; HEADER_LEN];
+    if read_full(input, &mut header)? != HEADER_LEN || &header[..3] != STREAM_MAGIC || header[3] != STREAM_VERSION {
+        return Err(Error::Decrypt);
+    }
+    let chunk = u32::from_be_bytes(header[4..8].try_into().unwrap());
+    if chunk == 0 || chunk > MAX_CHUNK {
+        return Err(Error::Decrypt);
+    }
+    let c = cipher(key, &header[8..24])?;
+    let mut ad = header.to_vec();
+    ad.extend_from_slice(aad);
+    let full = chunk as usize + TAG;
+    let mut cur = vec![0u8; full];
+    let mut next = vec![0u8; full];
+    let mut n = read_full(input, &mut cur)?;
+    let (mut i, mut total) = (0u32, 0u64);
+    loop {
+        if n < TAG {
+            return Err(Error::Decrypt);
+        }
+        let m = if n == full { read_full(input, &mut next)? } else { 0 };
+        let last = m == 0;
+        let pt = Zeroizing::new(
+            c.decrypt(&nonce(&header[24..43], i, last), Payload { msg: &cur[..n], aad: &ad })
+                .map_err(|_| Error::Decrypt)?,
+        );
+        output.write_all(&pt).map_err(|_| Error::InvalidArgument)?;
+        total += pt.len() as u64;
+        if last {
+            return Ok(total);
+        }
+        i = i.checked_add(1).ok_or(Error::Decrypt)?;
+        std::mem::swap(&mut cur, &mut next);
+        n = m;
+    }
+}

crates/encryptoglyph/tests/stream.rs

@@ -0,0 +1,66 @@
+use encryptoglyph::stream::{decrypt_stream, encrypt_stream, HEADER_LEN};
+use encryptoglyph::Error;
+
+const K: [u8; 32] = [7u8; 32];
+
+fn enc(pt: &[u8], chunk: u32) -> Vec<u8> {
+    let mut out = Vec::new();
+    encrypt_stream(&K, b"backup:42", chunk, &mut &pt[..], &mut out).unwrap();
+    out
+}
+fn dec(ct: &[u8]) -> Result<Vec<u8>, Error> {
+    let mut out = Vec::new();
+    decrypt_stream(&K, b"backup:42", &mut &ct[..], &mut out).map(|_| out)
+}
+
+#[test]
+fn roundtrip_sizes() {
+    for len in [0usize, 1, 63, 64, 65, 128, 1000] {
+        let pt: Vec<u8> = (0..len).map(|i| (i * 31) as u8).collect();
+        let ct = enc(&pt, 64);
+        let chunks = len.div_ceil(64).max(1); // empty input still gets one (empty, last) chunk
+        assert_eq!(ct.len(), HEADER_LEN + len + 16 * chunks, "len {len}");
+        assert_eq!(dec(&ct).unwrap(), pt);
+    }
+}
+
+#[test]
+fn random_salt_per_stream() {
+    assert_ne!(enc(b"same", 64), enc(b"same", 64));
+}
+
+#[test]
+fn tamper_truncate_reorder_append_fail() {
+    let pt = vec![9u8; 200];
+    let ct = enc(&pt, 64); // chunks: 64,64,64,8 -> each +16
+    let c = 64 + 16;
+    // Truncate at a chunk boundary: the new final chunk wasn't flagged last.
+    assert_eq!(dec(&ct[..HEADER_LEN + 3 * c]), Err(Error::Decrypt));
+    // Drop the last byte.
+    assert_eq!(dec(&ct[..ct.len() - 1]), Err(Error::Decrypt));
+    // Swap chunks 0 and 1.
+    let mut sw = ct.clone();
+    let (a, b) = (HEADER_LEN, HEADER_LEN + c);
+    let tmp = sw[a..a + c].to_vec();
+    sw.copy_within(b..b + c, a);
+    sw[b..b + c].copy_from_slice(&tmp);
+    assert_eq!(dec(&sw), Err(Error::Decrypt));
+    // Append junk.
+    let mut ap = ct.clone();
+    ap.extend_from_slice(&[0u8; 24]);
+    assert_eq!(dec(&ap), Err(Error::Decrypt));
+    // Flip a header bit (chunk size or salt).
+    let mut h = ct.clone();
+    h[10] ^= 1;
+    assert_eq!(dec(&h), Err(Error::Decrypt));
+}
+
+#[test]
+fn wrong_key_or_aad_fails() {
+    let ct = enc(b"hello", 64);
+    let mut out = Vec::new();
+    assert_eq!(decrypt_stream(&[8u8; 32], b"backup:42", &mut &ct[..], &mut out), Err(Error::Decrypt));
+    assert_eq!(decrypt_stream(&K, b"backup:43", &mut &ct[..], &mut out), Err(Error::Decrypt));
+    assert_eq!(encrypt_stream(&[1u8; 5], b"", 64, &mut &b""[..], &mut out), Err(Error::InvalidKey));
+    assert_eq!(encrypt_stream(&K, b"", 0, &mut &b""[..], &mut out), Err(Error::InvalidArgument));
+}