From b2e302152080962b489635276837b9254e1830b2 Mon Sep 17 00:00:00 2001 From: GoDeskio Date: Thu, 5 Nov 2026 18:43:11 +0800 Subject: [PATCH] feat(core): vaulted and vaultless tokenization --- crates/encryptoglyph/src/lib.rs | 1 + crates/encryptoglyph/src/token.rs | 126 ++++++++++++++++++++++++++++ crates/encryptoglyph/tests/token.rs | 28 +++++++ 3 files changed, 155 insertions(+) create mode 100644 crates/encryptoglyph/src/token.rs create mode 100644 crates/encryptoglyph/tests/token.rs diff --git a/crates/encryptoglyph/src/lib.rs b/crates/encryptoglyph/src/lib.rs index bcbfe32..a89af4f 100644 --- a/crates/encryptoglyph/src/lib.rs +++ b/crates/encryptoglyph/src/lib.rs @@ -30,6 +30,7 @@ pub mod pq; pub mod policy; pub mod tenancy; pub mod secrets; +pub mod token; pub const MAGIC: &[u8; 2] = b"EG"; pub const VERSION_LEGACY: u8 = 1; diff --git a/crates/encryptoglyph/src/token.rs b/crates/encryptoglyph/src/token.rs new file mode 100644 index 0000000..78c9839 --- /dev/null +++ b/crates/encryptoglyph/src/token.rs @@ -0,0 +1,126 @@ +//! Tokenization (backlog #8): vaulted random tokens and vaultless format-preserving tokens. +//! +//! * Vaulted: `tok_` tokens are random with no relation to the value. Values are stored +//! XChaCha20-Poly1305 encrypted, with AAD bound to the token. Optional deterministic mode +//! reuses a token for equal values via an HMAC fingerprint, so joins still work. A +//! `TokenStore` backend can swap in a database later. +//! * Vaultless: FF1 over digits that keeps a clear suffix (for example the last 4 of a PAN). +use crate::ff1::Ff1; +use crate::{Error, KEY_LEN}; +use chacha20poly1305::aead::{Aead, KeyInit, Payload}; +use chacha20poly1305::{XChaCha20Poly1305, XNonce}; +use hkdf::Hkdf; +use hmac::{Hmac, Mac}; +use sha2::Sha256; +use std::collections::HashMap; +use zeroize::Zeroizing; + +pub trait TokenStore { + fn put(&mut self, token: &str, sealed: Vec, fingerprint: Option<[u8; 32]>); + fn get(&self, token: &str) -> Option<&[u8]>; + fn by_fingerprint(&self, fp: &[u8; 32]) -> Option; + fn delete(&mut self, token: &str) -> bool; +} + +#[derive(Default)] +pub struct MemoryStore { + sealed: HashMap>, + fps: HashMap<[u8; 32], String>, +} + +impl TokenStore for MemoryStore { + fn put(&mut self, token: &str, sealed: Vec, fp: Option<[u8; 32]>) { + self.sealed.insert(token.to_string(), sealed); + if let Some(f) = fp { + self.fps.insert(f, token.to_string()); + } + } + fn get(&self, token: &str) -> Option<&[u8]> { + self.sealed.get(token).map(|v| v.as_slice()) + } + fn by_fingerprint(&self, fp: &[u8; 32]) -> Option { + self.fps.get(fp).cloned() + } + fn delete(&mut self, token: &str) -> bool { + self.fps.retain(|_, t| t != token); + self.sealed.remove(token).is_some() + } +} + +pub struct Vault { + aead: XChaCha20Poly1305, + fp_key: Zeroizing<[u8; 32]>, + store: S, +} + +impl Vault { + pub fn new(key: &[u8], store: S) -> Result { + if key.len() != KEY_LEN { + return Err(Error::InvalidKey); + } + let hk = Hkdf::::new(Some(b"encryptoglyph/token/v1"), key); + let mut ek = Zeroizing::new([0u8; 32]); + let mut fk = Zeroizing::new([0u8; 32]); + hk.expand(b"seal", &mut *ek).map_err(|_| Error::InvalidKey)?; + hk.expand(b"fingerprint", &mut *fk).map_err(|_| Error::InvalidKey)?; + Ok(Vault { aead: XChaCha20Poly1305::new_from_slice(&*ek).map_err(|_| Error::InvalidKey)?, fp_key: fk, store }) + } + + fn fingerprint(&self, value: &[u8]) -> [u8; 32] { + let mut m = as Mac>::new_from_slice(&*self.fp_key).expect("hmac key"); + m.update(value); + m.finalize().into_bytes().into() + } + + /// Tokenize. With `deterministic`, equal values get the same token. + pub fn tokenize(&mut self, value: &[u8], deterministic: bool) -> Result { + let fp = deterministic.then(|| self.fingerprint(value)); + if let Some(t) = fp.as_ref().and_then(|f| self.store.by_fingerprint(f)) { + return Ok(t); + } + let mut r = [0u8; 16 + 24]; + getrandom::getrandom(&mut r).map_err(|_| Error::InvalidArgument)?; + let token = format!("tok_{}", r[..16].iter().map(|b| format!("{b:02x}")).collect::()); + let mut sealed = r[16..].to_vec(); + let ct = self + .aead + .encrypt(XNonce::from_slice(&r[16..]), Payload { msg: value, aad: token.as_bytes() }) + .map_err(|_| Error::InvalidArgument)?; + sealed.extend(ct); + self.store.put(&token, sealed, fp); + Ok(token) + } + + pub fn detokenize(&self, token: &str) -> Result>, Error> { + let s = self.store.get(token).ok_or(Error::Decrypt)?; + if s.len() < 24 + 16 { + return Err(Error::Decrypt); + } + self.aead + .decrypt(XNonce::from_slice(&s[..24]), Payload { msg: &s[24..], aad: token.as_bytes() }) + .map(Zeroizing::new) + .map_err(|_| Error::Decrypt) + } + + /// Delete a token mapping (right-to-erasure). Returns whether it existed. + pub fn revoke(&mut self, token: &str) -> bool { + self.store.delete(token) + } +} + +const DIGITS: &str = "0123456789"; + +/// Vaultless digit token: FF1 over everything except the last `keep` digits, which stay clear. +pub fn vaultless_digits(ff1: &Ff1, tweak: &[u8], value: &str, keep: usize) -> Result { + let (head, tail) = value.split_at(value.len().checked_sub(keep).ok_or(Error::InvalidArgument)?); + let mut t = tweak.to_vec(); + t.extend_from_slice(tail.as_bytes()); // bind the clear suffix into the tweak + Ok(ff1.encrypt_str(&t, DIGITS, head)? + tail) +} + +pub fn vaultless_digits_reveal(ff1: &Ff1, tweak: &[u8], token: &str, keep: usize) -> Result { + let (head, tail) = token.split_at(token.len().checked_sub(keep).ok_or(Error::InvalidArgument)?); + let mut t = tweak.to_vec(); + t.extend_from_slice(tail.as_bytes()); + Ok(ff1.decrypt_str(&t, DIGITS, head)? + tail) +} diff --git a/crates/encryptoglyph/tests/token.rs b/crates/encryptoglyph/tests/token.rs new file mode 100644 index 0000000..7fa41d4 --- /dev/null +++ b/crates/encryptoglyph/tests/token.rs @@ -0,0 +1,28 @@ +use encryptoglyph::ff1::Ff1; +use encryptoglyph::token::{vaultless_digits, vaultless_digits_reveal, MemoryStore, Vault}; + +#[test] +fn vaulted_roundtrip_and_modes() { + let mut v = Vault::new(&[5u8; 32], MemoryStore::default()).unwrap(); + let a = v.tokenize(b"4111111111111111", false).unwrap(); + let b = v.tokenize(b"4111111111111111", false).unwrap(); + assert_ne!(a, b); + assert!(a.starts_with("tok_") && !a.contains("4111")); + assert_eq!(&v.detokenize(&a).unwrap()[..], b"4111111111111111"); + let d1 = v.tokenize(b"x@y.z", true).unwrap(); + assert_eq!(v.tokenize(b"x@y.z", true).unwrap(), d1); + assert!(v.revoke(&d1)); + assert!(v.detokenize(&d1).is_err()); + assert_ne!(v.tokenize(b"x@y.z", true).unwrap(), d1); + assert!(v.detokenize("tok_nope").is_err()); +} + +#[test] +fn vaultless_keeps_last4() { + let f = Ff1::new(&[9u8; 32], 10).unwrap(); + let t = vaultless_digits(&f, b"pan", "4111111111111111", 4).unwrap(); + assert_eq!(t.len(), 16); + assert!(t.ends_with("1111")); + assert_ne!(&t[..12], "411111111111"); + assert_eq!(vaultless_digits_reveal(&f, b"pan", &t, 4).unwrap(), "4111111111111111"); +}