diff --git a/Cargo.lock b/Cargo.lock index db2bdd0c74..d3d6e0886d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6080,6 +6080,7 @@ dependencies = [ name = "tracedecay-graph-db" version = "0.1.0" dependencies = [ + "base64 0.22.1", "criterion", "grafeo-adapters", "grafeo-common", diff --git a/crates/tracedecay-graph-db/Cargo.toml b/crates/tracedecay-graph-db/Cargo.toml index a3800bb84d..1636ce47ee 100644 --- a/crates/tracedecay-graph-db/Cargo.toml +++ b/crates/tracedecay-graph-db/Cargo.toml @@ -66,6 +66,7 @@ grafeo-common.workspace = true grafeo-core = { workspace = true, features = ["compact-store"] } grafeo-engine = { workspace = true, features = ["mmap", "compact-store"] } grafeo-storage.workspace = true +base64 = "0.22" hex = "0.4" hotpath.workspace = true parking_lot = { version = "0.12", features = ["arc_lock", "send_guard"] } diff --git a/crates/tracedecay-graph-db/src/location.rs b/crates/tracedecay-graph-db/src/location.rs index 5676dce9f8..02fd49142b 100644 --- a/crates/tracedecay-graph-db/src/location.rs +++ b/crates/tracedecay-graph-db/src/location.rs @@ -20,7 +20,7 @@ pub struct GraphFormatVersion(u32); impl GraphFormatVersion { #[must_use] pub const fn current() -> Self { - Self(3) + Self(4) } #[cfg(any(test, feature = "test-helpers", feature = "eval-helpers"))] diff --git a/crates/tracedecay-graph-db/src/projection.rs b/crates/tracedecay-graph-db/src/projection.rs index 6e3011ab49..f81422c268 100644 --- a/crates/tracedecay-graph-db/src/projection.rs +++ b/crates/tracedecay-graph-db/src/projection.rs @@ -12,6 +12,7 @@ use crate::limits::{ MAX_GRAPH_IDENTIFIER_BYTES, MAX_GRAPH_PROPERTIES, MAX_GRAPH_PROPERTY_AGGREGATE_BYTES, MAX_GRAPH_PROPERTY_VALUE_BYTES, }; +use crate::schema::COMPACT_IDENTITY_MARKER; use crate::{GraphBudgetKind, GraphDbError}; const RESERVED_PREFIX: &str = "__tracedecay_graph_db_"; @@ -98,6 +99,11 @@ fn validate_opaque(kind: &str, value: &str) -> Result<(), GraphDbError> { "{kind} exceeds {MAX_GRAPH_IDENTIFIER_BYTES} bytes" ))); } + if value.starts_with(COMPACT_IDENTITY_MARKER) { + return Err(GraphDbError::invalid(format!( + "{kind} starts with the reserved compact identity marker" + ))); + } if value.starts_with(RESERVED_PREFIX) { return Err(GraphDbError::invalid(format!( "{kind} uses the reserved graph database prefix" diff --git a/crates/tracedecay-graph-db/src/projection_identity_index.rs b/crates/tracedecay-graph-db/src/projection_identity_index.rs index a5022ace4c..723e2d0c26 100644 --- a/crates/tracedecay-graph-db/src/projection_identity_index.rs +++ b/crates/tracedecay-graph-db/src/projection_identity_index.rs @@ -24,12 +24,11 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, RwLock}; -use grafeo_common::types::Value; use grafeo_engine::GrafeoDB; use crate::projection::check_cancelled; use crate::projection_read::IdentityScope; -use crate::schema::{has_native_label, nodes_with_label}; +use crate::schema::{decode_identity, has_native_label, nodes_with_label}; use crate::{GraphCancellation, GraphDbError}; /// Identity bytes one cached index may retain. A projection whose identities @@ -200,12 +199,7 @@ fn build_identity_index( if !has_native_label(&record, record_label) { continue; } - let identity = record - .get_property(identity_property) - .and_then(Value::as_str) - .ok_or_else(|| GraphDbError::Corrupt { - message: format!("projection query returned a non-string `{identity_property}`"), - })?; + let identity = decode_identity(record.get_property(identity_property), identity_property)?; identity_bytes = identity_bytes.saturating_add(identity.len()); if identity_bytes > MAX_IDENTITY_INDEX_BYTES { return Ok(None); diff --git a/crates/tracedecay-graph-db/src/projection_read.rs b/crates/tracedecay-graph-db/src/projection_read.rs index d0779a8696..a310733bef 100644 --- a/crates/tracedecay-graph-db/src/projection_read.rs +++ b/crates/tracedecay-graph-db/src/projection_read.rs @@ -2,12 +2,11 @@ use std::collections::BTreeSet; use std::fmt; use std::sync::Arc; -use grafeo_common::types::Value; use grafeo_engine::GrafeoDB; use crate::projection::check_cancelled; use crate::schema::{ - ENTITY_ID_PROPERTY, ENTITY_LABEL, RELATION_ID_PROPERTY, RELATION_LABEL, + ENTITY_ID_PROPERTY, ENTITY_LABEL, RELATION_ID_PROPERTY, RELATION_LABEL, decode_identity, entity_projection_label, relation_projection_label, }; use crate::state::{labeled_projection_nodes, latest_projection, load_entity, load_relation}; @@ -411,25 +410,17 @@ fn streaming_identity_page( let Some(record) = store.get_node(node) else { continue; }; - let identity = record - .get_property(identity_property) - .and_then(Value::as_str) - .ok_or_else(|| GraphDbError::Corrupt { - message: format!("projection query returned a non-string `{identity_property}`"), - })?; - if after.is_some_and(|after| identity <= after) { + let identity = decode_identity(record.get_property(identity_property), identity_property)?; + if after.is_some_and(|after| identity.as_str() <= after) { continue; } if page.len() == limit { - if page - .last() - .is_some_and(|widest| identity >= widest.as_str()) - { + if page.last().is_some_and(|widest| identity >= *widest) { continue; } page.pop_last(); } - page.insert(identity.to_owned()); + page.insert(identity); } Ok(page.into_iter().collect()) } diff --git a/crates/tracedecay-graph-db/src/schema.rs b/crates/tracedecay-graph-db/src/schema.rs index 3152842ecf..81d01a3f66 100644 --- a/crates/tracedecay-graph-db/src/schema.rs +++ b/crates/tracedecay-graph-db/src/schema.rs @@ -1,6 +1,8 @@ use std::collections::{BTreeMap, BTreeSet}; -use grafeo_common::types::{EdgeId, NodeId, Value}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use grafeo_common::types::{EdgeId, NodeId, PropertyKey, Value}; use grafeo_core::graph::GraphStore; use grafeo_core::graph::lpg::{Edge, Node}; use sha2::{Digest, Sha256}; @@ -94,6 +96,9 @@ const NAMESPACE_KEY_ID_BYTES: usize = 8; const DIGEST_BYTES: usize = 32; const RAW_IDENTITY_TAG: u8 = 0; const DIGEST_IDENTITY_TAG: u8 = 1; +/// Leads a compact identity scalar; `validate_opaque` rejects graph +/// identifiers that start with it. +pub(crate) const COMPACT_IDENTITY_MARKER: char = '\u{1}'; /// The short id a namespace contributes to every unique key it owns: the /// leading bytes of the namespace's SHA-256. @@ -121,10 +126,7 @@ pub(crate) fn namespace_key_id(namespace: &GraphNamespace) -> NamespaceKeyId { pub(crate) fn stable_key(namespace_id: &NamespaceKeyId, identity: &str) -> Vec { let mut key = Vec::with_capacity(NAMESPACE_KEY_ID_BYTES + 1 + identity.len()); key.extend_from_slice(namespace_id); - match identity - .rsplit_once(':') - .and_then(|(kind, digest)| Some((kind, decode_lower_hex_digest(digest)?))) - { + match digest_identity(identity) { Some((kind, digest)) => { key.push(DIGEST_IDENTITY_TAG); key.extend_from_slice(kind.as_bytes()); @@ -138,6 +140,71 @@ pub(crate) fn stable_key(namespace_id: &NamespaceKeyId, identity: &str) -> Vec:<64 lowercase hex>` identity. +fn digest_identity(identity: &str) -> Option<(&str, [u8; DIGEST_BYTES])> { + identity + .rsplit_once(':') + .and_then(|(kind, digest)| Some((kind, decode_lower_hex_digest(digest)?))) +} + +/// Unpadded base64url width of a 32-byte digest. +const DIGEST_TEXT_BYTES: usize = 43; + +/// The stored scalar for a relation identity, source, or target. +/// +/// A `:<64 lowercase hex>` identity is stored as +/// [`COMPACT_IDENTITY_MARKER`], its kind, and its digest in unpadded +/// base64url; any other identity is stored verbatim. The value stays a +/// string because the sealed compact store keeps `Bytes` in its string +/// dictionary as marked hex, which would double the digest again. Only the +/// canonical spelling compacts and no graph identifier may start with the +/// marker, so [`decode_identity`] restores exactly the identity written. +pub(crate) fn encode_identity(identity: &str) -> Value { + match digest_identity(identity) { + Some((kind, digest)) => Value::from(format!( + "{COMPACT_IDENTITY_MARKER}{kind}{}", + URL_SAFE_NO_PAD.encode(digest) + )), + None => Value::from(identity), + } +} + +/// Reads back an identity written by [`encode_identity`]. +pub(crate) fn decode_identity( + value: Option<&Value>, + description: &str, +) -> Result { + let Some(Value::String(stored)) = value else { + return Err(GraphDbError::Corrupt { + message: format!("native {description} is missing or not a string"), + }); + }; + let identity = match stored.strip_prefix(COMPACT_IDENTITY_MARKER) { + None => stored.to_string(), + Some(compact) => { + let digest = compact + .len() + .checked_sub(DIGEST_TEXT_BYTES) + .filter(|split| compact.is_char_boundary(*split)) + .and_then(|split| { + let (kind, digest) = compact.split_at(split); + let mut bytes = [0; DIGEST_BYTES]; + let written = URL_SAFE_NO_PAD.decode_slice(digest, &mut bytes).ok()?; + (written == DIGEST_BYTES).then(|| format!("{kind}:{}", hex::encode(bytes))) + }); + digest.ok_or_else(|| GraphDbError::Corrupt { + message: format!("native {description} has a malformed compact digest"), + })? + } + }; + if identity.len() > MAX_GRAPH_IDENTIFIER_BYTES { + return Err(GraphDbError::Corrupt { + message: format!("native {description} exceeds its product bound"), + }); + } + Ok(identity) +} + /// Only the canonical lowercase spelling compacts, so every identity has /// exactly one key. fn decode_lower_hex_digest(digest: &str) -> Option<[u8; DIGEST_BYTES]> { @@ -153,8 +220,11 @@ fn decode_lower_hex_digest(digest: &str) -> Option<[u8; DIGEST_BYTES]> { Some(bytes) } +/// A unique key as the indexed scalar: unpadded base64url of +/// [`stable_key`]'s bytes, a string for the same reason as +/// [`encode_identity`]. pub(crate) fn key_value(namespace: &GraphNamespace, identity: &str) -> Value { - Value::Bytes(stable_key(&namespace_key_id(namespace), identity).into()) + Value::from(URL_SAFE_NO_PAD.encode(stable_key(&namespace_key_id(namespace), identity))) } /// The indexed unique-key value for one entity. @@ -368,15 +438,15 @@ pub(crate) fn relation_properties( ), ( RELATION_ID_PROPERTY.to_owned(), - Value::from(relation.identity.as_str()), + encode_identity(relation.identity.as_str()), ), ( RELATION_FROM_PROPERTY.to_owned(), - Value::from(relation.from.as_str()), + encode_identity(relation.from.as_str()), ), ( RELATION_TO_PROPERTY.to_owned(), - Value::from(relation.to.as_str()), + encode_identity(relation.to.as_str()), ), ( RELATION_KIND_PROPERTY.to_owned(), @@ -393,6 +463,9 @@ pub(crate) fn relation_properties( Ok(properties) } +/// A native edge carries its owner scalars and payload, never the relation's +/// identity or endpoints: those are owned by its locator node, which +/// [`edge_locator`] resolves through the `RELATION_EDGE` index. pub(crate) fn edge_properties( namespace: &GraphNamespace, projection: &GraphProjectionId, @@ -407,18 +480,6 @@ pub(crate) fn edge_properties( PROJECTION_PROPERTY.to_owned(), Value::from(projection.as_str()), ), - ( - RELATION_ID_PROPERTY.to_owned(), - Value::from(relation.identity.as_str()), - ), - ( - RELATION_FROM_PROPERTY.to_owned(), - Value::from(relation.from.as_str()), - ), - ( - RELATION_TO_PROPERTY.to_owned(), - Value::from(relation.to.as_str()), - ), ( RELATION_KIND_PROPERTY.to_owned(), Value::from(relation.kind.as_str()), @@ -547,17 +608,17 @@ pub(crate) fn decode_relation(locator: &Node, edge: &Edge) -> Result Result { @@ -620,11 +682,7 @@ pub(crate) fn decode_relation_identity( message: "traversal relation native type and kind disagree".to_owned(), }); } - let identity = GraphRelationId::new(required_string( - edge.get_property(RELATION_ID_PROPERTY), - "relation identity", - )?) - .map_err(|error| persisted_validation_error("relation identity", error))?; + let identity = edge_relation_identity(store, edge_locator(store, edge.id)?)?; Ok(DecodedRelationIdentity { identity, projection, @@ -632,6 +690,59 @@ pub(crate) fn decode_relation_identity( }) } +/// The locator node that owns `edge`'s identity and endpoints. +/// +/// Resolved through the `RELATION_EDGE` unique index. A locator deleted in +/// the live store keeps its index entry but loses its properties, so the +/// re-read of the indexed scalar discards such tombstones without +/// materializing the node. +pub(crate) fn edge_locator(store: &dyn GraphStore, edge: EdgeId) -> Result { + let value = relation_edge_value(edge)?; + let key = PropertyKey::new(RELATION_EDGE_PROPERTY); + let mut locators = store + .find_nodes_by_property(RELATION_EDGE_PROPERTY, &value) + .into_iter() + .filter(|node| store.get_node_property(*node, &key).as_ref() == Some(&value)); + match (locators.next(), locators.next()) { + (Some(locator), None) => Ok(locator), + (None, _) => Err(GraphDbError::Corrupt { + message: "native relation edge has no locator".to_owned(), + }), + (Some(_), Some(_)) => Err(GraphDbError::Corrupt { + message: "native relation edge has duplicate locators".to_owned(), + }), + } +} + +/// One identity scalar of a relation locator, read without materializing +/// the node. +pub(crate) fn locator_identity( + store: &dyn GraphStore, + locator: NodeId, + property: &str, + description: &str, +) -> Result { + decode_identity( + store + .get_node_property(locator, &PropertyKey::new(property)) + .as_ref(), + description, + ) +} + +pub(crate) fn edge_relation_identity( + store: &dyn GraphStore, + locator: NodeId, +) -> Result { + GraphRelationId::new(locator_identity( + store, + locator, + RELATION_ID_PROPERTY, + "relation identity", + )?) + .map_err(|error| persisted_validation_error("relation identity", error)) +} + pub(crate) fn decode_graph_properties( properties: impl IntoIterator, Value)>, ) -> Result, GraphDbError> { @@ -885,7 +996,11 @@ mod graph_stable_identity_tests { mod stable_key_tests { use std::collections::BTreeSet; - use super::{graph_stable_identity, namespace_key_id, stable_key}; + use grafeo_common::types::Value; + + use super::{ + decode_identity, encode_identity, graph_stable_identity, namespace_key_id, stable_key, + }; use crate::GraphNamespace; #[test] @@ -935,4 +1050,37 @@ mod stable_key_tests { .collect(); assert_eq!(keys.len(), 14); } + + #[test] + fn relation_identities_store_their_digest_compactly_and_read_back_exactly() { + let identity = graph_stable_identity("edge", "occ"); + assert_eq!( + identity, + "edge:a92cf2a4297d812859e387e8efac97838fc420befe35fa0f8b7e94ad9a139ff5" + ); + let digest = identity.strip_prefix("edge:").unwrap(); + let stored = encode_identity(&identity); + + assert_eq!( + stored, + Value::from("\u{1}edgeqSzypCl9gShZ44fo76yXg4_EIL7-NfoPi36UrZoTn_U") + ); + assert_eq!(stored.as_str().unwrap().len(), 48); + assert_eq!( + decode_identity(Some(&stored), "relation").unwrap(), + identity + ); + for verbatim in [ + format!("edge:{}", digest.to_uppercase()), + "relation:a-b".to_owned(), + format!("edge:{digest}0"), + ] { + let stored = encode_identity(&verbatim); + assert_eq!(stored, Value::from(verbatim.as_str())); + assert_eq!( + decode_identity(Some(&stored), "relation").unwrap(), + verbatim + ); + } + } } diff --git a/crates/tracedecay-graph-db/src/state.rs b/crates/tracedecay-graph-db/src/state.rs index 41c110f9da..3ef5a9ed22 100644 --- a/crates/tracedecay-graph-db/src/state.rs +++ b/crates/tracedecay-graph-db/src/state.rs @@ -11,17 +11,17 @@ use crate::limits::{ MAX_VERIFIED_GENERATION_RELATIONS, require_generation_capacity, }; use crate::schema::{ - COMMIT_SEQUENCE_PROPERTY, DIGEST_PROPERTY, ENTITY_ID_PROPERTY, ENTITY_KEY_PROPERTY, - ENTITY_LABEL, FORMAT_LABEL, GENERATION_DEPENDENCY_DIGEST_PROPERTY, IDEMPOTENCY_KEY_PROPERTY, - NAMESPACE_PROPERTY, PROJECTION_KEY_PROPERTY, PROJECTION_LABEL, PROJECTION_PROPERTY, - PUBLICATION_DIGEST_PROPERTY, PUBLICATION_INPUT_DIGEST_PROPERTY, PUBLICATION_KEY_PROPERTY, - PUBLICATION_LABEL, RELATION_EDGE_PROPERTY, RELATION_FROM_PROPERTY, RELATION_ID_PROPERTY, - RELATION_KEY_PROPERTY, RELATION_LABEL, RELATION_TO_PROPERTY, SEQUENCE_PROPERTY, - SOURCE_GENERATION_PROPERTY, WATERMARK_PROPERTY, decode_entity, decode_relation, - entity_key_value, entity_projection_label, has_native_label, namespace_key_id, - nodes_with_label, nodes_with_label_count, projection_state_key_value, publication_key_value, - relation_edge_value, relation_key_value, relation_projection_label, required_i64, - required_string, stable_key, + COMMIT_SEQUENCE_PROPERTY, COMPACT_IDENTITY_MARKER, DIGEST_PROPERTY, ENTITY_ID_PROPERTY, + ENTITY_KEY_PROPERTY, ENTITY_LABEL, FORMAT_LABEL, GENERATION_DEPENDENCY_DIGEST_PROPERTY, + IDEMPOTENCY_KEY_PROPERTY, NAMESPACE_PROPERTY, PROJECTION_KEY_PROPERTY, PROJECTION_LABEL, + PROJECTION_PROPERTY, PUBLICATION_DIGEST_PROPERTY, PUBLICATION_INPUT_DIGEST_PROPERTY, + PUBLICATION_KEY_PROPERTY, PUBLICATION_LABEL, RELATION_EDGE_PROPERTY, RELATION_FROM_PROPERTY, + RELATION_ID_PROPERTY, RELATION_KEY_PROPERTY, RELATION_LABEL, RELATION_TO_PROPERTY, + SEQUENCE_PROPERTY, SOURCE_GENERATION_PROPERTY, WATERMARK_PROPERTY, decode_entity, + decode_identity, decode_relation, entity_key_value, entity_projection_label, has_native_label, + namespace_key_id, nodes_with_label, nodes_with_label_count, projection_state_key_value, + publication_key_value, relation_edge_value, relation_key_value, relation_projection_label, + required_i64, required_string, stable_key, }; use crate::{ GraphCommit, GraphDbError, GraphEntity, GraphEntityId, GraphIdempotencyKey, GraphMutation, @@ -620,7 +620,7 @@ fn load_relation_reference_by_edge( .ok_or_else(|| GraphDbError::Corrupt { message: "indexed relation locator is unreadable".to_owned(), })?; - let identity = GraphRelationId::new(required_string( + let identity = GraphRelationId::new(decode_identity( locator.get_property(RELATION_ID_PROPERTY), "relation identity", )?) @@ -630,12 +630,12 @@ fn load_relation_reference_by_edge( "relation projection", )?) .map_err(|error| persisted_validation_error("relation projection", error))?; - let from = GraphEntityId::new(required_string( + let from = GraphEntityId::new(decode_identity( locator.get_property(RELATION_FROM_PROPERTY), "relation source", )?) .map_err(|error| persisted_validation_error("relation source", error))?; - let to = GraphEntityId::new(required_string( + let to = GraphEntityId::new(decode_identity( locator.get_property(RELATION_TO_PROPERTY), "relation target", )?) @@ -710,7 +710,7 @@ pub(crate) fn projection_entity_nodes_sorted_checked( message: "native graph entity disappeared during verification".to_owned(), })?; keyed.push(( - required_arc_string( + identity_arc( record.get_property(ENTITY_ID_PROPERTY), "native graph entity identity", )?, @@ -876,7 +876,7 @@ fn projection_identity_deletion_page_checked( else { continue; }; - let identity = required_arc_string( + let identity = identity_arc( record.get_property(identity_property), &format!("native graph {description} identity"), )?; @@ -926,7 +926,7 @@ pub(crate) fn projection_relation_nodes_sorted_checked( message: "native graph relation disappeared during verification".to_owned(), })?; keyed.push(( - required_arc_string( + identity_arc( record.get_property(crate::schema::RELATION_ID_PROPERTY), "native graph relation identity", )?, @@ -1111,16 +1111,17 @@ fn labeled_projection_nodes_checked( Ok(nodes) } -fn required_arc_string(value: Option<&Value>, description: &str) -> Result { +/// An identity scalar as a shared string: verbatim strings are shared as-is, +/// compact relation identities are decoded once. +fn identity_arc(value: Option<&Value>, description: &str) -> Result { match value { - Some(Value::String(value)) if value.len() <= MAX_GRAPH_IDENTIFIER_BYTES => { + Some(Value::String(value)) + if value.len() <= MAX_GRAPH_IDENTIFIER_BYTES + && !value.starts_with(COMPACT_IDENTITY_MARKER) => + { Ok(value.clone()) } - _ => Err(GraphDbError::Corrupt { - message: format!( - "native {description} is missing, not a string, or exceeds its product bound" - ), - }), + value => decode_identity(value, description).map(ArcStr::from), } } diff --git a/crates/tracedecay-graph-db/src/traversal.rs b/crates/tracedecay-graph-db/src/traversal.rs index b18559edec..d94dce1f75 100644 --- a/crates/tracedecay-graph-db/src/traversal.rs +++ b/crates/tracedecay-graph-db/src/traversal.rs @@ -13,9 +13,10 @@ use crate::adjacency_id_index::{AdjacencyIdIndexCache, AdjacencyIndexKey, page_i use crate::epoch_cache::LabelKeyCache; use crate::schema::{ ENTITY_ID_PROPERTY, ENTITY_KEY_PROPERTY, ENTITY_LABEL, NAMESPACE_PROPERTY, PROJECTION_PROPERTY, - RELATION_FROM_PROPERTY, RELATION_ID_PROPERTY, RELATION_KIND_PROPERTY, RELATION_TO_PROPERTY, - decode_entity, decode_graph_properties, decode_relation_identity, entity_key_value, - entity_projection_label, label_keys, relation_kind_from_type, relation_type_for_kind, + RELATION_FROM_PROPERTY, RELATION_KIND_PROPERTY, RELATION_TO_PROPERTY, decode_entity, + decode_graph_properties, decode_relation_identity, edge_locator, edge_relation_identity, + entity_key_value, entity_projection_label, label_keys, locator_identity, + relation_kind_from_type, relation_type_for_kind, }; use crate::{ GraphBudgetKind, GraphCancellation, GraphDbError, GraphEntity, GraphEntityId, GraphNamespace, @@ -136,14 +137,22 @@ pub(crate) fn traverse( let store = database.graph_store(); let start = node_for_entity(store.as_ref(), &request.namespace, &request.start)?; - let projected = relation_projection(store, &request.relation_kinds); + let projected = relation_projection(Arc::clone(&store), &request.relation_kinds); match request.direction { - GraphTraversalDirection::Outgoing => { - native_outgoing_traversal(&projected, start, &request, ensure_projection_readable) - } - GraphTraversalDirection::Incoming | GraphTraversalDirection::Both => { - directional_traversal(&projected, start, &request, ensure_projection_readable) - } + GraphTraversalDirection::Outgoing => native_outgoing_traversal( + &projected, + store.as_ref(), + start, + &request, + ensure_projection_readable, + ), + GraphTraversalDirection::Incoming | GraphTraversalDirection::Both => directional_traversal( + &projected, + store.as_ref(), + start, + &request, + ensure_projection_readable, + ), } } @@ -184,7 +193,7 @@ pub(crate) fn outgoing_relation_targets( })?; let target = decode_entity(&target)?; let relation = relation_for_edge( - &projected, + store.as_ref(), edge, namespace, ensure_projection_readable, @@ -240,7 +249,7 @@ pub(crate) fn visit_outgoing_relation_targets( })?; let target = decode_entity(&target)?; let relation = relation_for_edge( - &projected, + store.as_ref(), edge, namespace, ensure_projection_readable, @@ -431,7 +440,7 @@ fn collect_relation_ids( let stored = store.get_edge(edge).ok_or_else(|| GraphDbError::Corrupt { message: "outgoing relation references a missing native edge".to_owned(), })?; - let decoded = decode_relation_identity(&stored, namespace)?; + let decoded = decode_relation_identity(store.as_ref(), &stored, namespace)?; if !relation_kinds.is_empty() && !relation_kinds.contains(&decoded.kind) { return Err(GraphDbError::Corrupt { message: "relation kind escaped its projection filter".to_owned(), @@ -522,7 +531,7 @@ pub(crate) fn directed_relations( } } relations.push(relation_for_edge( - &projected, + store.as_ref(), edge, namespace, ensure_projection_readable, @@ -631,6 +640,7 @@ fn projection_relation_projection( #[hotpath::measure(label = "graph_db.compact.native_outgoing")] fn native_outgoing_traversal( store: &dyn GraphStore, + owners: &dyn GraphStore, start: NodeId, request: &TraversalRequest, ensure_projection_readable: &dyn Fn( @@ -709,7 +719,7 @@ fn native_outgoing_traversal( ))); }; let relation = match cached_relation_identity( - store, + owners, edge, &request.namespace, ensure_projection_readable, @@ -745,7 +755,7 @@ fn native_outgoing_traversal( }; if source_depth.checked_add(1) == Some(target_depth) { let relation = match cached_relation_identity( - store, + owners, edge, &request.namespace, ensure_projection_readable, @@ -835,6 +845,7 @@ enum NativeTraversalStop { #[hotpath::measure(label = "graph_db.compact.directional")] fn directional_traversal( store: &dyn GraphStore, + owners: &dyn GraphStore, start: NodeId, request: &TraversalRequest, ensure_projection_readable: &dyn Fn( @@ -886,7 +897,7 @@ fn directional_traversal( return Err(GraphDbError::Cancelled); } let relation = cached_relation_identity( - store, + owners, edge, &request.namespace, ensure_projection_readable, @@ -1160,15 +1171,7 @@ fn relation_identity( message: "traversal relation native type and kind disagree".to_owned(), }); } - let identity = stored - .get_property(RELATION_ID_PROPERTY) - .and_then(Value::as_str) - .ok_or_else(|| GraphDbError::Corrupt { - message: "traversal relation has no native identity".to_owned(), - })?; - GraphRelationId::new(identity).map_err(|error| GraphDbError::Corrupt { - message: format!("traversal relation has an invalid native identity: {error}"), - }) + edge_relation_identity(store, edge_locator(store, edge)?) } enum RelationEndpointCheck<'a> { @@ -1220,21 +1223,18 @@ fn relation_for_edge( message: "traversal relation native type and kind disagree".to_owned(), }); } - let identity = stored - .get_property(RELATION_ID_PROPERTY) - .and_then(Value::as_str) - .ok_or_else(|| GraphDbError::Corrupt { - message: "traversal relation has no native identity".to_owned(), - })?; - let identity = GraphRelationId::new(identity).map_err(|error| GraphDbError::Corrupt { - message: format!("traversal relation has an invalid native identity: {error}"), - })?; - let from = required_entity_property( - stored.get_property(RELATION_FROM_PROPERTY), + let locator = edge_locator(store, edge)?; + let identity = edge_relation_identity(store, locator)?; + let from = locator_entity( + store, + locator, + RELATION_FROM_PROPERTY, "outgoing relation source", )?; - let to = required_entity_property( - stored.get_property(RELATION_TO_PROPERTY), + let to = locator_entity( + store, + locator, + RELATION_TO_PROPERTY, "outgoing relation target", )?; let endpoints_match = match endpoint_check { @@ -1265,11 +1265,13 @@ fn relation_for_edge( }) } -fn required_entity_property( - value: Option<&Value>, +fn locator_entity( + store: &dyn GraphStore, + locator: NodeId, + property: &str, description: &str, ) -> Result { - GraphEntityId::new(required_string_property(value, description)?).map_err(|error| { + GraphEntityId::new(locator_identity(store, locator, property, description)?).map_err(|error| { GraphDbError::Corrupt { message: format!("{description} is invalid: {error}"), } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/open_error_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/open_error_contract.rs index e82f05817b..975285c33d 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/open_error_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/open_error_contract.rs @@ -1,8 +1,12 @@ +use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use grafeo_common::types::Value; use tempfile::TempDir; -use tracedecay_graph_db::{GraphDbError, GraphEntityId, GraphNamespace, NeverCancelled}; +use tracedecay_graph_db::{ + GraphDbError, GraphEntityId, GraphNamespace, GraphRelation, GraphRelationId, GraphRelationKind, + NeverCancelled, +}; use crate::support; @@ -25,40 +29,45 @@ fn foreign_grafeo_store_without_marker_is_reset_required() { ); } +fn raw_store(temp: &TempDir) -> grafeo_engine::GrafeoDB { + grafeo_engine::GrafeoDB::with_config( + grafeo_engine::Config::persistent(graph_path(temp.path())) + .with_storage_format(grafeo_engine::config::StorageFormat::SingleFile), + ) + .unwrap() +} + +const FORMAT_MARKER: [(&str, i64); 2] = [ + ("__tracedecay_graph_db_version", 4), + ("__tracedecay_graph_db_sequence", 0), +]; + +fn format_marker() -> Vec<(&'static str, Value)> { + let mut marker: Vec<(&str, Value)> = FORMAT_MARKER + .iter() + .map(|(name, value)| (*name, Value::from(*value))) + .collect(); + marker.push(("__tracedecay_graph_db_schema", "native-scalars-v1".into())); + marker +} + #[test] fn persisted_scalar_identity_mismatch_is_corrupt_on_point_read() { let temp = TempDir::new().unwrap(); - let path = graph_path(temp.path()); - let raw = grafeo_engine::GrafeoDB::with_config( - grafeo_engine::Config::persistent(&path) - .with_storage_format(grafeo_engine::config::StorageFormat::SingleFile), - ) - .unwrap(); + let raw = raw_store(&temp); let mut session = raw.session(); session.begin_transaction().unwrap(); session - .create_node_with_props( - &["__tracedecay_graph_db_format"], - [ - ("__tracedecay_graph_db_version", 3_i64.into()), - ("__tracedecay_graph_db_schema", "native-scalars-v1".into()), - ("__tracedecay_graph_db_sequence", 0_i64.into()), - ], - ) + .create_node_with_props(&["__tracedecay_graph_db_format"], format_marker()) .unwrap(); - // sha256("workspace")[..8], the raw-identity tag, then "entity". - let stable_key = [ - &[0x21, 0xa3, 0x23, 0x0e, 0x03, 0x77, 0x2a, 0x58, 0x00][..], - b"entity", - ] - .concat(); session .create_node_with_props( &["__tracedecay_graph_db_entity"], [ + // base64url(sha256("workspace")[..8] ‖ raw-identity tag ‖ "entity"). ( "__tracedecay_graph_db_entity_key", - Value::Bytes(stable_key.into()), + "IaMjDgN3KlgAZW50aXR5".into(), ), ("__tracedecay_graph_db_namespace", "workspace".into()), ("__tracedecay_graph_db_projection", "code".into()), @@ -78,3 +87,123 @@ fn persisted_scalar_identity_mismatch_is_corrupt_on_point_read() { Err(GraphDbError::Corrupt { .. }) )); } + +/// A format-4 relation as it lies on disk. Keys are base64url of +/// `sha256("workspace")[..8] ‖ digest-identity tag ‖ kind ‖ digest`. The +/// locator owns the identity, source, and target, each stored as U+0001, +/// kind, and base64url digest, and the native edge carries none of them. +/// Keyed reads and edge fan-outs both resolve the same relation. +#[test] +fn compact_relation_identities_read_back_through_keys_and_edges() { + let temp = TempDir::new().unwrap(); + let raw = raw_store(&temp); + let session = raw.session(); + session + .create_node_with_props(&["__tracedecay_graph_db_format"], format_marker()) + .unwrap(); + let symbol = |key: &str, identity: String| { + session + .create_node_with_props( + &["__tracedecay_graph_db_entity"], + [ + ("__tracedecay_graph_db_entity_key", Value::from(key)), + ("__tracedecay_graph_db_namespace", "workspace".into()), + ("__tracedecay_graph_db_projection", "code".into()), + ("__tracedecay_graph_db_entity_id", identity.into()), + ], + ) + .unwrap() + }; + let caller = symbol( + "IaMjDgN3KlgBc3ltYm9sERERERERERERERERERERERERERERERERERERERERERE", + format!("symbol:{}", "11".repeat(32)), + ); + let callee = symbol( + "IaMjDgN3KlgBc3ltYm9sIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiI", + format!("symbol:{}", "22".repeat(32)), + ); + let edge = session + .create_edge_with_props( + caller, + callee, + // `__tracedecay_graph_db_relation_` + hex("calls"). + "__tracedecay_graph_db_relation_63616c6c73", + [ + ("__tracedecay_graph_db_namespace", Value::from("workspace")), + ("__tracedecay_graph_db_projection", Value::from("code")), + ("__tracedecay_graph_db_relation_kind", Value::from("calls")), + ], + ) + .unwrap(); + session + .create_node_with_props( + &["__tracedecay_graph_db_relation_locator"], + [ + ( + "__tracedecay_graph_db_relation_key", + "IaMjDgN3KlgBZWRnZTMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMz".into(), + ), + ("__tracedecay_graph_db_namespace", "workspace".into()), + ("__tracedecay_graph_db_projection", "code".into()), + ( + "__tracedecay_graph_db_relation_id", + "\u{1}edgeMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM".into(), + ), + ( + "__tracedecay_graph_db_relation_from", + "\u{1}symbolERERERERERERERERERERERERERERERERERERERERERE".into(), + ), + ( + "__tracedecay_graph_db_relation_to", + "\u{1}symbolIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiI".into(), + ), + ("__tracedecay_graph_db_relation_kind", "calls".into()), + ( + "__tracedecay_graph_db_relation_edge", + i64::try_from(edge.as_u64()).unwrap().into(), + ), + ], + ) + .unwrap(); + raw.close().unwrap(); + + let (_, db) = RegisteredGraph::open_lease(temp.path()).unwrap(); + let workspace = GraphNamespace::new("workspace").unwrap(); + let relation_id = GraphRelationId::new(format!("edge:{}", "33".repeat(32))).unwrap(); + let expected = GraphRelation::new( + relation_id.clone(), + GraphEntityId::new(format!("symbol:{}", "11".repeat(32))).unwrap(), + GraphEntityId::new(format!("symbol:{}", "22".repeat(32))).unwrap(), + GraphRelationKind::new("calls").unwrap(), + BTreeMap::new(), + ) + .unwrap(); + assert_eq!( + db.relation(&workspace, &relation_id, Arc::new(NeverCancelled)) + .unwrap(), + Some(expected.clone()) + ); + let starts = [expected.from.clone()]; + assert_eq!( + db.outgoing_relations( + &workspace, + &starts, + &BTreeSet::new(), + 16, + Arc::new(NeverCancelled) + ) + .unwrap(), + vec![vec![expected]] + ); + assert_eq!( + db.outgoing_relation_ids( + &workspace, + &starts, + &BTreeSet::new(), + 16, + Arc::new(NeverCancelled) + ) + .unwrap(), + vec![vec![relation_id]] + ); +} diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs index c981150786..8aee1d119d 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs @@ -3,13 +3,14 @@ use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; +use grafeo_common::types::Value; use tempfile::TempDir; use tracedecay_graph_db::{ GraphBudgetKind, GraphCancellation, GraphDbError, GraphDbLeaseV1, GraphDbOwner, GraphEntity, - GraphEntityId, GraphFormatVersion, GraphIdempotencyKey, GraphLabel, GraphMutation, - GraphNamespace, GraphProjectionId, GraphPublication, GraphPublicationInputDigest, - GraphRelation, GraphRelationId, GraphRelationKind, GraphTraversalDirection, GraphWatermark, - GraphWriteBatch, NeverCancelled, ProjectionReplacement, SourceGeneration, TraversalRequest, + GraphEntityId, GraphIdempotencyKey, GraphLabel, GraphMutation, GraphNamespace, + GraphProjectionId, GraphPublication, GraphPublicationInputDigest, GraphRelation, + GraphRelationId, GraphRelationKind, GraphTraversalDirection, GraphWatermark, GraphWriteBatch, + NeverCancelled, ProjectionReplacement, SourceGeneration, TraversalRequest, }; use crate::support; @@ -1010,7 +1011,9 @@ fn wrong_tracedecay_format_requires_reset() { fn superseded_format_store_is_rebuilt_fresh_with_its_sealed_generations_discarded() { let temp = TempDir::new().unwrap(); let path = graph_path(temp.path()); - let previous_format = i64::from(GraphFormatVersion::current().get() - 1); + // Format 3 stored relation identities as strings on both the locator and + // the native edge; the row below is keyed the way format 3 keyed it. + let previous_format = 3_i64; let raw = grafeo_engine::GrafeoDB::with_config( grafeo_engine::Config::persistent(&path) .with_storage_format(grafeo_engine::config::StorageFormat::SingleFile), @@ -1032,7 +1035,15 @@ fn superseded_format_store_is_rebuilt_fresh_with_its_sealed_generations_discarde [ ( "__tracedecay_graph_db_entity_key", - "70726f6a656374:61".into(), + // sha256("project")[..8], the raw-identity tag, "stale". + Value::Bytes( + [ + &[0x24, 0x42, 0x10, 0xe4, 0x84, 0x37, 0xb6, 0x55, 0x00][..], + b"stale", + ] + .concat() + .into(), + ), ), ("__tracedecay_graph_db_namespace", "project".into()), ("__tracedecay_graph_db_projection", "code".into()), @@ -1045,7 +1056,7 @@ fn superseded_format_store_is_rebuilt_fresh_with_its_sealed_generations_discarde std::fs::create_dir_all(&sealed_generation).unwrap(); std::fs::write( sealed_generation.join("generation.grafeo"), - b"format 2 bytes", + b"format 3 bytes", ) .unwrap(); diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 09bf4af5fc..c848a79e7a 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1483,13 +1483,14 @@ fn stable_identity_manifest( .unwrap() } -/// Unique keys are written once per entity and relation locator, so their -/// encoding sets a sealed generation's size. Hex keys that repeat the -/// namespace sealed this 2,000-symbol generation to 3,150,837 bytes; binary -/// namespace-id keys seal it to 1,786,870, and the rows still resolve -/// through them. +/// Keys and relation identities are the bulk of a sealed generation's +/// bytes. Hex keys sealed this 2,000-symbol generation to 3,150,837 bytes, +/// and binary keys, which the compact dictionary stores as marked hex, to +/// 1,786,870. Base64url keys plus each relation identity, source, and target +/// stored once, on its locator, in compact form seal it to 1,037,302, and +/// the rows still resolve through keys and edges. #[test] -fn sealed_generation_bytes_stay_within_the_binary_key_budget() { +fn sealed_generation_bytes_stay_within_the_compact_identity_budget() { let temp = TempDir::new().unwrap(); let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); let mut authority = RelationalAuthority::default(); @@ -1508,7 +1509,7 @@ fn sealed_generation_bytes_stay_within_the_binary_key_budget() { let sealed_bytes = directory_bytes(&sealed_store_root(temp.path())); assert!( - sealed_bytes <= 1_900_000, + sealed_bytes <= 1_150_000, "sealed generation took {sealed_bytes} bytes" ); let last = GraphEntityId::new(graph_stable_identity("symbol", "1999")).unwrap();