From 97de4c3efd69e5167a53937ce80565a93a06f27e Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 26 Sep 2026 19:06:15 +0000 Subject: [PATCH] perf(graph-db): store relation identities once, on the locator Every relation stored its identity, source and target as 71-byte strings twice: on its locator node and again on its native edge. The locator now owns them. The edge keeps only its owner scalars and payload, and edge reads resolve the locator through the existing RELATION_EDGE unique index and read single columns from it. A `:<64 lowercase hex>` identity is stored as U+0001, its kind and its digest in unpadded base64url (50 bytes for a symbol id); any other identity is stored verbatim, and graph identifiers may no longer start with U+0001. Unique keys become base64url text of their binary form. Both stay strings because the sealed compact store keeps `Bytes` values in its string dictionary as marked hex, so the binary keys from #2257 took 102 bytes on disk, not 47; they now take 63. Graph format 3 -> 4 with the same typed `FormatSuperseded` rebuild. Paging and proofs still order by the decoded identity strings. Refs #1103 --- Cargo.lock | 1 + crates/tracedecay-graph-db/Cargo.toml | 1 + crates/tracedecay-graph-db/src/location.rs | 2 +- crates/tracedecay-graph-db/src/projection.rs | 6 + .../src/projection_identity_index.rs | 10 +- .../src/projection_read.rs | 19 +- crates/tracedecay-graph-db/src/schema.rs | 208 +++++++++++++++--- crates/tracedecay-graph-db/src/state.rs | 49 +++-- crates/tracedecay-graph-db/src/traversal.rs | 86 ++++---- .../graph_db_suite/open_error_contract.rs | 173 +++++++++++++-- .../tests/graph_db_suite/runtime_contract.rs | 25 ++- .../sealed_store.rs | 15 +- 12 files changed, 440 insertions(+), 155 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index db2bdd0c74..d3d6e0886d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -6080,6 +6080,7 @@ dependencies = [ name = "tracedecay-graph-db" version = "0.1.0" dependencies = [ + "base64 0.22.1", "criterion", "grafeo-adapters", "grafeo-common", diff --git a/crates/tracedecay-graph-db/Cargo.toml b/crates/tracedecay-graph-db/Cargo.toml index a3800bb84d..1636ce47ee 100644 --- a/crates/tracedecay-graph-db/Cargo.toml +++ b/crates/tracedecay-graph-db/Cargo.toml @@ -66,6 +66,7 @@ grafeo-common.workspace = true grafeo-core = { workspace = true, features = ["compact-store"] } grafeo-engine = { workspace = true, features = ["mmap", "compact-store"] } grafeo-storage.workspace = true +base64 = "0.22" hex = "0.4" hotpath.workspace = true parking_lot = { version = "0.12", features = ["arc_lock", "send_guard"] } diff --git a/crates/tracedecay-graph-db/src/location.rs b/crates/tracedecay-graph-db/src/location.rs index 5676dce9f8..02fd49142b 100644 --- a/crates/tracedecay-graph-db/src/location.rs +++ b/crates/tracedecay-graph-db/src/location.rs @@ -20,7 +20,7 @@ pub struct GraphFormatVersion(u32); impl GraphFormatVersion { #[must_use] pub const fn current() -> Self { - Self(3) + Self(4) } #[cfg(any(test, feature = "test-helpers", feature = "eval-helpers"))] diff --git a/crates/tracedecay-graph-db/src/projection.rs b/crates/tracedecay-graph-db/src/projection.rs index 6e3011ab49..f81422c268 100644 --- a/crates/tracedecay-graph-db/src/projection.rs +++ b/crates/tracedecay-graph-db/src/projection.rs @@ -12,6 +12,7 @@ use crate::limits::{ MAX_GRAPH_IDENTIFIER_BYTES, MAX_GRAPH_PROPERTIES, MAX_GRAPH_PROPERTY_AGGREGATE_BYTES, MAX_GRAPH_PROPERTY_VALUE_BYTES, }; +use crate::schema::COMPACT_IDENTITY_MARKER; use crate::{GraphBudgetKind, GraphDbError}; const RESERVED_PREFIX: &str = "__tracedecay_graph_db_"; @@ -98,6 +99,11 @@ fn validate_opaque(kind: &str, value: &str) -> Result<(), GraphDbError> { "{kind} exceeds {MAX_GRAPH_IDENTIFIER_BYTES} bytes" ))); } + if value.starts_with(COMPACT_IDENTITY_MARKER) { + return Err(GraphDbError::invalid(format!( + "{kind} starts with the reserved compact identity marker" + ))); + } if value.starts_with(RESERVED_PREFIX) { return Err(GraphDbError::invalid(format!( "{kind} uses the reserved graph database prefix" diff --git a/crates/tracedecay-graph-db/src/projection_identity_index.rs b/crates/tracedecay-graph-db/src/projection_identity_index.rs index a5022ace4c..723e2d0c26 100644 --- a/crates/tracedecay-graph-db/src/projection_identity_index.rs +++ b/crates/tracedecay-graph-db/src/projection_identity_index.rs @@ -24,12 +24,11 @@ use std::collections::HashMap; use std::sync::atomic::{AtomicU64, Ordering}; use std::sync::{Arc, RwLock}; -use grafeo_common::types::Value; use grafeo_engine::GrafeoDB; use crate::projection::check_cancelled; use crate::projection_read::IdentityScope; -use crate::schema::{has_native_label, nodes_with_label}; +use crate::schema::{decode_identity, has_native_label, nodes_with_label}; use crate::{GraphCancellation, GraphDbError}; /// Identity bytes one cached index may retain. A projection whose identities @@ -200,12 +199,7 @@ fn build_identity_index( if !has_native_label(&record, record_label) { continue; } - let identity = record - .get_property(identity_property) - .and_then(Value::as_str) - .ok_or_else(|| GraphDbError::Corrupt { - message: format!("projection query returned a non-string `{identity_property}`"), - })?; + let identity = decode_identity(record.get_property(identity_property), identity_property)?; identity_bytes = identity_bytes.saturating_add(identity.len()); if identity_bytes > MAX_IDENTITY_INDEX_BYTES { return Ok(None); diff --git a/crates/tracedecay-graph-db/src/projection_read.rs b/crates/tracedecay-graph-db/src/projection_read.rs index d0779a8696..a310733bef 100644 --- a/crates/tracedecay-graph-db/src/projection_read.rs +++ b/crates/tracedecay-graph-db/src/projection_read.rs @@ -2,12 +2,11 @@ use std::collections::BTreeSet; use std::fmt; use std::sync::Arc; -use grafeo_common::types::Value; use grafeo_engine::GrafeoDB; use crate::projection::check_cancelled; use crate::schema::{ - ENTITY_ID_PROPERTY, ENTITY_LABEL, RELATION_ID_PROPERTY, RELATION_LABEL, + ENTITY_ID_PROPERTY, ENTITY_LABEL, RELATION_ID_PROPERTY, RELATION_LABEL, decode_identity, entity_projection_label, relation_projection_label, }; use crate::state::{labeled_projection_nodes, latest_projection, load_entity, load_relation}; @@ -411,25 +410,17 @@ fn streaming_identity_page( let Some(record) = store.get_node(node) else { continue; }; - let identity = record - .get_property(identity_property) - .and_then(Value::as_str) - .ok_or_else(|| GraphDbError::Corrupt { - message: format!("projection query returned a non-string `{identity_property}`"), - })?; - if after.is_some_and(|after| identity <= after) { + let identity = decode_identity(record.get_property(identity_property), identity_property)?; + if after.is_some_and(|after| identity.as_str() <= after) { continue; } if page.len() == limit { - if page - .last() - .is_some_and(|widest| identity >= widest.as_str()) - { + if page.last().is_some_and(|widest| identity >= *widest) { continue; } page.pop_last(); } - page.insert(identity.to_owned()); + page.insert(identity); } Ok(page.into_iter().collect()) } diff --git a/crates/tracedecay-graph-db/src/schema.rs b/crates/tracedecay-graph-db/src/schema.rs index 3152842ecf..81d01a3f66 100644 --- a/crates/tracedecay-graph-db/src/schema.rs +++ b/crates/tracedecay-graph-db/src/schema.rs @@ -1,6 +1,8 @@ use std::collections::{BTreeMap, BTreeSet}; -use grafeo_common::types::{EdgeId, NodeId, Value}; +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use grafeo_common::types::{EdgeId, NodeId, PropertyKey, Value}; use grafeo_core::graph::GraphStore; use grafeo_core::graph::lpg::{Edge, Node}; use sha2::{Digest, Sha256}; @@ -94,6 +96,9 @@ const NAMESPACE_KEY_ID_BYTES: usize = 8; const DIGEST_BYTES: usize = 32; const RAW_IDENTITY_TAG: u8 = 0; const DIGEST_IDENTITY_TAG: u8 = 1; +/// Leads a compact identity scalar; `validate_opaque` rejects graph +/// identifiers that start with it. +pub(crate) const COMPACT_IDENTITY_MARKER: char = '\u{1}'; /// The short id a namespace contributes to every unique key it owns: the /// leading bytes of the namespace's SHA-256. @@ -121,10 +126,7 @@ pub(crate) fn namespace_key_id(namespace: &GraphNamespace) -> NamespaceKeyId { pub(crate) fn stable_key(namespace_id: &NamespaceKeyId, identity: &str) -> Vec { let mut key = Vec::with_capacity(NAMESPACE_KEY_ID_BYTES + 1 + identity.len()); key.extend_from_slice(namespace_id); - match identity - .rsplit_once(':') - .and_then(|(kind, digest)| Some((kind, decode_lower_hex_digest(digest)?))) - { + match digest_identity(identity) { Some((kind, digest)) => { key.push(DIGEST_IDENTITY_TAG); key.extend_from_slice(kind.as_bytes()); @@ -138,6 +140,71 @@ pub(crate) fn stable_key(namespace_id: &NamespaceKeyId, identity: &str) -> Vec:<64 lowercase hex>` identity. +fn digest_identity(identity: &str) -> Option<(&str, [u8; DIGEST_BYTES])> { + identity + .rsplit_once(':') + .and_then(|(kind, digest)| Some((kind, decode_lower_hex_digest(digest)?))) +} + +/// Unpadded base64url width of a 32-byte digest. +const DIGEST_TEXT_BYTES: usize = 43; + +/// The stored scalar for a relation identity, source, or target. +/// +/// A `:<64 lowercase hex>` identity is stored as +/// [`COMPACT_IDENTITY_MARKER`], its kind, and its digest in unpadded +/// base64url; any other identity is stored verbatim. The value stays a +/// string because the sealed compact store keeps `Bytes` in its string +/// dictionary as marked hex, which would double the digest again. Only the +/// canonical spelling compacts and no graph identifier may start with the +/// marker, so [`decode_identity`] restores exactly the identity written. +pub(crate) fn encode_identity(identity: &str) -> Value { + match digest_identity(identity) { + Some((kind, digest)) => Value::from(format!( + "{COMPACT_IDENTITY_MARKER}{kind}{}", + URL_SAFE_NO_PAD.encode(digest) + )), + None => Value::from(identity), + } +} + +/// Reads back an identity written by [`encode_identity`]. +pub(crate) fn decode_identity( + value: Option<&Value>, + description: &str, +) -> Result { + let Some(Value::String(stored)) = value else { + return Err(GraphDbError::Corrupt { + message: format!("native {description} is missing or not a string"), + }); + }; + let identity = match stored.strip_prefix(COMPACT_IDENTITY_MARKER) { + None => stored.to_string(), + Some(compact) => { + let digest = compact + .len() + .checked_sub(DIGEST_TEXT_BYTES) + .filter(|split| compact.is_char_boundary(*split)) + .and_then(|split| { + let (kind, digest) = compact.split_at(split); + let mut bytes = [0; DIGEST_BYTES]; + let written = URL_SAFE_NO_PAD.decode_slice(digest, &mut bytes).ok()?; + (written == DIGEST_BYTES).then(|| format!("{kind}:{}", hex::encode(bytes))) + }); + digest.ok_or_else(|| GraphDbError::Corrupt { + message: format!("native {description} has a malformed compact digest"), + })? + } + }; + if identity.len() > MAX_GRAPH_IDENTIFIER_BYTES { + return Err(GraphDbError::Corrupt { + message: format!("native {description} exceeds its product bound"), + }); + } + Ok(identity) +} + /// Only the canonical lowercase spelling compacts, so every identity has /// exactly one key. fn decode_lower_hex_digest(digest: &str) -> Option<[u8; DIGEST_BYTES]> { @@ -153,8 +220,11 @@ fn decode_lower_hex_digest(digest: &str) -> Option<[u8; DIGEST_BYTES]> { Some(bytes) } +/// A unique key as the indexed scalar: unpadded base64url of +/// [`stable_key`]'s bytes, a string for the same reason as +/// [`encode_identity`]. pub(crate) fn key_value(namespace: &GraphNamespace, identity: &str) -> Value { - Value::Bytes(stable_key(&namespace_key_id(namespace), identity).into()) + Value::from(URL_SAFE_NO_PAD.encode(stable_key(&namespace_key_id(namespace), identity))) } /// The indexed unique-key value for one entity. @@ -368,15 +438,15 @@ pub(crate) fn relation_properties( ), ( RELATION_ID_PROPERTY.to_owned(), - Value::from(relation.identity.as_str()), + encode_identity(relation.identity.as_str()), ), ( RELATION_FROM_PROPERTY.to_owned(), - Value::from(relation.from.as_str()), + encode_identity(relation.from.as_str()), ), ( RELATION_TO_PROPERTY.to_owned(), - Value::from(relation.to.as_str()), + encode_identity(relation.to.as_str()), ), ( RELATION_KIND_PROPERTY.to_owned(), @@ -393,6 +463,9 @@ pub(crate) fn relation_properties( Ok(properties) } +/// A native edge carries its owner scalars and payload, never the relation's +/// identity or endpoints: those are owned by its locator node, which +/// [`edge_locator`] resolves through the `RELATION_EDGE` index. pub(crate) fn edge_properties( namespace: &GraphNamespace, projection: &GraphProjectionId, @@ -407,18 +480,6 @@ pub(crate) fn edge_properties( PROJECTION_PROPERTY.to_owned(), Value::from(projection.as_str()), ), - ( - RELATION_ID_PROPERTY.to_owned(), - Value::from(relation.identity.as_str()), - ), - ( - RELATION_FROM_PROPERTY.to_owned(), - Value::from(relation.from.as_str()), - ), - ( - RELATION_TO_PROPERTY.to_owned(), - Value::from(relation.to.as_str()), - ), ( RELATION_KIND_PROPERTY.to_owned(), Value::from(relation.kind.as_str()), @@ -547,17 +608,17 @@ pub(crate) fn decode_relation(locator: &Node, edge: &Edge) -> Result Result { @@ -620,11 +682,7 @@ pub(crate) fn decode_relation_identity( message: "traversal relation native type and kind disagree".to_owned(), }); } - let identity = GraphRelationId::new(required_string( - edge.get_property(RELATION_ID_PROPERTY), - "relation identity", - )?) - .map_err(|error| persisted_validation_error("relation identity", error))?; + let identity = edge_relation_identity(store, edge_locator(store, edge.id)?)?; Ok(DecodedRelationIdentity { identity, projection, @@ -632,6 +690,59 @@ pub(crate) fn decode_relation_identity( }) } +/// The locator node that owns `edge`'s identity and endpoints. +/// +/// Resolved through the `RELATION_EDGE` unique index. A locator deleted in +/// the live store keeps its index entry but loses its properties, so the +/// re-read of the indexed scalar discards such tombstones without +/// materializing the node. +pub(crate) fn edge_locator(store: &dyn GraphStore, edge: EdgeId) -> Result { + let value = relation_edge_value(edge)?; + let key = PropertyKey::new(RELATION_EDGE_PROPERTY); + let mut locators = store + .find_nodes_by_property(RELATION_EDGE_PROPERTY, &value) + .into_iter() + .filter(|node| store.get_node_property(*node, &key).as_ref() == Some(&value)); + match (locators.next(), locators.next()) { + (Some(locator), None) => Ok(locator), + (None, _) => Err(GraphDbError::Corrupt { + message: "native relation edge has no locator".to_owned(), + }), + (Some(_), Some(_)) => Err(GraphDbError::Corrupt { + message: "native relation edge has duplicate locators".to_owned(), + }), + } +} + +/// One identity scalar of a relation locator, read without materializing +/// the node. +pub(crate) fn locator_identity( + store: &dyn GraphStore, + locator: NodeId, + property: &str, + description: &str, +) -> Result { + decode_identity( + store + .get_node_property(locator, &PropertyKey::new(property)) + .as_ref(), + description, + ) +} + +pub(crate) fn edge_relation_identity( + store: &dyn GraphStore, + locator: NodeId, +) -> Result { + GraphRelationId::new(locator_identity( + store, + locator, + RELATION_ID_PROPERTY, + "relation identity", + )?) + .map_err(|error| persisted_validation_error("relation identity", error)) +} + pub(crate) fn decode_graph_properties( properties: impl IntoIterator, Value)>, ) -> Result, GraphDbError> { @@ -885,7 +996,11 @@ mod graph_stable_identity_tests { mod stable_key_tests { use std::collections::BTreeSet; - use super::{graph_stable_identity, namespace_key_id, stable_key}; + use grafeo_common::types::Value; + + use super::{ + decode_identity, encode_identity, graph_stable_identity, namespace_key_id, stable_key, + }; use crate::GraphNamespace; #[test] @@ -935,4 +1050,37 @@ mod stable_key_tests { .collect(); assert_eq!(keys.len(), 14); } + + #[test] + fn relation_identities_store_their_digest_compactly_and_read_back_exactly() { + let identity = graph_stable_identity("edge", "occ"); + assert_eq!( + identity, + "edge:a92cf2a4297d812859e387e8efac97838fc420befe35fa0f8b7e94ad9a139ff5" + ); + let digest = identity.strip_prefix("edge:").unwrap(); + let stored = encode_identity(&identity); + + assert_eq!( + stored, + Value::from("\u{1}edgeqSzypCl9gShZ44fo76yXg4_EIL7-NfoPi36UrZoTn_U") + ); + assert_eq!(stored.as_str().unwrap().len(), 48); + assert_eq!( + decode_identity(Some(&stored), "relation").unwrap(), + identity + ); + for verbatim in [ + format!("edge:{}", digest.to_uppercase()), + "relation:a-b".to_owned(), + format!("edge:{digest}0"), + ] { + let stored = encode_identity(&verbatim); + assert_eq!(stored, Value::from(verbatim.as_str())); + assert_eq!( + decode_identity(Some(&stored), "relation").unwrap(), + verbatim + ); + } + } } diff --git a/crates/tracedecay-graph-db/src/state.rs b/crates/tracedecay-graph-db/src/state.rs index 41c110f9da..3ef5a9ed22 100644 --- a/crates/tracedecay-graph-db/src/state.rs +++ b/crates/tracedecay-graph-db/src/state.rs @@ -11,17 +11,17 @@ use crate::limits::{ MAX_VERIFIED_GENERATION_RELATIONS, require_generation_capacity, }; use crate::schema::{ - COMMIT_SEQUENCE_PROPERTY, DIGEST_PROPERTY, ENTITY_ID_PROPERTY, ENTITY_KEY_PROPERTY, - ENTITY_LABEL, FORMAT_LABEL, GENERATION_DEPENDENCY_DIGEST_PROPERTY, IDEMPOTENCY_KEY_PROPERTY, - NAMESPACE_PROPERTY, PROJECTION_KEY_PROPERTY, PROJECTION_LABEL, PROJECTION_PROPERTY, - PUBLICATION_DIGEST_PROPERTY, PUBLICATION_INPUT_DIGEST_PROPERTY, PUBLICATION_KEY_PROPERTY, - PUBLICATION_LABEL, RELATION_EDGE_PROPERTY, RELATION_FROM_PROPERTY, RELATION_ID_PROPERTY, - RELATION_KEY_PROPERTY, RELATION_LABEL, RELATION_TO_PROPERTY, SEQUENCE_PROPERTY, - SOURCE_GENERATION_PROPERTY, WATERMARK_PROPERTY, decode_entity, decode_relation, - entity_key_value, entity_projection_label, has_native_label, namespace_key_id, - nodes_with_label, nodes_with_label_count, projection_state_key_value, publication_key_value, - relation_edge_value, relation_key_value, relation_projection_label, required_i64, - required_string, stable_key, + COMMIT_SEQUENCE_PROPERTY, COMPACT_IDENTITY_MARKER, DIGEST_PROPERTY, ENTITY_ID_PROPERTY, + ENTITY_KEY_PROPERTY, ENTITY_LABEL, FORMAT_LABEL, GENERATION_DEPENDENCY_DIGEST_PROPERTY, + IDEMPOTENCY_KEY_PROPERTY, NAMESPACE_PROPERTY, PROJECTION_KEY_PROPERTY, PROJECTION_LABEL, + PROJECTION_PROPERTY, PUBLICATION_DIGEST_PROPERTY, PUBLICATION_INPUT_DIGEST_PROPERTY, + PUBLICATION_KEY_PROPERTY, PUBLICATION_LABEL, RELATION_EDGE_PROPERTY, RELATION_FROM_PROPERTY, + RELATION_ID_PROPERTY, RELATION_KEY_PROPERTY, RELATION_LABEL, RELATION_TO_PROPERTY, + SEQUENCE_PROPERTY, SOURCE_GENERATION_PROPERTY, WATERMARK_PROPERTY, decode_entity, + decode_identity, decode_relation, entity_key_value, entity_projection_label, has_native_label, + namespace_key_id, nodes_with_label, nodes_with_label_count, projection_state_key_value, + publication_key_value, relation_edge_value, relation_key_value, relation_projection_label, + required_i64, required_string, stable_key, }; use crate::{ GraphCommit, GraphDbError, GraphEntity, GraphEntityId, GraphIdempotencyKey, GraphMutation, @@ -620,7 +620,7 @@ fn load_relation_reference_by_edge( .ok_or_else(|| GraphDbError::Corrupt { message: "indexed relation locator is unreadable".to_owned(), })?; - let identity = GraphRelationId::new(required_string( + let identity = GraphRelationId::new(decode_identity( locator.get_property(RELATION_ID_PROPERTY), "relation identity", )?) @@ -630,12 +630,12 @@ fn load_relation_reference_by_edge( "relation projection", )?) .map_err(|error| persisted_validation_error("relation projection", error))?; - let from = GraphEntityId::new(required_string( + let from = GraphEntityId::new(decode_identity( locator.get_property(RELATION_FROM_PROPERTY), "relation source", )?) .map_err(|error| persisted_validation_error("relation source", error))?; - let to = GraphEntityId::new(required_string( + let to = GraphEntityId::new(decode_identity( locator.get_property(RELATION_TO_PROPERTY), "relation target", )?) @@ -710,7 +710,7 @@ pub(crate) fn projection_entity_nodes_sorted_checked( message: "native graph entity disappeared during verification".to_owned(), })?; keyed.push(( - required_arc_string( + identity_arc( record.get_property(ENTITY_ID_PROPERTY), "native graph entity identity", )?, @@ -876,7 +876,7 @@ fn projection_identity_deletion_page_checked( else { continue; }; - let identity = required_arc_string( + let identity = identity_arc( record.get_property(identity_property), &format!("native graph {description} identity"), )?; @@ -926,7 +926,7 @@ pub(crate) fn projection_relation_nodes_sorted_checked( message: "native graph relation disappeared during verification".to_owned(), })?; keyed.push(( - required_arc_string( + identity_arc( record.get_property(crate::schema::RELATION_ID_PROPERTY), "native graph relation identity", )?, @@ -1111,16 +1111,17 @@ fn labeled_projection_nodes_checked( Ok(nodes) } -fn required_arc_string(value: Option<&Value>, description: &str) -> Result { +/// An identity scalar as a shared string: verbatim strings are shared as-is, +/// compact relation identities are decoded once. +fn identity_arc(value: Option<&Value>, description: &str) -> Result { match value { - Some(Value::String(value)) if value.len() <= MAX_GRAPH_IDENTIFIER_BYTES => { + Some(Value::String(value)) + if value.len() <= MAX_GRAPH_IDENTIFIER_BYTES + && !value.starts_with(COMPACT_IDENTITY_MARKER) => + { Ok(value.clone()) } - _ => Err(GraphDbError::Corrupt { - message: format!( - "native {description} is missing, not a string, or exceeds its product bound" - ), - }), + value => decode_identity(value, description).map(ArcStr::from), } } diff --git a/crates/tracedecay-graph-db/src/traversal.rs b/crates/tracedecay-graph-db/src/traversal.rs index b18559edec..d94dce1f75 100644 --- a/crates/tracedecay-graph-db/src/traversal.rs +++ b/crates/tracedecay-graph-db/src/traversal.rs @@ -13,9 +13,10 @@ use crate::adjacency_id_index::{AdjacencyIdIndexCache, AdjacencyIndexKey, page_i use crate::epoch_cache::LabelKeyCache; use crate::schema::{ ENTITY_ID_PROPERTY, ENTITY_KEY_PROPERTY, ENTITY_LABEL, NAMESPACE_PROPERTY, PROJECTION_PROPERTY, - RELATION_FROM_PROPERTY, RELATION_ID_PROPERTY, RELATION_KIND_PROPERTY, RELATION_TO_PROPERTY, - decode_entity, decode_graph_properties, decode_relation_identity, entity_key_value, - entity_projection_label, label_keys, relation_kind_from_type, relation_type_for_kind, + RELATION_FROM_PROPERTY, RELATION_KIND_PROPERTY, RELATION_TO_PROPERTY, decode_entity, + decode_graph_properties, decode_relation_identity, edge_locator, edge_relation_identity, + entity_key_value, entity_projection_label, label_keys, locator_identity, + relation_kind_from_type, relation_type_for_kind, }; use crate::{ GraphBudgetKind, GraphCancellation, GraphDbError, GraphEntity, GraphEntityId, GraphNamespace, @@ -136,14 +137,22 @@ pub(crate) fn traverse( let store = database.graph_store(); let start = node_for_entity(store.as_ref(), &request.namespace, &request.start)?; - let projected = relation_projection(store, &request.relation_kinds); + let projected = relation_projection(Arc::clone(&store), &request.relation_kinds); match request.direction { - GraphTraversalDirection::Outgoing => { - native_outgoing_traversal(&projected, start, &request, ensure_projection_readable) - } - GraphTraversalDirection::Incoming | GraphTraversalDirection::Both => { - directional_traversal(&projected, start, &request, ensure_projection_readable) - } + GraphTraversalDirection::Outgoing => native_outgoing_traversal( + &projected, + store.as_ref(), + start, + &request, + ensure_projection_readable, + ), + GraphTraversalDirection::Incoming | GraphTraversalDirection::Both => directional_traversal( + &projected, + store.as_ref(), + start, + &request, + ensure_projection_readable, + ), } } @@ -184,7 +193,7 @@ pub(crate) fn outgoing_relation_targets( })?; let target = decode_entity(&target)?; let relation = relation_for_edge( - &projected, + store.as_ref(), edge, namespace, ensure_projection_readable, @@ -240,7 +249,7 @@ pub(crate) fn visit_outgoing_relation_targets( })?; let target = decode_entity(&target)?; let relation = relation_for_edge( - &projected, + store.as_ref(), edge, namespace, ensure_projection_readable, @@ -431,7 +440,7 @@ fn collect_relation_ids( let stored = store.get_edge(edge).ok_or_else(|| GraphDbError::Corrupt { message: "outgoing relation references a missing native edge".to_owned(), })?; - let decoded = decode_relation_identity(&stored, namespace)?; + let decoded = decode_relation_identity(store.as_ref(), &stored, namespace)?; if !relation_kinds.is_empty() && !relation_kinds.contains(&decoded.kind) { return Err(GraphDbError::Corrupt { message: "relation kind escaped its projection filter".to_owned(), @@ -522,7 +531,7 @@ pub(crate) fn directed_relations( } } relations.push(relation_for_edge( - &projected, + store.as_ref(), edge, namespace, ensure_projection_readable, @@ -631,6 +640,7 @@ fn projection_relation_projection( #[hotpath::measure(label = "graph_db.compact.native_outgoing")] fn native_outgoing_traversal( store: &dyn GraphStore, + owners: &dyn GraphStore, start: NodeId, request: &TraversalRequest, ensure_projection_readable: &dyn Fn( @@ -709,7 +719,7 @@ fn native_outgoing_traversal( ))); }; let relation = match cached_relation_identity( - store, + owners, edge, &request.namespace, ensure_projection_readable, @@ -745,7 +755,7 @@ fn native_outgoing_traversal( }; if source_depth.checked_add(1) == Some(target_depth) { let relation = match cached_relation_identity( - store, + owners, edge, &request.namespace, ensure_projection_readable, @@ -835,6 +845,7 @@ enum NativeTraversalStop { #[hotpath::measure(label = "graph_db.compact.directional")] fn directional_traversal( store: &dyn GraphStore, + owners: &dyn GraphStore, start: NodeId, request: &TraversalRequest, ensure_projection_readable: &dyn Fn( @@ -886,7 +897,7 @@ fn directional_traversal( return Err(GraphDbError::Cancelled); } let relation = cached_relation_identity( - store, + owners, edge, &request.namespace, ensure_projection_readable, @@ -1160,15 +1171,7 @@ fn relation_identity( message: "traversal relation native type and kind disagree".to_owned(), }); } - let identity = stored - .get_property(RELATION_ID_PROPERTY) - .and_then(Value::as_str) - .ok_or_else(|| GraphDbError::Corrupt { - message: "traversal relation has no native identity".to_owned(), - })?; - GraphRelationId::new(identity).map_err(|error| GraphDbError::Corrupt { - message: format!("traversal relation has an invalid native identity: {error}"), - }) + edge_relation_identity(store, edge_locator(store, edge)?) } enum RelationEndpointCheck<'a> { @@ -1220,21 +1223,18 @@ fn relation_for_edge( message: "traversal relation native type and kind disagree".to_owned(), }); } - let identity = stored - .get_property(RELATION_ID_PROPERTY) - .and_then(Value::as_str) - .ok_or_else(|| GraphDbError::Corrupt { - message: "traversal relation has no native identity".to_owned(), - })?; - let identity = GraphRelationId::new(identity).map_err(|error| GraphDbError::Corrupt { - message: format!("traversal relation has an invalid native identity: {error}"), - })?; - let from = required_entity_property( - stored.get_property(RELATION_FROM_PROPERTY), + let locator = edge_locator(store, edge)?; + let identity = edge_relation_identity(store, locator)?; + let from = locator_entity( + store, + locator, + RELATION_FROM_PROPERTY, "outgoing relation source", )?; - let to = required_entity_property( - stored.get_property(RELATION_TO_PROPERTY), + let to = locator_entity( + store, + locator, + RELATION_TO_PROPERTY, "outgoing relation target", )?; let endpoints_match = match endpoint_check { @@ -1265,11 +1265,13 @@ fn relation_for_edge( }) } -fn required_entity_property( - value: Option<&Value>, +fn locator_entity( + store: &dyn GraphStore, + locator: NodeId, + property: &str, description: &str, ) -> Result { - GraphEntityId::new(required_string_property(value, description)?).map_err(|error| { + GraphEntityId::new(locator_identity(store, locator, property, description)?).map_err(|error| { GraphDbError::Corrupt { message: format!("{description} is invalid: {error}"), } diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/open_error_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/open_error_contract.rs index e82f05817b..975285c33d 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/open_error_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/open_error_contract.rs @@ -1,8 +1,12 @@ +use std::collections::{BTreeMap, BTreeSet}; use std::sync::Arc; use grafeo_common::types::Value; use tempfile::TempDir; -use tracedecay_graph_db::{GraphDbError, GraphEntityId, GraphNamespace, NeverCancelled}; +use tracedecay_graph_db::{ + GraphDbError, GraphEntityId, GraphNamespace, GraphRelation, GraphRelationId, GraphRelationKind, + NeverCancelled, +}; use crate::support; @@ -25,40 +29,45 @@ fn foreign_grafeo_store_without_marker_is_reset_required() { ); } +fn raw_store(temp: &TempDir) -> grafeo_engine::GrafeoDB { + grafeo_engine::GrafeoDB::with_config( + grafeo_engine::Config::persistent(graph_path(temp.path())) + .with_storage_format(grafeo_engine::config::StorageFormat::SingleFile), + ) + .unwrap() +} + +const FORMAT_MARKER: [(&str, i64); 2] = [ + ("__tracedecay_graph_db_version", 4), + ("__tracedecay_graph_db_sequence", 0), +]; + +fn format_marker() -> Vec<(&'static str, Value)> { + let mut marker: Vec<(&str, Value)> = FORMAT_MARKER + .iter() + .map(|(name, value)| (*name, Value::from(*value))) + .collect(); + marker.push(("__tracedecay_graph_db_schema", "native-scalars-v1".into())); + marker +} + #[test] fn persisted_scalar_identity_mismatch_is_corrupt_on_point_read() { let temp = TempDir::new().unwrap(); - let path = graph_path(temp.path()); - let raw = grafeo_engine::GrafeoDB::with_config( - grafeo_engine::Config::persistent(&path) - .with_storage_format(grafeo_engine::config::StorageFormat::SingleFile), - ) - .unwrap(); + let raw = raw_store(&temp); let mut session = raw.session(); session.begin_transaction().unwrap(); session - .create_node_with_props( - &["__tracedecay_graph_db_format"], - [ - ("__tracedecay_graph_db_version", 3_i64.into()), - ("__tracedecay_graph_db_schema", "native-scalars-v1".into()), - ("__tracedecay_graph_db_sequence", 0_i64.into()), - ], - ) + .create_node_with_props(&["__tracedecay_graph_db_format"], format_marker()) .unwrap(); - // sha256("workspace")[..8], the raw-identity tag, then "entity". - let stable_key = [ - &[0x21, 0xa3, 0x23, 0x0e, 0x03, 0x77, 0x2a, 0x58, 0x00][..], - b"entity", - ] - .concat(); session .create_node_with_props( &["__tracedecay_graph_db_entity"], [ + // base64url(sha256("workspace")[..8] ‖ raw-identity tag ‖ "entity"). ( "__tracedecay_graph_db_entity_key", - Value::Bytes(stable_key.into()), + "IaMjDgN3KlgAZW50aXR5".into(), ), ("__tracedecay_graph_db_namespace", "workspace".into()), ("__tracedecay_graph_db_projection", "code".into()), @@ -78,3 +87,123 @@ fn persisted_scalar_identity_mismatch_is_corrupt_on_point_read() { Err(GraphDbError::Corrupt { .. }) )); } + +/// A format-4 relation as it lies on disk. Keys are base64url of +/// `sha256("workspace")[..8] ‖ digest-identity tag ‖ kind ‖ digest`. The +/// locator owns the identity, source, and target, each stored as U+0001, +/// kind, and base64url digest, and the native edge carries none of them. +/// Keyed reads and edge fan-outs both resolve the same relation. +#[test] +fn compact_relation_identities_read_back_through_keys_and_edges() { + let temp = TempDir::new().unwrap(); + let raw = raw_store(&temp); + let session = raw.session(); + session + .create_node_with_props(&["__tracedecay_graph_db_format"], format_marker()) + .unwrap(); + let symbol = |key: &str, identity: String| { + session + .create_node_with_props( + &["__tracedecay_graph_db_entity"], + [ + ("__tracedecay_graph_db_entity_key", Value::from(key)), + ("__tracedecay_graph_db_namespace", "workspace".into()), + ("__tracedecay_graph_db_projection", "code".into()), + ("__tracedecay_graph_db_entity_id", identity.into()), + ], + ) + .unwrap() + }; + let caller = symbol( + "IaMjDgN3KlgBc3ltYm9sERERERERERERERERERERERERERERERERERERERERERE", + format!("symbol:{}", "11".repeat(32)), + ); + let callee = symbol( + "IaMjDgN3KlgBc3ltYm9sIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiI", + format!("symbol:{}", "22".repeat(32)), + ); + let edge = session + .create_edge_with_props( + caller, + callee, + // `__tracedecay_graph_db_relation_` + hex("calls"). + "__tracedecay_graph_db_relation_63616c6c73", + [ + ("__tracedecay_graph_db_namespace", Value::from("workspace")), + ("__tracedecay_graph_db_projection", Value::from("code")), + ("__tracedecay_graph_db_relation_kind", Value::from("calls")), + ], + ) + .unwrap(); + session + .create_node_with_props( + &["__tracedecay_graph_db_relation_locator"], + [ + ( + "__tracedecay_graph_db_relation_key", + "IaMjDgN3KlgBZWRnZTMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMz".into(), + ), + ("__tracedecay_graph_db_namespace", "workspace".into()), + ("__tracedecay_graph_db_projection", "code".into()), + ( + "__tracedecay_graph_db_relation_id", + "\u{1}edgeMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzMzM".into(), + ), + ( + "__tracedecay_graph_db_relation_from", + "\u{1}symbolERERERERERERERERERERERERERERERERERERERERERE".into(), + ), + ( + "__tracedecay_graph_db_relation_to", + "\u{1}symbolIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiIiI".into(), + ), + ("__tracedecay_graph_db_relation_kind", "calls".into()), + ( + "__tracedecay_graph_db_relation_edge", + i64::try_from(edge.as_u64()).unwrap().into(), + ), + ], + ) + .unwrap(); + raw.close().unwrap(); + + let (_, db) = RegisteredGraph::open_lease(temp.path()).unwrap(); + let workspace = GraphNamespace::new("workspace").unwrap(); + let relation_id = GraphRelationId::new(format!("edge:{}", "33".repeat(32))).unwrap(); + let expected = GraphRelation::new( + relation_id.clone(), + GraphEntityId::new(format!("symbol:{}", "11".repeat(32))).unwrap(), + GraphEntityId::new(format!("symbol:{}", "22".repeat(32))).unwrap(), + GraphRelationKind::new("calls").unwrap(), + BTreeMap::new(), + ) + .unwrap(); + assert_eq!( + db.relation(&workspace, &relation_id, Arc::new(NeverCancelled)) + .unwrap(), + Some(expected.clone()) + ); + let starts = [expected.from.clone()]; + assert_eq!( + db.outgoing_relations( + &workspace, + &starts, + &BTreeSet::new(), + 16, + Arc::new(NeverCancelled) + ) + .unwrap(), + vec![vec![expected]] + ); + assert_eq!( + db.outgoing_relation_ids( + &workspace, + &starts, + &BTreeSet::new(), + 16, + Arc::new(NeverCancelled) + ) + .unwrap(), + vec![vec![relation_id]] + ); +} diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs index c981150786..8aee1d119d 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs @@ -3,13 +3,14 @@ use std::sync::Arc; use std::sync::atomic::{AtomicUsize, Ordering}; use std::time::Duration; +use grafeo_common::types::Value; use tempfile::TempDir; use tracedecay_graph_db::{ GraphBudgetKind, GraphCancellation, GraphDbError, GraphDbLeaseV1, GraphDbOwner, GraphEntity, - GraphEntityId, GraphFormatVersion, GraphIdempotencyKey, GraphLabel, GraphMutation, - GraphNamespace, GraphProjectionId, GraphPublication, GraphPublicationInputDigest, - GraphRelation, GraphRelationId, GraphRelationKind, GraphTraversalDirection, GraphWatermark, - GraphWriteBatch, NeverCancelled, ProjectionReplacement, SourceGeneration, TraversalRequest, + GraphEntityId, GraphIdempotencyKey, GraphLabel, GraphMutation, GraphNamespace, + GraphProjectionId, GraphPublication, GraphPublicationInputDigest, GraphRelation, + GraphRelationId, GraphRelationKind, GraphTraversalDirection, GraphWatermark, GraphWriteBatch, + NeverCancelled, ProjectionReplacement, SourceGeneration, TraversalRequest, }; use crate::support; @@ -1010,7 +1011,9 @@ fn wrong_tracedecay_format_requires_reset() { fn superseded_format_store_is_rebuilt_fresh_with_its_sealed_generations_discarded() { let temp = TempDir::new().unwrap(); let path = graph_path(temp.path()); - let previous_format = i64::from(GraphFormatVersion::current().get() - 1); + // Format 3 stored relation identities as strings on both the locator and + // the native edge; the row below is keyed the way format 3 keyed it. + let previous_format = 3_i64; let raw = grafeo_engine::GrafeoDB::with_config( grafeo_engine::Config::persistent(&path) .with_storage_format(grafeo_engine::config::StorageFormat::SingleFile), @@ -1032,7 +1035,15 @@ fn superseded_format_store_is_rebuilt_fresh_with_its_sealed_generations_discarde [ ( "__tracedecay_graph_db_entity_key", - "70726f6a656374:61".into(), + // sha256("project")[..8], the raw-identity tag, "stale". + Value::Bytes( + [ + &[0x24, 0x42, 0x10, 0xe4, 0x84, 0x37, 0xb6, 0x55, 0x00][..], + b"stale", + ] + .concat() + .into(), + ), ), ("__tracedecay_graph_db_namespace", "project".into()), ("__tracedecay_graph_db_projection", "code".into()), @@ -1045,7 +1056,7 @@ fn superseded_format_store_is_rebuilt_fresh_with_its_sealed_generations_discarde std::fs::create_dir_all(&sealed_generation).unwrap(); std::fs::write( sealed_generation.join("generation.grafeo"), - b"format 2 bytes", + b"format 3 bytes", ) .unwrap(); diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs index 09bf4af5fc..c848a79e7a 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/verified_generation_contract/sealed_store.rs @@ -1483,13 +1483,14 @@ fn stable_identity_manifest( .unwrap() } -/// Unique keys are written once per entity and relation locator, so their -/// encoding sets a sealed generation's size. Hex keys that repeat the -/// namespace sealed this 2,000-symbol generation to 3,150,837 bytes; binary -/// namespace-id keys seal it to 1,786,870, and the rows still resolve -/// through them. +/// Keys and relation identities are the bulk of a sealed generation's +/// bytes. Hex keys sealed this 2,000-symbol generation to 3,150,837 bytes, +/// and binary keys, which the compact dictionary stores as marked hex, to +/// 1,786,870. Base64url keys plus each relation identity, source, and target +/// stored once, on its locator, in compact form seal it to 1,037,302, and +/// the rows still resolve through keys and edges. #[test] -fn sealed_generation_bytes_stay_within_the_binary_key_budget() { +fn sealed_generation_bytes_stay_within_the_compact_identity_budget() { let temp = TempDir::new().unwrap(); let registered = RegisteredGraph::new_mounted(temp.path()).unwrap(); let mut authority = RelationalAuthority::default(); @@ -1508,7 +1509,7 @@ fn sealed_generation_bytes_stay_within_the_binary_key_budget() { let sealed_bytes = directory_bytes(&sealed_store_root(temp.path())); assert!( - sealed_bytes <= 1_900_000, + sealed_bytes <= 1_150_000, "sealed generation took {sealed_bytes} bytes" ); let last = GraphEntityId::new(graph_stable_identity("symbol", "1999")).unwrap();