From 31b2130c0001180575762edcfb23c6bbb42a808b Mon Sep 17 00:00:00 2001 From: ScriptedAlchemy Date: Sat, 26 Sep 2026 21:08:53 +0000 Subject: [PATCH] perf(graph-db): store each code edge as one relation row A code edge was an edge entity carrying its JSON record plus two relations (symbol -> edge, edge -> symbol), each with a locator node and a native edge. It is now one relation from its source symbol to its target symbol, kind `CodeEdge.`, whose native edge carries the edge record. Readers take the payload from the fan-out row: adjacency, key walks, degrees, edges-among, the evidence traversal, and the catalog warm read no edge entity and walk one hop instead of two. Graph-db relation payloads now live only on the native edge. Code-graph records are stored as compact text: structural JSON tokens and 64-hex digests become short marked codes; free-form values stay JSON. A superseded-format rebuild now discards only sealed generations whose receipt names an older graph format, so a generation sealed before the staging container is replaced keeps serving. Graph format 4 -> 5, projector v9 -> v10, same typed rebuild. Refs #1103 --- Cargo.lock | 1 + crates/tracedecay-code-index/Cargo.toml | 1 + .../src/graph_projection.rs | 76 +++--- .../src/graph_projection/builder.rs | 74 ++---- .../src/graph_projection/interactive.rs | 216 +++++------------ .../graph_projection/interactive/catalog.rs | 69 +++--- .../src/graph_projection/reader.rs | 73 +++--- .../src/graph_projection/schema.rs | 218 ++++++++++++++++-- .../graph_projection_publication.rs | 123 ++++++++-- .../tests/resident_accounting.rs | 6 +- .../tracedecay-graph-db/src/corrupt_store.rs | 28 ++- crates/tracedecay-graph-db/src/location.rs | 2 +- crates/tracedecay-graph-db/src/schema.rs | 18 +- .../tracedecay-graph-db/src/sealed_store.rs | 51 ++++ .../graph_db_suite/open_error_contract.rs | 29 ++- .../tests/graph_db_suite/runtime_contract.rs | 54 +++-- .../src/handlers/graph/search.rs | 5 +- 17 files changed, 632 insertions(+), 412 deletions(-) diff --git a/Cargo.lock b/Cargo.lock index d3d6e0886d..8bac0a425a 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -5733,6 +5733,7 @@ name = "tracedecay-code-index" version = "0.1.0" dependencies = [ "ast-grep-core", + "base64 0.22.1", "criterion", "flate2", "hex", diff --git a/crates/tracedecay-code-index/Cargo.toml b/crates/tracedecay-code-index/Cargo.toml index c34f0a263c..c285080449 100644 --- a/crates/tracedecay-code-index/Cargo.toml +++ b/crates/tracedecay-code-index/Cargo.toml @@ -73,6 +73,7 @@ lang-lean = ["tracedecay-code-extraction/lang-lean"] [dependencies] ast-grep-core = "0.44" flate2 = "1" +base64 = "0.22" hex = "0.4" hotpath.workspace = true ignore = "0.4" diff --git a/crates/tracedecay-code-index/src/graph_projection.rs b/crates/tracedecay-code-index/src/graph_projection.rs index 8b319cc925..8c2cf1a6ad 100644 --- a/crates/tracedecay-code-index/src/graph_projection.rs +++ b/crates/tracedecay-code-index/src/graph_projection.rs @@ -22,9 +22,9 @@ use tracedecay_graph_db::{ GraphCancellation, GraphConflictContextV1, GraphDbError, GraphEntity, GraphEntityId, GraphEntityRef, GraphGenerationId, GraphGenerationManifest, GraphGenerationManifestIdentity, GraphIdempotencyKey, GraphLabel, GraphNamespace, GraphProjectionId, GraphProjectionIdentity, - GraphProjectorRevision, GraphProperty, GraphPropertyName, GraphServingEnginePin, - GraphTraversalDirection, GraphWatermark, SourceGeneration, TraversalRequest, - VerifiedGraphSnapshot, + GraphProjectorRevision, GraphProperty, GraphPropertyName, GraphRelation, GraphRelationId, + GraphRelationKind, GraphServingEnginePin, GraphWatermark, MAX_VERIFIED_GENERATION_RELATIONS, + SourceGeneration, VerifiedGraphSnapshot, }; mod builder; @@ -61,9 +61,9 @@ const CURRENT_GENERATION_ENTITY: &str = "code-current-generation"; const CURRENT_GENERATION_PROPERTY: &str = "current-generation"; const PROJECTION_NODE_COUNT_PROPERTY: &str = "projection-node-count"; const EDGE_RECORD_PROPERTY: &str = "edge-record"; -const EDGE_LABEL: &str = "CodeRelationEvidence"; const FILE_SYMBOL_EDGE_KIND: &str = "CodeFileContainsSymbol"; -const TARGET_EDGE_KIND: &str = "CodeRelationTarget"; +/// Starts per adjacency fan-out batch; the store bounds one batch's starts. +const ADJACENCY_SEED_CHUNK: usize = 4_096; /// Names the shape of the rows this projector emits for one sealed code /// generation; the graph generation id is derived from it, so a revision /// bump seals a new graph generation from the same code generation rather @@ -73,8 +73,11 @@ const TARGET_EDGE_KIND: &str = "CodeRelationTarget"; /// v7 carries unresolved receiver-call limitations on each source symbol. v8 /// widens those limitations to bare TypeScript calls whose import the seal /// could not bind to project code. v9 names each source relation for its edge -/// kind, so adjacency filters kinds without decoding the edge. -pub const CODE_GRAPH_PROJECTOR_REVISION: &str = "code-graph-projector.v9"; +/// kind, so adjacency filters kinds without decoding the edge. v10 stores each +/// code edge as one relation from its source symbol to its target symbol that +/// carries the edge record, instead of an edge entity between two relations, +/// and stores every record as compact text (`schema::compact_record`). +pub const CODE_GRAPH_PROJECTOR_REVISION: &str = "code-graph-projector.v10"; /// Every semantic edge kind, at its [`relation_edge_kind_index`]. const RELATION_EDGE_KINDS: [RelationEdgeKindV1; 9] = [ @@ -113,28 +116,28 @@ const _: () = { } }; -/// The kind of the relation from an edge's source symbol to its edge entity: -/// one per edge kind, so a fan-out names the kinds it admits. -fn source_edge_kind(kind: RelationEdgeKindV1) -> &'static str { +/// The relation kind of a code edge row, from its source symbol to its +/// target symbol: one per edge kind, so a fan-out names the kinds it admits. +fn code_edge_kind(kind: RelationEdgeKindV1) -> &'static str { match kind { - RelationEdgeKindV1::Calls => "CodeRelationSource.calls", - RelationEdgeKindV1::Uses => "CodeRelationSource.uses", - RelationEdgeKindV1::TypeOf => "CodeRelationSource.type_of", - RelationEdgeKindV1::Contains => "CodeRelationSource.contains", - RelationEdgeKindV1::Implements => "CodeRelationSource.implements", - RelationEdgeKindV1::Extends => "CodeRelationSource.extends", - RelationEdgeKindV1::Annotates => "CodeRelationSource.annotates", - RelationEdgeKindV1::Returns => "CodeRelationSource.returns", - RelationEdgeKindV1::Receives => "CodeRelationSource.receives", + RelationEdgeKindV1::Calls => "CodeEdge.calls", + RelationEdgeKindV1::Uses => "CodeEdge.uses", + RelationEdgeKindV1::TypeOf => "CodeEdge.type_of", + RelationEdgeKindV1::Contains => "CodeEdge.contains", + RelationEdgeKindV1::Implements => "CodeEdge.implements", + RelationEdgeKindV1::Extends => "CodeEdge.extends", + RelationEdgeKindV1::Annotates => "CodeEdge.annotates", + RelationEdgeKindV1::Returns => "CodeEdge.returns", + RelationEdgeKindV1::Receives => "CodeEdge.receives", } } -/// The edge kind a source relation kind names, or `None` for any other +/// The edge kind a code edge relation kind names, or `None` for any other /// relation. -fn source_edge_kind_edge(relation_kind: &str) -> Option { +fn code_edge_kind_edge(relation_kind: &str) -> Option { RELATION_EDGE_KINDS .into_iter() - .find(|kind| source_edge_kind(*kind) == relation_kind) + .find(|kind| code_edge_kind(*kind) == relation_kind) } #[derive(Clone, Debug, Error, PartialEq, Eq)] @@ -1038,10 +1041,31 @@ fn symbol_entity_id( GraphEntityId::new(stable_identity("symbol", occurrence.as_str())).map_err(Into::into) } -fn edge_entity_id( +fn edge_relation_id( edge: &CanonicalRelationEdgeV1, -) -> Result { - GraphEntityId::new(stable_identity("edge", &hex::encode(serialize(edge)?))).map_err(Into::into) +) -> Result { + GraphRelationId::new(stable_identity("edge", &hex::encode(serialize(edge)?))) + .map_err(Into::into) +} + +/// The validated edge record a code edge relation row carries: its identity, +/// kind, and both endpoints must be the ones its payload derives. +fn edge_record( + relation: &GraphRelation, +) -> Result { + let edge: CanonicalRelationEdgeV1 = + deserialize_property(&relation.properties, EDGE_RECORD_PROPERTY)?; + validate_edge(&edge)?; + if edge_relation_id(&edge)? != relation.identity + || code_edge_kind(edge.kind) != relation.kind.as_str() + || symbol_entity_id(&edge.from_occurrence)? != relation.from + || symbol_entity_id(&edge.to_occurrence)? != relation.to + { + return Err(CodeGraphProjectionError::Corrupt( + "code graph edge row does not match its payload".to_owned(), + )); + } + Ok(edge) } #[cfg(any(feature = "test-helpers", feature = "eval-helpers"))] @@ -1217,7 +1241,7 @@ fn load_symbol_entity_record( "code graph symbol identity has the wrong label".to_owned(), )); } - let record: SymbolRecordV1 = deserialize_property(&entity, SYMBOL_RECORD_PROPERTY)?; + let record: SymbolRecordV1 = deserialize_property(&entity.properties, SYMBOL_RECORD_PROPERTY)?; validate_symbol_record(&record)?; if symbol_entity_id(&record.occurrence)? != *identity { return Err(CodeGraphProjectionError::Corrupt( diff --git a/crates/tracedecay-code-index/src/graph_projection/builder.rs b/crates/tracedecay-code-index/src/graph_projection/builder.rs index 6f67d69e01..1906e87c81 100644 --- a/crates/tracedecay-code-index/src/graph_projection/builder.rs +++ b/crates/tracedecay-code-index/src/graph_projection/builder.rs @@ -30,10 +30,10 @@ use super::schema::{ stable_identity, }; use super::{ - CodeGraphProjectionError, CodeGraphSymbolBindingV1, EDGE_LABEL, EDGE_RECORD_PROPERTY, - FILE_SYMBOL_EDGE_KIND, SealedCodeGraphRowsError, SymbolRecordV1, TARGET_EDGE_KIND, + CodeGraphProjectionError, CodeGraphSymbolBindingV1, EDGE_RECORD_PROPERTY, + FILE_SYMBOL_EDGE_KIND, SealedCodeGraphRowsError, SymbolRecordV1, code_edge_kind, code_graph_manifest_identity, compare_edges, current_generation_entity, projection, - source_edge_kind, symbol_entity, symbol_entity_id, validate_edge, + symbol_entity, symbol_entity_id, validate_edge, }; /// Builds a sealed generation's code graph from its on-disk file segments and @@ -532,23 +532,20 @@ fn emit_code_graph_rows( )) })?; // Chunks bind symbols to files and spans above; they are not graph rows. - // No reader addresses a chunk through the graph, traversal alternates - // symbol and edge-evidence entities, and a symbol's binding already - // names its chunk, so projecting one entity plus one relation per chunk - // only multiplied every graph artifact by the chunk count. + // No reader addresses a chunk through the graph and a symbol's binding + // already names its chunk, so projecting one entity plus one relation per + // chunk only multiplied every graph artifact by the chunk count. hotpath::measure_block!("code_index.seal.collect.emit", { let mut entities = Vec::with_capacity( batch .files .len() .saturating_add(batch.imports.len()) - .saturating_add(occurrences.len()) - .saturating_add(retained_edges.len()), + .saturating_add(occurrences.len()), ); let mut relations = Vec::with_capacity( retained_edges .len() - .saturating_mul(2) .saturating_add(bindings.len()) .saturating_add(batch.imports.len()), ); @@ -619,13 +616,9 @@ fn emit_code_graph_rows( } for window in retained_edges.chunks(row_window) { check()?; - for (entity, source, target) in collect_graph_rows_ordered(window, |edge| { - edge_artifacts(projection, edge, &symbol_ids) - })? { - entities.push(entity); - relations.push(source); - relations.push(target); - } + relations.extend(collect_graph_rows_ordered(window, |edge| { + edge_relation(projection, edge, &symbol_ids) + })?); } Ok(EmittedRows { entities, @@ -657,47 +650,28 @@ fn endpoint_symbol_id( } } -/// One retained edge's entity plus both endpoint relations, sharing a single -/// serialization and identity derivation of the edge payload. -fn edge_artifacts( +/// One retained edge as a single relation row from its source symbol to its +/// target symbol, carrying the edge record as its payload. +fn edge_relation( projection: &GraphProjectionIdentity, edge: &CanonicalRelationEdgeV1, symbol_ids: &BTreeMap, -) -> Result< - ( - GraphEntity, - GraphGenerationRelation, - GraphGenerationRelation, - ), - CodeGraphProjectionError, -> { +) -> Result { let payload = serialize(edge)?; - let identity = GraphEntityId::new(stable_identity("edge", &hex::encode(&payload)))?; - let entity = GraphEntity::new( - identity.clone(), - BTreeSet::from([GraphLabel::new(EDGE_LABEL)?]), - BTreeMap::from([( - GraphPropertyName::new(EDGE_RECORD_PROPERTY)?, - record_property(payload)?, - )]), - )?; + let identity = GraphRelationId::new(stable_identity("edge", &hex::encode(&payload)))?; let from = endpoint_symbol_id(symbol_ids, &edge.from_occurrence)?; let to = endpoint_symbol_id(symbol_ids, &edge.to_occurrence)?; - let source = GraphGenerationRelation::new( - GraphRelationId::new(stable_identity("source", identity.as_str()))?, + GraphGenerationRelation::new( + identity, GraphEntityRef::new(projection.clone(), from), - GraphEntityRef::new(projection.clone(), identity.clone()), - GraphRelationKind::new(source_edge_kind(edge.kind))?, - BTreeMap::new(), - )?; - let target = GraphGenerationRelation::new( - GraphRelationId::new(stable_identity("target", identity.as_str()))?, - GraphEntityRef::new(projection.clone(), identity), GraphEntityRef::new(projection.clone(), to), - GraphRelationKind::new(TARGET_EDGE_KIND)?, - BTreeMap::new(), - )?; - Ok((entity, source, target)) + GraphRelationKind::new(code_edge_kind(edge.kind))?, + BTreeMap::from([( + GraphPropertyName::new(EDGE_RECORD_PROPERTY)?, + record_property(payload)?, + )]), + ) + .map_err(Into::into) } fn file_entity( diff --git a/crates/tracedecay-code-index/src/graph_projection/interactive.rs b/crates/tracedecay-code-index/src/graph_projection/interactive.rs index 436ba6fb1b..50ceca8059 100644 --- a/crates/tracedecay-code-index/src/graph_projection/interactive.rs +++ b/crates/tracedecay-code-index/src/graph_projection/interactive.rs @@ -27,17 +27,16 @@ use tracedecay_domain::{ SanitizedCodeFileV1, SymbolOccurrenceId, repository_path_matches_scope, }; use tracedecay_graph_db::{ - GraphCancellation, GraphEntity, GraphEntityId, GraphProjectionIdentity, GraphReadMeter, - GraphRelation, GraphRelationKind, MAX_VERIFIED_GENERATION_RELATIONS, RelationFanoutOverflow, + GraphCancellation, GraphEntityId, GraphProjectionIdentity, GraphReadMeter, GraphRelation, + GraphRelationKind, MAX_VERIFIED_GENERATION_RELATIONS, RelationFanoutOverflow, VerifiedGraphSnapshot, }; use super::{ CodeGraphProjectionError, CodeGraphProjectionStore, CodeGraphReadCancellation, - CodeGraphSymbolBindingV1, EDGE_LABEL, EDGE_RECORD_PROPERTY, RELATION_EDGE_KINDS, - SymbolRecordV1, TARGET_EDGE_KIND, compare_edges, deserialize_property, edge_entity_id, - has_label, load_symbol_entity_record, load_symbol_record, source_edge_kind, - source_edge_kind_edge, symbol_entity_id, validate_edge, + CodeGraphSymbolBindingV1, RELATION_EDGE_KINDS, SymbolRecordV1, code_edge_kind, + code_edge_kind_edge, compare_edges, edge_record, load_symbol_entity_record, load_symbol_record, + symbol_entity_id, }; use crate::lineage::LineageSymbolRecordV1; @@ -494,26 +493,23 @@ impl CodeGraphInteractiveReader { let cancellation = self.read_cancellation(request_cancellation)?; require_positive(max_relations, "code graph relation key limit")?; let starts = seeds.iter().map(|seed| seed.0.clone()).collect::>(); - let target_kinds = target_relation_kinds()?; - let source_kinds = source_relation_kinds(kinds)?; - let (seed_kinds, far_kinds) = if reverse { - (&target_kinds, &source_kinds) - } else { - (&source_kinds, &target_kinds) - }; + // A reverse walk admits every kind at the store and filters after, + // so its truncation point is the one the unfiltered fan-out defines. + let admitted: BTreeSet = kinds.iter().copied().collect(); + let edge_kinds = code_relation_kinds(if reverse { &[] } else { kinds })?; let edge_rows = if reverse { self.snapshot.incoming_relations_truncated( &starts, - seed_kinds, + &edge_kinds, max_relations, - Arc::clone(&cancellation), + cancellation, )? } else { self.snapshot.outgoing_relations_truncated( &starts, - seed_kinds, + &edge_kinds, max_relations, - Arc::clone(&cancellation), + cancellation, )? }; if edge_rows.len() != seeds.len() { @@ -521,68 +517,24 @@ impl CodeGraphInteractiveReader { "code graph relation key batch shape does not match its seeds".to_owned(), )); } - let edges = edge_rows - .iter() - .flatten() - .map(|relation| { - if reverse { - relation.from.clone() - } else { - relation.to.clone() - } - }) - .collect::>(); - let truncated = edges.len() == max_relations; - // One far row per edge at most; the batch limit leaves room for one - // extra so a second far endpoint shows as corruption, not truncation. - let far_limit = edges.len().saturating_add(1); - let far_rows = if edges.is_empty() { - Vec::new() - } else if reverse { - self.snapshot.incoming_relations_truncated( - &edges, - far_kinds, - far_limit, - cancellation, - )? - } else { - self.snapshot.outgoing_relations_truncated( - &edges, - far_kinds, - far_limit, - cancellation, - )? - }; - if far_rows.len() != edges.len() { - return Err(CodeGraphProjectionError::Corrupt( - "code graph edge endpoint batch shape does not match its edges".to_owned(), - )); - } - let mut far_rows = far_rows.into_iter(); + let truncated = edge_rows.iter().map(Vec::len).sum::() == max_relations; let per_seed = edge_rows .into_iter() .map(|relations| { let mut keys = Vec::with_capacity(relations.len()); - for (edge, far) in relations.iter().zip(far_rows.by_ref()) { - let key = match (reverse, far.as_slice()) { - // A reverse walk names the admitted kinds on the far - // hop, so an edge of another kind has no far row. - (true, []) => continue, - (true, [source]) => CodeGraphRelationKeyV1 { - neighbor: CodeGraphSymbolRefV1(source.from.clone()), - kind: relation_edge_kind(source)?, - }, - (false, [target]) => CodeGraphRelationKeyV1 { - neighbor: CodeGraphSymbolRefV1(target.to.clone()), - kind: relation_edge_kind(edge)?, - }, - _ => { - return Err(CodeGraphProjectionError::Corrupt( - "code graph edge has no single far endpoint".to_owned(), - )); - } - }; - keys.push(key); + for relation in &relations { + let kind = relation_edge_kind(relation)?; + if reverse && !admitted.is_empty() && !admitted.contains(&kind) { + continue; + } + keys.push(CodeGraphRelationKeyV1 { + neighbor: CodeGraphSymbolRefV1(if reverse { + relation.from.clone() + } else { + relation.to.clone() + }), + kind, + }); } Ok(keys) }) @@ -805,15 +757,16 @@ impl CodeGraphInteractiveReader { ) -> Result, CodeGraphProjectionError> { let cancellation = self.read_cancellation(request_cancellation)?; let starts = entity_ids(occurrences)?; + let edge_kinds = code_relation_kinds(&[])?; let outgoing = self.snapshot.outgoing_relation_ids( &starts, - &source_relation_kinds(&[])?, + &edge_kinds, MAX_VERIFIED_GENERATION_RELATIONS, Arc::clone(&cancellation), )?; let incoming = self.snapshot.incoming_relation_ids( &starts, - &target_relation_kinds()?, + &edge_kinds, MAX_VERIFIED_GENERATION_RELATIONS, cancellation, )?; @@ -1055,9 +1008,9 @@ impl CodeGraphInteractiveReader { let mut edges: Vec = Vec::new(); for chunk in occurrences.chunks(SEMANTIC_NEIGHBOR_SEED_CHUNK) { let starts = entity_ids(chunk)?; - let per_seed = self.snapshot.outgoing_relation_targets( + let per_seed = self.snapshot.outgoing_relations( &starts, - &source_relation_kinds(kinds)?, + &code_relation_kinds(kinds)?, max_relations, Arc::clone(&cancellation), )?; @@ -1066,12 +1019,12 @@ impl CodeGraphInteractiveReader { "code graph adjacency batch shape does not match its seeds".to_owned(), )); } - for (seed, targets) in chunk.iter().zip(per_seed) { - for target in targets { + for (seed, relations) in chunk.iter().zip(per_seed) { + for relation in relations { if cancellation.is_cancelled() { return Err(CodeGraphProjectionError::Cancelled); } - let edge = load_edge_record(&target.target)?; + let edge = edge_record(&relation)?; if edge.from_occurrence != *seed { return Err(CodeGraphProjectionError::Corrupt( "code graph edge endpoint does not match its adjacency seed".to_owned(), @@ -1407,7 +1360,7 @@ impl CodeGraphInteractiveReader { (AdjacencyDirection::Outgoing, RelationFanoutOverflow::Refuse) => { self.snapshot.outgoing_relations( &starts, - &source_relation_kinds(kinds)?, + &code_relation_kinds(kinds)?, max_relations, Arc::clone(&cancellation), )? @@ -1415,7 +1368,7 @@ impl CodeGraphInteractiveReader { (AdjacencyDirection::Outgoing, RelationFanoutOverflow::Truncate) => { self.snapshot.outgoing_relations_truncated( &starts, - &source_relation_kinds(kinds)?, + &code_relation_kinds(kinds)?, max_relations, Arc::clone(&cancellation), )? @@ -1423,7 +1376,7 @@ impl CodeGraphInteractiveReader { (AdjacencyDirection::Incoming, RelationFanoutOverflow::Refuse) => { self.snapshot.incoming_relations( &starts, - &target_relation_kinds()?, + &code_relation_kinds(&[])?, max_relations, Arc::clone(&cancellation), )? @@ -1431,7 +1384,7 @@ impl CodeGraphInteractiveReader { (AdjacencyDirection::Incoming, RelationFanoutOverflow::Truncate) => { self.snapshot.incoming_relations_truncated( &starts, - &target_relation_kinds()?, + &code_relation_kinds(&[])?, max_relations, Arc::clone(&cancellation), )? @@ -1450,12 +1403,7 @@ impl CodeGraphInteractiveReader { if cancellation.is_cancelled() { return Err(CodeGraphProjectionError::Cancelled); } - let edge = self.hydrate_edge_record( - seed, - &relation, - direction, - Arc::clone(&cancellation), - )?; + let edge = seed_edge_record(seed, &relation, direction)?; if !admitted.is_empty() && !admitted.contains(&edge.kind) { continue; } @@ -1490,46 +1438,6 @@ impl CodeGraphInteractiveReader { } Ok(batches) } - - /// Loads one adjacency row up to its validated edge payload: the relation, - /// the edge entity, and the seed-endpoint check, no far-endpoint read. - fn hydrate_edge_record( - &self, - seed: &SymbolOccurrenceId, - relation: &GraphRelation, - direction: AdjacencyDirection, - cancellation: Arc, - ) -> Result { - let edge_reference = match direction { - AdjacencyDirection::Outgoing => &relation.to, - AdjacencyDirection::Incoming => &relation.from, - }; - let entity = self - .snapshot - .entity( - &tracedecay_graph_db::GraphEntityRef::new( - self.projection.clone(), - edge_reference.clone(), - ), - cancellation, - )? - .ok_or_else(|| { - CodeGraphProjectionError::Corrupt( - "code graph adjacency referenced a missing edge entity".to_owned(), - ) - })?; - let edge = load_edge_record(&entity)?; - let near = match direction { - AdjacencyDirection::Outgoing => &edge.from_occurrence, - AdjacencyDirection::Incoming => &edge.to_occurrence, - }; - if near != seed { - return Err(CodeGraphProjectionError::Corrupt( - "code graph edge endpoint does not match its adjacency seed".to_owned(), - )); - } - Ok(edge) - } } fn resolve_from_index( @@ -1561,32 +1469,32 @@ fn summary_from_record(record: SymbolRecordV1) -> CodeGraphSymbolSummaryV1 { } } -fn load_edge_record( - entity: &GraphEntity, +/// One adjacency row's validated edge payload, checked against the seed it +/// was read from. +fn seed_edge_record( + seed: &SymbolOccurrenceId, + relation: &GraphRelation, + direction: AdjacencyDirection, ) -> Result { - if !has_label(entity, EDGE_LABEL) { - return Err(CodeGraphProjectionError::Corrupt( - "code graph adjacency contains a non-edge entity".to_owned(), - )); - } - let edge: CanonicalRelationEdgeV1 = deserialize_property(entity, EDGE_RECORD_PROPERTY)?; - validate_edge(&edge)?; - if edge_entity_id(&edge)? != entity.identity { + let edge = edge_record(relation)?; + let near = match direction { + AdjacencyDirection::Outgoing => &edge.from_occurrence, + AdjacencyDirection::Incoming => &edge.to_occurrence, + }; + if near != seed { return Err(CodeGraphProjectionError::Corrupt( - "code graph edge identity does not match its payload".to_owned(), + "code graph edge endpoint does not match its adjacency seed".to_owned(), )); } Ok(edge) } -/// The edge kind a source relation row names. +/// The edge kind a code edge row names. fn relation_edge_kind( relation: &GraphRelation, ) -> Result { - source_edge_kind_edge(relation.kind.as_str()).ok_or_else(|| { - CodeGraphProjectionError::Corrupt( - "code graph source relation names no edge kind".to_owned(), - ) + code_edge_kind_edge(relation.kind.as_str()).ok_or_else(|| { + CodeGraphProjectionError::Corrupt("code graph edge row names no edge kind".to_owned()) }) } @@ -1609,9 +1517,9 @@ fn entity_ids( .collect() } -/// Source relation kinds for the admitted edge kinds; every kind when none -/// is named. -fn source_relation_kinds( +/// Code edge relation kinds for the admitted edge kinds; every kind when +/// none is named. +fn code_relation_kinds( kinds: &[RelationEdgeKindV1], ) -> Result, CodeGraphProjectionError> { let admitted = if kinds.is_empty() { @@ -1621,14 +1529,10 @@ fn source_relation_kinds( }; admitted .iter() - .map(|kind| GraphRelationKind::new(source_edge_kind(*kind)).map_err(Into::into)) + .map(|kind| GraphRelationKind::new(code_edge_kind(*kind)).map_err(Into::into)) .collect() } -fn target_relation_kinds() -> Result, CodeGraphProjectionError> { - Ok(BTreeSet::from([GraphRelationKind::new(TARGET_EDGE_KIND)?])) -} - fn contains_ignore_ascii_case(value: &str, query: &str) -> bool { query.is_empty() || value diff --git a/crates/tracedecay-code-index/src/graph_projection/interactive/catalog.rs b/crates/tracedecay-code-index/src/graph_projection/interactive/catalog.rs index acc73d151e..adaf02fa40 100644 --- a/crates/tracedecay-code-index/src/graph_projection/interactive/catalog.rs +++ b/crates/tracedecay-code-index/src/graph_projection/interactive/catalog.rs @@ -1,6 +1,6 @@ //! Bounded construction of the generation-pinned interactive catalog. -use std::borrow::{Borrow, Cow}; +use std::borrow::Borrow; use std::cmp::Ordering; use std::collections::{BTreeMap, HashMap, HashSet}; use std::sync::Arc; @@ -19,8 +19,8 @@ use super::super::schema::{ file_import_relation_id, has_label, import_entity_id, }; use super::super::{ - CodeGraphProjectionError, EDGE_LABEL, EDGE_RECORD_PROPERTY, SymbolRecordV1, TARGET_EDGE_KIND, - source_edge_kind_edge, symbol_entity_id, validate_symbol_record, + CodeGraphProjectionError, EDGE_RECORD_PROPERTY, SymbolRecordV1, code_edge_kind_edge, + symbol_entity_id, validate_symbol_record, }; use super::models::{CatalogSymbol, CodeGraphFileDependenciesV1, InteractiveCatalog}; use crate::chunks::CodeIndexImportEvidenceV1; @@ -106,12 +106,9 @@ struct CatalogScan { /// The fields of an edge record the file dependency fold reads, borrowed so /// edges of other kinds allocate nothing. #[derive(Deserialize)] -struct DependencyEdgeRecord<'entity> { - #[serde(borrow)] - from_occurrence: Cow<'entity, str>, - #[serde(borrow)] - to_occurrence: Cow<'entity, str>, - kind: RelationEdgeKindV1, +struct DependencyEdgeRecord { + from_occurrence: String, + to_occurrence: String, } impl CatalogScan { @@ -164,27 +161,12 @@ impl CatalogScan { if has_label(entity, IMPORT_LABEL) { self.record_import(entity)?; } - if has_label(entity, EDGE_LABEL) { - let edge: DependencyEdgeRecord = deserialize_property(entity, EDGE_RECORD_PROPERTY)?; - if matches!( - edge.kind, - RelationEdgeKindV1::Calls | RelationEdgeKindV1::Uses - ) { - let endpoint = |occurrence: Cow<'_, str>| { - SymbolOccurrenceId::new(occurrence.into_owned()) - .map_err(|error| CodeGraphProjectionError::Corrupt(error.to_string())) - }; - self.dependency_edges.push(( - endpoint(edge.from_occurrence)?, - endpoint(edge.to_occurrence)?, - )); - } - } Ok(()) } fn record_file(&mut self, entity: &GraphEntity) -> Result<(), CodeGraphProjectionError> { - let record: SanitizedCodeFileV1 = deserialize_property(entity, FILE_RECORD_PROPERTY)?; + let record: SanitizedCodeFileV1 = + deserialize_property(&entity.properties, FILE_RECORD_PROPERTY)?; record .validate() .map_err(|error| CodeGraphProjectionError::Contract(error.to_string()))?; @@ -219,7 +201,8 @@ impl CatalogScan { } fn record_symbol(&mut self, entity: &GraphEntity) -> Result<(), CodeGraphProjectionError> { - let record: SymbolRecordV1 = deserialize_property(entity, SYMBOL_RECORD_PROPERTY)?; + let record: SymbolRecordV1 = + deserialize_property(&entity.properties, SYMBOL_RECORD_PROPERTY)?; validate_symbol_record(&record)?; if symbol_entity_id(&record.occurrence)? != entity.identity { return Err(CodeGraphProjectionError::Corrupt( @@ -246,7 +229,7 @@ impl CatalogScan { fn record_import(&mut self, entity: &GraphEntity) -> Result<(), CodeGraphProjectionError> { let record: CodeIndexImportEvidenceV1 = - deserialize_property(entity, IMPORT_RECORD_PROPERTY)?; + deserialize_property(&entity.properties, IMPORT_RECORD_PROPERTY)?; record.validate().map_err(|error| { CodeGraphProjectionError::Corrupt(format!( "code graph import row is not canonical: {error}" @@ -279,16 +262,38 @@ impl CatalogScan { self.count_relation()?; match relation.kind.as_str() { FILE_IMPORT_EDGE_KIND => self.record_import_link(relation.clone())?, - TARGET_EDGE_KIND => self.degrees.record_incoming(relation.to.clone()), - kind if source_edge_kind_edge(kind).is_some() => { - self.degrees.record_outgoing(relation.from.clone()); + kind => { + if let Some(edge_kind) = code_edge_kind_edge(kind) { + self.record_code_edge(relation, edge_kind)?; + } } - _ => {} } } Ok(()) } + fn record_code_edge( + &mut self, + relation: &GraphRelation, + kind: RelationEdgeKindV1, + ) -> Result<(), CodeGraphProjectionError> { + self.degrees.record_outgoing(relation.from.clone()); + self.degrees.record_incoming(relation.to.clone()); + if matches!(kind, RelationEdgeKindV1::Calls | RelationEdgeKindV1::Uses) { + let edge: DependencyEdgeRecord = + deserialize_property(&relation.properties, EDGE_RECORD_PROPERTY)?; + let endpoint = |occurrence: String| { + SymbolOccurrenceId::new(occurrence) + .map_err(|error| CodeGraphProjectionError::Corrupt(error.to_string())) + }; + self.dependency_edges.push(( + endpoint(edge.from_occurrence)?, + endpoint(edge.to_occurrence)?, + )); + } + Ok(()) + } + fn count_relation(&mut self) -> Result<(), CodeGraphProjectionError> { self.scanned_relations = self.scanned_relations.checked_add(1).ok_or_else(|| { CodeGraphProjectionError::Corrupt( diff --git a/crates/tracedecay-code-index/src/graph_projection/reader.rs b/crates/tracedecay-code-index/src/graph_projection/reader.rs index 48be149a10..7cdf1384f3 100644 --- a/crates/tracedecay-code-index/src/graph_projection/reader.rs +++ b/crates/tracedecay-code-index/src/graph_projection/reader.rs @@ -147,54 +147,37 @@ impl CodeGraphEvidenceReader { cancellation: Arc, ) -> Result>, CodeGraphProjectionError> { - let graph_depth = usize::try_from(max_depth) - .ok() - .and_then(|depth| depth.checked_mul(2)) - .ok_or_else(|| { - CodeGraphProjectionError::Contract( - "code graph traversal depth overflowed".to_owned(), - ) - })?; - let result = self.snapshot.traverse(TraversalRequest { - namespace: self.projection.namespace.clone(), - start: symbol_entity_id(seed)?, - relation_kinds: BTreeSet::new(), - direction: GraphTraversalDirection::Outgoing, - max_depth: graph_depth, - max_visits: self.projection_node_count, - max_results: self.projection_node_count, - cancellation: Arc::clone(&cancellation), - })?; + let kinds = RELATION_EDGE_KINDS + .iter() + .map(|kind| GraphRelationKind::new(code_edge_kind(*kind)).map_err(Into::into)) + .collect::, CodeGraphProjectionError>>()?; let mut adjacency = BTreeMap::>::new(); - for visit in result.visits { - if visit.depth % 2 == 0 { - continue; + let mut reached = BTreeSet::from([symbol_entity_id(seed)?]); + let mut frontier = vec![symbol_entity_id(seed)?]; + for _ in 0..max_depth { + if frontier.is_empty() { + break; } - let entity = self - .snapshot - .entity(&visit.entity, Arc::clone(&cancellation))? - .ok_or_else(|| { - CodeGraphProjectionError::Corrupt( - "graph traversal referenced a missing edge entity".to_owned(), - ) - })?; - if !has_label(&entity, EDGE_LABEL) { - return Err(CodeGraphProjectionError::Corrupt( - "code graph alternation contains a non-edge entity".to_owned(), - )); - } - let edge: CanonicalRelationEdgeV1 = - deserialize_property(&entity, EDGE_RECORD_PROPERTY)?; - validate_edge(&edge)?; - if edge_entity_id(&edge)? != entity.identity { - return Err(CodeGraphProjectionError::Corrupt( - "code graph edge identity does not match its payload".to_owned(), - )); + let mut next = Vec::new(); + for starts in frontier.chunks(ADJACENCY_SEED_CHUNK) { + let per_start = self.snapshot.outgoing_relations( + starts, + &kinds, + MAX_VERIFIED_GENERATION_RELATIONS, + Arc::clone(&cancellation), + )?; + for relation in per_start.into_iter().flatten() { + let edge = edge_record(&relation)?; + if reached.insert(relation.to.clone()) { + next.push(relation.to); + } + adjacency + .entry(edge.from_occurrence.clone()) + .or_default() + .push(edge); + } } - adjacency - .entry(edge.from_occurrence.clone()) - .or_default() - .push(edge); + frontier = next; } for edges in adjacency.values_mut() { edges.sort_by(compare_edges); diff --git a/crates/tracedecay-code-index/src/graph_projection/schema.rs b/crates/tracedecay-code-index/src/graph_projection/schema.rs index fbdc0dbb31..1b679de5cb 100644 --- a/crates/tracedecay-code-index/src/graph_projection/schema.rs +++ b/crates/tracedecay-code-index/src/graph_projection/schema.rs @@ -1,6 +1,11 @@ //! Durable labels, properties, and identities for the code-graph projection. -use serde::{Deserialize, Serialize}; +use std::collections::BTreeMap; + +use base64::Engine; +use base64::engine::general_purpose::URL_SAFE_NO_PAD; +use serde::Serialize; +use serde::de::DeserializeOwned; use tracedecay_domain::FileOccurrenceId; pub(super) use tracedecay_graph_db::graph_stable_identity as stable_identity; use tracedecay_graph_db::{ @@ -59,44 +64,209 @@ pub(super) fn serialize(value: &impl Serialize) -> Result, CodeGraphProj serde_json::to_vec(value).map_err(|error| CodeGraphProjectionError::Contract(error.to_string())) } -/// A record's serialized JSON as the entity property that carries it. +/// A record's serialized JSON as the compact text property that carries it. /// -/// The record is stored as a string, not as bytes: the sealed compact store -/// keeps byte payloads in its string dictionary as marked hex, which doubles -/// every record on disk, while the JSON text is already valid UTF-8 and is -/// stored as-is. Readers accept both forms. +/// The record stays a string, not bytes: the sealed compact store keeps byte +/// payloads in its string dictionary as marked hex. [`compact_record`] turns +/// the JSON's structural tokens and 64-hex digests into short marked codes; +/// free-form values stay JSON text. pub(super) fn record_property(payload: Vec) -> Result { - String::from_utf8(payload) - .map(GraphProperty::String) - .map_err(|error| CodeGraphProjectionError::Contract(error.to_string())) + let json = String::from_utf8(payload) + .map_err(|error| CodeGraphProjectionError::Contract(error.to_string()))?; + Ok(GraphProperty::String(compact_record(&json))) } -pub(super) fn deserialize_property<'entity, T>( - entity: &'entity GraphEntity, +pub(super) fn deserialize_property( + properties: &BTreeMap, name: &str, ) -> Result where - T: Deserialize<'entity>, + T: DeserializeOwned, { - let property = entity - .properties + let property = properties .get(&GraphPropertyName::new(name)?) .ok_or_else(|| { - CodeGraphProjectionError::Corrupt(format!("code graph entity is missing {name}")) + CodeGraphProjectionError::Corrupt(format!("code graph row is missing {name}")) })?; - let bytes = match property { - GraphProperty::String(text) => text.as_bytes(), - GraphProperty::Bytes(bytes) => bytes.as_slice(), - GraphProperty::Bool(_) | GraphProperty::I64(_) | GraphProperty::F64(_) => { - return Err(CodeGraphProjectionError::Corrupt(format!( - "code graph entity {name} has the wrong type" - ))); - } + let GraphProperty::String(stored) = property else { + return Err(CodeGraphProjectionError::Corrupt(format!( + "code graph row {name} has the wrong type" + ))); }; - serde_json::from_slice(bytes) + serde_json::from_str(&expand_record(stored)?) .map_err(|error| CodeGraphProjectionError::Corrupt(error.to_string())) } +/// Leads a [`RECORD_TOKENS`] code in stored record text. +const TOKEN_MARKER: char = '\u{1}'; +/// Leads a 64-hex digest stored as unpadded base64url. +const DIGEST_MARKER: char = '\u{2}'; +const DIGEST_HEX_CHARS: usize = 64; +const DIGEST_TEXT_CHARS: usize = 43; +const _: () = assert!(RECORD_TOKENS.len() <= TOKEN_CODES.len()); +const TOKEN_CODES: &[u8; 64] = b"ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz0123456789-_"; + +/// Structural JSON the projector's records repeat: field keys with their +/// colon, enum values, and identifier prefixes, each starting at a quote. +/// A stored code is the token's index here, so entries are only appended +/// under a projector revision bump. +const RECORD_TOKENS: &[&str] = &[ + "\"occurrence\":", + "\"binding\":", + "\"metadata\":", + "\"unresolved_calls\":", + "\"file\":", + "\"logical_path\":", + "\"source_span\":", + "\"chunk\":", + "\"language_descriptor_revision\":", + "\"start_byte\":", + "\"end_byte\":", + "\"identity\":", + "\"qualified_name\":", + "\"simple_name\":", + "\"kind\":", + "\"visibility\":", + "\"branches\":", + "\"loops\":", + "\"max_nesting\":", + "\"complexity_analysis\":", + "\"line_span\":", + "\"start_line\":", + "\"signature\":", + "\"docstring\":", + "\"is_async\":", + "\"derives\":", + "\"skip_test_coverage\":", + "\"file_identity\":", + "\"content_digest\":", + "\"from_occurrence\":", + "\"to_occurrence\":", + "\"authority\":", + "\"evidence_span\":", + "\"reference_name\":", + "\"file_occurrence_id\":", + "\"language\":", + "\"disposition\":", + "\"module_specifier\":", + "\"imported_name\":", + "\"local_name\":", + "\"is_public\":", + "\"reexport_scope\":", + "\"is_glob\":", + "\"namespace\":", + "\"module_kind\":", + "\"span\":", + "\"start_column\":", + "\"symbol.v1.sha256:", + "\"chunk.v1.sha256:", + "\"descriptor.", + "\"sha256:", + "\"function\"", + "\"method\"", + "\"public\"", + "\"private\"", + "\"syntax_exact\"", + "\"name_resolved\"", + "\"calls\"", + "\"uses\"", + "\"present\"", + ":null", + ":false", + ":true", + ":[]", +]; + +/// Stored text for record JSON: every [`RECORD_TOKENS`] entry becomes +/// [`TOKEN_MARKER`] and its code, and every maximal run of exactly 64 +/// lowercase hex characters becomes [`DIGEST_MARKER`] and base64url. +/// `serde_json` escapes every control character, so neither marker occurs in +/// the JSON and [`expand_record`] inverts this exactly. +pub(super) fn compact_record(json: &str) -> String { + let bytes = json.as_bytes(); + let mut out = String::with_capacity(json.len()); + let mut index = 0; + let mut run_start = None::; + let flush = |out: &mut String, start: usize, end: usize| { + let run = &json[start..end]; + let mut digest = [0_u8; DIGEST_HEX_CHARS / 2]; + if run.len() == DIGEST_HEX_CHARS && hex::decode_to_slice(run, &mut digest).is_ok() { + out.push(DIGEST_MARKER); + out.push_str(&URL_SAFE_NO_PAD.encode(digest)); + } else { + out.push_str(run); + } + }; + while index < bytes.len() { + let byte = bytes[index]; + if byte.is_ascii_digit() || (b'a'..=b'f').contains(&byte) { + run_start.get_or_insert(index); + index += 1; + continue; + } + if let Some(start) = run_start.take() { + flush(&mut out, start, index); + } + if matches!(byte, b'"' | b':') + && let Some((code, token)) = RECORD_TOKENS + .iter() + .enumerate() + .filter(|(_, token)| json[index..].starts_with(**token)) + .max_by_key(|(_, token)| token.len()) + { + out.push(TOKEN_MARKER); + out.push(char::from(TOKEN_CODES[code])); + index += token.len(); + continue; + } + let Some(character) = json[index..].chars().next() else { + break; + }; + out.push(character); + index += character.len_utf8(); + } + if let Some(start) = run_start { + flush(&mut out, start, bytes.len()); + } + out +} + +/// Inverts [`compact_record`]. +pub(super) fn expand_record(stored: &str) -> Result { + let malformed = + || CodeGraphProjectionError::Corrupt("code graph record text is malformed".to_owned()); + let mut out = String::with_capacity(stored.len() * 2); + let mut rest = stored; + while let Some(position) = rest.find([TOKEN_MARKER, DIGEST_MARKER]) { + out.push_str(&rest[..position]); + let marker = rest[position..].chars().next().ok_or_else(malformed)?; + let body = &rest[position + marker.len_utf8()..]; + if marker == TOKEN_MARKER { + let code = *body.as_bytes().first().ok_or_else(malformed)?; + let token = TOKEN_CODES + .iter() + .position(|candidate| *candidate == code) + .and_then(|index| RECORD_TOKENS.get(index)) + .ok_or_else(malformed)?; + out.push_str(token); + rest = &body[1..]; + } else { + let encoded = body.get(..DIGEST_TEXT_CHARS).ok_or_else(malformed)?; + let mut digest = [0_u8; DIGEST_HEX_CHARS / 2]; + let written = URL_SAFE_NO_PAD + .decode_slice(encoded, &mut digest) + .map_err(|_| malformed())?; + if written != digest.len() { + return Err(malformed()); + } + out.push_str(&hex::encode(digest)); + rest = &body[DIGEST_TEXT_CHARS..]; + } + } + out.push_str(rest); + Ok(out) +} + pub(super) fn has_label(entity: &GraphEntity, label: &str) -> bool { entity .labels diff --git a/crates/tracedecay-code-index/tests/code_index_suite/graph_projection_publication.rs b/crates/tracedecay-code-index/tests/code_index_suite/graph_projection_publication.rs index 8389f05ae8..db065449f6 100644 --- a/crates/tracedecay-code-index/tests/code_index_suite/graph_projection_publication.rs +++ b/crates/tracedecay-code-index/tests/code_index_suite/graph_projection_publication.rs @@ -29,7 +29,6 @@ const IMPORT_SOURCE: &str = concat!( "import type { Foo as LocalFoo } from \"pkg\";\n", "export function local() { return 1; }\n", ); -const IMPORT_RECORD_PROPERTY: &str = "import-record"; const IMPORT_LABEL: &str = "CodeImport"; const FILE_LABEL: &str = "CodeFile"; const SYMBOL_LABEL: &str = "CodeSymbol"; @@ -163,21 +162,6 @@ fn has_label(entity: &GraphEntity, label: &str) -> bool { .any(|candidate| candidate.as_str() == label) } -fn projected_import(entity: &GraphEntity) -> CodeIndexImportEvidenceV1 { - let property = entity - .properties - .iter() - .find(|(name, _)| name.as_str() == IMPORT_RECORD_PROPERTY) - .map(|(_, value)| value) - .expect("CodeImport carries its exact parser-backed record"); - // Records travel as JSON text: the sealed compact store keeps byte - // payloads as marked hex in its dictionary, which would double them. - let GraphProperty::String(record) = property else { - panic!("CodeImport record must use the JSON string property"); - }; - serde_json::from_str(record).expect("CodeImport record decodes") -} - fn verified_store( manifest: GraphGenerationManifest, generation: &CodeIndexPublishedGenerationV1, @@ -217,7 +201,6 @@ fn published_generation_imports_survive_verified_projection_and_reader_open() { assert_eq!(import_entities.len(), 1); let import_entity = import_entities[0]; assert!(!has_label(import_entity, SYMBOL_LABEL)); - assert_eq!(projected_import(import_entity), expected); let file_entity = manifest .entities @@ -317,3 +300,109 @@ fn current_projector_changes_generation_identity_without_a_v4_alias() { .expect_err("a v4 graph snapshot cannot serve the current generation authority"); assert_eq!(error, CodeGraphProjectionError::GenerationMismatch); } + +const CALL_SOURCE: &str = concat!( + "pub fn alpha() -> u32 { beta() + gamma() }\n", + "pub fn beta() -> u32 { gamma() }\n", + "pub fn gamma() -> u32 { 1 }\n", +); + +fn published_call_generation() -> Arc { + let mut owner = CodeIndexProductionOwnerV1::new( + config(), + SharedPublicationStore::default(), + ApplyingProjectionSink, + ) + .expect("production owner"); + owner + .build_and_publish( + request_with_source( + "file.graph-calls", + 1_600_000, + "commit.graph-calls", + "tree.graph-calls", + CALL_SOURCE, + ), + &ActiveControl, + ) + .expect("parser-backed call generation publishes") +} + +/// Each code edge is one relation row from its source symbol to its target +/// symbol carrying the edge record, and every record is stored as compact +/// text, so a generation's rows and record bytes stay within this budget. +/// The previous projector added an edge entity plus two relations per edge +/// and stored records as JSON: 8 entities, 9 relations, 4,259 record bytes. +#[test] +fn graph_manifest_stores_each_code_edge_as_one_row_within_the_byte_budget() { + let generation = published_call_generation(); + let manifest = projection_manifest(&generation, ¤t_projector_revision()); + let record_bytes = manifest + .entities + .iter() + .flat_map(|entity| entity.properties.values()) + .chain( + manifest + .relations + .iter() + .flat_map(|relation| relation.properties.values()), + ) + .map(|property| match property { + GraphProperty::String(text) => text.len(), + _ => 0, + }) + .sum::(); + let mut relation_kinds = manifest + .relations + .iter() + .map(|relation| relation.kind.as_str().to_owned()) + .collect::>(); + relation_kinds.sort(); + assert_eq!(manifest.entities.len(), 5); + assert_eq!( + relation_kinds, + [ + "CodeEdge.calls", + "CodeEdge.calls", + "CodeEdge.calls", + "CodeFileContainsSymbol", + "CodeFileContainsSymbol", + "CodeFileContainsSymbol", + ] + ); + assert!(record_bytes <= 2_200, "records took {record_bytes} bytes"); + + let names = generation + .symbols() + .symbols + .iter() + .map(|symbol| (symbol.occurrence.clone(), symbol.simple_name.clone())) + .collect::>(); + let occurrences = names.keys().cloned().collect::>(); + let reader = verified_store(manifest, &generation) + .interactive_reader_with_cancellation( + &generation.manifest().generation_id, + Arc::new(NeverCancelled), + ) + .expect("generation-pinned reader"); + let mut edges = reader + .edges_among(&occurrences, &[], 64, Arc::new(NeverCancelled)) + .expect("edges among the fixture symbols") + .into_iter() + .map(|edge| { + ( + names[&edge.from_occurrence].as_str().to_owned(), + names[&edge.to_occurrence].as_str().to_owned(), + ) + }) + .collect::>(); + edges.sort(); + assert_eq!( + edges, + vec![ + ("alpha".to_owned(), "beta".to_owned()), + ("alpha".to_owned(), "gamma".to_owned()), + ("beta".to_owned(), "gamma".to_owned()), + ] + ); +} diff --git a/crates/tracedecay-code-index/tests/resident_accounting.rs b/crates/tracedecay-code-index/tests/resident_accounting.rs index e7dd298fa3..347d9648f7 100644 --- a/crates/tracedecay-code-index/tests/resident_accounting.rs +++ b/crates/tracedecay-code-index/tests/resident_accounting.rs @@ -278,7 +278,8 @@ fn decode(manifest: &[u8], segments: &BTreeMap>) -> CodeIndexPub /// at a time: the most the build ever holds above the sealed input stays /// within a fixed budget, below the 22.3 MB decoding this generation alone /// leaves live. Decoding the whole generation and projecting it in one piece -/// peaked at 38,827,295 bytes for the same 4,201 entities and 5,100 relations. +/// peaked at 38,827,295 bytes when the generation projected 4,201 entities and +/// 5,100 relations, an edge entity plus two relations per code edge. #[test] fn a_sealed_graph_build_holds_windows_not_the_decoded_generation() { const PEAK_BUDGET_BYTES: usize = 19_000_000; @@ -317,7 +318,8 @@ fn a_sealed_graph_build_holds_windows_not_the_decoded_generation() { tracedecay_code_index::parallelism::clear_forced_indexing_workers_for_test(); eprintln!("GRAPH ROWS peak {peak}"); - assert_eq!(spilled.row_counts(), (4_201, 5_100)); + // Each of the fixture's 1,200 code edges is one relation row. + assert_eq!(spilled.row_counts(), (3_001, 3_900)); assert!( peak <= PEAK_BUDGET_BYTES, "the graph build held {peak} bytes at peak, over its {PEAK_BUDGET_BYTES}-byte budget" diff --git a/crates/tracedecay-graph-db/src/corrupt_store.rs b/crates/tracedecay-graph-db/src/corrupt_store.rs index 3d4384d742..bd78b28292 100644 --- a/crates/tracedecay-graph-db/src/corrupt_store.rs +++ b/crates/tracedecay-graph-db/src/corrupt_store.rs @@ -38,6 +38,7 @@ use tracedecay_domain::canonical_text::sha256_hex; use tracedecay_private_fs::FileLease; use tracedecay_private_fs::framed_log::{DirectorySyncPolicy, sync_directory}; +use crate::sealed_store::discard_superseded_sealed_generations; use crate::{GraphDb, GraphDbError}; const CORRUPTION_DECISION_LOCK_SUFFIX: &str = ".corruption-lock"; @@ -89,7 +90,7 @@ pub(crate) fn recover_deterministically_corrupt_container_with( }); } - delete_container_family(container, &[])?; + delete_container_family(container)?; tracing::warn!( event = "store_corrupt_deleted", container = %container.display(), @@ -105,9 +106,9 @@ pub(crate) fn recover_deterministically_corrupt_container_with( /// /// Under the same decision lock as corruption, the open is re-run: another /// authority may already have replaced the store. A second superseded -/// verdict deletes the family together with the sealed generation root, -/// because every artifact under it was written in the superseded format. -/// The caller reopens the vacant path fresh. +/// verdict deletes the family and every sealed generation written in a +/// superseded format; a generation this build already sealed is kept. The +/// caller reopens the vacant path fresh. pub(crate) fn replace_superseded_container( container: &Path, verification_open: &dyn Fn() -> Result, @@ -118,7 +119,8 @@ pub(crate) fn replace_superseded_container( Err(GraphDbError::FormatSuperseded { found, expected }) => (found, expected), Err(other) => return Err(other), }; - delete_container_family(container, &[container.with_extension("sealed")])?; + delete_container_family(container)?; + discard_superseded_sealed_generations(container)?; tracing::info!( event = "store_superseded_format_deleted", container = %container.display(), @@ -180,11 +182,10 @@ fn acquire_corruption_decision_lock( } } -/// Deletes the container family and any `derived` siblings. The container -/// goes last: it is the fault authority, so an interruption mid-delete leaves -/// it in place for the next deciding authority rather than a vacant path -/// beside stranded sidecars. -fn delete_container_family(container: &Path, derived: &[PathBuf]) -> Result<(), GraphDbError> { +/// Deletes the container family. The container goes last: it is the fault +/// authority, so an interruption mid-delete leaves it in place for the next +/// deciding authority rather than a vacant path beside stranded sidecars. +fn delete_container_family(container: &Path) -> Result<(), GraphDbError> { match container.symlink_metadata() { Ok(metadata) if metadata.is_file() => {} Ok(_) => { @@ -205,11 +206,8 @@ fn delete_container_family(container: &Path, derived: &[PathBuf]) -> Result<(), wal_sidecar_path(container), container.with_extension("verified"), container.with_extension("spill"), - ] - .iter() - .chain(derived) - { - remove_family_member(sidecar)?; + ] { + remove_family_member(&sidecar)?; } remove_family_member(container)?; if let Some(parent) = container.parent() { diff --git a/crates/tracedecay-graph-db/src/location.rs b/crates/tracedecay-graph-db/src/location.rs index 02fd49142b..96915018f5 100644 --- a/crates/tracedecay-graph-db/src/location.rs +++ b/crates/tracedecay-graph-db/src/location.rs @@ -20,7 +20,7 @@ pub struct GraphFormatVersion(u32); impl GraphFormatVersion { #[must_use] pub const fn current() -> Self { - Self(4) + Self(5) } #[cfg(any(test, feature = "test-helpers", feature = "eval-helpers"))] diff --git a/crates/tracedecay-graph-db/src/schema.rs b/crates/tracedecay-graph-db/src/schema.rs index 81d01a3f66..f1eb3c4f41 100644 --- a/crates/tracedecay-graph-db/src/schema.rs +++ b/crates/tracedecay-graph-db/src/schema.rs @@ -423,7 +423,7 @@ pub(crate) fn relation_properties( let edge = i64::try_from(edge.as_u64()).map_err(|_| GraphDbError::Corrupt { message: "Grafeo edge identity exceeds the persisted scalar range".to_owned(), })?; - let mut properties = vec![ + let properties = vec![ ( RELATION_KEY_PROPERTY.to_owned(), key_value(namespace, relation.identity.as_str()), @@ -454,18 +454,13 @@ pub(crate) fn relation_properties( ), (RELATION_EDGE_PROPERTY.to_owned(), Value::from(edge)), ]; - properties.extend( - relation - .properties - .iter() - .map(|(name, property)| encode_graph_property(name, property)), - ); Ok(properties) } -/// A native edge carries its owner scalars and payload, never the relation's -/// identity or endpoints: those are owned by its locator node, which -/// [`edge_locator`] resolves through the `RELATION_EDGE` index. +/// A native edge carries its owner scalars and the relation's payload +/// properties, never its identity or endpoints: those are owned by its +/// locator node, which [`edge_locator`] resolves through the `RELATION_EDGE` +/// index. The payload has no second copy on the locator. pub(crate) fn edge_properties( namespace: &GraphNamespace, projection: &GraphProjectionId, @@ -639,8 +634,7 @@ pub(crate) fn decode_relation(locator: &Node, edge: &Edge) -> Result bool { self.version == SEALED_STORE_RECEIPT_VERSION + && self.graph_format == GraphFormatVersion::current().get() && self.recovered_digest == expected_digest && self.physical_namespace == physical_namespace && self.namespace == locator.projection.namespace.as_str() @@ -443,6 +449,50 @@ fn directory_bytes(directory: &Path) -> std::io::Result { Ok(total) } +/// Removes every sealed generation beside `database_path` whose receipt does +/// not name the current graph format: after a superseded-format rebuild those +/// containers can never serve, while a generation this build already sealed +/// keeps serving. In-flight `.staging-*` seals are left to their builder. +pub(crate) fn discard_superseded_sealed_generations( + database_path: &Path, +) -> Result<(), GraphDbError> { + let root = sealed_store_root(database_path); + let entries = match std::fs::read_dir(&root) { + Ok(entries) => entries, + Err(error) if error.kind() == std::io::ErrorKind::NotFound => return Ok(()), + Err(error) => return Err(sealed_store_io_failure("sealed root read failed", error)), + }; + for entry in entries { + let entry = + entry.map_err(|error| sealed_store_io_failure("sealed root read failed", error))?; + let path = entry.path(); + if entry + .file_name() + .to_str() + .is_some_and(|name| name.starts_with(".staging-")) + { + continue; + } + let current = load_sealed_store_receipt(&path) + .ok() + .flatten() + .is_some_and(|receipt| receipt.graph_format == GraphFormatVersion::current().get()); + if !current { + match std::fs::remove_dir_all(&path) { + Ok(()) => {} + Err(error) if error.kind() == std::io::ErrorKind::NotFound => {} + Err(error) => { + return Err(sealed_store_io_failure( + "superseded sealed generation removal failed", + error, + )); + } + } + } + } + Ok(()) +} + /// Removes every `.staging-*` directory a seal left behind under the store's /// sealed root. /// @@ -1399,6 +1449,7 @@ fn build_or_open_sealed_store( recovered_digest: expected.as_str().to_owned(), entities, relations, + graph_format: GraphFormatVersion::current().get(), }; let encoded = serde_json::to_vec_pretty(&receipt) .map_err(|error| GraphDbError::unavailable(format!("sealed receipt encode: {error}")))?; diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/open_error_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/open_error_contract.rs index 975285c33d..c1ca1a5c68 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/open_error_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/open_error_contract.rs @@ -4,8 +4,8 @@ use std::sync::Arc; use grafeo_common::types::Value; use tempfile::TempDir; use tracedecay_graph_db::{ - GraphDbError, GraphEntityId, GraphNamespace, GraphRelation, GraphRelationId, GraphRelationKind, - NeverCancelled, + GraphDbError, GraphEntityId, GraphNamespace, GraphProperty, GraphPropertyName, GraphRelation, + GraphRelationId, GraphRelationKind, NeverCancelled, }; use crate::support; @@ -38,7 +38,7 @@ fn raw_store(temp: &TempDir) -> grafeo_engine::GrafeoDB { } const FORMAT_MARKER: [(&str, i64); 2] = [ - ("__tracedecay_graph_db_version", 4), + ("__tracedecay_graph_db_version", 5), ("__tracedecay_graph_db_sequence", 0), ]; @@ -88,13 +88,14 @@ fn persisted_scalar_identity_mismatch_is_corrupt_on_point_read() { )); } -/// A format-4 relation as it lies on disk. Keys are base64url of -/// `sha256("workspace")[..8] ‖ digest-identity tag ‖ kind ‖ digest`. The -/// locator owns the identity, source, and target, each stored as U+0001, -/// kind, and base64url digest, and the native edge carries none of them. -/// Keyed reads and edge fan-outs both resolve the same relation. +/// A format-5 relation as it lies on disk: one native edge row carrying the +/// relation's kind and payload, and its locator carrying the key, identity, +/// source, and target. Keys are base64url of +/// `sha256("workspace")[..8] ‖ digest-identity tag ‖ kind ‖ digest`; +/// identities are U+0001, kind, and base64url digest. Keyed reads and edge +/// fan-outs both resolve the same relation, payload included. #[test] -fn compact_relation_identities_read_back_through_keys_and_edges() { +fn single_row_edge_reads_back_its_payload_through_keys_and_fanouts() { let temp = TempDir::new().unwrap(); let raw = raw_store(&temp); let session = raw.session(); @@ -132,6 +133,11 @@ fn compact_relation_identities_read_back_through_keys_and_edges() { ("__tracedecay_graph_db_namespace", Value::from("workspace")), ("__tracedecay_graph_db_projection", Value::from("code")), ("__tracedecay_graph_db_relation_kind", Value::from("calls")), + // `__tracedecay_graph_db_property_str_` + hex("edge-record"). + ( + "__tracedecay_graph_db_property_str_656467652d7265636f7264", + Value::from("{\"span\":[3,9]}"), + ), ], ) .unwrap(); @@ -175,7 +181,10 @@ fn compact_relation_identities_read_back_through_keys_and_edges() { GraphEntityId::new(format!("symbol:{}", "11".repeat(32))).unwrap(), GraphEntityId::new(format!("symbol:{}", "22".repeat(32))).unwrap(), GraphRelationKind::new("calls").unwrap(), - BTreeMap::new(), + BTreeMap::from([( + GraphPropertyName::new("edge-record").unwrap(), + GraphProperty::String("{\"span\":[3,9]}".to_owned()), + )]), ) .unwrap(); assert_eq!( diff --git a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs index 8aee1d119d..6737209fa7 100644 --- a/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs +++ b/crates/tracedecay-graph-db/tests/graph_db_suite/runtime_contract.rs @@ -7,10 +7,10 @@ use grafeo_common::types::Value; use tempfile::TempDir; use tracedecay_graph_db::{ GraphBudgetKind, GraphCancellation, GraphDbError, GraphDbLeaseV1, GraphDbOwner, GraphEntity, - GraphEntityId, GraphIdempotencyKey, GraphLabel, GraphMutation, GraphNamespace, - GraphProjectionId, GraphPublication, GraphPublicationInputDigest, GraphRelation, - GraphRelationId, GraphRelationKind, GraphTraversalDirection, GraphWatermark, GraphWriteBatch, - NeverCancelled, ProjectionReplacement, SourceGeneration, TraversalRequest, + GraphEntityId, GraphFormatVersion, GraphIdempotencyKey, GraphLabel, GraphMutation, + GraphNamespace, GraphProjectionId, GraphPublication, GraphPublicationInputDigest, + GraphRelation, GraphRelationId, GraphRelationKind, GraphTraversalDirection, GraphWatermark, + GraphWriteBatch, NeverCancelled, ProjectionReplacement, SourceGeneration, TraversalRequest, }; use crate::support; @@ -1008,12 +1008,12 @@ fn wrong_tracedecay_format_requires_reset() { } #[test] -fn superseded_format_store_is_rebuilt_fresh_with_its_sealed_generations_discarded() { +fn superseded_format_store_is_rebuilt_fresh_keeping_only_current_sealed_generations() { let temp = TempDir::new().unwrap(); let path = graph_path(temp.path()); - // Format 3 stored relation identities as strings on both the locator and - // the native edge; the row below is keyed the way format 3 keyed it. - let previous_format = 3_i64; + // Format 4 kept a relation's payload properties on its locator as well as + // its native edge; the row below is keyed the way format 4 keyed it. + let previous_format = 4_i64; let raw = grafeo_engine::GrafeoDB::with_config( grafeo_engine::Config::persistent(&path) .with_storage_format(grafeo_engine::config::StorageFormat::SingleFile), @@ -1035,15 +1035,8 @@ fn superseded_format_store_is_rebuilt_fresh_with_its_sealed_generations_discarde [ ( "__tracedecay_graph_db_entity_key", - // sha256("project")[..8], the raw-identity tag, "stale". - Value::Bytes( - [ - &[0x24, 0x42, 0x10, 0xe4, 0x84, 0x37, 0xb6, 0x55, 0x00][..], - b"stale", - ] - .concat() - .into(), - ), + // base64url(sha256("project")[..8] ‖ raw-identity tag ‖ "stale"). + Value::from("JEIQ5IQ3tlUAc3RhbGU"), ), ("__tracedecay_graph_db_namespace", "project".into()), ("__tracedecay_graph_db_projection", "code".into()), @@ -1056,13 +1049,36 @@ fn superseded_format_store_is_rebuilt_fresh_with_its_sealed_generations_discarde std::fs::create_dir_all(&sealed_generation).unwrap(); std::fs::write( sealed_generation.join("generation.grafeo"), - b"format 3 bytes", + b"format 4 bytes", + ) + .unwrap(); + + // A generation this build sealed before the staging container was + // replaced, as a publication racing the rebuild leaves it. + let current_generation = temp.path().join("graph.sealed").join("1".repeat(64)); + std::fs::create_dir_all(¤t_generation).unwrap(); + std::fs::write( + current_generation.join("sealed.json"), + serde_json::json!({ + "version": 1, + "form": "compact", + "namespace": "project", + "projection": "code", + "generation": "g1", + "physical_namespace": format!("generation:{}", "1".repeat(64)), + "recovered_digest": "sha256:1", + "entities": 1, + "relations": 0, + "graph_format": GraphFormatVersion::current().get(), + }) + .to_string(), ) .unwrap(); let (_registered, db) = RegisteredGraph::open_lease(temp.path()).unwrap(); - assert!(!temp.path().join("graph.sealed").exists()); + assert!(!sealed_generation.exists()); + assert!(current_generation.join("sealed.json").exists()); assert_eq!( db.entity(&namespace(), &entity_id("stale"), live()) .unwrap(), diff --git a/crates/tracedecay-mcp/src/handlers/graph/search.rs b/crates/tracedecay-mcp/src/handlers/graph/search.rs index 6285ca664c..9d4436453c 100644 --- a/crates/tracedecay-mcp/src/handlers/graph/search.rs +++ b/crates/tracedecay-mcp/src/handlers/graph/search.rs @@ -540,9 +540,8 @@ fn render_search_md(value: &Value) -> String { /// tool uses a 50k refuse budget; context used to keep that budget, hydrate /// every edge, then discard all but `max_nodes`. That walk is CPU-bound and /// shows no warm benefit. Cap examination at a small multiple of the kept -/// page. Semantic kind lives on the edge entity (not the physical -/// SOURCE/TARGET relation type), so the page is all-kinds, the same -/// neighborhood the previous complete walk returned, just a prefix. +/// page. The page admits every edge kind: the same neighborhood a complete +/// walk returns, cut to a prefix in store order. fn context_related_relation_budget(max_nodes: usize) -> usize { max_nodes.saturating_mul(4).clamp(16, 64) }