diff --git a/database/src/main/java/io/hstore/db/query/AccessPath.java b/database/src/main/java/io/hstore/db/query/AccessPath.java index 30e1b76..35992b9 100644 --- a/database/src/main/java/io/hstore/db/query/AccessPath.java +++ b/database/src/main/java/io/hstore/db/query/AccessPath.java @@ -8,12 +8,19 @@ import io.hstore.db.schema.TypeDef; import io.hstore.db.semantic.SemanticPlane; import io.hstore.db.value.Value; -import io.hstore.engine.tree.Tree; -import io.hstore.engine.tree.TreeAlgebra; +import io.hstore.engine.Dictionary; +import io.hstore.engine.index.IndexValues; +import io.hstore.engine.index.PostingCursor; import java.util.Arrays; +import java.util.BitSet; +import java.util.LinkedHashMap; +import java.util.LinkedHashSet; import java.util.List; +import java.util.Map; import java.util.Optional; +import java.util.Set; +import java.util.function.Predicate; import java.util.stream.LongStream; public sealed interface AccessPath { @@ -52,21 +59,51 @@ public LongStream scan(Reader reader) { } } - record Incident(List atoms) implements AccessPath { + record Incident(List legs) implements AccessPath { + public record Leg(long atom, List roles) { + public Leg { + roles = List.copyOf(roles); + } + + String describe() { + return "@" + atom + (roles.isEmpty() ? "" : " AS " + String.join("&", roles)); + } + } + + public Incident { + legs = List.copyOf(legs); + } + public String kind() { return "incident"; } public String describe() { - return "IncidentIntersection" + atoms.stream().map(atom -> "@" + atom).toList(); + return "IncidentIntersection" + legs.stream().map(Leg::describe).toList(); } public LongStream scan(Reader reader) { - if (!atoms.stream().allMatch(reader::visible)) { + if (legs.isEmpty() || !legs.stream().allMatch(leg -> reader.visible(leg.atom()))) { return LongStream.empty(); } - List> postings = atoms.stream().>map(atom -> reader.view().incidentTree(atom)).toList(); - return TreeAlgebra.intersectKeys(postings); + Dictionary dictionary = reader.database().engine().dictionary(); + List> cursors = legs.stream() + .map(leg -> leg.roles().isEmpty() + ? reader.view().incidentCursor(leg.atom()) + : reader.view().incidentCursor(leg.atom(), accepting(dictionary, leg.roles()))) + .toList(); + return PostingCursor.intersect(cursors); + } + + private static Predicate accepting(Dictionary dictionary, List roles) { + BitSet accepted = dictionary.roleSetsWithAll(roles); + return incident -> accepted.get(incident.roleSet()); + } + + static Incident merge(List legs) { + Map> roles = new LinkedHashMap<>(); + legs.forEach(leg -> roles.computeIfAbsent(leg.atom(), _ -> new LinkedHashSet<>()).addAll(leg.roles())); + return new Incident(roles.entrySet().stream().map(entry -> new Leg(entry.getKey(), List.copyOf(entry.getValue()))).toList()); } } diff --git a/database/src/main/java/io/hstore/db/query/Planner.java b/database/src/main/java/io/hstore/db/query/Planner.java index 4ea56ca..f2964aa 100644 --- a/database/src/main/java/io/hstore/db/query/Planner.java +++ b/database/src/main/java/io/hstore/db/query/Planner.java @@ -86,8 +86,16 @@ Plan plan(Ast.Match match) { candidates.add(new Candidate(catalogScan.path(), List.of(), catalogScan.rows(), catalogScan.cost() + VERIFY_WEIGHT * catalogScan.rows())); type.ifPresent(found -> candidates.add(candidate(new AccessPath.TypeScan(found), List.of(), population, conjuncts))); List exactParts = new ArrayList<>(); + List legs = new ArrayList<>(); + List incidentSources = new ArrayList<>(); for (Expr conjunct : conjuncts) { - accessPath(match, type, conjunct).or(() -> union(match, type, conjunct)).ifPresent(path -> { + Optional direct = accessPath(match, type, conjunct); + if (direct.isPresent() && direct.get() instanceof AccessPath.Incident(List found)) { + legs.addAll(found); + incidentSources.add(conjunct); + continue; + } + direct.or(() -> union(match, type, conjunct)).ifPresent(path -> { List consumed = exact(path, conjunct) ? List.of(conjunct) : List.of(); Candidate single = candidate(path, consumed, estimate(path, catalogSize), conjuncts); candidates.add(single); @@ -96,6 +104,12 @@ Plan plan(Ast.Match match) { } }); } + if (!legs.isEmpty()) { + AccessPath merged = AccessPath.Incident.merge(legs); + Candidate incident = candidate(merged, List.copyOf(incidentSources), estimate(merged, catalogSize), conjuncts); + candidates.add(incident); + exactParts.add(incident); + } exactParts.sort(Comparator.comparingDouble(Candidate::rows)); for (int size = 2; size <= Math.min(MAX_INTERSECTED, exactParts.size()); size++) { candidates.add(intersection(exactParts.subList(0, size), population, conjuncts)); @@ -168,8 +182,8 @@ private double estimate(AccessPath path, double population) { private double unrecordedEstimate(AccessPath path, double population) { return switch (path) { - case AccessPath.Incident(List atoms) -> { - double[] degrees = atoms.stream().mapToDouble(reader::degree).sorted().toArray(); + case AccessPath.Incident(List legs) -> { + double[] degrees = legs.stream().mapToDouble(leg -> reader.degree(leg.atom())).sorted().toArray(); double rows = degrees.length == 0 ? 0 : degrees[0]; for (int i = 1; i < degrees.length; i++) { rows *= Math.min(1.0, degrees[i] / Math.max(1.0, population)); @@ -201,10 +215,11 @@ private double indexEstimate(AccessPath.IndexRange range) { private Optional accessPath(Ast.Match match, Optional type, Expr conjunct) { String variable = match.variable(); return switch (conjunct) { - case Expr.Contains(String target, List members) when target.equals(variable) && match.edges() -> - Optional.of(new AccessPath.Incident(members.stream().map(resolver::applyAsLong).toList())); - case Expr.Has(String target, Ast.Ref member, Optional _) when target.equals(variable) && match.edges() -> - Optional.of(new AccessPath.Incident(List.of(resolver.applyAsLong(member)))); + case Expr.Contains(String target, List members) when target.equals(variable) && match.edges() && !members.isEmpty() -> + Optional.of(AccessPath.Incident.merge(members.stream() + .map(member -> new AccessPath.Incident.Leg(resolver.applyAsLong(member), List.of())).toList())); + case Expr.Has(String target, Ast.Ref member, Optional role) when target.equals(variable) && match.edges() -> + Optional.of(new AccessPath.Incident(List.of(new AccessPath.Incident.Leg(resolver.applyAsLong(member), role.stream().toList())))); case Expr.In(String target, Ast.Ref edge) when target.equals(variable) -> Optional.of(new AccessPath.Members(resolver.applyAsLong(edge))); case Expr.Similar(String target, String text, int k, var consistency) when target.equals(variable) -> @@ -259,8 +274,7 @@ private static Comparison flip(Comparison op) { private static boolean exact(AccessPath path, Expr conjunct) { return switch (path) { case AccessPath.Semantic _, AccessPath.TypeScan _, AccessPath.AllAtoms _ -> false; - case AccessPath.Incident _ -> !(conjunct instanceof Expr.Has(String _, Ast.Ref _, Optional role) && role.isPresent()); - case AccessPath.Members _, AccessPath.IndexRange _, AccessPath.Fixed _ -> true; + case AccessPath.Incident _, AccessPath.Members _, AccessPath.IndexRange _, AccessPath.Fixed _ -> true; case AccessPath.Union(List parts) -> conjunct instanceof Expr.Or(List terms) && terms.size() == parts.size() && IntStream.range(0, parts.size()).allMatch(i -> exact(parts.get(i), terms.get(i))); case AccessPath.Intersection _ -> false; diff --git a/database/src/test/java/io/hstore/db/SecurityTest.java b/database/src/test/java/io/hstore/db/SecurityTest.java index 883bdff..9bb6203 100644 --- a/database/src/test/java/io/hstore/db/SecurityTest.java +++ b/database/src/test/java/io/hstore/db/SecurityTest.java @@ -301,6 +301,31 @@ void membershipsAcrossTenantsAreNotFollowed() { } } + @Test + void roleFilteredMatchesStayInsideTheTenant() { + try (Session alice = login("alice", "s3cret-pass"); Session bob = login("bob", "hunter2-pass")) { + alice.executeAll("INSERT NODE Patient 'p1' {mrn: 'A-1'} AS $p; INSERT EDGE Encounter MEMBERS ($p AS patient) AS $e;"); + bob.executeAll("INSERT NODE Patient 'b1' {mrn: 'B-1'} AS $p; INSERT EDGE Encounter MEMBERS ($p AS patient) AS $e;"); + long acmePatient = alice.variables().get("p"); + long acmeEdge = alice.variables().get("e"); + long globexPatient = bob.variables().get("p"); + long globexEdge = bob.variables().get("e"); + int patient = database.engine().dictionary().roleSet(List.of("patient")); + database.write(writer -> { + writer.transaction().upsert(globexEdge, acmePatient, _ -> new Incidence(acmePatient, patient, Weight.ONE, + Long.MIN_VALUE, Long.MAX_VALUE, 0, 0)); + return null; + }); + String query = "MATCH EDGE e:Encounter WHERE e HAS @" + acmePatient + " AS patient RETURN e"; + assertEquals(List.of("@" + acmeEdge), alice.execute(query).rows().stream() + .map(row -> row.getFirst().render().split(" ")[0]).toList()); + assertEquals("1", cell(alice.execute("MATCH EDGE e WHERE e HAS @" + acmePatient + " AS patient RETURN count(*)"))); + assertEquals("0", cell(bob.execute("MATCH EDGE e:Encounter WHERE e HAS @" + acmePatient + " AS patient RETURN count(*)"))); + assertEquals("0", cell(bob.execute("MATCH EDGE e:Encounter WHERE e HAS @" + acmePatient + " AS patient AND e HAS @" + + globexPatient + " AS patient RETURN count(*)"))); + } + } + @Test void qualifyOnlyAcceptsVisibleMembers() { try (Session alice = login("alice", "s3cret-pass"); Session bob = login("bob", "hunter2-pass")) { diff --git a/database/src/test/java/io/hstore/db/query/RoleRetrievalTest.java b/database/src/test/java/io/hstore/db/query/RoleRetrievalTest.java new file mode 100644 index 0000000..ba59563 --- /dev/null +++ b/database/src/test/java/io/hstore/db/query/RoleRetrievalTest.java @@ -0,0 +1,189 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.query; + +import io.hstore.db.DatabaseOptions; +import io.hstore.db.HypergraphDatabase; +import io.hstore.db.Member; +import io.hstore.engine.EngineOptions; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.List; +import java.util.Map; +import java.util.function.Predicate; +import java.util.random.RandomGenerator; +import java.util.random.RandomGeneratorFactory; +import java.util.stream.Collectors; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class RoleRetrievalTest { + + private static final List ROLES = List.of("actor", "reviewer", "target"); + + @TempDir + Path directory; + + HypergraphDatabase database; + Session session; + + @BeforeEach + void open() { + database = HypergraphDatabase.open(directory, DatabaseOptions.defaults() + .withEngine(EngineOptions.defaults().withPageSize(4096))); + session = new Session(database); + session.executeAll(""" + CREATE NODE TYPE Person; + CREATE SET EDGE TYPE Review ROLES (actor, reviewer, target); + """); + } + + @AfterEach + void close() { + session.close(); + database.close(); + } + + private record Condition(String text, Predicate>> holds) { + } + + private static List ids(QueryResult result) { + return result.rows().stream().map(row -> Long.parseLong(row.getFirst().render().substring(1).split(" ")[0])).sorted().toList(); + } + + private Condition has(long atom, String role) { + return role == null + ? new Condition("e HAS @" + atom, members -> members.containsKey(atom)) + : new Condition("e HAS @" + atom + " AS " + role, + members -> members.containsKey(atom) && members.get(atom).contains(role)); + } + + private Condition random(RandomGenerator random, List people) { + long a = people.get(random.nextInt(people.size())); + long b = people.get(random.nextInt(people.size())); + String role = ROLES.get(random.nextInt(ROLES.size())); + String other = ROLES.get(random.nextInt(ROLES.size())); + return switch (random.nextInt(6)) { + case 0 -> new Condition("e CONTAINS (@" + a + ", @" + b + ")", members -> members.containsKey(a) && members.containsKey(b)); + case 1 -> has(a, null); + case 2, 3 -> has(a, role); + case 4 -> { + Condition left = has(a, role); + Condition right = has(b, other); + yield new Condition("(" + left.text() + " OR " + right.text() + ")", members -> left.holds().test(members) || right.holds().test(members)); + } + default -> { + Condition inner = has(a, role); + yield new Condition("NOT " + inner.text(), members -> !inner.holds().test(members)); + } + }; + } + + @Test + void roleFilteredMatchesAgreeWithMembers() { + RandomGenerator random = RandomGeneratorFactory.of("L64X128MixRandom").create(11); + List people = new ArrayList<>(); + database.write(writer -> { + for (int i = 0; i < 30; i++) { + people.add(writer.node("Person", "p" + i)); + } + for (int e = 0; e < 600; e++) { + long edge = writer.edge("Review"); + int size = 1 + random.nextInt(4); + for (int k = 0; k < size; k++) { + long person = k == 0 && random.nextInt(3) > 0 ? people.get(random.nextInt(3)) : people.get(random.nextInt(people.size())); + List roles = ROLES.stream().filter(_ -> random.nextInt(3) == 0).toList(); + writer.add(edge, person, roles.toArray(String[]::new)); + } + } + return null; + }); + Map>> edges = database.read(reader -> reader.atoms("Review").boxed() + .collect(Collectors.toMap(edge -> edge, edge -> reader.members(edge) + .collect(Collectors.toMap(Member::atom, Member::roles))))); + for (int round = 0; round < 250; round++) { + List conditions = new ArrayList<>(); + for (int i = 0, n = 1 + random.nextInt(3); i < n; i++) { + conditions.add(random(random, people)); + } + String where = conditions.stream().map(Condition::text).collect(Collectors.joining(" AND ")); + List expected = edges.entrySet().stream() + .filter(entry -> conditions.stream().allMatch(condition -> condition.holds().test(entry.getValue()))) + .map(Map.Entry::getKey).sorted().toList(); + assertEquals(expected, ids(session.execute("MATCH EDGE e:Review WHERE " + where + " RETURN e")), where); + assertEquals(String.valueOf(expected.size()), + session.execute("MATCH EDGE e:Review WHERE " + where + " RETURN count(*)").rows().getFirst().getFirst().render(), where); + } + } + + @Test + void roleRequirementsOnTheSameAnchorShareOneExactLeg() { + session.executeAll(""" + INSERT NODE Person 'alice' AS $alice; + INSERT NODE Person 'bob' AS $bob; + INSERT NODE Person 'doc' AS $doc; + INSERT EDGE Review MEMBERS ($alice AS actor, $bob AS reviewer, $doc AS target) AS $e10; + INSERT EDGE Review MEMBERS ($alice AS actor|reviewer, $bob AS actor, $doc AS target) AS $e11; + INSERT EDGE Review MEMBERS ($alice AS reviewer, $bob AS reviewer) AS $e13; + """); + long alice = session.variables().get("alice"); + long bob = session.variables().get("bob"); + String query = "MATCH EDGE e:Review WHERE e HAS $alice AS actor AND e HAS $bob AS reviewer"; + String plan = session.execute("EXPLAIN " + query).message(); + assertTrue(plan.startsWith("IncidentIntersection[@" + alice + " AS actor, @" + bob + " AS reviewer]"), plan); + assertFalse(plan.contains("Verify"), plan); + assertEquals(List.of(session.variables().get("e10")), ids(session.execute(query + " RETURN e"))); + String both = "MATCH EDGE e:Review WHERE e HAS $alice AS actor AND e HAS $alice AS reviewer AND e CONTAINS ($alice)"; + String merged = session.execute("EXPLAIN " + both).message(); + assertTrue(merged.startsWith("IncidentIntersection[@" + alice + " AS actor&reviewer]"), merged); + assertEquals(List.of(session.variables().get("e11")), ids(session.execute(both + " RETURN e"))); + assertEquals(List.of(), ids(session.execute("MATCH EDGE e:Review WHERE e HAS $alice AS approver RETURN e"))); + } + + @Test + void transactionsMatchTheirOwnRoles() { + session.executeAll(""" + INSERT NODE Person 'alice' AS $alice; + INSERT NODE Person 'bob' AS $bob; + INSERT EDGE Review MEMBERS ($alice AS actor) AS $committed; + BEGIN; + INSERT EDGE Review MEMBERS ($alice AS actor, $bob AS target) AS $pending; + """); + List inside = ids(session.execute("MATCH EDGE e:Review WHERE e HAS $alice AS actor RETURN e")); + assertEquals(List.of(session.variables().get("committed"), session.variables().get("pending")), inside); + session.execute("ROLLBACK"); + assertEquals(List.of(session.variables().get("committed")), + ids(session.execute("MATCH EDGE e:Review WHERE e HAS $alice AS actor RETURN e"))); + } + + @Test + void olderGenerationsAndRestartsKeepRoleMatches() { + session.executeAll(""" + INSERT NODE Person 'alice' AS $alice; + INSERT EDGE Review MEMBERS ($alice AS actor) AS $first; + """); + long alice = session.variables().get("alice"); + long first = session.variables().get("first"); + long generation = database.read(reader -> reader.generation()); + session.execute("INSERT EDGE Review MEMBERS ($alice AS actor|target) AS $second"); + long second = session.variables().get("second"); + assertEquals(List.of(first), ids(session.execute("AT GENERATION " + generation + " MATCH EDGE e:Review WHERE e HAS $alice AS actor RETURN e"))); + assertEquals(List.of(), ids(session.execute("AT GENERATION " + generation + " MATCH EDGE e:Review WHERE e HAS $alice AS target RETURN e"))); + assertEquals(List.of(first, second), ids(session.execute("MATCH EDGE e:Review WHERE e HAS $alice AS actor RETURN e"))); + session.close(); + database.close(); + database = HypergraphDatabase.open(directory, DatabaseOptions.defaults().withEngine(EngineOptions.defaults().withPageSize(4096))); + session = new Session(database); + assertEquals(List.of(second), ids(session.execute("MATCH EDGE e:Review WHERE e HAS @" + alice + " AS target RETURN e"))); + assertEquals(List.of(second), ids(session.execute("MATCH EDGE e:Review WHERE e HAS @" + alice + " AS actor AND e HAS @" + alice + " AS target RETURN e"))); + } +} diff --git a/database/src/test/java/io/hstore/db/query/SerializableTest.java b/database/src/test/java/io/hstore/db/query/SerializableTest.java index 655c26c..c02f1db 100644 --- a/database/src/test/java/io/hstore/db/query/SerializableTest.java +++ b/database/src/test/java/io/hstore/db/query/SerializableTest.java @@ -227,6 +227,21 @@ void phantomThroughIncidenceIsRejected() { assertEquals("1", count(BOOKED_R1)); } + @Test + void phantomThroughARoleFilteredMatchIsRejected() { + long bx = admin.variables().get("bx"); + long by = admin.variables().get("by"); + String booked = "MATCH EDGE e:Booking WHERE e HAS @Room:'r1' AS room AND e HAS @Room:'r1' RETURN count(*)"; + begin(true); + assertEquals("0", cell(a.execute(booked))); + assertEquals("0", cell(b.execute(booked))); + a.execute("ADD @Room:'r1' TO @" + bx + " AS room"); + b.execute("ADD @Room:'r1' TO @" + by + " AS room"); + String message = onlySecondFails(commitBoth()); + assertTrue(message.contains("incident edges of atom"), message); + assertEquals("1", count(booked)); + } + @Test void countThenInsertThroughIncidenceIsRejected() { begin(true); diff --git a/docs/database/planner.md b/docs/database/planner.md index fd14a63..8bd356f 100644 --- a/docs/database/planner.md +++ b/docs/database/planner.md @@ -2,7 +2,7 @@ This page describes how a `MATCH` statement ([HQL](hql.md#match)) is planned and executed: `query/Planner.java`, `query/AccessPath.java`, `query/Evaluator.java`, `query/Executor.java#query`, -`stats/Statistics.java`, `engine/.../tree/TreeAlgebra.java` and `engine/.../page/IoTrace.java`. +`stats/Statistics.java`, `engine/.../index/PostingCursor.java` and `engine/.../page/IoTrace.java`. ## Pipeline @@ -11,7 +11,7 @@ flowchart TD P["Parser: Ast.Match"] --> PL["Planner.plan"] PL --> C1["candidate: CatalogScan"] PL --> C2["candidate: TypeScan"] - PL --> C3["candidate per sargable conjunct:
IncidentIntersection / MemberScan / IndexRange / Lookup / SemanticCandidates"] + PL --> C3["candidate per sargable conjunct:
IncidentIntersection (all CONTAINS / HAS merged) / MemberScan / IndexRange / Lookup / SemanticCandidates"] C1 & C2 & C3 --> CH["choose min cost"] CH --> RS["residual = conjuncts not exactly consumed,
sorted by cost class"] RS --> EX["Executor.query (inside IoTrace with page budget)"] @@ -40,7 +40,7 @@ drive an access path; a predicate inside `OR` or `NOT` is always evaluated as a |---|---|---|---|---| | `AllAtoms(edges)` | `catalog-scan` | always | `Reader.allAtoms` (full catalog scan, tenant-filtered) | no | | `TypeScan(type)` | `type-scan` | `MATCH x:Type` | `Reader.atomsOfType` (type posting list of the tenant) | no | -| `Incident(atoms)` | `incident` | `x CONTAINS (r1..rn)`, `x HAS r [AS role]` on an edge match | leapfrog intersection of the incidence trees of `r1..rn` | yes, except `HAS … AS role` | +| `Incident(legs)` | `incident` | every `x CONTAINS (r1..rn)` and `x HAS r [AS role]` conjunct of an edge match, merged into one path with one leg per distinct atom | leapfrog intersection of the legs' reverse-index postings, each filtered by the roles its leg requires | yes | | `Members(edge)` | `members` | `x IN r` | keys of `r`'s membership tree | yes | | `IndexRange(type, p, lo, hi, …)` | `property-index` | `x.p op literal`, `literal op x.p`, `x.p BETWEEN a AND b` on an `INDEXED` declared property; the same forms with `json(x, '$.path')` in place of `x.p` when the type has a JSON index on exactly `'$.path'` (`Planner.indexed`) | `Reader.range` | yes | | `Semantic(text, k, consistency)` | `semantic` | `x SIMILAR TO 'text' TOP k` | `Reader.similar` | no | @@ -48,43 +48,64 @@ drive an access path; a predicate inside `OR` or `NOT` is always evaluated as a `!=` on an indexed property maps to a `TypeScan` (`Planner.range`, case `NE`). A path is **exact** when every atom it yields satisfies the conjunct that produced it (`Planner.exact`); an exact conjunct is dropped from the residual -filters. `HAS r AS role` is not exact because the incidence tree only says that `r` is a member, not in which role. +filters. `HAS r AS role` is exact because every reverse-index posting carries the role set of its incidence +(`IndexValues.Incident(roleSet, locator)`), kept in step with the membership by the derivation that maintains the +index, so the role check runs on the posting. Paths other than `TypeScan` may yield atoms of the wrong kind or type (an incidence set contains hyperedges of every type), so `Executor.admits` re-checks `isEdge` and the type id for them. -### Incidence intersection (`CONTAINS`) - -`Incident.scan` obtains, for every referenced atom, its reverse-index tree `View.incidentTree(atom)` — the sorted -set of hyperedge ids the atom belongs to (see [topology](../storage/topology.md)) — and calls -`TreeAlgebra.intersectKeys(trees)`: - -1. If any tree is empty the result is empty. -2. The key window is `[max(tree.summary().min), min(tree.summary().max)]`; an empty window returns immediately - without touching leaves. -3. Cursors are opened over each tree ordered by ascending size, with a summary predicate - `summary.overlaps(low, high)` that prunes subtrees outside the window. -4. A leapfrog join (`TreeAlgebra.Leapfrog`) repeatedly `seek`s each cursor to the current candidate: if the - cursor lands on the candidate, one more tree agrees; otherwise the landed key becomes the new candidate with one - agreeing tree. After either case the join checks whether all trees agree and emits the key, so a single tree - (`CONTAINS (one ref)`, `HAS`) degenerates to an ordered scan of that tree. +### Incidence intersection (`CONTAINS`, `HAS`) + +`Planner.plan` collects every `CONTAINS` and `HAS` conjunct on the match variable into one `Incident` path. Each +distinct atom becomes a leg, and a leg requires every role that any of its conjuncts names: +`c HAS @1 AS actor AND c HAS @1 AS reviewer AND c CONTAINS (@1, @2)` has the legs `@1 AS actor&reviewer` and `@2`. +All of those conjuncts are consumed exactly. `OR` of such conjuncts becomes a `Union` of single-conjunct paths and is +exact when each part is. + +`Incident.scan` returns nothing unless every leg's atom is visible to the reader. Otherwise it opens one cursor per +leg with `View.incidentCursor(atom)`, or `View.incidentCursor(atom, filter)` when the leg names roles, and calls +`PostingCursor.intersect`: + +1. Role names become the set of role-set ids that contain all of them (`Dictionary.roleSetsWithAll`). Role sets + are interned for the whole database, and symbol ids only grow, so the dictionary keeps a role-to-role-sets index + and on each call reads only symbols above the highest id it has seen. An unknown role name gives an empty set, + and the leg matches nothing. +2. A cursor reads the posting list in place: the key and value arrays of an inline list, or a `TreeCursor` over a + promoted postings tree. `seek` moves on keys only and never decodes a value. +3. Cursors are ordered by ascending posting count. A leapfrog join `seek`s each cursor to the current candidate: if + the cursor lands on the candidate, one more leg agrees; otherwise the landed key becomes the new candidate with + one agreeing leg. When all legs agree, each leg with roles checks the role set of its posting for that key. The + key is emitted if every leg accepts it, and the search continues at the key plus one either way, which a cursor + reaches by stepping instead of searching. A single leg (`CONTAINS (one ref)`, `HAS`) is an ordered scan of that + leg's postings. + +Role sets are checked only where all legs agree, never while a cursor skips. A hub atom with a million postings and +a rare role, intersected with a leaf atom in two hyperedges, costs two role checks and the seeks of the leaf's two +keys, the same work as scanning the leaf and verifying the hub's role on each candidate. ```mermaid sequenceDiagram - participant A as incidentTree(@ines) [smallest] - participant B as incidentTree(@ada) - Note over A,B: candidate = max(minA, minB) + participant A as incidentCursor(@ines, patient) [smallest] + participant B as incidentCursor(@ada, provider) + Note over A,B: candidate = first accepted key of A A->>A: seek(candidate) → key k1 - B->>B: seek(k1) → key k2 (> k1) + B->>B: seek(k1) → next accepted key k2 (> k1) A->>A: seek(k2) → k2 (agree 1) B->>B: seek(k2) → k2 (agree 2 = all) ⇒ emit k2 - Note over A,B: candidate = k2 + 1, repeat until a seek runs past high + Note over A,B: candidate = k2 + 1, repeat until a cursor runs out ``` Each `seek` descends the B+-tree from the cursor's position using the counted-tree structure, so the cost is `O(Σ_i min(|T_i|, k · log|T_i|))` for `k` emitted keys plus skipped runs, instead of the -`O(Σ|T_i|)` of a merge join. Results are emitted in ascending edge-id order, which is the order of all other -access paths too. +`O(Σ|T_i|)` of a merge join. Role checks add one value read per key on which all legs agree. Results are emitted in ascending edge-id +order, which is the order of all other access paths too. + +The estimate for the path uses each leg's degree, so a leg with roles is counted as if every posting had the role. +Cardinality feedback for the `incident` kind corrects that over time. + +The reverse index is keyed by member, not by tenant, so an edge id from a posting is only used after +`Executor.admits` has read it through the tenant-filtered `Reader`. ### Property index ranges @@ -181,7 +202,7 @@ estimate always wins. | `Fixed` | 1 | | `Semantic` | `k` | | `IndexRange` | `PostingIndex.countRange` over the order-key window of the tree selected by `Reader.indexedTag` — exact count of postings in the window, assembled from subtree weights without visiting leaves | -| `Incident(r1..rn)` | `d1 × Π_{i≥2} min(1, d_i / N)`, with degrees `d` sorted ascending and `N` the catalog size (independence assumption) | +| `Incident(legs)` | `d1 × Π_{i≥2} min(1, d_i / N)`, with the legs' degrees `d` sorted ascending and `N` the catalog size (independence assumption); role filters are not counted | The independence estimate is a lower bound for correlated members; in the demo dataset `CONTAINS (@ines, @ada)` is estimated at 1 row and produces 3. diff --git a/engine/src/main/java/io/hstore/engine/Dictionary.java b/engine/src/main/java/io/hstore/engine/Dictionary.java index 921289f..57493aa 100644 --- a/engine/src/main/java/io/hstore/engine/Dictionary.java +++ b/engine/src/main/java/io/hstore/engine/Dictionary.java @@ -10,11 +10,15 @@ import io.hstore.engine.index.Postings; import io.hstore.engine.tree.Entry; import io.hstore.engine.tree.Tree; +import io.hstore.engine.tree.TreeCursor; import io.hstore.engine.txn.Snapshot; import io.hstore.engine.txn.TransactionManager; +import java.util.BitSet; import java.util.Collection; +import java.util.HashMap; import java.util.List; +import java.util.Map; import java.util.Optional; import java.util.OptionalInt; import java.util.concurrent.ConcurrentHashMap; @@ -30,6 +34,9 @@ public final class Dictionary { private final ConcurrentHashMap ids = new ConcurrentHashMap<>(); private final ConcurrentHashMap symbols = new ConcurrentHashMap<>(); private final ConcurrentHashMap> roleLists = new ConcurrentHashMap<>(); + private final Object roleSetsLock = new Object(); + private final Map rememberedRoleSets = new HashMap<>(); + private volatile Map roleSetsByRole; Dictionary(TransactionManager transactions) { this.transactions = transactions; @@ -116,16 +123,52 @@ public OptionalInt roleId(String role) { } public List roleSetsContaining(String role) { - OptionalInt roleId = roleId(role); - if (roleId.isEmpty()) { - return List.of(); + return roleSetsWithAll(List.of(role)).stream().boxed().toList(); + } + + public BitSet roleSetsWithAll(Collection roles) { + if (roles.isEmpty()) { + throw new IllegalArgumentException("at least one role is required"); } - try (Snapshot snapshot = transactions.snapshot(Branch.MAIN)) { - return snapshot.scan(EngineSlots.SYMBOLS).stream() - .filter(entry -> entry.value() instanceof Symbol.Group(int namespace, List members) - && namespace == ROLE_SET && members.contains(roleId.getAsInt())) - .map(entry -> (int) entry.key()) - .toList(); + List roleIds = roles.stream().distinct().map(this::roleId).toList(); + if (roleIds.stream().anyMatch(OptionalInt::isEmpty)) { + return new BitSet(); + } + Map byRole = roleSetsByRole(); + BitSet accepted = (BitSet) byRole.getOrDefault(roleIds.getFirst().getAsInt(), new BitSet()).clone(); + roleIds.stream().skip(1).forEach(id -> accepted.and(byRole.getOrDefault(id.getAsInt(), new BitSet()))); + return accepted; + } + + private Map roleSetsByRole() { + Map loaded = roleSetsByRole; + if (loaded != null) { + return loaded; + } + synchronized (roleSetsLock) { + if (roleSetsByRole == null) { + Map byRole = new HashMap<>(); + try (Snapshot snapshot = transactions.snapshot(Branch.MAIN)) { + TreeCursor cursor = snapshot.scan(EngineSlots.SYMBOLS).cursor(); + for (boolean found = cursor.seek(0); found && cursor.hasNext(); cursor.advance()) { + addRoleSet(byRole, (int) cursor.key(), cursor.value()); + } + } + rememberedRoleSets.forEach((id, symbol) -> addRoleSet(byRole, id, symbol)); + rememberedRoleSets.clear(); + roleSetsByRole = Map.copyOf(byRole); + } + return roleSetsByRole; + } + } + + private static void addRoleSet(Map byRole, int id, Symbol symbol) { + if (symbol instanceof Symbol.Group(int namespace, List members) && namespace == ROLE_SET) { + members.forEach(role -> byRole.compute(role, (_, current) -> { + BitSet updated = current == null ? new BitSet() : (BitSet) current.clone(); + updated.set(id); + return updated; + })); } } @@ -152,5 +195,16 @@ private static OptionalInt find(Tree> lookup, Tree symb private void remember(int id, Symbol symbol) { ids.put(symbol, id); symbols.put(id, symbol); + if (symbol instanceof Symbol.Group group && group.namespace() == ROLE_SET) { + synchronized (roleSetsLock) { + if (roleSetsByRole == null) { + rememberedRoleSets.put(id, symbol); + } else { + Map byRole = new HashMap<>(roleSetsByRole); + addRoleSet(byRole, id, symbol); + roleSetsByRole = Map.copyOf(byRole); + } + } + } } } diff --git a/engine/src/main/java/io/hstore/engine/index/PostingCursor.java b/engine/src/main/java/io/hstore/engine/index/PostingCursor.java new file mode 100644 index 0000000..9fda2c0 --- /dev/null +++ b/engine/src/main/java/io/hstore/engine/index/PostingCursor.java @@ -0,0 +1,214 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.engine.index; + +import io.hstore.engine.tree.TreeCursor; + +import java.util.Arrays; +import java.util.Comparator; +import java.util.List; +import java.util.NoSuchElementException; +import java.util.PrimitiveIterator; +import java.util.Spliterator; +import java.util.Spliterators; +import java.util.function.Predicate; +import java.util.stream.LongStream; +import java.util.stream.StreamSupport; + +public sealed interface PostingCursor

permits PostingCursor.Inline, PostingCursor.Promoted { + + boolean seek(long target); + + long key(); + + P value(); + + boolean accepted(); + + long size(); + + static LongStream intersect(List> cursors) { + if (cursors.isEmpty()) { + return LongStream.empty(); + } + List> ordered = cursors.stream().sorted(Comparator.comparingLong(PostingCursor::size)).toList(); + return StreamSupport.longStream(Spliterators.spliteratorUnknownSize(new Leapfrog(ordered), + Spliterator.ORDERED | Spliterator.DISTINCT | Spliterator.SORTED | Spliterator.NONNULL), false); + } + + final class Inline

implements PostingCursor

{ + private final long[] keys; + private final List

values; + private final Predicate accept; + private int position; + + Inline(long[] keys, List

values, Predicate accept) { + this.keys = keys; + this.values = values; + this.accept = accept; + } + + @Override + public boolean seek(long target) { + if (position < keys.length && keys[position] < target) { + if (keys[position] + 1 == target) { + position++; + } else { + int found = Arrays.binarySearch(keys, position, keys.length, target); + position = found >= 0 ? found : -found - 1; + } + } + return position < keys.length; + } + + @Override + public boolean accepted() { + return accept == null || accept.test(value()); + } + + @Override + public long key() { + if (position >= keys.length) { + throw new NoSuchElementException(); + } + return keys[position]; + } + + @Override + public P value() { + if (position >= keys.length) { + throw new NoSuchElementException(); + } + return values.get(position); + } + + @Override + public long size() { + return keys.length; + } + } + + final class Promoted

implements PostingCursor

{ + private final TreeCursor

cursor; + private final Predicate accept; + private final long size; + + Promoted(TreeCursor

cursor, Predicate accept, long size) { + this.cursor = cursor; + this.accept = accept; + this.size = size; + } + + @Override + public boolean seek(long target) { + if (!cursor.hasNext()) { + return false; + } + long key = cursor.key(); + if (key < target && key + 1 == target) { + cursor.advance(); + } else if (key < target && !cursor.seek(target)) { + return false; + } + return cursor.hasNext(); + } + + @Override + public boolean accepted() { + return accept == null || accept.test(cursor.value()); + } + + @Override + public long key() { + return cursor.key(); + } + + @Override + public P value() { + return cursor.value(); + } + + @Override + public long size() { + return size; + } + } + + final class Leapfrog implements PrimitiveIterator.OfLong { + private final List> cursors; + private boolean ready; + private boolean done; + private long current; + private long target = Long.MIN_VALUE; + + private Leapfrog(List> cursors) { + this.cursors = cursors; + } + + @Override + public boolean hasNext() { + if (!ready && !done) { + search(); + } + return ready; + } + + @Override + public long nextLong() { + if (!hasNext()) { + throw new NoSuchElementException(); + } + ready = false; + if (current == Long.MAX_VALUE) { + done = true; + } else { + target = current + 1; + } + return current; + } + + private void search() { + int agreeing = 0; + int index = 0; + long candidate = target; + while (true) { + PostingCursor cursor = cursors.get(index); + if (!cursor.seek(candidate)) { + done = true; + return; + } + long key = cursor.key(); + if (key == candidate) { + agreeing++; + } else { + candidate = key; + agreeing = 1; + } + if (agreeing == cursors.size()) { + if (allAccepted()) { + current = candidate; + ready = true; + return; + } + if (candidate == Long.MAX_VALUE) { + done = true; + return; + } + candidate++; + agreeing = 0; + } + index = (index + 1) % cursors.size(); + } + } + + private boolean allAccepted() { + for (PostingCursor cursor : cursors) { + if (!cursor.accepted()) { + return false; + } + } + return true; + } + } +} diff --git a/engine/src/main/java/io/hstore/engine/index/PostingIndex.java b/engine/src/main/java/io/hstore/engine/index/PostingIndex.java index c7b8c02..ce8103e 100644 --- a/engine/src/main/java/io/hstore/engine/index/PostingIndex.java +++ b/engine/src/main/java/io/hstore/engine/index/PostingIndex.java @@ -19,6 +19,7 @@ import java.util.List; import java.util.Optional; import java.util.function.LongConsumer; +import java.util.function.Predicate; import java.util.function.ToLongFunction; import java.util.stream.IntStream; import java.util.stream.Stream; @@ -124,6 +125,17 @@ public void forEachPosting(Tree> index, long key, LongConsumer actio }); } + public PostingCursor

cursor(Tree> index, long key) { + return cursor(index, key, null); + } + + public PostingCursor

cursor(Tree> index, long key, Predicate accept) { + return index.get(key).>map(value -> switch (value) { + case Postings.Inline

(long[] keys, List

values, long _, int _) -> new PostingCursor.Inline<>(keys, values, accept); + case Postings.Promoted

(Ref root) -> new PostingCursor.Promoted<>(subtree(index, root).cursor(), accept, root.count()); + }).orElseGet(() -> new PostingCursor.Inline<>(new long[0], List.of(), accept)); + } + public Tree

tree(Tree> index, long key) { return index.get(key).map(value -> switch (value) { case Postings.Inline

(long[] keys, List

values, long _, int _) -> { diff --git a/engine/src/main/java/io/hstore/engine/txn/View.java b/engine/src/main/java/io/hstore/engine/txn/View.java index 5158c2a..ea89f1d 100644 --- a/engine/src/main/java/io/hstore/engine/txn/View.java +++ b/engine/src/main/java/io/hstore/engine/txn/View.java @@ -9,6 +9,8 @@ import io.hstore.engine.catalog.EngineSlots; import io.hstore.engine.catalog.Slot; import io.hstore.engine.catalog.Symbol; +import io.hstore.engine.index.IndexValues; +import io.hstore.engine.index.PostingCursor; import io.hstore.engine.topology.Hyperedge; import io.hstore.engine.topology.Incidence; import io.hstore.engine.tree.Entry; @@ -19,6 +21,7 @@ import java.util.Optional; import java.util.OptionalLong; import java.util.function.LongConsumer; +import java.util.function.Predicate; import java.util.stream.LongStream; import java.util.stream.Stream; @@ -119,6 +122,16 @@ public void forEachIncidentEdge(long atom, LongConsumer action) { EngineSlots.REVERSE_INDEX.forEachPosting(lazyTree(EngineSlots.REVERSE), atom, action); } + public PostingCursor incidentCursor(long atom) { + observeKey(EngineSlots.REVERSE, atom); + return EngineSlots.REVERSE_INDEX.cursor(lazyTree(EngineSlots.REVERSE), atom); + } + + public PostingCursor incidentCursor(long atom, Predicate accept) { + observeKey(EngineSlots.REVERSE, atom); + return EngineSlots.REVERSE_INDEX.cursor(lazyTree(EngineSlots.REVERSE), atom, accept); + } + public Tree incidentTree(long atom) { observeKey(EngineSlots.REVERSE, atom); return EngineSlots.REVERSE_INDEX.tree(lazyTree(EngineSlots.REVERSE), atom); diff --git a/engine/src/test/java/io/hstore/engine/PostingCursorTest.java b/engine/src/test/java/io/hstore/engine/PostingCursorTest.java new file mode 100644 index 0000000..a280bb8 --- /dev/null +++ b/engine/src/test/java/io/hstore/engine/PostingCursorTest.java @@ -0,0 +1,280 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.engine; + +import io.hstore.engine.index.IndexValues; +import io.hstore.engine.index.PostingCursor; +import io.hstore.engine.topology.EdgeKind; +import io.hstore.engine.topology.Incidence; +import io.hstore.engine.topology.Weight; +import io.hstore.engine.txn.CrashPoint; +import io.hstore.engine.txn.Durability; +import io.hstore.engine.txn.IncidentEdge; +import io.hstore.engine.txn.View; + +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.BitSet; +import java.util.List; +import java.util.Set; +import java.util.concurrent.CompletableFuture; +import java.util.concurrent.CountDownLatch; +import java.util.concurrent.TimeUnit; +import java.util.concurrent.atomic.AtomicBoolean; +import java.util.concurrent.atomic.AtomicInteger; +import java.util.function.Predicate; +import java.util.random.RandomGenerator; +import java.util.random.RandomGeneratorFactory; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertFalse; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class PostingCursorTest { + + static final int NODE = 1; + static final int EDGE = 2; + + @TempDir + Path directory; + + private static Incidence member(long atom, int roleSet) { + return new Incidence(atom, roleSet, Weight.ONE, Long.MIN_VALUE, Long.MAX_VALUE, 0, 0); + } + + private static List naive(View view, List anchors, List> filters) { + List> sets = IntStream.range(0, anchors.size()) + .mapToObj(i -> Set.copyOf(view.incident(anchors.get(i)) + .filter(edge -> filters.get(i).test(new IndexValues.Incident(edge.roleSet(), edge.locator()))) + .map(IncidentEdge::edge).toList())) + .toList(); + return sets.getFirst().stream().filter(edge -> sets.stream().allMatch(set -> set.contains(edge))).sorted().toList(); + } + + @Test + void filteredLeapfrogMatchesNaiveIntersection() { + RandomGenerator random = RandomGeneratorFactory.of("L64X128MixRandom").create(7); + try (StorageEngine engine = StorageEngine.open(directory, EngineTest.small())) { + Dictionary dictionary = engine.dictionary(); + int[] roleSets = { + Dictionary.NO_ROLES, + dictionary.roleSet(List.of("actor")), + dictionary.roleSet(List.of("reviewer")), + dictionary.roleSet(List.of("actor", "reviewer")), + dictionary.roleSet(List.of("target")) + }; + List nodes = new ArrayList<>(); + engine.write(txn -> { + for (int i = 0; i < 40; i++) { + nodes.add(txn.createNode(NODE, "n" + i)); + } + for (int e = 0; e < 400; e++) { + long edge = txn.createEdge(EDGE, EdgeKind.SET); + txn.upsert(edge, nodes.getFirst(), _ -> member(nodes.getFirst(), roleSets[random.nextInt(roleSets.length)])); + if (e % 2 == 0) { + txn.upsert(edge, nodes.get(1), _ -> member(nodes.get(1), roleSets[random.nextInt(roleSets.length)])); + } + for (int k = 0; k < 3; k++) { + long node = nodes.get(2 + random.nextInt(nodes.size() - 2)); + txn.upsert(edge, node, _ -> member(node, roleSets[random.nextInt(roleSets.length)])); + } + } + return null; + }); + engine.read(snapshot -> { + assertTrue(snapshot.degree(nodes.getFirst()) > 48); + assertTrue(snapshot.degree(nodes.get(5)) <= 48); + for (int round = 0; round < 300; round++) { + int legs = 1 + random.nextInt(3); + List anchors = new ArrayList<>(); + List> filters = new ArrayList<>(); + for (int i = 0; i < legs; i++) { + anchors.add(random.nextInt(4) == 0 ? nodes.get(random.nextInt(nodes.size())) : nodes.get(random.nextInt(2))); + BitSet accepted = new BitSet(); + IntStream.of(roleSets).filter(_ -> random.nextBoolean()).forEach(accepted::set); + filters.add(random.nextInt(3) == 0 ? _ -> true : incident -> accepted.get(incident.roleSet())); + } + List> cursors = IntStream.range(0, legs) + .mapToObj(i -> snapshot.incidentCursor(anchors.get(i), filters.get(i))).toList(); + assertEquals(naive(snapshot, anchors, filters), PostingCursor.intersect(cursors).boxed().toList(), + "anchors " + anchors); + } + return null; + }); + } + } + + @Test + void cursorsOnlyMoveForward() { + try (StorageEngine engine = StorageEngine.open(directory, EngineTest.small())) { + int actor = engine.dictionary().roleSet(List.of("actor")); + int reviewer = engine.dictionary().roleSet(List.of("reviewer")); + List edges = new ArrayList<>(); + long node = engine.write(txn -> { + long created = txn.createNode(NODE, "hub"); + for (int i = 0; i < 120; i++) { + long edge = txn.createEdge(EDGE, EdgeKind.SET); + edges.add(edge); + txn.upsert(edge, created, _ -> member(created, edges.size() % 3 == 0 ? actor : reviewer)); + } + return created; + }); + engine.read(snapshot -> { + PostingCursor cursor = snapshot.incidentCursor(node, incident -> incident.roleSet() == actor); + assertTrue(cursor.seek(Long.MIN_VALUE)); + assertEquals(edges.getFirst(), cursor.key()); + assertFalse(cursor.accepted()); + assertTrue(cursor.seek(edges.get(2))); + assertTrue(cursor.accepted()); + assertTrue(cursor.seek(edges.get(50))); + assertEquals(edges.get(50), cursor.key()); + assertTrue(cursor.seek(edges.get(3))); + assertEquals(edges.get(50), cursor.key()); + assertTrue(cursor.seek(edges.get(50) + 1)); + assertEquals(edges.get(51), cursor.key()); + assertEquals(reviewer, cursor.value().roleSet()); + assertFalse(cursor.seek(edges.getLast() + 1)); + assertThrows(RuntimeException.class, cursor::key); + List actors = IntStream.range(0, edges.size()).filter(i -> (i + 1) % 3 == 0).mapToObj(edges::get).toList(); + assertEquals(actors, PostingCursor.intersect(List.of(snapshot.incidentCursor(node, incident -> incident.roleSet() == actor))) + .boxed().toList()); + PostingCursor missing = snapshot.incidentCursor(node + 1_000_000, _ -> true); + assertFalse(missing.seek(Long.MIN_VALUE)); + assertEquals(0, PostingCursor.intersect(List.of(missing, snapshot.incidentCursor(node))).count()); + return null; + }); + } + } + + @Test + void rolesAreOnlyCheckedWhereEveryLegAgrees() { + try (StorageEngine engine = StorageEngine.open(directory, EngineTest.small())) { + int common = engine.dictionary().roleSet(List.of("author")); + int rare = engine.dictionary().roleSet(List.of("editor")); + long[] ids = engine.write(txn -> { + long hub = txn.createNode(NODE, "hub"); + long leaf = txn.createNode(NODE, "leaf"); + long last = 0; + for (int i = 0; i < 3_000; i++) { + last = txn.createEdge(EDGE, EdgeKind.SET); + long edge = last; + int roleSet = i == 2_999 ? rare : common; + txn.upsert(edge, hub, _ -> member(hub, roleSet)); + if (i < 2) { + txn.upsert(edge, leaf, _ -> member(leaf, Dictionary.NO_ROLES)); + } + } + return new long[]{hub, leaf, last}; + }); + engine.read(snapshot -> { + AtomicInteger checked = new AtomicInteger(); + Predicate editor = incident -> { + checked.incrementAndGet(); + return incident.roleSet() == rare; + }; + assertEquals(0, PostingCursor.intersect(List.of(snapshot.incidentCursor(ids[0], editor), snapshot.incidentCursor(ids[1]))).count()); + assertTrue(checked.get() <= 2, "role checks " + checked.get()); + checked.set(0); + assertEquals(List.of(ids[2]), PostingCursor.intersect(List.of(snapshot.incidentCursor(ids[0], editor))).boxed().toList()); + assertEquals(3_000, checked.get()); + return null; + }); + } + } + + @Test + void transactionsSeeTheirOwnPostings() { + try (StorageEngine engine = StorageEngine.open(directory, EngineTest.small())) { + int actor = engine.dictionary().roleSet(List.of("actor")); + engine.write(txn -> { + long node = txn.createNode(NODE, "n"); + long edge = txn.createEdge(EDGE, EdgeKind.SET); + txn.upsert(edge, node, _ -> member(node, actor)); + PostingCursor cursor = txn.incidentCursor(node, incident -> incident.roleSet() == actor); + assertTrue(cursor.seek(Long.MIN_VALUE)); + assertEquals(edge, cursor.key()); + return null; + }); + } + } + + @Test + void roleSetIndexFollowsNewSymbols() { + try (StorageEngine engine = StorageEngine.open(directory, EngineTest.small())) { + Dictionary dictionary = engine.dictionary(); + assertTrue(dictionary.roleSetsWithAll(List.of("actor")).isEmpty()); + int actor = dictionary.roleSet(List.of("actor")); + int both = dictionary.roleSet(List.of("actor", "reviewer")); + dictionary.roleSet(List.of("reviewer")); + BitSet actorSets = new BitSet(); + actorSets.set(actor); + actorSets.set(both); + assertEquals(actorSets, dictionary.roleSetsWithAll(List.of("actor"))); + int later = dictionary.roleSet(List.of("actor", "target")); + actorSets.set(later); + assertEquals(actorSets, dictionary.roleSetsWithAll(List.of("actor"))); + BitSet bothSets = new BitSet(); + bothSets.set(both); + assertEquals(bothSets, dictionary.roleSetsWithAll(List.of("reviewer", "actor"))); + assertTrue(dictionary.roleSetsWithAll(List.of("actor", "unknown")).isEmpty()); + assertEquals(actorSets.stream().boxed().toList(), dictionary.roleSetsContaining("actor")); + assertThrows(IllegalArgumentException.class, () -> dictionary.roleSetsWithAll(List.of())); + engine.checkpoint(); + } + try (StorageEngine engine = StorageEngine.open(directory, EngineTest.small())) { + assertEquals(3, engine.dictionary().roleSetsWithAll(List.of("actor")).cardinality()); + } + } + + @Test + void aRoleSetStillBeingCommittedIsInTheFirstLoadedMap() throws Exception { + CountDownLatch syncing = new CountDownLatch(1); + CountDownLatch release = new CountDownLatch(1); + AtomicBoolean armed = new AtomicBoolean(false); + EngineOptions options = EngineTest.small().withDurability(Durability.SYNC).withFaults(reached -> { + if (reached == CrashPoint.WAL_SYNC && armed.compareAndSet(true, false)) { + syncing.countDown(); + try { + release.await(10, TimeUnit.SECONDS); + } catch (InterruptedException e) { + throw new IllegalStateException(e); + } + } + }); + try (StorageEngine engine = StorageEngine.open(directory, options)) { + Dictionary dictionary = engine.dictionary(); + dictionary.roleSet(List.of("actor")); + dictionary.roleSet(List.of("extra")); + armed.set(true); + CompletableFuture first = CompletableFuture.supplyAsync(() -> dictionary.roleSet(List.of("actor", "extra"))); + assertTrue(syncing.await(10, TimeUnit.SECONDS)); + int id = dictionary.roleSet(List.of("actor", "extra")); + boolean seen = dictionary.roleSetsWithAll(List.of("actor")).get(id); + release.countDown(); + assertEquals(id, first.get(10, TimeUnit.SECONDS)); + assertTrue(seen, "role set " + id + " missing from the first load"); + } + } + + @Test + void everyNewRoleSetIsVisibleToTheThreadThatCreatedIt() throws Exception { + try (StorageEngine engine = StorageEngine.open(directory, EngineTest.small())) { + Dictionary dictionary = engine.dictionary(); + dictionary.roleSet(List.of("actor")); + List> threads = IntStream.range(0, 8).mapToObj(thread -> CompletableFuture.runAsync(() -> { + for (int i = 0; i < 50; i++) { + int created = dictionary.roleSet(List.of("actor", "role-" + thread + "-" + i)); + assertTrue(dictionary.roleSetsWithAll(List.of("actor")).get(created)); + } + })).toList(); + CompletableFuture.allOf(threads.toArray(CompletableFuture[]::new)).get(60, TimeUnit.SECONDS); + assertEquals(401, dictionary.roleSetsWithAll(List.of("actor")).cardinality()); + } + } +}