diff --git a/benchmarks/src/main/java/io/hstore/bench/HStoreStore.java b/benchmarks/src/main/java/io/hstore/bench/HStoreStore.java index 64c0ae9..cb0333f 100644 --- a/benchmarks/src/main/java/io/hstore/bench/HStoreStore.java +++ b/benchmarks/src/main/java/io/hstore/bench/HStoreStore.java @@ -154,8 +154,8 @@ public long coMembership(int[][] pairs, int from, int to) { return database.read(reader -> { long total = 0; for (int i = from; i < to; i++) { - total += TreeAlgebra.intersectKeys(reader.view().incidentTree(nodes[pairs[i][0]]), - reader.view().incidentTree(nodes[pairs[i][1]])).count(); + total += TreeAlgebra.intersectKeys(reader.engineView().incidentTree(nodes[pairs[i][0]]), + reader.engineView().incidentTree(nodes[pairs[i][1]])).count(); } return total; }); @@ -209,7 +209,7 @@ public long probeLargeEdge(int[] probes, int from, int to) { return database.read(reader -> { long found = 0; for (int i = from; i < to; i++) { - found += reader.view().incidence(large, nodes[probes[i]]).isPresent() ? 1 : 0; + found += reader.engineView().incidence(large, nodes[probes[i]]).isPresent() ? 1 : 0; } return found; }); diff --git a/database/src/main/java/io/hstore/db/HypergraphDatabase.java b/database/src/main/java/io/hstore/db/HypergraphDatabase.java index d36f736..3f9dc44 100644 --- a/database/src/main/java/io/hstore/db/HypergraphDatabase.java +++ b/database/src/main/java/io/hstore/db/HypergraphDatabase.java @@ -14,6 +14,7 @@ import io.hstore.db.security.Security; import io.hstore.db.security.Scram; import io.hstore.db.security.SignIns; +import io.hstore.db.security.TenantMemberships; import io.hstore.db.semantic.Embedding; import io.hstore.db.semantic.SemanticPlane; import io.hstore.db.signal.Signals; @@ -31,6 +32,7 @@ import java.nio.file.Path; import java.time.Duration; +import java.util.ArrayList; import java.util.List; import java.util.Map; import java.util.Optional; @@ -77,15 +79,42 @@ private HypergraphDatabase(Path directory, DatabaseOptions options) { } private byte[] prepareSignIns() { - if (read(reader -> Security.scramSecret(reader.view())).isEmpty()) { + if (read(reader -> Security.scramSecret(reader.engineView())).isEmpty()) { write(writer -> { if (Security.scramSecret(writer.transaction()).isEmpty()) { Security.createScramSecret(writer.transaction()); } + if (!TenantMemberships.checked(writer.transaction()) && writer.transaction().atoms().findAny().isEmpty()) { + TenantMemberships.markChecked(writer.transaction()); + } return null; }); } - return read(reader -> Security.scramSecret(reader.view())).orElseThrow(); + return read(reader -> Security.scramSecret(reader.engineView())).orElseThrow(); + } + + public boolean tenantMembershipsChecked() { + return read(reader -> TenantMemberships.checked(reader.engineView())); + } + + public List repairTenantMemberships() { + List removed = new ArrayList<>(); + for (int round = 0; round < 5 && (round == 0 || !TenantMemberships.find(engine).isEmpty()); round++) { + for (Branch branch : engine.branches().stream().filter(Branch::isActive).toList()) { + removed.addAll(write(options.transaction().onBranch(branch.id()), Principal.SYSTEM, + writer -> TenantMemberships.repair(writer.transaction(), branch.name()))); + } + } + if (!TenantMemberships.find(engine).isEmpty()) { + throw HStoreException.invalid("memberships across tenants kept appearing while they were being removed; run the repair again"); + } + write(writer -> { + if (!TenantMemberships.checked(writer.transaction())) { + TenantMemberships.markChecked(writer.transaction()); + } + return null; + }); + return removed; } public Scram.Server scram(Optional channelBinding) { @@ -93,7 +122,7 @@ public Scram.Server scram(Optional channelBinding) { } public Scram.Server scram(Optional channelBinding, Consumer used) { - return new Scram.Server(name -> read(reader -> Security.user(reader.view(), name)).flatMap(user -> { + return new Scram.Server(name -> read(reader -> Security.user(reader.engineView(), name)).flatMap(user -> { if (Security.legacy(user)) { warnLegacy(user.name()); return Optional.empty(); @@ -114,7 +143,7 @@ private void warnLegacy(String user) { } public List usersWithOldPasswords() { - return read(reader -> Security.legacyUsers(reader.view())); + return read(reader -> Security.legacyUsers(reader.engineView())); } public SignIns signIns() { @@ -213,12 +242,12 @@ public T write(TxnOptions txnOptions, Function work) { } public TxnOptions transactionOptions(Principal principal) { - long tenantPages = read(reader -> Security.quota(reader.view(), principal.tenant()).maxTransactionPages()); + long tenantPages = read(reader -> Security.quota(reader.engineView(), principal.tenant()).maxTransactionPages()); return options.transaction().withMaxPages(Math.min(options.transactionPages(), tenantPages)); } public long queryPageBudget(Principal principal) { - long tenantPages = read(reader -> Security.quota(reader.view(), principal.tenant()).maxQueryPages()); + long tenantPages = read(reader -> Security.quota(reader.engineView(), principal.tenant()).maxQueryPages()); return Math.min(options.defaultQueryPages(), tenantPages); } @@ -231,7 +260,7 @@ public Optional authenticate(String user, String password, Optional signIn(String user, String password, Optional origin) { - return signIns.attempt(user, origin, () -> read(reader -> Security.signIn(reader.view(), user, password))) + return signIns.attempt(user, origin, () -> read(reader -> Security.signIn(reader.engineView(), user, password))) .map(found -> Security.legacy(found) ? convert(found, password) : found); } @@ -255,7 +284,7 @@ void chargePayload(int tenant, long bytes) { } public boolean requiresAuthentication() { - return read(reader -> Security.hasUsers(reader.view())); + return read(reader -> Security.hasUsers(reader.engineView())); } public T write(TxnOptions txnOptions, Principal principal, Function work) { diff --git a/database/src/main/java/io/hstore/db/Reader.java b/database/src/main/java/io/hstore/db/Reader.java index cd05f18..e41e748 100644 --- a/database/src/main/java/io/hstore/db/Reader.java +++ b/database/src/main/java/io/hstore/db/Reader.java @@ -18,11 +18,15 @@ import io.hstore.engine.HStoreException; import io.hstore.engine.catalog.AtomRecord; import io.hstore.engine.catalog.EngineSlots; +import io.hstore.engine.index.IndexValues; +import io.hstore.engine.index.PostingCursor; import io.hstore.engine.topology.EdgeKind; import io.hstore.engine.topology.Hyperedge; import io.hstore.engine.topology.Incidence; import io.hstore.engine.topology.Weight; import io.hstore.engine.tree.Entry; +import io.hstore.engine.tree.Tree; +import io.hstore.engine.txn.IncidentEdge; import io.hstore.engine.txn.View; import java.util.LinkedHashMap; @@ -79,10 +83,57 @@ public boolean owns(long atom) { return onlyDefaultTenant() || visible(atom); } - public View view() { + public View engineView() { return view; } + public View.Unrecorded unrecorded() { + return view.unrecorded(); + } + + public boolean isEdge(long atom) { + return view.atom(atom).map(record -> record.tenant() == tenant() && record instanceof AtomRecord.EdgeRecord).orElse(false); + } + + public Optional hyperedge(long id) { + return view.atom(id) + .filter(record -> record.tenant() == tenant()) + .flatMap(record -> record instanceof AtomRecord.EdgeRecord edge ? Optional.of(view.edge(id, edge)) : Optional.empty()); + } + + public Optional incidence(long edge, long member) { + return owns(member) ? view.incidence(edge, member, tenant()) : Optional.empty(); + } + + public boolean contains(long edge, long member) { + return owns(member) && view.contains(edge, member, tenant()); + } + + public Stream incidentEdges(long atom) { + if (!onlyDefaultTenant() && !visible(atom)) { + return Stream.empty(); + } + return view.incident(atom).filter(incident -> owns(incident.edge())); + } + + public Tree incidentTree(long atom) { + return onlyDefaultTenant() || visible(atom) + ? view.incidentTree(atom) + : Tree.empty(EngineSlots.REVERSE_INDEX.postingSchema(), view.source()); + } + + public PostingCursor incidentCursor(long atom) { + return onlyDefaultTenant() || visible(atom) ? view.incidentCursor(atom) : nothing(); + } + + public PostingCursor incidentCursor(long atom, Predicate accept) { + return onlyDefaultTenant() || visible(atom) ? view.incidentCursor(atom, accept) : nothing(); + } + + private PostingCursor nothing() { + return EngineSlots.REVERSE_INDEX.cursor(Tree.empty(EngineSlots.REVERSE_INDEX.schema(), view.source()), 0); + } + public HypergraphDatabase database() { return database; } diff --git a/database/src/main/java/io/hstore/db/Writer.java b/database/src/main/java/io/hstore/db/Writer.java index 1abaa36..bc142ba 100644 --- a/database/src/main/java/io/hstore/db/Writer.java +++ b/database/src/main/java/io/hstore/db/Writer.java @@ -76,7 +76,7 @@ public TypeDef createIndex(String type, String property) { .orElseThrow(() -> HStoreException.invalid(type + " has no declared property '" + property + "'")); TypeDef updated = schema().alter(txn, current.withProperty(new PropertyDef(property, existing.type(), true, existing.required()))); int key = schema().propertyKey(property); - List tenants = Security.tenants(view()).stream().map(Security.Tenant::id).toList(); + List tenants = Security.tenants(engineView()).stream().map(Security.Tenant::id).toList(); txn.apply("backfill " + type + "." + property, workspace -> tenants.forEach(tenant -> EngineSlots.TYPE_INDEX.stream(workspace.tree(EngineSlots.TYPES), EngineSlots.typeKey(tenant, updated.id())).toList() .forEach(entry -> workspace.get(PropertySlots.PROPERTIES, entry.key()) @@ -90,8 +90,8 @@ public TypeDef createJsonIndex(String type, String path, TypeTag tag) { TypeDef current = requireType(type); TypeDef updated = schema().alter(txn, current.withJsonIndex(new JsonIndex(path, tag))); int key = schema().propertyKey(path); - for (Security.Tenant tenant : Security.tenants(view())) { - Reader scoped = new Reader(database, view(), principal().inTenant(tenant.id())); + for (Security.Tenant tenant : Security.tenants(engineView())) { + Reader scoped = new Reader(database, engineView(), principal().inTenant(tenant.id())); for (long owner : scoped.atoms(type).boxed().toList()) { scoped.document(owner).ifPresent(json -> json.select(path) .filter(value -> !(value instanceof Value.Null)) @@ -309,7 +309,7 @@ public void embed(long atom, String model, int version, float[] vector) { } return semantic.model(model); }); - semantic.validate(view(), model, id, vector); + semantic.validate(engineView(), model, id, vector); long ref = writePayload(Embedding.encode(vector)); Embedding embedding = new Embedding(id, version, vector.length, ref); txn.put(Embedding.EMBEDDINGS, atom, embedding); diff --git a/database/src/main/java/io/hstore/db/evidence/Provenance.java b/database/src/main/java/io/hstore/db/evidence/Provenance.java index 64b870c..c902271 100644 --- a/database/src/main/java/io/hstore/db/evidence/Provenance.java +++ b/database/src/main/java/io/hstore/db/evidence/Provenance.java @@ -83,11 +83,11 @@ private static void requireReferable(Reader reader, long id) { } public static Optional qualifier(Reader reader, long id) { - return id == 0 ? Optional.empty() : reader.view().get(QUALIFIERS, id).filter(found -> found.tenant() == reader.tenant()); + return id == 0 ? Optional.empty() : reader.engineView().get(QUALIFIERS, id).filter(found -> found.tenant() == reader.tenant()); } public static Optional evidence(Reader reader, long id) { - return reader.view().get(EVIDENCE, id).filter(found -> found.tenant() == reader.tenant()); + return reader.engineView().get(EVIDENCE, id).filter(found -> found.tenant() == reader.tenant()); } public static Stream effective(Reader reader, long edge, EvidencePolicy policy) { diff --git a/database/src/main/java/io/hstore/db/hora/Hora.java b/database/src/main/java/io/hstore/db/hora/Hora.java index 7b1e439..7e5f3f4 100644 --- a/database/src/main/java/io/hstore/db/hora/Hora.java +++ b/database/src/main/java/io/hstore/db/hora/Hora.java @@ -67,7 +67,7 @@ public Stream incidence(long atom) { if (reader.require(atom).isEdge()) { return reader.members(atom).map(member -> tuple(atom, member)); } - return reader.view().incident(atom).filter(incident -> reader.owns(incident.edge())).flatMap(incident -> reader.view().incidence(incident.edge(), atom).stream() + return reader.incidentEdges(atom).flatMap(incident -> reader.incidence(incident.edge(), atom).stream() .map(found -> new IncidenceTuple(incident.edge(), atom, reader.roles(found.roleSet()), found.weightValue(), found.validity(), reader.edge(incident.edge()).positionOfLocator(incident.locator())))); } @@ -178,8 +178,8 @@ public double jaccard(long left, long right) { public boolean adjacent(long u, long v, int threshold) { reader.require(u); reader.require(v); - Tree left = reader.view().incidentTree(u); - Tree right = reader.view().incidentTree(v); + Tree left = reader.incidentTree(u); + Tree right = reader.incidentTree(v); return TreeAlgebra.intersectKeys(left, right).filter(reader::owns).limit(threshold).count() >= threshold; } @@ -187,7 +187,7 @@ public LongStream neighbors(long node, int threshold, Budget budget) { reader.require(node); Budget.Meter meter = budget.start(); Map counts = new HashMap<>(); - reader.view().incident(node).map(IncidentEdge::edge).filter(reader::owns).forEach(edge -> reader.edge(edge).stream() + reader.incidentEdges(node).map(IncidentEdge::edge).forEach(edge -> reader.edge(edge).stream() .mapToLong(Incidence::member) .filter(member -> member != node) .forEach(member -> { @@ -201,7 +201,7 @@ public LongStream edgeNeighbors(long edge, int threshold, Budget budget) { Budget.Meter meter = budget.start(); Tree membership = reader.edge(edge).membership(); Set candidates = new HashSet<>(); - reader.edge(edge).stream().forEach(incidence -> reader.view().incident(incidence.member()) + reader.edge(edge).stream().forEach(incidence -> reader.incidentEdges(incidence.member()) .mapToLong(IncidentEdge::edge) .filter(other -> other != edge && reader.owns(other)) .forEach(other -> { @@ -224,7 +224,7 @@ public Stream overlapJoin(List edges, long threshold, Budget continue; } Set partners = new HashSet<>(); - a.stream().forEach(incidence -> reader.view().incident(incidence.member()) + a.stream().forEach(incidence -> reader.incidentEdges(incidence.member()) .mapToLong(IncidentEdge::edge) .filter(right -> right > left && scope.contains(right) && reader.owns(right)) .forEach(partners::add)); diff --git a/database/src/main/java/io/hstore/db/hora/PatternMatcher.java b/database/src/main/java/io/hstore/db/hora/PatternMatcher.java index 95895eb..32a61cf 100644 --- a/database/src/main/java/io/hstore/db/hora/PatternMatcher.java +++ b/database/src/main/java/io/hstore/db/hora/PatternMatcher.java @@ -96,7 +96,7 @@ private long estimate(Pattern pattern, String name, Set bound) { private long typeDomain(Variable variable) { Optional type = variable.type().flatMap(reader::type); - try (View.Unrecorded _ = reader.view().unrecorded()) { + try (View.Unrecorded _ = reader.unrecorded()) { return type.map(found -> reader.countOfType(found.id())).orElse(UNKNOWN_DOMAIN); } } @@ -120,9 +120,9 @@ private LongStream edgeCandidates(Pattern pattern, String name, Map - incidentSets.add(reader.view().incidentTree(binding.get(member))); + incidentSets.add(reader.incidentTree(binding.get(member))); case Constraint.HasRole(String edge, String member, String _) when edge.equals(name) && binding.containsKey(member) -> - incidentSets.add(reader.view().incidentTree(binding.get(member))); + incidentSets.add(reader.incidentTree(binding.get(member))); case Constraint.Shares(String left, String right, long _, long _) when left.equals(name) && binding.containsKey(right) -> { relatedConstraint = true; related.addAll(neighbourEdges(binding.get(right))); @@ -156,7 +156,7 @@ private LongStream edgeCandidates(Pattern pattern, String name, Map neighbourEdges(long edge) { Set edges = new HashSet<>(); reader.edge(edge).stream().forEach(incidence -> - reader.view().incident(incidence.member()).map(IncidentEdge::edge).filter(reader::owns).forEach(edges::add)); + reader.incidentEdges(incidence.member()).map(IncidentEdge::edge).forEach(edges::add)); return edges; } @@ -208,7 +208,7 @@ private boolean holds(Constraint constraint, Map binding, Pattern } private Optional incidence(long edge, long member, Pattern pattern, String edgeVariable) { - Optional found = reader.view().edge(edge).flatMap(hyperedge -> hyperedge.get(member)); + Optional found = reader.hyperedge(edge).flatMap(hyperedge -> hyperedge.get(member)); OptionalLong instant = validAt(pattern, edgeVariable); return instant.isPresent() ? found.filter(incidence -> incidence.validAt(instant.getAsLong())) : found; } diff --git a/database/src/main/java/io/hstore/db/query/AccessPath.java b/database/src/main/java/io/hstore/db/query/AccessPath.java index 35992b9..6cd3d8a 100644 --- a/database/src/main/java/io/hstore/db/query/AccessPath.java +++ b/database/src/main/java/io/hstore/db/query/AccessPath.java @@ -89,8 +89,8 @@ public LongStream scan(Reader reader) { Dictionary dictionary = reader.database().engine().dictionary(); List> cursors = legs.stream() .map(leg -> leg.roles().isEmpty() - ? reader.view().incidentCursor(leg.atom()) - : reader.view().incidentCursor(leg.atom(), accepting(dictionary, leg.roles()))) + ? reader.incidentCursor(leg.atom()) + : reader.incidentCursor(leg.atom(), accepting(dictionary, leg.roles()))) .toList(); return PostingCursor.intersect(cursors); } @@ -118,7 +118,7 @@ public String describe() { public LongStream scan(Reader reader) { return reader.visible(edge) - ? reader.view().edge(edge).map(found -> found.membership().keys()).orElseGet(LongStream::empty) + ? reader.hyperedge(edge).map(found -> found.membership().keys()).orElseGet(LongStream::empty) : LongStream.empty(); } } diff --git a/database/src/main/java/io/hstore/db/query/Evaluator.java b/database/src/main/java/io/hstore/db/query/Evaluator.java index a0130de..bec0cde 100644 --- a/database/src/main/java/io/hstore/db/query/Evaluator.java +++ b/database/src/main/java/io/hstore/db/query/Evaluator.java @@ -12,7 +12,6 @@ import io.hstore.db.temporal.StateBindings; import io.hstore.db.value.Value; import io.hstore.engine.HStoreException; -import io.hstore.engine.catalog.AtomRecord.EdgeRecord; import io.hstore.engine.topology.Incidence; import io.hstore.engine.tree.TreeAlgebra; @@ -46,14 +45,13 @@ boolean test(Expr expr, Map binding) { case Expr.Or(List terms) -> terms.stream().anyMatch(term -> test(term, binding)); case Expr.Not(Expr term) -> !test(term, binding); case Expr.Contains(String variable, List members) -> isEdge(atom(variable, binding)) - && members.stream().map(context::resolve).allMatch(member -> reader().owns(member) && reader().view().contains(atom(variable, binding), member)); - case Expr.Has(String variable, Ast.Ref member, Optional role) -> Optional.of(context.resolve(member)).filter(reader()::owns) - .flatMap(resolved -> reader().view().incidence(atom(variable, binding), resolved)) + && members.stream().map(context::resolve).allMatch(member -> reader().contains(atom(variable, binding), member)); + case Expr.Has(String variable, Ast.Ref member, Optional role) -> reader().incidence(atom(variable, binding), context.resolve(member)) .map(incidence -> role.map(name -> reader().roles(incidence.roleSet()).contains(name)).orElse(true)) .orElse(false); case Expr.In(String variable, Ast.Ref edge) -> { long resolved = context.resolve(edge); - yield reader().owns(resolved) && reader().view().contains(resolved, atom(variable, binding)); + yield reader().contains(resolved, atom(variable, binding)); } case Expr.Similar(String variable, String text, int k, var consistency) -> context.similar(text, k, consistency).contains(atom(variable, binding)); case Expr.ValidAt(String variable, Expr instant) -> validAt(atom(variable, binding), (long) number(instant, binding)); @@ -70,7 +68,7 @@ private boolean validAt(long atom, long instant) { if (isEdge(atom)) { return reader().edge(atom).validAt(instant).findAny().isPresent(); } - return reader().view().incident(atom).filter(incident -> reader().owns(incident.edge())).anyMatch(incident -> reader().view().incidence(incident.edge(), atom) + return reader().incidentEdges(atom).anyMatch(incident -> reader().incidence(incident.edge(), atom) .map(incidence -> incidence.validAt(instant)).orElse(false)); } @@ -226,7 +224,7 @@ private long atom(String variable, Map binding) { } private boolean isEdge(long atom) { - return reader().view().atom(atom).map(record -> record instanceof EdgeRecord).orElse(false); + return reader().isEdge(atom); } private Reader reader() { diff --git a/database/src/main/java/io/hstore/db/query/Executor.java b/database/src/main/java/io/hstore/db/query/Executor.java index 8bdeea4..8cfe2e0 100644 --- a/database/src/main/java/io/hstore/db/query/Executor.java +++ b/database/src/main/java/io/hstore/db/query/Executor.java @@ -136,7 +136,7 @@ QueryResult run(Statement statement) { return QueryResult.message("created tenant " + tenant.name() + " #" + tenant.id()); }); case Statement.AlterTenant(String name, Map quota) -> write(writer -> { - Security.Tenant tenant = Security.requireTenant(writer.view(), name); + Security.Tenant tenant = Security.requireTenant(writer.engineView(), name); Security.setQuota(writer.transaction(), tenant, tenant.quota().with(quota)); return QueryResult.message("updated quota of tenant " + name); }); @@ -151,20 +151,20 @@ QueryResult run(Statement statement) { }); case Statement.DropUser(String name) -> write(writer -> { governedUser(writer, name); - if (!session.principal().global() && Security.users(writer.view()).size() <= 1) { + if (!session.principal().global() && Security.users(writer.engineView()).size() <= 1) { throw HStoreException.invalid("cannot drop the last user: the server would stop asking for credentials"); } Security.dropUser(writer.transaction(), name); return QueryResult.message("dropped user " + name); }); case Statement.ShowTenants _ -> read(reader -> table(List.of("id", "name", "atoms", "edges", "payload bytes", "max atoms", "max edges"), - Security.tenants(reader.view()).stream().filter(tenant -> session.principal().governs(tenant.id())).map(tenant -> { - Quota.Usage usage = Security.usage(reader.view(), tenant.id()); + Security.tenants(reader.engineView()).stream().filter(tenant -> session.principal().governs(tenant.id())).map(tenant -> { + Quota.Usage usage = Security.usage(reader.engineView(), tenant.id()); return List.of(QueryResult.number(tenant.id()), QueryResult.text(tenant.name()), QueryResult.number(usage.atoms()), QueryResult.number(usage.edges()), QueryResult.number(usage.payloadBytes()), limit(tenant.quota().maxAtoms()), limit(tenant.quota().maxEdges())); }))); - case Statement.ShowUsers _ -> read(reader -> table(List.of("user", "tenant", "role"), Security.users(reader.view()).stream() + case Statement.ShowUsers _ -> read(reader -> table(List.of("user", "tenant", "role"), Security.users(reader.engineView()).stream() .filter(user -> session.principal().governs(user.tenant())) .map(user -> List.of(QueryResult.text(user.name()), QueryResult.number(user.tenant()), QueryResult.text(user.role().name()))))); case Statement.DropBranch(String name) -> { @@ -189,7 +189,7 @@ QueryResult run(Statement statement) { long memberId = session.resolve(writer, member.member()); if (at.isPresent()) { writer.insertAt(edgeId, at.getAsLong(), memberId, spec(member)); - } else if (writer.edge(edgeId).kind() == EdgeKind.ORDERED && !writer.view().contains(edgeId, memberId)) { + } else if (writer.edge(edgeId).kind() == EdgeKind.ORDERED && !writer.contains(edgeId, memberId)) { writer.append(edgeId, memberId, spec(member)); } else { writer.add(edgeId, memberId, spec(member)); @@ -318,7 +318,7 @@ QueryResult run(Statement statement) { case Statement.History(OptionalLong limit) -> history(limit); case Statement.Stats _ -> stats(); case Statement.ShowTypes _ -> read(reader -> table(List.of("id", "name", "kind", "properties", "roles", "version"), - reader.schema().types(reader.view()).stream().map(type -> List.of(QueryResult.number(type.id()), QueryResult.text(type.name()), + reader.schema().types(reader.engineView()).stream().map(type -> List.of(QueryResult.number(type.id()), QueryResult.text(type.name()), QueryResult.text(type.kind().name()), QueryResult.text(type.properties().stream() .map(property -> property.name() + " " + property.type() + (property.indexed() ? " INDEXED" : "")) .collect(Collectors.joining(", "))), @@ -369,7 +369,7 @@ private int governedTenant(Reader reader, Optional name) { if (name.isEmpty()) { return principal.tenant(); } - Optional found = Security.tenant(reader.view(), name.get()); + Optional found = Security.tenant(reader.engineView(), name.get()); if (principal.global()) { return found.orElseThrow(() -> HStoreException.invalid("unknown tenant '" + name.get() + "'")).id(); } @@ -378,7 +378,7 @@ private int governedTenant(Reader reader, Optional name) { } private void governedUser(Reader reader, String name) { - Security.user(reader.view(), name).filter(user -> session.principal().governs(user.tenant())) + Security.user(reader.engineView(), name).filter(user -> session.principal().governs(user.tenant())) .orElseThrow(() -> HStoreException.invalid("unknown user '" + name + "'")); } @@ -744,7 +744,7 @@ private double numeric(Reader reader, long atom, Field field) { case Field.Property(String name) -> reader.property(atom, name).flatMap(value -> value.number().stream().boxed().findFirst()).orElse(0.0); case Field.State _ -> StateBindings.state(reader, atom).flatMap(binding -> binding.value().number().stream().boxed().findFirst()).orElse(0.0); case Field.Degree _ -> reader.degree(atom); - case Field.Weight _ -> reader.view().edge(atom).map(edge -> edge.summary().weightSum() / 1e9).orElse(1.0); + case Field.Weight _ -> reader.hyperedge(atom).map(edge -> edge.summary().weightSum() / 1e9).orElse(1.0); }; } diff --git a/database/src/main/java/io/hstore/db/query/Planner.java b/database/src/main/java/io/hstore/db/query/Planner.java index f2964aa..623f0ba 100644 --- a/database/src/main/java/io/hstore/db/query/Planner.java +++ b/database/src/main/java/io/hstore/db/query/Planner.java @@ -77,7 +77,7 @@ Plan plan(Ast.Match match) { List conjuncts = match.where().map(Planner::conjuncts).orElse(List.of()); List candidates = new ArrayList<>(); double population; - try (View.Unrecorded _ = reader.view().unrecorded()) { + try (View.Unrecorded _ = reader.unrecorded()) { universe = reader.population(match.edges()); population = type.map(found -> (double) reader.countOfType(found.id())).orElse(universe); } @@ -175,7 +175,7 @@ private double scanCost(AccessPath path, double rows) { } private double estimate(AccessPath path, double population) { - try (View.Unrecorded _ = reader.view().unrecorded()) { + try (View.Unrecorded _ = reader.unrecorded()) { return unrecordedEstimate(path, population); } } @@ -208,7 +208,7 @@ private double indexEstimate(AccessPath.IndexRange range) { if (key.isEmpty()) { return 0; } - return PropertyIndex.estimate(reader.view().scan(PropertySlots.PROPERTY_INDEX), reader.tenant(), key.getAsInt(), tag, + return PropertyIndex.estimate(reader.engineView().scan(PropertySlots.PROPERTY_INDEX), reader.tenant(), key.getAsInt(), tag, PropertyIndex.lowerKey(tag, range.low()), PropertyIndex.upperKey(tag, range.high())); } diff --git a/database/src/main/java/io/hstore/db/query/Session.java b/database/src/main/java/io/hstore/db/query/Session.java index 3f87723..a5e6fae 100644 --- a/database/src/main/java/io/hstore/db/query/Session.java +++ b/database/src/main/java/io/hstore/db/query/Session.java @@ -114,7 +114,7 @@ public Optional scramFinal(String clientFinal) { throw malformed; } Optional found = outcome instanceof Scram.Outcome.Accepted(String user, String _) - ? database.read(reader -> Security.user(reader.view(), user)).filter(current -> attempt.user().equals(Optional.of(current))) + ? database.read(reader -> Security.user(reader.engineView(), user)).filter(current -> attempt.user().equals(Optional.of(current))) : Optional.empty(); if (found.isEmpty()) { attempt.reservation().failed(); @@ -164,7 +164,7 @@ public boolean credentialsCurrent() { if (!authenticated) { return true; } - Optional stored = database.read(reader -> Security.user(reader.view(), principal.user())); + Optional stored = database.read(reader -> Security.user(reader.engineView(), principal.user())); boolean current = stored.isPresent() && credentials.isPresent() && stored.get().tenant() == credentials.get().tenant() && stored.get().role() == credentials.get().role() && (renewing || stored.get().hash().equals(credentials.get().hash())); @@ -217,7 +217,7 @@ private QueryResult dispatch(Statement statement) { if (transaction.isPresent()) { throw HStoreException.invalid("cannot switch tenants inside a transaction"); } - Security.Tenant tenant = database.read(reader -> Security.tenant(reader.view(), name)) + Security.Tenant tenant = database.read(reader -> Security.tenant(reader.engineView(), name)) .filter(found -> principal.global() || found.id() == principal.tenant()) .orElseThrow(() -> principal.global() ? HStoreException.invalid("unknown tenant '" + name + "'") @@ -236,7 +236,7 @@ private QueryResult dispatch(Statement statement) { } case Statement.WhoAmI _ -> QueryResult.table(List.of("user", "tenant", "role", "branch"), List.of(List.of( QueryResult.text(principal.user()), - QueryResult.text(database.read(reader -> Security.tenant(reader.view(), principal.tenant()).map(Security.Tenant::name).orElse("#" + principal.tenant()))), + QueryResult.text(database.read(reader -> Security.tenant(reader.engineView(), principal.tenant()).map(Security.Tenant::name).orElse("#" + principal.tenant()))), QueryResult.text(principal.role().name()), QueryResult.number(branch)))); case Statement.SetFormat(String name) -> { format = Arrays.stream(Format.values()).filter(candidate -> candidate.name().equalsIgnoreCase(name)).findFirst() diff --git a/database/src/main/java/io/hstore/db/retrieve/Ranking.java b/database/src/main/java/io/hstore/db/retrieve/Ranking.java index aa2e2ff..62e9d8b 100644 --- a/database/src/main/java/io/hstore/db/retrieve/Ranking.java +++ b/database/src/main/java/io/hstore/db/retrieve/Ranking.java @@ -103,10 +103,10 @@ private LongStream candidates(RoleTimeScorer scorer) { Slot slot = query.slots().get(i); long atom = ((Target.Atom) slot.target()).id(); if (slot.roles().isEmpty()) { - return reader.view().incidentCursor(atom); + return reader.incidentCursor(atom); } BitSet accepted = scorer.accepted(i); - return reader.view().incidentCursor(atom, incident -> accepted.get(incident.roleSet())); + return reader.incidentCursor(atom, incident -> accepted.get(incident.roleSet())); }).toList(); return PostingCursor.intersect(cursors); } diff --git a/database/src/main/java/io/hstore/db/retrieve/RoleTimeScorer.java b/database/src/main/java/io/hstore/db/retrieve/RoleTimeScorer.java index 0d70ab0..0e8c0d8 100644 --- a/database/src/main/java/io/hstore/db/retrieve/RoleTimeScorer.java +++ b/database/src/main/java/io/hstore/db/retrieve/RoleTimeScorer.java @@ -129,7 +129,7 @@ private OptionalInt memberType(long member) { } private Fill fixed(long edge, int slot, long atom) { - return reader.view().incidence(edge, atom).map(incidence -> fill(slot, incidence)).orElse(Fill.EMPTY); + return reader.incidence(edge, atom).map(incidence -> fill(slot, incidence)).orElse(Fill.EMPTY); } private Fill fill(int slot, Incidence incidence) { diff --git a/database/src/main/java/io/hstore/db/security/TenantMemberships.java b/database/src/main/java/io/hstore/db/security/TenantMemberships.java new file mode 100644 index 0000000..38ff9d2 --- /dev/null +++ b/database/src/main/java/io/hstore/db/security/TenantMemberships.java @@ -0,0 +1,69 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.security; + +import io.hstore.engine.StorageEngine; +import io.hstore.engine.catalog.AtomRecord; +import io.hstore.engine.catalog.Branch; +import io.hstore.engine.catalog.EngineSlots; +import io.hstore.engine.txn.Snapshot; +import io.hstore.engine.txn.Transaction; +import io.hstore.engine.txn.View; + +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.stream.Collectors; +import java.util.stream.Stream; + +public final class TenantMemberships { + + private static final long CHECKED = 0x4D45_4D42_4552_5301L; + + public record Crossing(String branch, long edge, long member, int edgeTenant, String edgeTenantName, int memberTenant) { + public String repair() { + return "USE TENANT " + edgeTenantName + "; " + (branch.equals("main") ? "" : "USE BRANCH " + branch + "; ") + + "REMOVE @" + member + " FROM @" + edge + ";"; + } + } + + public static List find(StorageEngine engine) { + return engine.branches().stream().filter(Branch::isActive).flatMap(branch -> { + try (Snapshot snapshot = engine.snapshot(branch.id())) { + return find(snapshot, branch.name()).stream(); + } + }).toList(); + } + + private TenantMemberships() { + } + + public static List find(View view, String branch) { + Map names = Security.tenants(view).stream().collect(Collectors.toMap(Security.Tenant::id, Security.Tenant::name)); + return view.atoms() + .flatMap(entry -> entry.value() instanceof AtomRecord.EdgeRecord edge + ? view.edge(entry.key(), edge).memberIds().boxed() + .flatMap(member -> view.atom(member) + .filter(record -> record.tenant() != edge.tenant()) + .map(record -> new Crossing(branch, entry.key(), member, edge.tenant(), + names.getOrDefault(edge.tenant(), "#" + edge.tenant()), record.tenant())) + .stream()) + : Stream.empty()) + .toList(); + } + + public static boolean checked(View view) { + return view.get(EngineSlots.SETTINGS, CHECKED).isPresent(); + } + + public static void markChecked(Transaction txn) { + txn.merge(EngineSlots.SETTINGS, CHECKED, Optional::isEmpty, _ -> Optional.of(1L)); + } + + public static List repair(Transaction txn, String branch) { + List crossings = find(txn, branch); + crossings.forEach(crossing -> txn.remove(crossing.edge(), crossing.member())); + return crossings; + } +} diff --git a/database/src/main/java/io/hstore/db/signal/Signals.java b/database/src/main/java/io/hstore/db/signal/Signals.java index 457530f..08ef451 100644 --- a/database/src/main/java/io/hstore/db/signal/Signals.java +++ b/database/src/main/java/io/hstore/db/signal/Signals.java @@ -86,9 +86,9 @@ public static Stream of(Reader reader, long atom, long from, long to) if (!reader.visible(atom)) { return Stream.empty(); } - return BY_ATOM.stream(reader.view().scan(SIGNAL_INDEX), atom) + return BY_ATOM.stream(reader.engineView().scan(SIGNAL_INDEX), atom) .map(Entry::key) - .flatMap(id -> reader.view().get(SIGNALS, id).filter(signal -> signal.overlaps(from, to)) + .flatMap(id -> reader.engineView().get(SIGNALS, id).filter(signal -> signal.overlaps(from, to)) .map(signal -> new Resolved(id, signal)).stream()); } diff --git a/database/src/main/java/io/hstore/db/temporal/BranchMerge.java b/database/src/main/java/io/hstore/db/temporal/BranchMerge.java index 04608da..659f08d 100644 --- a/database/src/main/java/io/hstore/db/temporal/BranchMerge.java +++ b/database/src/main/java/io/hstore/db/temporal/BranchMerge.java @@ -92,7 +92,7 @@ public static Outcome merge(HypergraphDatabase database, int source, int target, try (Snapshot base = database.engine().snapshotAt(branch.baseGeneration(), branch.parent()); Snapshot head = database.engine().snapshot(source)) { Side changes = side(database.reader(base), database.reader(head), base, head); - List conflicts = conflicts(changes, side(database.reader(base), writer, base, writer.view())); + List conflicts = conflicts(changes, side(database.reader(base), writer, base, writer.engineView())); requireMergeable(branch, conflicts, policy); Set skipped = policy == Policy.TARGET ? new HashSet<>(conflicts) : Set.of(); return apply(writer.transaction(), database.reader(head), changes, skipped, source, target, conflicts); @@ -197,10 +197,10 @@ private static Outcome apply(Transaction txn, Reader source, Side incoming, Set< if (!txn.exists(edge)) { continue; } - if (source.view().requireEdge(edge).kind() == EdgeKind.ORDERED) { + if (source.engineView().requireEdge(edge).kind() == EdgeKind.ORDERED) { boolean contested = changes.stream().anyMatch(change -> skipped.contains(new Conflict("membership", edge, change.member()))); if (!contested) { - txn.load(edge, source.view().requireEdge(edge).stream().toList()); + txn.load(edge, source.engineView().requireEdge(edge).stream().toList()); members += changes.size(); } continue; diff --git a/database/src/main/java/io/hstore/db/temporal/StateBindings.java b/database/src/main/java/io/hstore/db/temporal/StateBindings.java index e2b0cd2..ee5698f 100644 --- a/database/src/main/java/io/hstore/db/temporal/StateBindings.java +++ b/database/src/main/java/io/hstore/db/temporal/StateBindings.java @@ -64,7 +64,7 @@ public static void bind(Writer writer, long atom, String schema, Value value, Va } public static Optional state(Reader reader, long atom) { - return reader.visible(atom) ? reader.view().get(STATE, atom) : Optional.empty(); + return reader.visible(atom) ? reader.engineView().get(STATE, atom) : Optional.empty(); } public static Optional stateAt(Reader reader, long atom, long instant) { diff --git a/database/src/main/java/io/hstore/db/temporal/Temporal.java b/database/src/main/java/io/hstore/db/temporal/Temporal.java index 7443253..181cebd 100644 --- a/database/src/main/java/io/hstore/db/temporal/Temporal.java +++ b/database/src/main/java/io/hstore/db/temporal/Temporal.java @@ -48,15 +48,14 @@ public static LongStream activeEdges(Reader reader, long atom, long instant) { if (!reader.visible(atom)) { return LongStream.empty(); } - return reader.view().incident(atom) - .filter(incident -> reader.owns(incident.edge())) - .filter(incident -> reader.view().incidence(incident.edge(), atom).map(found -> found.validAt(instant)).orElse(false)) + return reader.incidentEdges(atom) + .filter(incident -> reader.incidence(incident.edge(), atom).map(found -> found.validAt(instant)).orElse(false)) .mapToLong(IncidentEdge::edge); } public static Stream diff(Reader before, Reader after) { - Tree left = before.view().scan(EngineSlots.CATALOG); - Tree right = after.view().scan(EngineSlots.CATALOG); + Tree left = before.engineView().scan(EngineSlots.CATALOG); + Tree right = after.engineView().scan(EngineSlots.CATALOG); return TreeDiff.diff(left, right).flatMap(delta -> switch (delta) { case TreeDiff.Delta.Added(long atom, AtomRecord record) -> Stream.concat( Stream.of(new Delta.AtomCreated(atom, record)), membersOf(after, atom, record)); @@ -69,13 +68,13 @@ private static Stream membersOf(Reader reader, long atom, AtomRecord reco if (!(record instanceof EdgeRecord edge) || edge.members() instanceof Ref.Empty) { return Stream.empty(); } - Hyperedge empty = Hyperedge.empty(atom, edge.kind(), reader.view().source()); - return changes(empty, reader.view().edge(atom, edge), atom); + Hyperedge empty = Hyperedge.empty(atom, edge.kind(), reader.engineView().source()); + return changes(empty, reader.engineView().edge(atom, edge), atom); } private static Stream changedMembers(Reader before, Reader after, long atom, AtomRecord was, AtomRecord now) { if (was instanceof EdgeRecord from && now instanceof EdgeRecord to) { - return changes(before.view().edge(atom, from), after.view().edge(atom, to), atom); + return changes(before.engineView().edge(atom, from), after.engineView().edge(atom, to), atom); } return Stream.empty(); } diff --git a/database/src/main/java/io/hstore/db/view/MaterializedViews.java b/database/src/main/java/io/hstore/db/view/MaterializedViews.java index b37fcfe..76fe912 100644 --- a/database/src/main/java/io/hstore/db/view/MaterializedViews.java +++ b/database/src/main/java/io/hstore/db/view/MaterializedViews.java @@ -93,7 +93,7 @@ public MaterializedViews(HypergraphDatabase database) { this.database = database; this.subscription = database.engine().feed().subscribe(database.engine().transactions().current().id(), this::onCommit); this.hold = database.engine().feed().hold(() -> database.read(reader -> - reader.view().scan(VIEWS).values().filter(descriptor -> descriptor.refresh() == Refresh.CONTINUOUS) + reader.engineView().scan(VIEWS).values().filter(descriptor -> descriptor.refresh() == Refresh.CONTINUOUS) .mapToLong(Descriptor::lastGeneration).min().orElse(Long.MAX_VALUE))); } @@ -115,7 +115,7 @@ public Descriptor create(Principal principal, String name, Kind kind, Refresh re if (existing.size() >= MAX_VIEWS_PER_TENANT) { throw HStoreException.limit("a tenant can have at most " + MAX_VIEWS_PER_TENANT + " views; drop one first"); } - int id = writer.view().scan(VIEWS).last().map(entry -> (int) entry.key() + 1).orElse(1); + int id = writer.engineView().scan(VIEWS).last().map(entry -> (int) entry.key() + 1).orElse(1); Descriptor descriptor = new Descriptor(id, name, kind, refresh, parameter, writer.generation(), writer.tenant()); writer.transaction().put(VIEWS, id, descriptor); build(writer, descriptor); @@ -164,7 +164,7 @@ public Optional find(Reader reader, String name) { } public List list(Reader reader) { - return reader.view().scan(VIEWS).values().filter(descriptor -> descriptor.tenant() == reader.tenant()).toList(); + return reader.engineView().scan(VIEWS).values().filter(descriptor -> descriptor.tenant() == reader.tenant()).toList(); } public Descriptor refresh(Principal principal, String name) { @@ -200,11 +200,11 @@ private Descriptor rebuild(Writer writer, Descriptor descriptor) { } private boolean owns(Writer writer, Descriptor descriptor, long atom) { - return writer.view().get(VIEW_DATA, cellKey(descriptor.id(), atom)).isPresent() || inTenant(writer, descriptor, atom); + return writer.engineView().get(VIEW_DATA, cellKey(descriptor.id(), atom)).isPresent() || inTenant(writer, descriptor, atom); } private static boolean inTenant(Reader reader, Descriptor descriptor, long atom) { - return reader.view().atom(atom).map(record -> record.tenant() == descriptor.tenant()).orElse(false); + return reader.engineView().atom(atom).map(record -> record.tenant() == descriptor.tenant()).orElse(false); } private static boolean touches(Reader reader, Descriptor descriptor, MemberChange change) { @@ -214,7 +214,7 @@ private static boolean touches(Reader reader, Descriptor descriptor, MemberChang public Optional read(Reader reader, String name, long key) { reader.principal().requireAdmin(); Descriptor descriptor = find(reader, name).orElseThrow(() -> unknown(name)); - return reader.view().get(VIEW_DATA, cellKey(descriptor.id(), key)) + return reader.engineView().get(VIEW_DATA, cellKey(descriptor.id(), key)) .map(cell -> new Reading(cell, staleness(reader, descriptor, reader.generation()))); } @@ -240,25 +240,25 @@ public Stream> scan(Reader reader, String name) { } private static Stream> cells(Reader reader, Descriptor descriptor) { - return reader.view().scan(VIEW_DATA).range(cellKey(descriptor.id(), 0), cellKey(descriptor.id(), (1L << KEY_BITS) - 1)); + return reader.engineView().scan(VIEW_DATA).range(cellKey(descriptor.id(), 0), cellKey(descriptor.id(), (1L << KEY_BITS) - 1)); } private void build(Writer writer, Descriptor descriptor) { switch (descriptor.kind()) { - case DEGREE -> writer.view().atoms().filter(entry -> entry.value().tenant() == descriptor.tenant()).map(Entry::key).forEach(atom -> { + case DEGREE -> writer.engineView().atoms().filter(entry -> entry.value().tenant() == descriptor.tenant()).map(Entry::key).forEach(atom -> { long degree = writer.degree(atom); if (degree > 0) { put(writer, descriptor, atom, new ViewCell.Count(degree)); } }); - case CARDINALITY -> writer.view().atoms() + case CARDINALITY -> writer.engineView().atoms() .filter(entry -> entry.value() instanceof EdgeRecord && entry.value().tenant() == descriptor.tenant()) .forEach(entry -> put(writer, descriptor, entry.key(), new ViewCell.Count(((EdgeRecord) entry.value()).cardinality()))); case ACTIVITY -> apply(writer, descriptor, database.engine().feed().replay(0) .takeWhile(event -> event.generation() <= descriptor.lastGeneration()) .filter(event -> event.branch() == Branch.MAIN) .toList()); - case OVERLAP_TOP_K -> writer.view().atoms() + case OVERLAP_TOP_K -> writer.engineView().atoms() .filter(entry -> entry.value() instanceof EdgeRecord && entry.value().tenant() == descriptor.tenant()) .map(Entry::key) .toList() @@ -286,7 +286,7 @@ private void apply(Writer writer, Descriptor descriptor, List event long added = changes.stream().filter(change -> change instanceof MemberChange.Added).count(); long removed = changes.stream().filter(change -> change instanceof MemberChange.Removed).count(); if (added + removed > 0) { - ViewCell.Activity current = writer.view().get(VIEW_DATA, cellKey(descriptor.id(), bucket)) + ViewCell.Activity current = writer.engineView().get(VIEW_DATA, cellKey(descriptor.id(), bucket)) .map(ViewCell.Activity.class::cast).orElse(new ViewCell.Activity(0, 0)); put(writer, descriptor, bucket, new ViewCell.Activity(current.added() + added, current.removed() + removed)); } @@ -296,13 +296,13 @@ private void apply(Writer writer, Descriptor descriptor, List event events.forEach(event -> event.members().forEach(change -> { affected.add(change.edge()); if (touches(writer, descriptor, change)) { - writer.view().incident(change.member()).map(IncidentEdge::edge).forEach(affected::add); + writer.engineView().incident(change.member()).map(IncidentEdge::edge).forEach(affected::add); } })); affected.forEach(edge -> { - if (writer.view().edge(edge).isPresent() && inTenant(writer, descriptor, edge)) { + if (writer.engineView().edge(edge).isPresent() && inTenant(writer, descriptor, edge)) { put(writer, descriptor, edge, topK(writer, descriptor, edge)); - } else if (writer.view().get(VIEW_DATA, cellKey(descriptor.id(), edge)).isPresent()) { + } else if (writer.engineView().get(VIEW_DATA, cellKey(descriptor.id(), edge)).isPresent()) { writer.transaction().delete(VIEW_DATA, cellKey(descriptor.id(), edge)); } }); @@ -326,7 +326,7 @@ private static Map deltas(List events, ToLongFunction ((ViewCell.Count) cell).value()).orElse(0L); long next = current + delta; if (next == 0) { @@ -343,7 +343,7 @@ private void put(Writer writer, Descriptor descriptor, long key, ViewCell cell) private static ViewCell.Ranked topK(Reader reader, Descriptor descriptor, long edge) { Tree membership = reader.edge(edge).membership(); Set candidates = new HashSet<>(); - reader.edge(edge).stream().map(Incidence::member).forEach(member -> reader.view().incident(member).map(IncidentEdge::edge) + reader.edge(edge).stream().map(Incidence::member).forEach(member -> reader.engineView().incident(member).map(IncidentEdge::edge) .filter(other -> other != edge && inTenant(reader, descriptor, other)).forEach(candidates::add)); return new ViewCell.Ranked(candidates.stream() .map(other -> new ViewCell.Neighbor(other, TreeAlgebra.countIntersect(membership, reader.edge(other).membership()))) @@ -358,7 +358,7 @@ private void onCommit(CommitEvent event) { return; } try { - database.read(reader -> reader.view().scan(VIEWS).values() + database.read(reader -> reader.engineView().scan(VIEWS).values() .filter(descriptor -> descriptor.refresh() == Refresh.CONTINUOUS) .toList()).forEach(this::refreshContinuous); } catch (RuntimeException e) { @@ -369,7 +369,7 @@ private void onCommit(CommitEvent event) { private void refreshContinuous(Descriptor descriptor) { try { database.write(TxnOptions.defaults(), Principal.SYSTEM.inTenant(descriptor.tenant()), writer -> - writer.view().get(VIEWS, descriptor.id()).map(current -> refresh(writer, current))); + writer.engineView().get(VIEWS, descriptor.id()).map(current -> refresh(writer, current))); } catch (RuntimeException e) { LOG.log(System.Logger.Level.WARNING, "continuous refresh of view " + descriptor.id() + " failed", e); } diff --git a/database/src/test/java/io/hstore/db/SecurityTest.java b/database/src/test/java/io/hstore/db/SecurityTest.java index acf138f..c1821a1 100644 --- a/database/src/test/java/io/hstore/db/SecurityTest.java +++ b/database/src/test/java/io/hstore/db/SecurityTest.java @@ -217,7 +217,7 @@ void dropViewIsTenantScopedAndNeedsAdmin() { carol.execute("DROP VIEW shared"); assertTrue(carol.execute("SHOW VIEWS").rows().isEmpty()); assertEquals(1, keys(dave.execute("VIEW shared")).size()); - assertEquals(1L, (long) database.read(reader -> reader.view().scan(MaterializedViews.VIEW_DATA).values().count())); + assertEquals(1L, (long) database.read(reader -> reader.engineView().scan(MaterializedViews.VIEW_DATA).values().count())); String again = assertThrows(HStoreException.InvalidSchema.class, () -> carol.execute("DROP VIEW shared")).getMessage(); String missing = assertThrows(HStoreException.InvalidSchema.class, () -> carol.execute("DROP VIEW nothing")).getMessage(); assertEquals(missing.replace("nothing", "shared"), again); @@ -367,6 +367,52 @@ void rankStaysInsideTheTenant() { } } + @Test + void queriesNeverFollowMembershipsIntoAnotherTenant() { + try (Session alice = login("alice", "s3cret-pass"); Session bob = login("bob", "hunter2-pass")) { + alice.executeAll(""" + INSERT NODE Patient 'p1' {mrn: 'A-1'} AS $p; + INSERT NODE Patient 'p2' {mrn: 'A-2'} AS $o; + INSERT EDGE Encounter MEMBERS ($p AS patient, $o AS clinician) AS $e; + """); + bob.executeAll("INSERT NODE Patient 'b1' {mrn: 'B-1'} AS $q; INSERT EDGE Encounter MEMBERS ($q AS patient) AS $g;"); + long p = alice.variables().get("p"); + long e = alice.variables().get("e"); + long q = bob.variables().get("q"); + long g = bob.variables().get("g"); + int patient = database.engine().dictionary().roleSet(List.of("patient")); + database.write(writer -> { + writer.transaction().upsert(g, p, _ -> new Incidence(p, patient, Weight.ONE, Long.MIN_VALUE, Long.MAX_VALUE, 0, 0)); + return null; + }); + List statements = List.of( + "MATCH EDGE x:Encounter WHERE x HAS @" + p + " RETURN x", + "MATCH EDGE x:Encounter WHERE x HAS @" + p + " AS patient RETURN x", + "MATCH EDGE x:Encounter WHERE x CONTAINS (@" + p + ") RETURN x", + "MATCH EDGE x WHERE x CONTAINS (@" + p + ") RETURN x", + "MATCH NODE n:Patient WHERE n VALID AT 5 RETURN n", + "INCIDENT TO @" + p, + "INCIDENT TO @" + p + " VALID AT 5", + "NEIGHBORS OF @" + p, + "NEIGHBORS OF @" + e + " THRESHOLD 1", + "OVERLAP JOIN Encounter THRESHOLD 1", + "PATTERN (x: EDGE Encounter, n: NODE Patient) WHERE x CONTAINS n AND n = @" + p, + "PATTERN (x: EDGE Encounter, n: NODE Patient) WHERE x HAS n AS patient AND n = @" + p, + "PATTERN (x: EDGE Encounter, y: EDGE Encounter) WHERE SHARED(x, y) >= 1", + "RANK EDGE x:Encounter LIKE (@" + p + " AS patient) TOP 10", + "RANK EDGE x LIKE (@" + p + " REQUIRED) TOP 10", + "EXPAND @" + e + " DEPTH 3", + "MATCH NODE n:Patient WHERE n = @" + p + " RETURN degree(n)"); + for (String statement : statements) { + String rendered = alice.execute(statement).render(); + assertFalse(rendered.contains("@" + g) || rendered.contains("@" + q), statement + " returned another tenant's atom:\n" + rendered); + } + assertEquals("1", cell(alice.execute("MATCH NODE n:Patient WHERE n = @" + p + " RETURN degree(n)"))); + assertEquals(1, alice.execute("INCIDENT TO @" + p).rows().size()); + assertEquals(1, alice.execute("MATCH EDGE x WHERE x CONTAINS (@" + p + ") RETURN x").rows().size()); + } + } + @Test void qualifyOnlyAcceptsVisibleMembers() { try (Session alice = login("alice", "s3cret-pass"); Session bob = login("bob", "hunter2-pass")) { @@ -796,7 +842,7 @@ private Optional scram(String user, String password) { @Test void anOldPasswordHashIsNeverEnoughToSignIn() throws Exception { - Security.User alice = database.read(reader -> Security.user(reader.view(), "alice")).orElseThrow(); + Security.User alice = database.read(reader -> Security.user(reader.engineView(), "alice")).orElseThrow(); assertTrue(alice.hash().startsWith("SCRAM-SHA-256$600000:"), alice.hash()); Scram.Verifier aliceVerifier = Security.verifier(alice).orElseThrow(); try (Session session = new Session(database)) { @@ -816,23 +862,23 @@ void anOldPasswordHashIsNeverEnoughToSignIn() throws Exception { database.close(); database = HypergraphDatabase.open(directory, options()); admin = new Session(database); - assertEquals(legacy, database.read(reader -> Security.user(reader.view(), "old")).orElseThrow(), "opening converts nothing"); - assertEquals(List.of("old"), database.read(reader -> Security.legacyUsers(reader.view()))); + assertEquals(legacy, database.read(reader -> Security.user(reader.engineView(), "old")).orElseThrow(), "opening converts nothing"); + assertEquals(List.of("old"), database.read(reader -> Security.legacyUsers(reader.engineView()))); try (Session session = new Session(database)) { assertEquals(Optional.empty(), signInWithSaltedPassword(session, "old", stolen)); } assertEquals(Optional.empty(), scram("old", "legacy-pass"), "an old hash cannot be used with SCRAM, even with the password"); assertFalse(new Session(database).authenticate("old", "wrong-pass")); - assertEquals(legacy, database.read(reader -> Security.user(reader.view(), "old")).orElseThrow()); + assertEquals(legacy, database.read(reader -> Security.user(reader.engineView(), "old")).orElseThrow()); assertTrue(new Session(database).authenticate("old", "legacy-pass")); - Security.User converted = database.read(reader -> Security.user(reader.view(), "old")).orElseThrow(); + Security.User converted = database.read(reader -> Security.user(reader.engineView(), "old")).orElseThrow(); assertFalse(Security.legacy(converted)); Scram.Verifier fresh = Security.verifier(converted).orElseThrow(); assertFalse(Arrays.equals(salt, fresh.salt()), "the conversion picks a new salt"); assertEquals(Security.ITERATIONS, fresh.iterations()); - assertEquals(List.of(), database.read(reader -> Security.legacyUsers(reader.view()))); + assertEquals(List.of(), database.read(reader -> Security.legacyUsers(reader.engineView()))); try (Session session = new Session(database)) { assertEquals(Optional.empty(), signInWithSaltedPassword(session, "old", stolen)); } @@ -1002,7 +1048,7 @@ void qualifiersMustBeTheWritersOwnAssertions() { QUALIFY $b IN $e AS OBSERVED; """); } - long foreignAssertion = database.read(reader -> reader.view().scan(io.hstore.db.evidence.Provenance.QUALIFIERS).keys().findFirst()).orElseThrow(); + long foreignAssertion = database.read(reader -> reader.engineView().scan(io.hstore.db.evidence.Provenance.QUALIFIERS).keys().findFirst()).orElseThrow(); try (Session alice = login("alice", "s3cret-pass")) { alice.executeAll("INSERT NODE Patient 'a' {mrn: 'A'} AS $a; INSERT EDGE Encounter MEMBERS ($a AS patient) AS $e;"); long patient = alice.variables().get("a"); @@ -1017,6 +1063,6 @@ void qualifiersMustBeTheWritersOwnAssertions() { } private int tenant(String name) { - return database.read(reader -> io.hstore.db.security.Security.tenant(reader.view(), name)).orElseThrow().id(); + return database.read(reader -> io.hstore.db.security.Security.tenant(reader.engineView(), name)).orElseThrow().id(); } } diff --git a/database/src/test/java/io/hstore/db/TenantBoundaryTest.java b/database/src/test/java/io/hstore/db/TenantBoundaryTest.java new file mode 100644 index 0000000..b66f1f2 --- /dev/null +++ b/database/src/test/java/io/hstore/db/TenantBoundaryTest.java @@ -0,0 +1,60 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db; + +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static java.util.Map.entry; +import static org.junit.jupiter.api.Assertions.assertEquals; + +class TenantBoundaryTest { + + static final Map ENGINE_WIDE = Map.ofEntries( + entry("io/hstore/db/Reader.java", "defines engineView and the tenant-checked lookups built on it"), + entry("io/hstore/db/Writer.java", "index backfills and embedding checks that run for every tenant or on shared structures"), + entry("io/hstore/db/HypergraphDatabase.java", "users, tenants, quotas and the SCRAM secret, which are not tenant data"), + entry("io/hstore/db/query/Session.java", "the session's own user and tenant records"), + entry("io/hstore/db/query/Executor.java", "user, tenant and quota administration and the shared schema listing"), + entry("io/hstore/db/query/Planner.java", "property index estimates keyed by the reader's tenant"), + entry("io/hstore/db/evidence/Provenance.java", "qualifier and evidence records filtered by their stamped tenant"), + entry("io/hstore/db/signal/Signals.java", "signals of an atom that Reader.visible accepted"), + entry("io/hstore/db/temporal/StateBindings.java", "state of an atom that Reader.visible accepted"), + entry("io/hstore/db/temporal/Temporal.java", "DIFF over whole generations, for global admins only"), + entry("io/hstore/db/temporal/BranchMerge.java", "branch merges, for global admins only"), + entry("io/hstore/db/view/MaterializedViews.java", "view maintenance, filtered by the view's tenant")); + + static List rawViewUsers(Path root) throws IOException { + try (Stream files = Files.walk(root)) { + return files.filter(file -> file.toString().endsWith(".java")) + .filter(file -> { + try { + return Files.readString(file).contains("engineView()"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }) + .map(file -> root.relativize(file).toString().replace(File.separatorChar, '/')) + .sorted() + .toList(); + } + } + + @Test + void onlyEngineWideCodeReadsTheRawView() throws IOException { + List users = rawViewUsers(Path.of("src/main/java")); + assertEquals(List.of(), users.stream().filter(file -> !ENGINE_WIDE.containsKey(file)).toList(), + "query code must use the tenant-checked methods of Reader; engine-wide code belongs in ENGINE_WIDE with a reason"); + assertEquals(List.of(), ENGINE_WIDE.keySet().stream().filter(file -> !users.contains(file)).sorted().toList(), + "these files no longer read the raw view; remove them from ENGINE_WIDE"); + } +} diff --git a/docs/database/security.md b/docs/database/security.md index 3d0de34..ec992e9 100644 --- a/docs/database/security.md +++ b/docs/database/security.md @@ -79,17 +79,29 @@ Every `Reader` method that accepts an atom id calls `visible` (or filters by ten in the default tenant and the type index holds no atom of another tenant, every atom is visible and a missing atom has no incidence entries anyway, so the extra catalog lookup is skipped. The reader checks this once per snapshot. -Code that needs the raw engine `View` (`reader.view()`) may only pass it ids that are already known to belong to the -reader's tenant, such as bound query variables or atoms from `atomsOfType`. Ids that come from a statement go -through `Reader` first. The `MemberScan` and `IncidentIntersection` access paths return nothing for an id the -reader cannot see. +Query code reads through `Reader` only. Besides the methods above, `Reader` offers the tenant-checked lookups that +query paths need: `incidence(edge, member)` and `contains(edge, member)` (the edge must be the reader's and the +member owned, checked inside the same catalog lookup), `incidentEdges(atom)` (edges the reader owns), +`hyperedge(id)`, `isEdge(id)`, and `incidentTree(atom)` / `incidentCursor(atom)` (empty for an atom the reader +cannot see; the edge ids they yield still go through `Executor.admits` or `owns` before use). `MATCH`, `RANK`, +`PATTERN`, HORA operators and temporal queries use nothing else. + +The raw engine `View` is available as `reader.engineView()` for code that works across tenants by design: user, +tenant and quota records, the SCRAM secret, the shared schema listing, branch merges and `DIFF` (global admins +only), materialized view maintenance (filtered by the view's tenant), and records that carry their own tenant +stamp (evidence, qualifiers, state, signals). `TenantBoundaryTest` in the database and server modules lists the +files allowed to call it, each with its reason, and fails when any other file does, or when a listed file no +longer needs it. Adding a raw read therefore means stating why it is safe in that list. Writes call `require`, so a principal cannot modify an atom it cannot see. A hyperedge may only gain members that are visible to the writer (`Writer.incidence`, also used by `Writer.qualify`), so a tenant cannot reference another tenant's atoms. Evidence sources, assertion sources and the evidence list of `QUALIFY` must be atoms, evidence or -assertions of the writer's tenant (`Provenance.record/assertion`). `hstore check` lists any membership on main -whose member belongs to a different tenant than its edge, with the `REMOVE` statement that undoes it. Databases -written before these checks existed could have such memberships from `QUALIFY`. +assertions of the writer's tenant (`Provenance.record/assertion`). `hstore check` lists any membership on an active branch +whose member belongs to a different tenant than its edge, with the statement that undoes it, and +`hstore check --repair-tenants` removes them in one transaction per branch. Databases written before these checks +existed could have such memberships from `QUALIFY`. A new database, and one that `hstore check` found clean or +repaired, records that in `EngineSlots.SETTINGS`; `hstore serve` logs a WARNING at startup for a database without +that record. The planner only uses numbers from the reader's tenant: untyped matches are estimated from `Reader.population` (the sum of the tenant's per-type counts), and the feedback that corrects estimates after each query is kept per diff --git a/docs/operations/cli.md b/docs/operations/cli.md index 19575ce..b4d307b 100644 --- a/docs/operations/cli.md +++ b/docs/operations/cli.md @@ -14,7 +14,7 @@ usage: hstore [options] ping HOST:PORT exit 0 when a server accepts connections verifier [--password PW] print a SCRAM-SHA-256 verifier to use as a CREATE USER password bench [--scenario NAME] [--scale N] adversarial storage benchmarks - check verify checksums and tree invariants + check [--repair-tenants] verify checksums and tree invariants; repair memberships across tenants config [] print effective settings version ``` @@ -213,6 +213,22 @@ reference counts match the trees: 20 pages in use, 423 waiting for older readers A violated invariant aborts the check with the offending page and a reason such as `leaf keys out of order in ` or `unbalanced tree in `. +The report ends with a tenant isolation line. Older versions could leave a hyperedge with a member from another +tenant; the check lists each such membership, on every active branch, with the statement that undoes it. With `--repair-tenants` +(after the directory), `check` first removes all of them, in one transaction per branch, and lists what it removed: + +``` +$ hstore check /srv/hstore --repair-tenants +tenant isolation: removed 1 memberships that crossed tenants: + USE TENANT globex; REMOVE @14 FROM @12; -- member of tenant 1 in an edge of tenant 2 +generation 160, recovered 0 commits from the log +… +tenant isolation: no hyperedge on any branch has a member from another tenant +``` + +A check that finds no such membership, and every repair, records that the database is clean, which silences the +startup warning of `hstore serve` (see [logging](logging.md)). + ### `bench` The storage-engine stress suite from the paper's adversarial evaluation. It runs against a scratch diff --git a/docs/operations/logging.md b/docs/operations/logging.md index 53480d0..378d2a3 100644 --- a/docs/operations/logging.md +++ b/docs/operations/logging.md @@ -144,6 +144,7 @@ and in Docker it is the last line of `docker logs`. The storage-integrity failur | WARNING | `user "{user}" failed to sign in 50 times within 15 minutes; someone may be guessing its password from several addresses` (logger `hstore.security`). | | WARNING | `user "{user}" has a password stored by HStore 0.1, which cannot be used with SCRAM-SHA-256; it can sign in once through Studio to convert it, or an administrator can set a new one with ALTER USER` (logger `hstore.security`, at most once every 10 minutes per user): the client only sees `authentication failed`. | | INFO | `the password of user "{user}" is now stored as a SCRAM-SHA-256 verifier` (logger `hstore.security`): a user with a password from HStore 0.1 signed in through Studio or an `AUTHENTICATE` statement. | +| WARNING | at startup, `this database was written by a version that did not keep hyperedge members inside their tenant; …`: run `hstore check` and `hstore check --repair-tenants` ([cli](cli.md)). | | WARNING | at startup, `authentication = off and the server listens on {address}: …` or `the database has no users, so connections from other hosts are refused until CREATE USER creates one`, when the listen address is not loopback. | | WARNING | at startup, `{n} users have passwords stored by HStore 0.1 and cannot sign in over this port until each signs in once through Studio or gets a new password with ALTER USER: {names}`. | | LOG | `studio session opened for user={user} role={ROLE} from /{ip}:{port}` | diff --git a/docs/storage/maintenance.md b/docs/storage/maintenance.md index 67f7945..cf8919f 100644 --- a/docs/storage/maintenance.md +++ b/docs/storage/maintenance.md @@ -319,12 +319,11 @@ branch main catalog 458 entries 1 pages 85 nested trees height 1 verified 20 pages: checksums, ordering, balance and summaries are consistent reference counts match the trees: 20 pages in use, 423 waiting for older readers -tenant isolation: no hyperedge on main has a member from another tenant +tenant isolation: no hyperedge on any branch has a member from another tenant ``` -The last line comes from a scan of every hyperedge on main that compares each member's tenant with the edge's. If -some differ, it prints them as `REMOVE @member FROM @edge` statements for the edge's tenant to run (up to 50 of -them). Such memberships can only exist in databases written before `QUALIFY` checked its member +The last line comes from a scan of every hyperedge on every active branch that compares each member's tenant with +the edge's. If some differ, it prints statements that remove them, `USE TENANT ; [USE BRANCH ;] REMOVE @member FROM @edge;`, for an admin of the default tenant to run as printed (up to 50 of them). `hstore check --repair-tenants` removes them all, repeating until a fresh scan finds none, and only then records the database as checked. Such memberships can only exist in databases written before `QUALIFY` checked its member ([security](../database/security.md#tenants)). They don't make the command fail. A failure names the page, for example `HStoreException.corrupt(pageId, "checksum mismatch")` or `"separator does not route child 3 in atoms#1"`, and the command exits non-zero. diff --git a/engine/src/main/java/io/hstore/engine/txn/View.java b/engine/src/main/java/io/hstore/engine/txn/View.java index ea89f1d..6f788b6 100644 --- a/engine/src/main/java/io/hstore/engine/txn/View.java +++ b/engine/src/main/java/io/hstore/engine/txn/View.java @@ -108,6 +108,16 @@ public Optional incidence(long edge, long member) { : Optional.empty()); } + public Optional incidence(long edge, long member, int tenant) { + return atom(edge).flatMap(record -> record instanceof EdgeRecord found && found.tenant() == tenant + ? hyperedge(edge, found, false).get(member) + : Optional.empty()); + } + + public boolean contains(long edge, long member, int tenant) { + return incidence(edge, member, tenant).isPresent(); + } + public long cardinality(long edge) { return atom(edge).map(record -> record instanceof EdgeRecord found ? found.cardinality() : 0L).orElse(0L); } diff --git a/server/src/main/java/io/hstore/server/Check.java b/server/src/main/java/io/hstore/server/Check.java index fad147a..de972e7 100644 --- a/server/src/main/java/io/hstore/server/Check.java +++ b/server/src/main/java/io/hstore/server/Check.java @@ -3,19 +3,19 @@ package io.hstore.server; +import io.hstore.db.HypergraphDatabase; +import io.hstore.db.security.TenantMemberships; import io.hstore.engine.StorageEngine; -import io.hstore.engine.catalog.AtomRecord; import io.hstore.engine.catalog.Branch; import io.hstore.engine.catalog.Generation; import io.hstore.engine.catalog.Slot; import io.hstore.engine.maintenance.ReferenceCounter; import io.hstore.engine.tree.TreeVerifier; import io.hstore.engine.txn.Snapshot; -import io.hstore.engine.txn.View; import java.util.List; import java.util.Map; -import java.util.stream.Stream; +import java.util.stream.Collectors; final class Check { @@ -25,9 +25,32 @@ private Check() { } static String run(StorageEngine engine) { + String report; + boolean clean; + boolean checked; try (Snapshot pinned = engine.snapshot()) { - return run(engine, pinned.catalog()) + "\n" + isolation(pinned); + List crossings = TenantMemberships.find(engine); + report = run(engine, pinned.catalog()) + "\n" + isolation(crossings); + clean = crossings.isEmpty(); + checked = TenantMemberships.checked(pinned); } + if (clean && !checked) { + engine.write(txn -> { + TenantMemberships.markChecked(txn); + return null; + }); + } + return report; + } + + static String repair(HypergraphDatabase database) { + List removed = database.repairTenantMemberships(); + if (removed.isEmpty()) { + return "tenant isolation: no hyperedge on any branch had a member from another tenant; nothing to repair"; + } + return "tenant isolation: removed " + removed.size() + " memberships that crossed tenants:\n" + + removed.stream().map(crossing -> " " + crossing.repair() + " -- member of tenant " + crossing.memberTenant() + + " in an edge of tenant " + crossing.edgeTenant()).collect(Collectors.joining("\n")); } private static String run(StorageEngine engine, Generation generation) { @@ -56,19 +79,14 @@ private static String run(StorageEngine engine, Generation generation) { return report.toString(); } - private static String isolation(View view) { - List crossings = view.atoms() - .flatMap(entry -> entry.value() instanceof AtomRecord.EdgeRecord edge - ? view.edge(entry.key(), edge).memberIds() - .filter(member -> view.atom(member).map(record -> record.tenant() != edge.tenant()).orElse(false)) - .mapToObj(member -> " REMOVE @" + member + " FROM @" + entry.key() + " -- member of tenant " - + view.atom(member).orElseThrow().tenant() + " in an edge of tenant " + edge.tenant()) - : Stream.empty()) - .toList(); + private static String isolation(List found) { + List crossings = found.stream().map(crossing -> " " + crossing.repair() + " -- member of tenant " + + crossing.memberTenant() + " in an edge of tenant " + crossing.edgeTenant()).toList(); if (crossings.isEmpty()) { - return "tenant isolation: no hyperedge on main has a member from another tenant"; + return "tenant isolation: no hyperedge on any branch has a member from another tenant"; } - return "tenant isolation: " + crossings.size() + " memberships on main cross tenants; run as the edge's tenant:\n" + return "tenant isolation: " + crossings.size() + " memberships cross tenants; remove them with " + + "hstore check --repair-tenants, or run these as an admin of the default tenant:\n" + String.join("\n", crossings.subList(0, Math.min(MAX_LISTED, crossings.size()))) + (crossings.size() > MAX_LISTED ? "\n ... " + (crossings.size() - MAX_LISTED) + " more" : ""); } diff --git a/server/src/main/java/io/hstore/server/Main.java b/server/src/main/java/io/hstore/server/Main.java index cb066e0..d064667 100644 --- a/server/src/main/java/io/hstore/server/Main.java +++ b/server/src/main/java/io/hstore/server/Main.java @@ -43,7 +43,7 @@ public final class Main { ping HOST:PORT exit 0 when a server accepts connections verifier [--password PW] print a SCRAM-SHA-256 verifier to use as a CREATE USER password bench [--scenario NAME] [--scale N] adversarial storage benchmarks - check verify checksums and tree invariants + check [--repair-tenants] verify checksums and tree invariants; repair memberships across tenants config [] print effective settings version settings are read from /hstore.conf, then HSTORE_ variables, then -- flags. @@ -74,7 +74,7 @@ record Verifier(Optional password) implements Command { record Bench(Path directory, String scenario, int scale) implements Command { } - record Check(Path directory) implements Command { + record Check(Path directory, boolean repairTenants) implements Command { } record Config(Optional directory) implements Command { @@ -159,7 +159,7 @@ private static Command command(Arguments arguments) { case "verifier" -> new Command.Verifier(arguments.option("password").or(() -> Optional.ofNullable(environment.get("HSTORE_PASSWORD")))); case "bench" -> new Command.Bench(directory(arguments), arguments.option("scenario").orElse("all"), Integer.parseInt(arguments.option("scale").orElse("1"))); - case "check" -> new Command.Check(directory(arguments)); + case "check" -> new Command.Check(directory(arguments), arguments.option("repair-tenants").isPresent()); case "config" -> new Command.Config(arguments.positional().size() > 1 ? Optional.of(directory(arguments)) : Optional.empty()); case "version", "--version" -> new Command.Version(); case "help", "--help", "-h" -> new Command.Help(); @@ -250,9 +250,14 @@ private static int run(Command command, Arguments arguments) { new Bench(directory, config(Optional.empty(), arguments).engineOptions(), scale).run(scenario); yield 0; } - case Command.Check(Path directory) -> { + case Command.Check(Path directory, boolean repairTenants) -> { ServerConfig config = config(Optional.of(directory), arguments); Logging.configure(Level.WARNING, "", 1, 1); + if (repairTenants) { + try (HypergraphDatabase database = HypergraphDatabase.open(directory, config.databaseOptions())) { + IO.println(Check.repair(database)); + } + } try (StorageEngine engine = StorageEngine.open(directory, config.engineOptions().withExtensions( HypergraphDatabase.extensionSlots(), List.of()))) { IO.println(Check.run(engine)); @@ -309,6 +314,10 @@ private static int serve(Path directory, Arguments arguments) { } Optional tls = config.serverTls(); HypergraphDatabase database = HypergraphDatabase.open(directory, config.databaseOptions()); + if (!database.tenantMembershipsChecked()) { + log.log(System.Logger.Level.WARNING, "this database was written by a version that did not keep hyperedge members inside their " + + "tenant; run hstore check on it to list memberships that cross tenants, and hstore check --repair-tenants to remove them"); + } Server.Policy policy = Server.Policy.from(config, tls); Server server = new Server(database, config.address(), policy); Optional studio = config.flag(Setting.STUDIO) ? Optional.of(new Studio(database, new Studio.Options( diff --git a/server/src/main/java/io/hstore/server/studio/Neighborhood.java b/server/src/main/java/io/hstore/server/studio/Neighborhood.java index 6671242..22cbb73 100644 --- a/server/src/main/java/io/hstore/server/studio/Neighborhood.java +++ b/server/src/main/java/io/hstore/server/studio/Neighborhood.java @@ -65,7 +65,7 @@ static Json.Obj sample(Reader reader, Optional type, Limits limits) { if (type.isPresent()) { return around(reader, reader.atoms(type.get()).limit(limits.atoms() / 2), 0, true, limits); } - List types = reader.schema().types(reader.view()); + List types = reader.schema().types(reader.engineView()); List edges = types.stream().filter(candidate -> candidate.kind().isEdge()).toList(); List chosen = edges.isEmpty() ? types : edges; int perType = Math.max(1, limits.atoms() / 4 / Math.max(1, chosen.size())); diff --git a/server/src/main/java/io/hstore/server/studio/Studio.java b/server/src/main/java/io/hstore/server/studio/Studio.java index 21b14af..01b264c 100644 --- a/server/src/main/java/io/hstore/server/studio/Studio.java +++ b/server/src/main/java/io/hstore/server/studio/Studio.java @@ -282,7 +282,7 @@ private boolean current(Sessions.Entry entry) { private Json describe(Session session) { Principal principal = session.principal(); - String tenant = database.read(reader -> Security.tenant(reader.view(), principal.tenant()) + String tenant = database.read(reader -> Security.tenant(reader.engineView(), principal.tenant()) .map(Security.Tenant::name).orElse("#" + principal.tenant())); String branch = database.engine().branches().stream().filter(candidate -> candidate.id() == session.branch()) .map(Branch::name).findFirst().orElse("main"); @@ -431,7 +431,7 @@ private Response schema(Request request) { Session session = session(request).session(); return Response.json(database.read(session.branch(), session.principal(), reader -> JsonFields.object() .put("generation", reader.generation()) - .put("types", JsonFields.array(reader.schema().types(reader.view()).stream() + .put("types", JsonFields.array(reader.schema().types(reader.engineView()).stream() .sorted(Comparator.comparing(TypeDef::kind).thenComparing(TypeDef::name)) .map(type -> type(reader, type)))) .put("branches", JsonFields.array(database.engine().branches().stream() diff --git a/server/src/test/java/io/hstore/server/ServerTest.java b/server/src/test/java/io/hstore/server/ServerTest.java index f09d4c7..fc579a7 100644 --- a/server/src/test/java/io/hstore/server/ServerTest.java +++ b/server/src/test/java/io/hstore/server/ServerTest.java @@ -7,8 +7,12 @@ import io.hstore.db.HypergraphDatabase; import io.hstore.db.query.Parser; import io.hstore.db.query.Session; +import io.hstore.db.security.Principal; +import io.hstore.db.security.Role; import io.hstore.engine.Dictionary; import io.hstore.engine.EngineOptions; +import io.hstore.engine.StorageEngine; +import io.hstore.engine.catalog.Branch; import io.hstore.engine.topology.Incidence; import io.hstore.engine.topology.Validity; import io.hstore.engine.topology.Weight; @@ -37,6 +41,7 @@ import java.util.List; import java.util.Map; import java.util.Optional; +import java.util.Set; import java.util.concurrent.Callable; import java.util.concurrent.ExecutorService; import java.util.concurrent.Executors; @@ -470,6 +475,49 @@ void clearTextPasswordsAreRefusedWithAnUpgradeMessage() throws Exception { } } + @Test + void checkFindsAndRepairsMembershipsAcrossTenants() throws Exception { + DatabaseOptions options = DatabaseOptions.defaults().withEngine(EngineOptions.defaults().withPageSize(4096)); + long[] ids; + try (HypergraphDatabase database = HypergraphDatabase.open(directory, options)) { + assertTrue(database.tenantMembershipsChecked()); + Session admin = new Session(database); + admin.executeAll(""" + CREATE NODE TYPE Patient; + CREATE SET EDGE TYPE Encounter ROLES (patient); + CREATE TENANT acme; + CREATE TENANT globex; + """); + Session acme = new Session(database, new Principal("a", 1, Role.ADMIN)); + Session globex = new Session(database, new Principal("g", 2, Role.ADMIN)); + acme.executeAll("INSERT NODE Patient 'p' AS $p; INSERT EDGE Encounter MEMBERS ($p AS patient) AS $e;"); + globex.executeAll("INSERT NODE Patient 'q' AS $q; INSERT EDGE Encounter MEMBERS ($q AS patient) AS $g;"); + long p = acme.variables().get("p"); + long g = globex.variables().get("g"); + database.write(writer -> { + writer.transaction().upsert(g, p, _ -> Incidence.of(p)); + return null; + }); + database.engine().createBranch("side", Branch.MAIN); + ids = new long[]{p, g}; + } + try (StorageEngine engine = StorageEngine.open(directory, options.engine().withExtensions(HypergraphDatabase.extensionSlots(), List.of()))) { + String report = Check.run(engine); + assertTrue(report.contains("2 memberships cross tenants"), report); + assertTrue(report.contains("USE TENANT globex; USE BRANCH side; REMOVE @" + ids[0] + " FROM @" + ids[1] + ";"), report); + } + try (HypergraphDatabase database = HypergraphDatabase.open(directory, options)) { + String repaired = Check.repair(database); + assertTrue(repaired.contains("removed 2 memberships") && repaired.contains("REMOVE @" + ids[0] + " FROM @" + ids[1]), repaired); + assertTrue(database.tenantMembershipsChecked()); + assertTrue(Check.repair(database).contains("nothing to repair")); + assertEquals(1L, (long) database.read(reader -> reader.engineView().cardinality(ids[1]))); + } + try (StorageEngine engine = StorageEngine.open(directory, options.engine().withExtensions(HypergraphDatabase.extensionSlots(), List.of()))) { + assertTrue(Check.run(engine).contains("no hyperedge on any branch has a member from another tenant")); + } + } + private static List exchange(InetSocketAddress address, String payload) throws Exception { try (Socket socket = new Socket(address.getHostString(), address.getPort())) { socket.setSoTimeout(10_000); @@ -514,15 +562,25 @@ void checkListsMembershipsThatCrossTenants() { """); long patient = admin.variables().get("a"); long edge = admin.variables().get("e"); - assertTrue(Check.run(database.engine()).contains("no hyperedge on main has a member from another tenant")); + assertTrue(Check.run(database.engine()).contains("no hyperedge on any branch has a member from another tenant")); database.write(writer -> { writer.transaction().insert(edge, new Incidence(patient, Dictionary.NO_ROLES, Weight.ONE, Validity.ALWAYS.from(), Validity.ALWAYS.to(), 0, 0)); return null; }); + database.engine().createBranch("side", Branch.MAIN); String report = Check.run(database.engine()); - assertTrue(report.contains("1 memberships on main cross tenants"), report); - assertTrue(report.contains("REMOVE @" + patient + " FROM @" + edge), report); + assertTrue(report.contains("2 memberships cross tenants"), report); + List statements = report.lines().filter(line -> line.startsWith(" USE TENANT")) + .map(line -> line.substring(2, line.indexOf(" -- "))).toList(); + assertEquals(Set.of("USE TENANT globex; REMOVE @" + patient + " FROM @" + edge + ";", + "USE TENANT globex; USE BRANCH side; REMOVE @" + patient + " FROM @" + edge + ";"), Set.copyOf(statements)); + for (String statement : statements) { + try (Session global = new Session(database, new Principal("root", Principal.DEFAULT_TENANT, Role.ADMIN))) { + global.executeAll(statement); + } + } + assertTrue(Check.run(database.engine()).contains("no hyperedge on any branch has a member from another tenant")); } } diff --git a/server/src/test/java/io/hstore/server/TenantBoundaryTest.java b/server/src/test/java/io/hstore/server/TenantBoundaryTest.java new file mode 100644 index 0000000..8de6389 --- /dev/null +++ b/server/src/test/java/io/hstore/server/TenantBoundaryTest.java @@ -0,0 +1,47 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.server; + +import org.junit.jupiter.api.Test; + +import java.io.File; +import java.io.IOException; +import java.io.UncheckedIOException; +import java.nio.file.Files; +import java.nio.file.Path; +import java.util.List; +import java.util.Map; +import java.util.stream.Stream; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class TenantBoundaryTest { + + private static final Map ENGINE_WIDE = Map.of( + "io/hstore/server/studio/Studio.java", "the shared schema listing and the session's tenant name", + "io/hstore/server/studio/Neighborhood.java", "the shared schema listing"); + + @Test + void onlyEngineWideCodeReadsTheRawView() throws IOException { + Path root = Path.of("src/main/java"); + List users; + try (Stream files = Files.walk(root)) { + users = files.filter(file -> file.toString().endsWith(".java")) + .filter(file -> { + try { + return Files.readString(file).contains("engineView()"); + } catch (IOException e) { + throw new UncheckedIOException(e); + } + }) + .map(file -> root.relativize(file).toString().replace(File.separatorChar, '/')) + .sorted() + .toList(); + } + assertEquals(List.of(), users.stream().filter(file -> !ENGINE_WIDE.containsKey(file)).toList(), + "server code must use the tenant-checked methods of Reader; engine-wide code belongs in ENGINE_WIDE with a reason"); + assertEquals(List.of(), ENGINE_WIDE.keySet().stream().filter(file -> !users.contains(file)).sorted().toList(), + "these files no longer read the raw view; remove them from ENGINE_WIDE"); + } +}