Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
6 changes: 3 additions & 3 deletions benchmarks/src/main/java/io/hstore/bench/HStoreStore.java
Original file line number Diff line number Diff line change
Expand Up @@ -154,8 +154,8 @@ public long coMembership(int[][] pairs, int from, int to) {
return database.read(reader -> {
long total = 0;
for (int i = from; i < to; i++) {
total += TreeAlgebra.intersectKeys(reader.view().incidentTree(nodes[pairs[i][0]]),
reader.view().incidentTree(nodes[pairs[i][1]])).count();
total += TreeAlgebra.intersectKeys(reader.engineView().incidentTree(nodes[pairs[i][0]]),
reader.engineView().incidentTree(nodes[pairs[i][1]])).count();
}
return total;
});
Expand Down Expand Up @@ -209,7 +209,7 @@ public long probeLargeEdge(int[] probes, int from, int to) {
return database.read(reader -> {
long found = 0;
for (int i = from; i < to; i++) {
found += reader.view().incidence(large, nodes[probes[i]]).isPresent() ? 1 : 0;
found += reader.engineView().incidence(large, nodes[probes[i]]).isPresent() ? 1 : 0;
}
return found;
});
Expand Down
45 changes: 37 additions & 8 deletions database/src/main/java/io/hstore/db/HypergraphDatabase.java
Original file line number Diff line number Diff line change
Expand Up @@ -14,6 +14,7 @@
import io.hstore.db.security.Security;
import io.hstore.db.security.Scram;
import io.hstore.db.security.SignIns;
import io.hstore.db.security.TenantMemberships;
import io.hstore.db.semantic.Embedding;
import io.hstore.db.semantic.SemanticPlane;
import io.hstore.db.signal.Signals;
Expand All @@ -31,6 +32,7 @@

import java.nio.file.Path;
import java.time.Duration;
import java.util.ArrayList;
import java.util.List;
import java.util.Map;
import java.util.Optional;
Expand Down Expand Up @@ -77,23 +79,50 @@ private HypergraphDatabase(Path directory, DatabaseOptions options) {
}

private byte[] prepareSignIns() {
if (read(reader -> Security.scramSecret(reader.view())).isEmpty()) {
if (read(reader -> Security.scramSecret(reader.engineView())).isEmpty()) {
write(writer -> {
if (Security.scramSecret(writer.transaction()).isEmpty()) {
Security.createScramSecret(writer.transaction());
}
if (!TenantMemberships.checked(writer.transaction()) && writer.transaction().atoms().findAny().isEmpty()) {
TenantMemberships.markChecked(writer.transaction());
}
return null;
});
}
return read(reader -> Security.scramSecret(reader.view())).orElseThrow();
return read(reader -> Security.scramSecret(reader.engineView())).orElseThrow();
}

public boolean tenantMembershipsChecked() {
return read(reader -> TenantMemberships.checked(reader.engineView()));
}

public List<TenantMemberships.Crossing> repairTenantMemberships() {
List<TenantMemberships.Crossing> removed = new ArrayList<>();
for (int round = 0; round < 5 && (round == 0 || !TenantMemberships.find(engine).isEmpty()); round++) {
for (Branch branch : engine.branches().stream().filter(Branch::isActive).toList()) {
removed.addAll(write(options.transaction().onBranch(branch.id()), Principal.SYSTEM,
writer -> TenantMemberships.repair(writer.transaction(), branch.name())));
}
}
if (!TenantMemberships.find(engine).isEmpty()) {
throw HStoreException.invalid("memberships across tenants kept appearing while they were being removed; run the repair again");
}
write(writer -> {
if (!TenantMemberships.checked(writer.transaction())) {
TenantMemberships.markChecked(writer.transaction());
}
return null;
});
return removed;
}

public Scram.Server scram(Optional<byte[]> channelBinding) {
return scram(channelBinding, _ -> { });
}

public Scram.Server scram(Optional<byte[]> channelBinding, Consumer<Security.User> used) {
return new Scram.Server(name -> read(reader -> Security.user(reader.view(), name)).flatMap(user -> {
return new Scram.Server(name -> read(reader -> Security.user(reader.engineView(), name)).flatMap(user -> {
if (Security.legacy(user)) {
warnLegacy(user.name());
return Optional.empty();
Expand All @@ -114,7 +143,7 @@ private void warnLegacy(String user) {
}

public List<String> usersWithOldPasswords() {
return read(reader -> Security.legacyUsers(reader.view()));
return read(reader -> Security.legacyUsers(reader.engineView()));
}

public SignIns signIns() {
Expand Down Expand Up @@ -213,12 +242,12 @@ public <T> T write(TxnOptions txnOptions, Function<? super Writer, T> work) {
}

public TxnOptions transactionOptions(Principal principal) {
long tenantPages = read(reader -> Security.quota(reader.view(), principal.tenant()).maxTransactionPages());
long tenantPages = read(reader -> Security.quota(reader.engineView(), principal.tenant()).maxTransactionPages());
return options.transaction().withMaxPages(Math.min(options.transactionPages(), tenantPages));
}

public long queryPageBudget(Principal principal) {
long tenantPages = read(reader -> Security.quota(reader.view(), principal.tenant()).maxQueryPages());
long tenantPages = read(reader -> Security.quota(reader.engineView(), principal.tenant()).maxQueryPages());
return Math.min(options.defaultQueryPages(), tenantPages);
}

Expand All @@ -231,7 +260,7 @@ public Optional<Principal> authenticate(String user, String password, Optional<S
}

public Optional<Security.User> signIn(String user, String password, Optional<String> origin) {
return signIns.attempt(user, origin, () -> read(reader -> Security.signIn(reader.view(), user, password)))
return signIns.attempt(user, origin, () -> read(reader -> Security.signIn(reader.engineView(), user, password)))
.map(found -> Security.legacy(found) ? convert(found, password) : found);
}

Expand All @@ -255,7 +284,7 @@ void chargePayload(int tenant, long bytes) {
}

public boolean requiresAuthentication() {
return read(reader -> Security.hasUsers(reader.view()));
return read(reader -> Security.hasUsers(reader.engineView()));
}

public <T> T write(TxnOptions txnOptions, Principal principal, Function<? super Writer, T> work) {
Expand Down
53 changes: 52 additions & 1 deletion database/src/main/java/io/hstore/db/Reader.java
Original file line number Diff line number Diff line change
Expand Up @@ -18,11 +18,15 @@
import io.hstore.engine.HStoreException;
import io.hstore.engine.catalog.AtomRecord;
import io.hstore.engine.catalog.EngineSlots;
import io.hstore.engine.index.IndexValues;
import io.hstore.engine.index.PostingCursor;
import io.hstore.engine.topology.EdgeKind;
import io.hstore.engine.topology.Hyperedge;
import io.hstore.engine.topology.Incidence;
import io.hstore.engine.topology.Weight;
import io.hstore.engine.tree.Entry;
import io.hstore.engine.tree.Tree;
import io.hstore.engine.txn.IncidentEdge;
import io.hstore.engine.txn.View;

import java.util.LinkedHashMap;
Expand Down Expand Up @@ -79,10 +83,57 @@ public boolean owns(long atom) {
return onlyDefaultTenant() || visible(atom);
}

public View view() {
public View engineView() {
return view;
}

public View.Unrecorded unrecorded() {
return view.unrecorded();
}

public boolean isEdge(long atom) {
return view.atom(atom).map(record -> record.tenant() == tenant() && record instanceof AtomRecord.EdgeRecord).orElse(false);
}

public Optional<Hyperedge> hyperedge(long id) {
return view.atom(id)
.filter(record -> record.tenant() == tenant())
.flatMap(record -> record instanceof AtomRecord.EdgeRecord edge ? Optional.of(view.edge(id, edge)) : Optional.empty());
}

public Optional<Incidence> incidence(long edge, long member) {
return owns(member) ? view.incidence(edge, member, tenant()) : Optional.empty();
}

public boolean contains(long edge, long member) {
return owns(member) && view.contains(edge, member, tenant());
}

public Stream<IncidentEdge> incidentEdges(long atom) {
if (!onlyDefaultTenant() && !visible(atom)) {
return Stream.empty();
}
return view.incident(atom).filter(incident -> owns(incident.edge()));
}

public Tree<?> incidentTree(long atom) {
return onlyDefaultTenant() || visible(atom)
? view.incidentTree(atom)
: Tree.empty(EngineSlots.REVERSE_INDEX.postingSchema(), view.source());
}

public PostingCursor<IndexValues.Incident> incidentCursor(long atom) {
return onlyDefaultTenant() || visible(atom) ? view.incidentCursor(atom) : nothing();
}

public PostingCursor<IndexValues.Incident> incidentCursor(long atom, Predicate<? super IndexValues.Incident> accept) {
return onlyDefaultTenant() || visible(atom) ? view.incidentCursor(atom, accept) : nothing();
}

private PostingCursor<IndexValues.Incident> nothing() {
return EngineSlots.REVERSE_INDEX.cursor(Tree.empty(EngineSlots.REVERSE_INDEX.schema(), view.source()), 0);
}

public HypergraphDatabase database() {
return database;
}
Expand Down
8 changes: 4 additions & 4 deletions database/src/main/java/io/hstore/db/Writer.java
Original file line number Diff line number Diff line change
Expand Up @@ -76,7 +76,7 @@ public TypeDef createIndex(String type, String property) {
.orElseThrow(() -> HStoreException.invalid(type + " has no declared property '" + property + "'"));
TypeDef updated = schema().alter(txn, current.withProperty(new PropertyDef(property, existing.type(), true, existing.required())));
int key = schema().propertyKey(property);
List<Integer> tenants = Security.tenants(view()).stream().map(Security.Tenant::id).toList();
List<Integer> tenants = Security.tenants(engineView()).stream().map(Security.Tenant::id).toList();
txn.apply("backfill " + type + "." + property, workspace -> tenants.forEach(tenant ->
EngineSlots.TYPE_INDEX.stream(workspace.tree(EngineSlots.TYPES), EngineSlots.typeKey(tenant, updated.id())).toList()
.forEach(entry -> workspace.get(PropertySlots.PROPERTIES, entry.key())
Expand All @@ -90,8 +90,8 @@ public TypeDef createJsonIndex(String type, String path, TypeTag tag) {
TypeDef current = requireType(type);
TypeDef updated = schema().alter(txn, current.withJsonIndex(new JsonIndex(path, tag)));
int key = schema().propertyKey(path);
for (Security.Tenant tenant : Security.tenants(view())) {
Reader scoped = new Reader(database, view(), principal().inTenant(tenant.id()));
for (Security.Tenant tenant : Security.tenants(engineView())) {
Reader scoped = new Reader(database, engineView(), principal().inTenant(tenant.id()));
for (long owner : scoped.atoms(type).boxed().toList()) {
scoped.document(owner).ifPresent(json -> json.select(path)
.filter(value -> !(value instanceof Value.Null))
Expand Down Expand Up @@ -309,7 +309,7 @@ public void embed(long atom, String model, int version, float[] vector) {
}
return semantic.model(model);
});
semantic.validate(view(), model, id, vector);
semantic.validate(engineView(), model, id, vector);
long ref = writePayload(Embedding.encode(vector));
Embedding embedding = new Embedding(id, version, vector.length, ref);
txn.put(Embedding.EMBEDDINGS, atom, embedding);
Expand Down
4 changes: 2 additions & 2 deletions database/src/main/java/io/hstore/db/evidence/Provenance.java
Original file line number Diff line number Diff line change
Expand Up @@ -83,11 +83,11 @@ private static void requireReferable(Reader reader, long id) {
}

public static Optional<Qualifier> qualifier(Reader reader, long id) {
return id == 0 ? Optional.empty() : reader.view().get(QUALIFIERS, id).filter(found -> found.tenant() == reader.tenant());
return id == 0 ? Optional.empty() : reader.engineView().get(QUALIFIERS, id).filter(found -> found.tenant() == reader.tenant());
}

public static Optional<Evidence> evidence(Reader reader, long id) {
return reader.view().get(EVIDENCE, id).filter(found -> found.tenant() == reader.tenant());
return reader.engineView().get(EVIDENCE, id).filter(found -> found.tenant() == reader.tenant());
}

public static Stream<Member> effective(Reader reader, long edge, EvidencePolicy policy) {
Expand Down
12 changes: 6 additions & 6 deletions database/src/main/java/io/hstore/db/hora/Hora.java
Original file line number Diff line number Diff line change
Expand Up @@ -67,7 +67,7 @@ public Stream<IncidenceTuple> incidence(long atom) {
if (reader.require(atom).isEdge()) {
return reader.members(atom).map(member -> tuple(atom, member));
}
return reader.view().incident(atom).filter(incident -> reader.owns(incident.edge())).flatMap(incident -> reader.view().incidence(incident.edge(), atom).stream()
return reader.incidentEdges(atom).flatMap(incident -> reader.incidence(incident.edge(), atom).stream()
.map(found -> new IncidenceTuple(incident.edge(), atom, reader.roles(found.roleSet()), found.weightValue(),
found.validity(), reader.edge(incident.edge()).positionOfLocator(incident.locator()))));
}
Expand Down Expand Up @@ -178,16 +178,16 @@ public double jaccard(long left, long right) {
public boolean adjacent(long u, long v, int threshold) {
reader.require(u);
reader.require(v);
Tree<?> left = reader.view().incidentTree(u);
Tree<?> right = reader.view().incidentTree(v);
Tree<?> left = reader.incidentTree(u);
Tree<?> right = reader.incidentTree(v);
return TreeAlgebra.intersectKeys(left, right).filter(reader::owns).limit(threshold).count() >= threshold;
}

public LongStream neighbors(long node, int threshold, Budget budget) {
reader.require(node);
Budget.Meter meter = budget.start();
Map<Long, Integer> counts = new HashMap<>();
reader.view().incident(node).map(IncidentEdge::edge).filter(reader::owns).forEach(edge -> reader.edge(edge).stream()
reader.incidentEdges(node).map(IncidentEdge::edge).forEach(edge -> reader.edge(edge).stream()
.mapToLong(Incidence::member)
.filter(member -> member != node)
.forEach(member -> {
Expand All @@ -201,7 +201,7 @@ public LongStream edgeNeighbors(long edge, int threshold, Budget budget) {
Budget.Meter meter = budget.start();
Tree<?> membership = reader.edge(edge).membership();
Set<Long> candidates = new HashSet<>();
reader.edge(edge).stream().forEach(incidence -> reader.view().incident(incidence.member())
reader.edge(edge).stream().forEach(incidence -> reader.incidentEdges(incidence.member())
.mapToLong(IncidentEdge::edge)
.filter(other -> other != edge && reader.owns(other))
.forEach(other -> {
Expand All @@ -224,7 +224,7 @@ public Stream<OverlapPair> overlapJoin(List<Long> edges, long threshold, Budget
continue;
}
Set<Long> partners = new HashSet<>();
a.stream().forEach(incidence -> reader.view().incident(incidence.member())
a.stream().forEach(incidence -> reader.incidentEdges(incidence.member())
.mapToLong(IncidentEdge::edge)
.filter(right -> right > left && scope.contains(right) && reader.owns(right))
.forEach(partners::add));
Expand Down
10 changes: 5 additions & 5 deletions database/src/main/java/io/hstore/db/hora/PatternMatcher.java
Original file line number Diff line number Diff line change
Expand Up @@ -96,7 +96,7 @@ private long estimate(Pattern pattern, String name, Set<String> bound) {

private long typeDomain(Variable variable) {
Optional<TypeDef> type = variable.type().flatMap(reader::type);
try (View.Unrecorded _ = reader.view().unrecorded()) {
try (View.Unrecorded _ = reader.unrecorded()) {
return type.map(found -> reader.countOfType(found.id())).orElse(UNKNOWN_DOMAIN);
}
}
Expand All @@ -120,9 +120,9 @@ private LongStream edgeCandidates(Pattern pattern, String name, Map<String, Long
for (Constraint constraint : pattern.constraints()) {
switch (constraint) {
case Constraint.Contains(String edge, String member) when edge.equals(name) && binding.containsKey(member) ->
incidentSets.add(reader.view().incidentTree(binding.get(member)));
incidentSets.add(reader.incidentTree(binding.get(member)));
case Constraint.HasRole(String edge, String member, String _) when edge.equals(name) && binding.containsKey(member) ->
incidentSets.add(reader.view().incidentTree(binding.get(member)));
incidentSets.add(reader.incidentTree(binding.get(member)));
case Constraint.Shares(String left, String right, long _, long _) when left.equals(name) && binding.containsKey(right) -> {
relatedConstraint = true;
related.addAll(neighbourEdges(binding.get(right)));
Expand Down Expand Up @@ -156,7 +156,7 @@ private LongStream edgeCandidates(Pattern pattern, String name, Map<String, Long
private Set<Long> neighbourEdges(long edge) {
Set<Long> edges = new HashSet<>();
reader.edge(edge).stream().forEach(incidence ->
reader.view().incident(incidence.member()).map(IncidentEdge::edge).filter(reader::owns).forEach(edges::add));
reader.incidentEdges(incidence.member()).map(IncidentEdge::edge).forEach(edges::add));
return edges;
}

Expand Down Expand Up @@ -208,7 +208,7 @@ private boolean holds(Constraint constraint, Map<String, Long> binding, Pattern
}

private Optional<Incidence> incidence(long edge, long member, Pattern pattern, String edgeVariable) {
Optional<Incidence> found = reader.view().edge(edge).flatMap(hyperedge -> hyperedge.get(member));
Optional<Incidence> found = reader.hyperedge(edge).flatMap(hyperedge -> hyperedge.get(member));
OptionalLong instant = validAt(pattern, edgeVariable);
return instant.isPresent() ? found.filter(incidence -> incidence.validAt(instant.getAsLong())) : found;
}
Expand Down
6 changes: 3 additions & 3 deletions database/src/main/java/io/hstore/db/query/AccessPath.java
Original file line number Diff line number Diff line change
Expand Up @@ -89,8 +89,8 @@ public LongStream scan(Reader reader) {
Dictionary dictionary = reader.database().engine().dictionary();
List<PostingCursor<IndexValues.Incident>> cursors = legs.stream()
.map(leg -> leg.roles().isEmpty()
? reader.view().incidentCursor(leg.atom())
: reader.view().incidentCursor(leg.atom(), accepting(dictionary, leg.roles())))
? reader.incidentCursor(leg.atom())
: reader.incidentCursor(leg.atom(), accepting(dictionary, leg.roles())))
.toList();
return PostingCursor.intersect(cursors);
}
Expand Down Expand Up @@ -118,7 +118,7 @@ public String describe() {

public LongStream scan(Reader reader) {
return reader.visible(edge)
? reader.view().edge(edge).map(found -> found.membership().keys()).orElseGet(LongStream::empty)
? reader.hyperedge(edge).map(found -> found.membership().keys()).orElseGet(LongStream::empty)
: LongStream.empty();
}
}
Expand Down
Loading
Loading