Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
31 changes: 31 additions & 0 deletions database/src/main/java/io/hstore/db/query/Ast.java
Original file line number Diff line number Diff line change
Expand Up @@ -110,6 +110,33 @@ public record Match(boolean edges, String variable, Optional<String> type, Optio
List<Projection> projections, Optional<Expr> orderBy, boolean descending, OptionalLong limit) {
}

public sealed interface RankTarget {
record Atom(Ref ref) implements RankTarget {
}

record OfType(String type) implements RankTarget {
}
}

public record RankSlot(RankTarget target, List<String> roles, OptionalDouble weight, boolean required) {
}

public sealed interface RankTime {
record At(Expr instant, OptionalDouble tau) implements RankTime {
}

record During(long from, long to) implements RankTime {
}
}

public sealed interface RankPreference {
record Equals(String variable, String property, Value target, OptionalDouble weight) implements RankPreference {
}

record Near(String variable, String property, double target, double scale, OptionalDouble weight) implements RankPreference {
}
}

public sealed interface Statement {
record Begin(boolean serializable) implements Statement {
}
Expand Down Expand Up @@ -265,6 +292,10 @@ record SetAlgebra(SetOp op, Ref left, Ref right, Optional<String> into) implemen
record Expand(Ref atom, int depth, OptionalLong limit) implements Statement {
}

record Rank(String variable, Optional<String> type, List<RankSlot> slots, Optional<RankTime> time,
List<RankPreference> preferences, OptionalLong size, Map<String, Double> weights, long top) implements Statement {
}

record Gather(Field field, Ref edge) implements Statement {
}

Expand Down
67 changes: 67 additions & 0 deletions database/src/main/java/io/hstore/db/query/Executor.java
Original file line number Diff line number Diff line change
Expand Up @@ -23,6 +23,9 @@
import io.hstore.db.query.Ast.MemberClause;
import io.hstore.db.query.Ast.Statement;
import io.hstore.db.query.QueryResult.Cell;
import io.hstore.db.retrieve.RankQuery;
import io.hstore.db.retrieve.Ranking;
import io.hstore.db.retrieve.RoleTimeScorer;
import io.hstore.db.schema.TypeDef;
import io.hstore.db.security.Principal;
import io.hstore.db.security.Quota;
Expand Down Expand Up @@ -61,6 +64,7 @@
import java.util.Optional;
import java.util.OptionalLong;
import java.util.Set;
import java.util.TreeSet;
import java.util.concurrent.atomic.AtomicLong;
import java.util.function.Function;
import java.util.stream.Collectors;
Expand Down Expand Up @@ -268,6 +272,7 @@ QueryResult run(Statement statement) {
case Statement.IncidentTo(Ast.Ref atom, Optional<Expr> validAt) -> read(reader -> incident(reader, session.resolve(reader, atom), validAt));
case Statement.Describe(Ast.Ref atom) -> read(reader -> describe(reader, session.resolve(reader, atom)));
case Statement.SetAlgebra algebra -> algebra.into().isPresent() ? write(writer -> setAlgebra(writer, algebra)) : read(reader -> setAlgebra(reader, algebra));
case Statement.Rank rank -> read(reader -> rank(reader, rank));
case Statement.Expand(Ast.Ref atom, int depth, OptionalLong limit) -> read(reader -> table(List.of("path", "atom", "status"),
Hora.on(reader).expand(session.resolve(reader, atom), depth(depth), budget(limit).withDepth(depth(depth)))
.map(expansion -> List.<Cell>of(new Cell.Items(expansion.path().stream().map(id -> (Cell) new Cell.AtomCell(id, "")).toList()),
Expand Down Expand Up @@ -466,6 +471,68 @@ private QueryResult query(Reader reader, Ast.Match match) {
rows.size(), trace.pagesRead(), trace.cacheHits(), System.nanoTime() - started));
}

private QueryResult rank(Reader reader, Statement.Rank statement) {
long started = System.nanoTime();
IoTrace trace = IoTrace.withBudget(database().queryPageBudget(session.principal()));
Ranking ranking = Ranking.of(reader, rankQuery(reader, statement));
List<RoleTimeScorer.EdgeScore> scores = trace.call(() -> ranking.top(Budget.DEFAULT.start()));
List<List<Cell>> rows = scores.stream().map(scored -> List.of(atomCell(reader, scored.edge()), QueryResult.number(scored.score()),
component(scored.components().members()), component(scored.components().roles()), component(scored.components().attributes()),
component(scored.components().time()), component(scored.components().topology()))).toList();
return QueryResult.table(List.of("edge", "score", "members", "roles", "attributes", "time", "topology"), rows)
.withTrace(new QueryResult.Trace(0, reader.generation(), ranking.access(), Double.NaN, rows.size(),
trace.pagesRead(), trace.cacheHits(), System.nanoTime() - started));
}

private static Cell component(double value) {
return Double.isNaN(value) ? QueryResult.scalar(Value.NULL) : QueryResult.number(value);
}

private RankQuery rankQuery(Reader reader, Statement.Rank statement) {
Optional<TypeDef> type = statement.type().map(reader::requireType);
List<RankQuery.Slot> slots = statement.slots().stream().map(slot -> new RankQuery.Slot(switch (slot.target()) {
case Ast.RankTarget.Atom(Ast.Ref ref) -> new RankQuery.Target.Atom(reader.require(session.resolve(reader, ref)).id());
case Ast.RankTarget.OfType(String name) -> new RankQuery.Target.OfType(reader.requireType(name));
}, Set.copyOf(slot.roles()), slot.weight().orElse(1), slot.required())).toList();
Evaluator evaluator = evaluator(reader);
Optional<RankQuery.TimeSpec> time = statement.time().map(spec -> switch (spec) {
case Ast.RankTime.At(Expr instant, var tau) -> new RankQuery.TimeSpec.At(
(long) evaluator.value(instant, Map.of()).number().orElseThrow(() -> HStoreException.invalid("VALID AT needs a number or a timestamp")),
tau.orElse(0));
case Ast.RankTime.During(long from, long to) -> new RankQuery.TimeSpec.During(from, to);
});
List<RankQuery.Preference> preferences = statement.preferences().stream().map(preference -> switch (preference) {
case Ast.RankPreference.Equals(String variable, String property, Value target, var weight) -> {
requireRankVariable(statement, variable);
yield (RankQuery.Preference) new RankQuery.Preference.Equals(property, target, weight.orElse(1));
}
case Ast.RankPreference.Near(String variable, String property, double target, double scale, var weight) -> {
requireRankVariable(statement, variable);
yield new RankQuery.Preference.Near(property, target, scale, weight.orElse(1));
}
}).toList();
RankQuery.Weights defaults = RankQuery.Weights.DEFAULT;
Map<String, Double> given = statement.weights();
Set<String> unknown = new TreeSet<>(given.keySet());
unknown.removeAll(Set.of("members", "roles", "attributes", "time", "topology"));
if (!unknown.isEmpty()) {
throw HStoreException.invalid("unknown component " + unknown + "; expected members, roles, attributes, time or topology");
}
RankQuery.Weights weights = new RankQuery.Weights(given.getOrDefault("members", defaults.members()),
given.getOrDefault("roles", defaults.roles()), given.getOrDefault("attributes", defaults.attributes()),
given.getOrDefault("time", defaults.time()), given.getOrDefault("topology", defaults.topology()));
if (statement.top() > RankQuery.MAX_TOP) {
throw HStoreException.invalid("TOP must be between 1 and " + RankQuery.MAX_TOP);
}
return new RankQuery(type, slots, time, preferences, statement.size(), weights, (int) Math.max(0, statement.top()));
}

private static void requireRankVariable(Statement.Rank statement, String variable) {
if (!variable.equals(statement.variable())) {
throw HStoreException.invalid("unknown variable " + variable + "; RANK binds only " + statement.variable());
}
}

private static boolean admits(Reader reader, Planner.Plan plan, long atom) {
if (plan.chosen().path() instanceof AccessPath.TypeScan) {
return true;
Expand Down
89 changes: 89 additions & 0 deletions database/src/main/java/io/hstore/db/query/Parser.java
Original file line number Diff line number Diff line change
Expand Up @@ -15,6 +15,7 @@
import io.hstore.db.query.Ast.Ref;
import io.hstore.db.query.Ast.SetOp;
import io.hstore.db.query.Ast.Statement;
import io.hstore.db.retrieve.RankQuery;
import io.hstore.db.schema.AtomKind;
import io.hstore.db.schema.TypeDef.PropertyDef;
import io.hstore.db.security.Role;
Expand Down Expand Up @@ -314,6 +315,9 @@ private Statement statement() {
return new Statement.SetAlgebra(op, left, right, accept("INTO") ? Optional.of(variable()) : Optional.empty());
}
}
if (accept("RANK")) {
return rank();
}
if (accept("EXPAND")) {
Ref atom = ref();
int depth = accept("DEPTH") ? intValue() : 4;
Expand Down Expand Up @@ -651,6 +655,88 @@ private Bounds bounds() {
return new Bounds(exact, exact);
}

private Statement.Rank rank() {
if (!accept("EDGE") && !accept("EDGES")) {
throw error("expected EDGE");
}
String variable = identifier();
Optional<String> type = acceptSymbol(":") ? Optional.of(identifier()) : Optional.empty();
expect("LIKE");
expectSymbol("(");
List<Ast.RankSlot> slots = new ArrayList<>();
do {
if (slots.size() == RankQuery.MAX_SLOTS) {
throw error("RANK accepts at most " + RankQuery.MAX_SLOTS + " slots");
}
slots.add(rankSlot());
} while (acceptSymbol(","));
expectSymbol(")");
Optional<Ast.RankTime> time = Optional.empty();
if (accept("VALID")) {
if (accept("DURING")) {
Validity interval = interval();
time = Optional.of(new Ast.RankTime.During(interval.from(), interval.to()));
} else {
expect("AT");
Expr instant = additive();
time = Optional.of(new Ast.RankTime.At(instant, accept("TAU") ? OptionalDouble.of(number().doubleValue()) : OptionalDouble.empty()));
}
}
List<Ast.RankPreference> preferences = new ArrayList<>();
if (accept("PREFER")) {
do {
if (preferences.size() == RankQuery.MAX_PREFERENCES) {
throw error("RANK accepts at most " + RankQuery.MAX_PREFERENCES + " preferences");
}
preferences.add(preference());
} while (acceptSymbol(","));
}
OptionalLong size = accept("SIZE") ? OptionalLong.of(longValue()) : OptionalLong.empty();
Map<String, Double> weights = new LinkedHashMap<>();
if (accept("WEIGHTS")) {
expectSymbol("(");
do {
String component = identifier().toLowerCase(Locale.ROOT);
if (weights.put(component, number().doubleValue()) != null) {
throw error("the weight of " + component + " is given twice");
}
} while (acceptSymbol(","));
expectSymbol(")");
}
expect("TOP");
return new Statement.Rank(variable, type, slots, time, preferences, size, weights, longValue());
}

private Ast.RankSlot rankSlot() {
Ast.RankTarget target = accept("ANY") ? new Ast.RankTarget.OfType(identifier()) : new Ast.RankTarget.Atom(ref());
List<String> roles = new ArrayList<>();
if (accept("AS")) {
do {
roles.add(identifier());
} while (accept("OR"));
}
OptionalDouble weight = accept("WEIGHT") ? OptionalDouble.of(number().doubleValue()) : OptionalDouble.empty();
return new Ast.RankSlot(target, roles, weight, accept("REQUIRED"));
}

private Ast.RankPreference preference() {
String variable = identifier();
expectSymbol(".");
String property = identifier();
if (accept("NEAR")) {
double target = number().doubleValue();
expect("SCALE");
double scale = number().doubleValue();
return new Ast.RankPreference.Near(variable, property, target, scale, preferenceWeight());
}
expectSymbol("=");
return new Ast.RankPreference.Equals(variable, property, literal(), preferenceWeight());
}

private OptionalDouble preferenceWeight() {
return accept("WEIGHT") ? OptionalDouble.of(number().doubleValue()) : OptionalDouble.empty();
}

private Ast.Match match() {
boolean edges;
if (accept("EDGE") || accept("EDGES")) {
Expand Down Expand Up @@ -867,6 +953,9 @@ private Validity interval() {
expectSymbol(",");
long to = acceptSymbol("*") ? Long.MAX_VALUE : instant();
expectSymbol(")");
if (from >= to) {
throw error("the interval [" + from + ", " + to + ") is empty");
}
return new Validity(from, to);
}

Expand Down
128 changes: 128 additions & 0 deletions database/src/main/java/io/hstore/db/retrieve/RankQuery.java
Original file line number Diff line number Diff line change
@@ -0,0 +1,128 @@
// SPDX-FileCopyrightText: MetaCognition India Pvt Limited
// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0

package io.hstore.db.retrieve;

import io.hstore.db.schema.TypeDef;
import io.hstore.db.value.Value;
import io.hstore.engine.HStoreException;

import java.util.List;
import java.util.Optional;
import java.util.OptionalLong;
import java.util.Set;

public record RankQuery(Optional<TypeDef> type, List<Slot> slots, Optional<TimeSpec> time, List<Preference> preferences,
OptionalLong size, Weights weights, int top) {

public static final int MAX_SLOTS = 16;
public static final int MAX_PREFERENCES = 16;
public static final int MAX_TOP = 10_000;

public RankQuery {
slots = List.copyOf(slots);
preferences = List.copyOf(preferences);
if (slots.isEmpty() || slots.size() > MAX_SLOTS) {
throw HStoreException.invalid("RANK needs between 1 and " + MAX_SLOTS + " slots");
}
if (preferences.size() > MAX_PREFERENCES) {
throw HStoreException.invalid("RANK accepts at most " + MAX_PREFERENCES + " preferences");
}
if (top < 1 || top > MAX_TOP) {
throw HStoreException.invalid("TOP must be between 1 and " + MAX_TOP);
}
if (size.isPresent() && size.getAsLong() < 1) {
throw HStoreException.invalid("SIZE must be at least 1");
}
type.ifPresent(found -> {
if (!found.kind().isEdge()) {
throw HStoreException.invalid(found.name() + " is not a hyperedge type");
}
});
}

public Weights effectiveWeights() {
return weights.restrictedTo(!preferences.isEmpty(), time.isPresent(), size.isPresent());
}

public sealed interface Target {
record Atom(long id) implements Target {
}

record OfType(TypeDef type) implements Target {
}
}

public record Slot(Target target, Set<String> roles, double weight, boolean required) {
public Slot {
roles = Set.copyOf(roles);
requireWeight("slot", weight);
}
}

public sealed interface TimeSpec {
record At(long instant, double tau) implements TimeSpec {
public At {
if (!Double.isFinite(tau) || tau < 0) {
throw HStoreException.invalid("TAU must be a number of at least 0");
}
}
}

record During(long from, long to) implements TimeSpec {
public During {
if (from >= to || from == Long.MIN_VALUE || to == Long.MAX_VALUE) {
throw HStoreException.invalid("VALID DURING needs a finite, non-empty interval");
}
}
}
}

public sealed interface Preference {
String property();

double weight();

record Equals(String property, Value target, double weight) implements Preference {
public Equals {
requireWeight("preference", weight);
}
}

record Near(String property, double target, double scale, double weight) implements Preference {
public Near {
requireWeight("preference", weight);
if (!Double.isFinite(target) || !Double.isFinite(scale) || scale <= 0) {
throw HStoreException.invalid("NEAR needs a finite target and a positive SCALE");
}
}
}
}

public record Weights(double members, double roles, double attributes, double time, double topology) {
public static final Weights DEFAULT = new Weights(0.3, 0.2, 0.2, 0.2, 0.1);

public Weights {
for (double weight : new double[]{members, roles, attributes, time, topology}) {
if (!Double.isFinite(weight) || weight < 0) {
throw HStoreException.invalid("component weights must be finite and not negative");
}
}
}

Weights restrictedTo(boolean useAttributes, boolean useTime, boolean useTopology) {
Weights used = new Weights(members, roles, useAttributes ? attributes : 0, useTime ? time : 0, useTopology ? topology : 0);
double total = used.members + used.roles + used.attributes + used.time + used.topology;
if (total <= 0) {
throw HStoreException.invalid("the weights of the components this query uses add up to zero");
}
return new Weights(used.members / total, used.roles / total, used.attributes / total, used.time / total, used.topology / total);
}
}

private static void requireWeight(String kind, double weight) {
if (!Double.isFinite(weight) || weight <= 0 || weight > 1_000_000) {
throw HStoreException.invalid("a " + kind + " WEIGHT must be a positive number up to 1000000");
}
}
}
Loading
Loading