diff --git a/database/src/main/java/io/hstore/db/query/Ast.java b/database/src/main/java/io/hstore/db/query/Ast.java index ec695d6..774c704 100644 --- a/database/src/main/java/io/hstore/db/query/Ast.java +++ b/database/src/main/java/io/hstore/db/query/Ast.java @@ -110,6 +110,33 @@ public record Match(boolean edges, String variable, Optional type, Optio List projections, Optional orderBy, boolean descending, OptionalLong limit) { } + public sealed interface RankTarget { + record Atom(Ref ref) implements RankTarget { + } + + record OfType(String type) implements RankTarget { + } + } + + public record RankSlot(RankTarget target, List roles, OptionalDouble weight, boolean required) { + } + + public sealed interface RankTime { + record At(Expr instant, OptionalDouble tau) implements RankTime { + } + + record During(long from, long to) implements RankTime { + } + } + + public sealed interface RankPreference { + record Equals(String variable, String property, Value target, OptionalDouble weight) implements RankPreference { + } + + record Near(String variable, String property, double target, double scale, OptionalDouble weight) implements RankPreference { + } + } + public sealed interface Statement { record Begin(boolean serializable) implements Statement { } @@ -265,6 +292,10 @@ record SetAlgebra(SetOp op, Ref left, Ref right, Optional into) implemen record Expand(Ref atom, int depth, OptionalLong limit) implements Statement { } + record Rank(String variable, Optional type, List slots, Optional time, + List preferences, OptionalLong size, Map weights, long top) implements Statement { + } + record Gather(Field field, Ref edge) implements Statement { } diff --git a/database/src/main/java/io/hstore/db/query/Executor.java b/database/src/main/java/io/hstore/db/query/Executor.java index 1c75e05..8bdeea4 100644 --- a/database/src/main/java/io/hstore/db/query/Executor.java +++ b/database/src/main/java/io/hstore/db/query/Executor.java @@ -23,6 +23,9 @@ import io.hstore.db.query.Ast.MemberClause; import io.hstore.db.query.Ast.Statement; import io.hstore.db.query.QueryResult.Cell; +import io.hstore.db.retrieve.RankQuery; +import io.hstore.db.retrieve.Ranking; +import io.hstore.db.retrieve.RoleTimeScorer; import io.hstore.db.schema.TypeDef; import io.hstore.db.security.Principal; import io.hstore.db.security.Quota; @@ -61,6 +64,7 @@ import java.util.Optional; import java.util.OptionalLong; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.stream.Collectors; @@ -268,6 +272,7 @@ QueryResult run(Statement statement) { case Statement.IncidentTo(Ast.Ref atom, Optional validAt) -> read(reader -> incident(reader, session.resolve(reader, atom), validAt)); case Statement.Describe(Ast.Ref atom) -> read(reader -> describe(reader, session.resolve(reader, atom))); case Statement.SetAlgebra algebra -> algebra.into().isPresent() ? write(writer -> setAlgebra(writer, algebra)) : read(reader -> setAlgebra(reader, algebra)); + case Statement.Rank rank -> read(reader -> rank(reader, rank)); case Statement.Expand(Ast.Ref atom, int depth, OptionalLong limit) -> read(reader -> table(List.of("path", "atom", "status"), Hora.on(reader).expand(session.resolve(reader, atom), depth(depth), budget(limit).withDepth(depth(depth))) .map(expansion -> List.of(new Cell.Items(expansion.path().stream().map(id -> (Cell) new Cell.AtomCell(id, "")).toList()), @@ -466,6 +471,68 @@ private QueryResult query(Reader reader, Ast.Match match) { rows.size(), trace.pagesRead(), trace.cacheHits(), System.nanoTime() - started)); } + private QueryResult rank(Reader reader, Statement.Rank statement) { + long started = System.nanoTime(); + IoTrace trace = IoTrace.withBudget(database().queryPageBudget(session.principal())); + Ranking ranking = Ranking.of(reader, rankQuery(reader, statement)); + List scores = trace.call(() -> ranking.top(Budget.DEFAULT.start())); + List> rows = scores.stream().map(scored -> List.of(atomCell(reader, scored.edge()), QueryResult.number(scored.score()), + component(scored.components().members()), component(scored.components().roles()), component(scored.components().attributes()), + component(scored.components().time()), component(scored.components().topology()))).toList(); + return QueryResult.table(List.of("edge", "score", "members", "roles", "attributes", "time", "topology"), rows) + .withTrace(new QueryResult.Trace(0, reader.generation(), ranking.access(), Double.NaN, rows.size(), + trace.pagesRead(), trace.cacheHits(), System.nanoTime() - started)); + } + + private static Cell component(double value) { + return Double.isNaN(value) ? QueryResult.scalar(Value.NULL) : QueryResult.number(value); + } + + private RankQuery rankQuery(Reader reader, Statement.Rank statement) { + Optional type = statement.type().map(reader::requireType); + List slots = statement.slots().stream().map(slot -> new RankQuery.Slot(switch (slot.target()) { + case Ast.RankTarget.Atom(Ast.Ref ref) -> new RankQuery.Target.Atom(reader.require(session.resolve(reader, ref)).id()); + case Ast.RankTarget.OfType(String name) -> new RankQuery.Target.OfType(reader.requireType(name)); + }, Set.copyOf(slot.roles()), slot.weight().orElse(1), slot.required())).toList(); + Evaluator evaluator = evaluator(reader); + Optional time = statement.time().map(spec -> switch (spec) { + case Ast.RankTime.At(Expr instant, var tau) -> new RankQuery.TimeSpec.At( + (long) evaluator.value(instant, Map.of()).number().orElseThrow(() -> HStoreException.invalid("VALID AT needs a number or a timestamp")), + tau.orElse(0)); + case Ast.RankTime.During(long from, long to) -> new RankQuery.TimeSpec.During(from, to); + }); + List preferences = statement.preferences().stream().map(preference -> switch (preference) { + case Ast.RankPreference.Equals(String variable, String property, Value target, var weight) -> { + requireRankVariable(statement, variable); + yield (RankQuery.Preference) new RankQuery.Preference.Equals(property, target, weight.orElse(1)); + } + case Ast.RankPreference.Near(String variable, String property, double target, double scale, var weight) -> { + requireRankVariable(statement, variable); + yield new RankQuery.Preference.Near(property, target, scale, weight.orElse(1)); + } + }).toList(); + RankQuery.Weights defaults = RankQuery.Weights.DEFAULT; + Map given = statement.weights(); + Set unknown = new TreeSet<>(given.keySet()); + unknown.removeAll(Set.of("members", "roles", "attributes", "time", "topology")); + if (!unknown.isEmpty()) { + throw HStoreException.invalid("unknown component " + unknown + "; expected members, roles, attributes, time or topology"); + } + RankQuery.Weights weights = new RankQuery.Weights(given.getOrDefault("members", defaults.members()), + given.getOrDefault("roles", defaults.roles()), given.getOrDefault("attributes", defaults.attributes()), + given.getOrDefault("time", defaults.time()), given.getOrDefault("topology", defaults.topology())); + if (statement.top() > RankQuery.MAX_TOP) { + throw HStoreException.invalid("TOP must be between 1 and " + RankQuery.MAX_TOP); + } + return new RankQuery(type, slots, time, preferences, statement.size(), weights, (int) Math.max(0, statement.top())); + } + + private static void requireRankVariable(Statement.Rank statement, String variable) { + if (!variable.equals(statement.variable())) { + throw HStoreException.invalid("unknown variable " + variable + "; RANK binds only " + statement.variable()); + } + } + private static boolean admits(Reader reader, Planner.Plan plan, long atom) { if (plan.chosen().path() instanceof AccessPath.TypeScan) { return true; diff --git a/database/src/main/java/io/hstore/db/query/Parser.java b/database/src/main/java/io/hstore/db/query/Parser.java index 45e0950..a710b51 100644 --- a/database/src/main/java/io/hstore/db/query/Parser.java +++ b/database/src/main/java/io/hstore/db/query/Parser.java @@ -15,6 +15,7 @@ import io.hstore.db.query.Ast.Ref; import io.hstore.db.query.Ast.SetOp; import io.hstore.db.query.Ast.Statement; +import io.hstore.db.retrieve.RankQuery; import io.hstore.db.schema.AtomKind; import io.hstore.db.schema.TypeDef.PropertyDef; import io.hstore.db.security.Role; @@ -314,6 +315,9 @@ private Statement statement() { return new Statement.SetAlgebra(op, left, right, accept("INTO") ? Optional.of(variable()) : Optional.empty()); } } + if (accept("RANK")) { + return rank(); + } if (accept("EXPAND")) { Ref atom = ref(); int depth = accept("DEPTH") ? intValue() : 4; @@ -651,6 +655,88 @@ private Bounds bounds() { return new Bounds(exact, exact); } + private Statement.Rank rank() { + if (!accept("EDGE") && !accept("EDGES")) { + throw error("expected EDGE"); + } + String variable = identifier(); + Optional type = acceptSymbol(":") ? Optional.of(identifier()) : Optional.empty(); + expect("LIKE"); + expectSymbol("("); + List slots = new ArrayList<>(); + do { + if (slots.size() == RankQuery.MAX_SLOTS) { + throw error("RANK accepts at most " + RankQuery.MAX_SLOTS + " slots"); + } + slots.add(rankSlot()); + } while (acceptSymbol(",")); + expectSymbol(")"); + Optional time = Optional.empty(); + if (accept("VALID")) { + if (accept("DURING")) { + Validity interval = interval(); + time = Optional.of(new Ast.RankTime.During(interval.from(), interval.to())); + } else { + expect("AT"); + Expr instant = additive(); + time = Optional.of(new Ast.RankTime.At(instant, accept("TAU") ? OptionalDouble.of(number().doubleValue()) : OptionalDouble.empty())); + } + } + List preferences = new ArrayList<>(); + if (accept("PREFER")) { + do { + if (preferences.size() == RankQuery.MAX_PREFERENCES) { + throw error("RANK accepts at most " + RankQuery.MAX_PREFERENCES + " preferences"); + } + preferences.add(preference()); + } while (acceptSymbol(",")); + } + OptionalLong size = accept("SIZE") ? OptionalLong.of(longValue()) : OptionalLong.empty(); + Map weights = new LinkedHashMap<>(); + if (accept("WEIGHTS")) { + expectSymbol("("); + do { + String component = identifier().toLowerCase(Locale.ROOT); + if (weights.put(component, number().doubleValue()) != null) { + throw error("the weight of " + component + " is given twice"); + } + } while (acceptSymbol(",")); + expectSymbol(")"); + } + expect("TOP"); + return new Statement.Rank(variable, type, slots, time, preferences, size, weights, longValue()); + } + + private Ast.RankSlot rankSlot() { + Ast.RankTarget target = accept("ANY") ? new Ast.RankTarget.OfType(identifier()) : new Ast.RankTarget.Atom(ref()); + List roles = new ArrayList<>(); + if (accept("AS")) { + do { + roles.add(identifier()); + } while (accept("OR")); + } + OptionalDouble weight = accept("WEIGHT") ? OptionalDouble.of(number().doubleValue()) : OptionalDouble.empty(); + return new Ast.RankSlot(target, roles, weight, accept("REQUIRED")); + } + + private Ast.RankPreference preference() { + String variable = identifier(); + expectSymbol("."); + String property = identifier(); + if (accept("NEAR")) { + double target = number().doubleValue(); + expect("SCALE"); + double scale = number().doubleValue(); + return new Ast.RankPreference.Near(variable, property, target, scale, preferenceWeight()); + } + expectSymbol("="); + return new Ast.RankPreference.Equals(variable, property, literal(), preferenceWeight()); + } + + private OptionalDouble preferenceWeight() { + return accept("WEIGHT") ? OptionalDouble.of(number().doubleValue()) : OptionalDouble.empty(); + } + private Ast.Match match() { boolean edges; if (accept("EDGE") || accept("EDGES")) { @@ -867,6 +953,9 @@ private Validity interval() { expectSymbol(","); long to = acceptSymbol("*") ? Long.MAX_VALUE : instant(); expectSymbol(")"); + if (from >= to) { + throw error("the interval [" + from + ", " + to + ") is empty"); + } return new Validity(from, to); } diff --git a/database/src/main/java/io/hstore/db/retrieve/RankQuery.java b/database/src/main/java/io/hstore/db/retrieve/RankQuery.java new file mode 100644 index 0000000..35b7d0e --- /dev/null +++ b/database/src/main/java/io/hstore/db/retrieve/RankQuery.java @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.retrieve; + +import io.hstore.db.schema.TypeDef; +import io.hstore.db.value.Value; +import io.hstore.engine.HStoreException; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; + +public record RankQuery(Optional type, List slots, Optional time, List preferences, + OptionalLong size, Weights weights, int top) { + + public static final int MAX_SLOTS = 16; + public static final int MAX_PREFERENCES = 16; + public static final int MAX_TOP = 10_000; + + public RankQuery { + slots = List.copyOf(slots); + preferences = List.copyOf(preferences); + if (slots.isEmpty() || slots.size() > MAX_SLOTS) { + throw HStoreException.invalid("RANK needs between 1 and " + MAX_SLOTS + " slots"); + } + if (preferences.size() > MAX_PREFERENCES) { + throw HStoreException.invalid("RANK accepts at most " + MAX_PREFERENCES + " preferences"); + } + if (top < 1 || top > MAX_TOP) { + throw HStoreException.invalid("TOP must be between 1 and " + MAX_TOP); + } + if (size.isPresent() && size.getAsLong() < 1) { + throw HStoreException.invalid("SIZE must be at least 1"); + } + type.ifPresent(found -> { + if (!found.kind().isEdge()) { + throw HStoreException.invalid(found.name() + " is not a hyperedge type"); + } + }); + } + + public Weights effectiveWeights() { + return weights.restrictedTo(!preferences.isEmpty(), time.isPresent(), size.isPresent()); + } + + public sealed interface Target { + record Atom(long id) implements Target { + } + + record OfType(TypeDef type) implements Target { + } + } + + public record Slot(Target target, Set roles, double weight, boolean required) { + public Slot { + roles = Set.copyOf(roles); + requireWeight("slot", weight); + } + } + + public sealed interface TimeSpec { + record At(long instant, double tau) implements TimeSpec { + public At { + if (!Double.isFinite(tau) || tau < 0) { + throw HStoreException.invalid("TAU must be a number of at least 0"); + } + } + } + + record During(long from, long to) implements TimeSpec { + public During { + if (from >= to || from == Long.MIN_VALUE || to == Long.MAX_VALUE) { + throw HStoreException.invalid("VALID DURING needs a finite, non-empty interval"); + } + } + } + } + + public sealed interface Preference { + String property(); + + double weight(); + + record Equals(String property, Value target, double weight) implements Preference { + public Equals { + requireWeight("preference", weight); + } + } + + record Near(String property, double target, double scale, double weight) implements Preference { + public Near { + requireWeight("preference", weight); + if (!Double.isFinite(target) || !Double.isFinite(scale) || scale <= 0) { + throw HStoreException.invalid("NEAR needs a finite target and a positive SCALE"); + } + } + } + } + + public record Weights(double members, double roles, double attributes, double time, double topology) { + public static final Weights DEFAULT = new Weights(0.3, 0.2, 0.2, 0.2, 0.1); + + public Weights { + for (double weight : new double[]{members, roles, attributes, time, topology}) { + if (!Double.isFinite(weight) || weight < 0) { + throw HStoreException.invalid("component weights must be finite and not negative"); + } + } + } + + Weights restrictedTo(boolean useAttributes, boolean useTime, boolean useTopology) { + Weights used = new Weights(members, roles, useAttributes ? attributes : 0, useTime ? time : 0, useTopology ? topology : 0); + double total = used.members + used.roles + used.attributes + used.time + used.topology; + if (total <= 0) { + throw HStoreException.invalid("the weights of the components this query uses add up to zero"); + } + return new Weights(used.members / total, used.roles / total, used.attributes / total, used.time / total, used.topology / total); + } + } + + private static void requireWeight(String kind, double weight) { + if (!Double.isFinite(weight) || weight <= 0 || weight > 1_000_000) { + throw HStoreException.invalid("a " + kind + " WEIGHT must be a positive number up to 1000000"); + } + } +} diff --git a/database/src/main/java/io/hstore/db/retrieve/Ranking.java b/database/src/main/java/io/hstore/db/retrieve/Ranking.java new file mode 100644 index 0000000..aa2e2ff --- /dev/null +++ b/database/src/main/java/io/hstore/db/retrieve/Ranking.java @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.retrieve; + +import io.hstore.db.Atom; +import io.hstore.db.Reader; +import io.hstore.db.hora.Budget; +import io.hstore.db.retrieve.RankQuery.Slot; +import io.hstore.db.retrieve.RankQuery.Target; +import io.hstore.db.retrieve.RoleTimeScorer.EdgeScore; +import io.hstore.engine.index.IndexValues; +import io.hstore.engine.index.PostingCursor; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.PriorityQueue; +import java.util.stream.IntStream; +import java.util.stream.LongStream; + +public final class Ranking { + + private static final double TOLERANCE = 1e-9; + private static final Comparator BEST_FIRST = + Comparator.comparingDouble(EdgeScore::score).reversed().thenComparingLong(EdgeScore::edge); + + private final Reader reader; + private final RankQuery query; + + private Ranking(Reader reader, RankQuery query) { + this.reader = reader; + this.query = query; + } + + public static Ranking of(Reader reader, RankQuery query) { + return new Ranking(reader, query); + } + + public String access() { + List anchors = requiredAnchors(); + if (!anchors.isEmpty()) { + return "IncidentIntersection" + anchors.stream().map(i -> describe(query.slots().get(i))).toList(); + } + return query.type().map(type -> "TypeScan(" + type.name() + ")").orElse("CatalogScan(edges)"); + } + + public List top(Budget.Meter meter) { + RoleTimeScorer scorer = new RoleTimeScorer(reader, query, meter); + long[] edges = new long[64]; + double[] bounds = new double[64]; + int count = 0; + for (var candidates = candidates(scorer).iterator(); candidates.hasNext(); ) { + long edge = candidates.nextLong(); + meter.consume(1); + Optional admitted = RoleTimeScorer.admitted(reader, edge, query); + if (admitted.isEmpty()) { + continue; + } + if (count == edges.length) { + edges = Arrays.copyOf(edges, count * 2); + bounds = Arrays.copyOf(bounds, count * 2); + } + edges[count] = edge; + bounds[count] = scorer.bound(edge); + count++; + } + long[] ids = edges; + double[] limits = bounds; + Integer[] order = IntStream.range(0, count).boxed().toArray(Integer[]::new); + Arrays.sort(order, Comparator.comparingDouble(i -> limits[i]).reversed().thenComparingLong(i -> ids[i])); + PriorityQueue best = new PriorityQueue<>(BEST_FIRST.reversed()); + for (int i : order) { + if (best.size() == query.top() && limits[i] + TOLERANCE < best.peek().score()) { + break; + } + meter.consume(1); + scorer.score(ids[i]).ifPresent(scored -> { + best.add(scored); + if (best.size() > query.top()) { + best.poll(); + } + }); + } + List result = new ArrayList<>(best); + result.sort(BEST_FIRST); + return result; + } + + private List requiredAnchors() { + return IntStream.range(0, query.slots().size()) + .filter(i -> query.slots().get(i).required() && query.slots().get(i).target() instanceof Target.Atom) + .boxed().toList(); + } + + private LongStream candidates(RoleTimeScorer scorer) { + List anchors = requiredAnchors(); + if (!anchors.isEmpty()) { + List> cursors = anchors.stream().map(i -> { + Slot slot = query.slots().get(i); + long atom = ((Target.Atom) slot.target()).id(); + if (slot.roles().isEmpty()) { + return reader.view().incidentCursor(atom); + } + BitSet accepted = scorer.accepted(i); + return reader.view().incidentCursor(atom, incident -> accepted.get(incident.roleSet())); + }).toList(); + return PostingCursor.intersect(cursors); + } + return query.type().map(type -> reader.atomsOfType(type.id())) + .orElseGet(() -> reader.allAtoms(true).mapToLong(Atom::id)); + } + + private static String describe(Slot slot) { + long atom = ((Target.Atom) slot.target()).id(); + return "@" + atom + (slot.roles().isEmpty() ? "" : " AS " + String.join(" OR ", slot.roles().stream().sorted().toList())); + } +} diff --git a/database/src/main/java/io/hstore/db/retrieve/RoleTimeScorer.java b/database/src/main/java/io/hstore/db/retrieve/RoleTimeScorer.java new file mode 100644 index 0000000..0d70ab0 --- /dev/null +++ b/database/src/main/java/io/hstore/db/retrieve/RoleTimeScorer.java @@ -0,0 +1,230 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.retrieve; + +import io.hstore.db.Atom; +import io.hstore.db.Reader; +import io.hstore.db.hora.Budget; +import io.hstore.db.retrieve.RankQuery.Preference; +import io.hstore.db.retrieve.RankQuery.Slot; +import io.hstore.db.retrieve.RankQuery.Target; +import io.hstore.db.retrieve.RankQuery.TimeSpec; +import io.hstore.db.retrieve.RankQuery.Weights; +import io.hstore.db.value.Value; +import io.hstore.engine.Dictionary; +import io.hstore.engine.topology.Incidence; + +import java.util.BitSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; + +public final class RoleTimeScorer { + + private static final int MEMBER_TYPE_CACHE = 65_536; + + public record Components(double members, double roles, double attributes, double time, double topology) { + } + + public record EdgeScore(long edge, double score, Components components, double[] slotCover) { + } + + private record Fill(int value, double time) { + static final Fill EMPTY = new Fill(0, 0); + + boolean better(Fill other) { + return value > other.value || value == other.value && time > other.time; + } + } + + private final Reader reader; + private final RankQuery query; + private final Weights alpha; + private final List accepted; + private final double slotWeights; + private final double preferenceWeights; + private final Map memberTypes = new HashMap<>(); + private final Budget.Meter meter; + + public RoleTimeScorer(Reader reader, RankQuery query, Budget.Meter meter) { + this.reader = reader; + this.query = query; + this.meter = meter; + this.alpha = query.effectiveWeights(); + Dictionary dictionary = reader.database().engine().dictionary(); + this.accepted = query.slots().stream().map(slot -> anyOf(dictionary, slot)).toList(); + this.slotWeights = query.slots().stream().mapToDouble(Slot::weight).sum(); + this.preferenceWeights = query.preferences().stream().mapToDouble(Preference::weight).sum(); + } + + static BitSet anyOf(Dictionary dictionary, Slot slot) { + BitSet sets = new BitSet(); + slot.roles().forEach(role -> sets.or(dictionary.roleSetsWithAll(List.of(role)))); + return sets; + } + + BitSet accepted(int slot) { + return accepted.get(slot); + } + + public double bound(long edge) { + Fill[] fills = new Fill[query.slots().size()]; + for (int i = 0; i < fills.length; i++) { + fills[i] = query.slots().get(i).target() instanceof Target.Atom(long atom) + ? fixed(edge, i, atom) + : new Fill(2, 1); + } + return combine(edge, fills, 1).score(); + } + + public Optional score(long edge) { + Fill[] fills = new Fill[query.slots().size()]; + boolean typed = false; + for (int i = 0; i < fills.length; i++) { + if (query.slots().get(i).target() instanceof Target.Atom(long atom)) { + fills[i] = fixed(edge, i, atom); + } else { + fills[i] = Fill.EMPTY; + typed = true; + } + } + if (typed) { + reader.edge(edge).stream().forEach(incidence -> { + meter.consume(1); + OptionalInt type = memberType(incidence.member()); + if (type.isEmpty()) { + return; + } + for (int i = 0; i < fills.length; i++) { + if (query.slots().get(i).target() instanceof Target.OfType(var slotType) && slotType.id() == type.getAsInt()) { + Fill candidate = fill(i, incidence); + if (candidate.better(fills[i])) { + fills[i] = candidate; + } + } + } + }); + } + for (int i = 0; i < fills.length; i++) { + if (query.slots().get(i).required() && fills[i].value() < 2) { + return Optional.empty(); + } + } + return Optional.of(combine(edge, fills, attributes(edge))); + } + + private OptionalInt memberType(long member) { + OptionalInt cached = memberTypes.get(member); + if (cached != null) { + return cached; + } + OptionalInt type = reader.atom(member).map(found -> OptionalInt.of(found.type().id())).orElse(OptionalInt.empty()); + if (memberTypes.size() < MEMBER_TYPE_CACHE) { + memberTypes.put(member, type); + } + return type; + } + + private Fill fixed(long edge, int slot, long atom) { + return reader.view().incidence(edge, atom).map(incidence -> fill(slot, incidence)).orElse(Fill.EMPTY); + } + + private Fill fill(int slot, Incidence incidence) { + boolean inRole = query.slots().get(slot).roles().isEmpty() || accepted.get(slot).get(incidence.roleSet()); + return new Fill(inRole ? 2 : 1, query.time().map(spec -> time(spec, incidence)).orElse(1.0)); + } + + static double time(TimeSpec spec, Incidence incidence) { + return switch (spec) { + case TimeSpec.At(long instant, double tau) -> { + if (incidence.validAt(instant)) { + yield 1; + } + double distance = instant < incidence.validFrom() + ? gap(incidence.validFrom(), instant) + : gap(instant, incidence.validTo() - 1); + yield tau == 0 ? 0 : Math.exp(-distance / tau); + } + case TimeSpec.During(long start, long end) -> { + long low = Math.max(incidence.validFrom(), start); + long high = Math.min(incidence.validTo(), end); + yield high <= low ? 0 : Math.min(1, gap(high, low) / gap(end, start)); + } + }; + } + + private static double gap(long high, long low) { + long difference = high - low; + return ((high ^ low) & (high ^ difference)) < 0 ? (double) high - (double) low : difference; + } + + private double attributes(long edge) { + if (query.preferences().isEmpty()) { + return 0; + } + double total = 0; + for (Preference preference : query.preferences()) { + Optional value = reader.property(edge, preference.property()); + total += preference.weight() * value.map(found -> similarity(preference, found)).orElse(0.0); + } + return total / preferenceWeights; + } + + static double similarity(Preference preference, Value value) { + return switch (preference) { + case Preference.Equals(String _, Value target, double _) -> + comparable(value) && comparable(target) && value.compareTo(target) == 0 ? 1 : 0; + case Preference.Near(String _, double target, double scale, double _) -> { + double number = value.tag().numeric() ? value.number().orElse(Double.NaN) : Double.NaN; + yield Double.isFinite(number) ? Math.max(0, 1 - Math.abs(number - target) / scale) : 0; + } + }; + } + + private static boolean comparable(Value value) { + return !(value instanceof Value.Null) && !(value instanceof Value.Real(double number) && !Double.isFinite(number)); + } + + private EdgeScore combine(long edge, Fill[] fills, double attributes) { + double members = 0; + double roles = 0; + double filledWeight = 0; + double time = 0; + double[] cover = new double[fills.length]; + for (int i = 0; i < fills.length; i++) { + double weight = query.slots().get(i).weight(); + Fill fill = fills[i]; + if (fill.value() > 0) { + members += weight; + filledWeight += weight; + time += weight * fill.time(); + } + if (fill.value() == 2) { + roles += weight; + } + cover[i] = fill.value() / 2.0; + } + Components components = new Components(members / slotWeights, roles / slotWeights, + query.preferences().isEmpty() ? Double.NaN : attributes, + query.time().isEmpty() ? Double.NaN : filledWeight == 0 ? 0 : time / filledWeight, + query.size().isEmpty() ? Double.NaN : topology(edge)); + double score = alpha.members() * components.members() + alpha.roles() * components.roles() + + (query.preferences().isEmpty() ? 0 : alpha.attributes() * components.attributes()) + + (query.time().isEmpty() ? 0 : alpha.time() * components.time()) + + (query.size().isEmpty() ? 0 : alpha.topology() * components.topology()); + return new EdgeScore(edge, score, components, cover); + } + + private double topology(long edge) { + double size = reader.cardinality(edge); + double target = query.size().orElseThrow(); + return size == 0 ? 0 : Math.min(size, target) / Math.max(size, target); + } + + static Optional admitted(Reader reader, long edge, RankQuery query) { + return reader.atom(edge).filter(Atom::isEdge).filter(atom -> query.type().map(type -> type.id() == atom.type().id()).orElse(true)); + } +} diff --git a/database/src/main/java/module-info.java b/database/src/main/java/module-info.java index 3396d01..cbd0c23 100644 --- a/database/src/main/java/module-info.java +++ b/database/src/main/java/module-info.java @@ -11,6 +11,7 @@ exports io.hstore.db.payload; exports io.hstore.db.property; exports io.hstore.db.query; + exports io.hstore.db.retrieve; exports io.hstore.db.schema; exports io.hstore.db.security; exports io.hstore.db.semantic; diff --git a/database/src/test/java/io/hstore/db/SecurityTest.java b/database/src/test/java/io/hstore/db/SecurityTest.java index 9bb6203..81b07f7 100644 --- a/database/src/test/java/io/hstore/db/SecurityTest.java +++ b/database/src/test/java/io/hstore/db/SecurityTest.java @@ -326,6 +326,36 @@ void roleFilteredMatchesStayInsideTheTenant() { } } + @Test + void rankStaysInsideTheTenant() { + try (Session alice = login("alice", "s3cret-pass"); Session bob = login("bob", "hunter2-pass")) { + alice.executeAll("INSERT NODE Patient 'p1' {mrn: 'A-1'} AS $p; INSERT EDGE Encounter MEMBERS ($p AS patient) AS $e;"); + bob.executeAll(""" + INSERT NODE Patient 'b1' {mrn: 'B-1'} AS $p; + INSERT NODE Patient 'b2' {mrn: 'B-2'} AS $q; + INSERT EDGE Encounter MEMBERS ($q AS clinician) AS $e; + """); + long acmePatient = alice.variables().get("p"); + long acmeEdge = alice.variables().get("e"); + long globexEdge = bob.variables().get("e"); + int patient = database.engine().dictionary().roleSet(List.of("patient")); + database.write(writer -> { + writer.transaction().upsert(globexEdge, acmePatient, _ -> new Incidence(acmePatient, patient, Weight.ONE, + Long.MIN_VALUE, Long.MAX_VALUE, 0, 0)); + return null; + }); + String failure = assertThrows(HStoreException.InvalidSchema.class, + () -> bob.execute("RANK EDGE e:Encounter LIKE (@" + acmePatient + " AS patient) TOP 5")).getMessage(); + assertEquals("atom " + acmePatient + " does not exist", failure); + QueryResult ranked = bob.execute("RANK EDGE e:Encounter LIKE (ANY Patient AS patient) TOP 5"); + assertEquals(List.of("@" + globexEdge), ranked.rows().stream().map(row -> row.getFirst().render().split(" ")[0]).toList()); + assertEquals("0.6", ranked.rows().getFirst().get(1).render()); + assertEquals("0", ranked.rows().getFirst().get(3).render()); + QueryResult own = alice.execute("RANK EDGE e:Encounter LIKE (@" + acmePatient + " AS patient REQUIRED) TOP 5"); + assertEquals(List.of("@" + acmeEdge), own.rows().stream().map(row -> row.getFirst().render().split(" ")[0]).toList()); + } + } + @Test void qualifyOnlyAcceptsVisibleMembers() { try (Session alice = login("alice", "s3cret-pass"); Session bob = login("bob", "hunter2-pass")) { diff --git a/database/src/test/java/io/hstore/db/query/RankTest.java b/database/src/test/java/io/hstore/db/query/RankTest.java new file mode 100644 index 0000000..995dcc2 --- /dev/null +++ b/database/src/test/java/io/hstore/db/query/RankTest.java @@ -0,0 +1,269 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.query; + +import io.hstore.db.DatabaseOptions; +import io.hstore.db.HypergraphDatabase; +import io.hstore.db.MemberSpec; +import io.hstore.db.hora.Budget; +import io.hstore.db.retrieve.RankQuery; +import io.hstore.db.retrieve.Ranking; +import io.hstore.db.retrieve.RoleTimeScorer; +import io.hstore.db.schema.TypeDef; +import io.hstore.db.value.Value; +import io.hstore.engine.EngineOptions; +import io.hstore.engine.HStoreException; +import io.hstore.engine.topology.Validity; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; +import java.util.random.RandomGenerator; +import java.util.random.RandomGeneratorFactory; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class RankTest { + + private static final List ROLES = List.of("actor", "reviewer", "target", "observer"); + + @TempDir + Path directory; + + HypergraphDatabase database; + Session session; + + @BeforeEach + void open() { + database = HypergraphDatabase.open(directory, DatabaseOptions.defaults() + .withEngine(EngineOptions.defaults().withPageSize(4096))); + session = new Session(database); + session.executeAll(""" + CREATE NODE TYPE Person; + CREATE NODE TYPE Document; + CREATE SET EDGE TYPE Review (status STRING, amount FLOAT) ROLES (actor, reviewer, target, observer); + """); + } + + @AfterEach + void close() { + session.close(); + database.close(); + } + + private static List column(QueryResult result, int index) { + return result.rows().stream().map(row -> row.get(index).render()).toList(); + } + + @Test + void workedExampleRanksByTheFiveComponents() { + session.executeAll(""" + INSERT NODE Person 'alice' AS $alice; + INSERT NODE Person 'bob' AS $bob; + INSERT NODE Person 'carol' AS $carol; + INSERT NODE Person 'dave' AS $dave; + INSERT NODE Document 'd1' AS $d1; + INSERT NODE Document 'd2' AS $d2; + INSERT NODE Document 'd3' AS $d3; + INSERT EDGE Review {status: 'approved'} MEMBERS ($alice AS actor, $bob AS reviewer VALID [0, 50), $d1 AS target) AS $e1; + INSERT EDGE Review {status: 'draft'} MEMBERS ($alice AS actor, $bob AS actor, $d2 AS target) AS $e2; + INSERT EDGE Review {status: 'approved'} MEMBERS ($alice AS actor, $carol AS reviewer, $d3 AS target, $dave AS observer) AS $e3; + """); + String query = """ + RANK EDGE e:Review LIKE ($alice AS actor REQUIRED, $bob AS reviewer, ANY Document AS target) + VALID AT 100 TAU 100 PREFER e.status = 'approved' SIZE 3 TOP %d"""; + QueryResult all = session.execute(query.formatted(3)); + List order = all.rows().stream().map(row -> Long.parseLong(row.getFirst().render().substring(1).split(" ")[0])).toList(); + assertEquals(List.of(session.variables().get("e1"), session.variables().get("e3"), session.variables().get("e2")), order); + List scores = all.rows().stream().map(row -> Double.parseDouble(row.get(1).render())).toList(); + double tau = Math.exp(-51.0 / 100); + assertEquals(0.3 + 0.2 + 0.2 + 0.2 * (2 + tau) / 3 + 0.1, scores.get(0), 1e-9); + assertEquals(0.3 * 2 / 3 + 0.2 * 2 / 3 + 0.2 + 0.2 + 0.1 * 0.75, scores.get(1), 1e-9); + assertEquals(0.3 + 0.2 * 2 / 3 + 0.2 + 0.1, scores.get(2), 1e-9); + assertEquals(List.of(session.variables().get("e1")), session.execute(query.formatted(1)).rows().stream() + .map(row -> Long.parseLong(row.getFirst().render().substring(1).split(" ")[0])).toList()); + QueryResult plain = session.execute("RANK EDGE e:Review LIKE ($bob AS reviewer) TOP 5"); + assertEquals(List.of("NULL", "NULL", "NULL"), List.of(plain.rows().getFirst().get(4).render().toUpperCase(), + plain.rows().getFirst().get(5).render().toUpperCase(), plain.rows().getFirst().get(6).render().toUpperCase())); + assertEquals("1", plain.rows().getFirst().get(1).render()); + assertEquals(3, plain.rows().size()); + assertTrue(all.trace().orElseThrow().plan().startsWith("IncidentIntersection[@" + session.variables().get("alice") + " AS actor]")); + } + + @Test + void earlyStoppingAgreesWithScoringEverything() { + RandomGenerator random = RandomGeneratorFactory.of("L64X128MixRandom").create(5); + List people = new ArrayList<>(); + List documents = new ArrayList<>(); + database.write(writer -> { + for (int i = 0; i < 25; i++) { + people.add(writer.node("Person", "p" + i)); + documents.add(writer.node("Document", "d" + i)); + } + for (int e = 0; e < 500; e++) { + long edge = writer.edge("Review", Map.of("status", random.nextBoolean() ? "approved" : "draft", + "amount", random.nextInt(200) * 1.0)); + for (int k = 0, size = 1 + random.nextInt(5); k < size; k++) { + long member = random.nextInt(3) == 0 ? documents.get(random.nextInt(documents.size())) + : people.get(random.nextInt(k == 0 ? 3 : people.size())); + long from = random.nextInt(100); + Validity validity = random.nextBoolean() ? Validity.ALWAYS : new Validity(from, from + 1 + random.nextInt(60)); + writer.add(edge, member, MemberSpec.roles(ROLES.stream().filter(_ -> random.nextInt(3) == 0).toArray(String[]::new)) + .withValidity(validity)); + } + } + return null; + }); + database.read(reader -> { + TypeDef review = reader.requireType("Review"); + TypeDef document = reader.requireType("Document"); + for (int round = 0; round < 300; round++) { + List slots = new ArrayList<>(); + for (int i = 0, n = 1 + random.nextInt(4); i < n; i++) { + RankQuery.Target target = random.nextInt(4) == 0 + ? new RankQuery.Target.OfType(document) + : new RankQuery.Target.Atom(people.get(random.nextInt(i == 0 ? 3 : people.size()))); + Set roles = Set.copyOf(ROLES.stream().filter(_ -> random.nextInt(3) == 0).toList()); + slots.add(new RankQuery.Slot(target, roles, 0.5 + random.nextInt(3), random.nextInt(4) == 0)); + } + Optional time = switch (random.nextInt(3)) { + case 0 -> Optional.empty(); + case 1 -> Optional.of(new RankQuery.TimeSpec.At(random.nextInt(150), random.nextInt(3) * 25.0)); + default -> { + long from = random.nextInt(120); + yield Optional.of(new RankQuery.TimeSpec.During(from, from + 1 + random.nextInt(80))); + } + }; + List preferences = random.nextBoolean() ? List.of() + : List.of(new RankQuery.Preference.Equals("status", new Value.Text("approved"), 1), + new RankQuery.Preference.Near("amount", 100, 50, 2)); + OptionalLong size = random.nextBoolean() ? OptionalLong.empty() : OptionalLong.of(1 + random.nextInt(4)); + int top = 1 + random.nextInt(12); + RankQuery.Weights weights = random.nextBoolean() ? RankQuery.Weights.DEFAULT : new RankQuery.Weights(0.05 + random.nextDouble(), + random.nextDouble(), random.nextDouble(), random.nextDouble(), random.nextDouble()); + RankQuery query = new RankQuery(Optional.of(review), slots, time, preferences, size, weights, top); + RoleTimeScorer scorer = new RoleTimeScorer(reader, query, Budget.UNBOUNDED.start()); + List everything = reader.atomsOfType(review.id()).boxed() + .flatMap(edge -> { + Optional scored = scorer.score(edge); + scored.ifPresent(found -> assertTrue(scorer.bound(edge) + 1e-9 >= found.score(), + "bound below score for " + edge + " in " + query)); + return scored.stream(); + }) + .sorted(Comparator.comparingDouble(RoleTimeScorer.EdgeScore::score).reversed() + .thenComparingLong(RoleTimeScorer.EdgeScore::edge)) + .limit(top) + .toList(); + List ranked = Ranking.of(reader, query).top(Budget.DEFAULT.start()); + assertEquals(everything.stream().map(RoleTimeScorer.EdgeScore::edge).toList(), + ranked.stream().map(RoleTimeScorer.EdgeScore::edge).toList(), query.toString()); + IntStream.range(0, ranked.size()).forEach(i -> assertEquals(everything.get(i).score(), ranked.get(i).score(), 1e-12)); + } + return null; + }); + } + + @Test + void requiredTypedSlotsAreHardConstraints() { + session.executeAll(""" + INSERT NODE Person 'alice' AS $alice; + INSERT NODE Document 'd1' AS $d1; + INSERT EDGE Review MEMBERS ($alice AS actor, $d1 AS target) AS $with; + INSERT EDGE Review MEMBERS ($alice AS actor, $d1 AS observer) AS $wrongRole; + INSERT EDGE Review MEMBERS ($alice AS actor) AS $without; + """); + QueryResult result = session.execute("RANK EDGE e:Review LIKE ($alice, ANY Document AS target REQUIRED) TOP 10"); + assertEquals(1, result.rows().size()); + assertTrue(result.rows().getFirst().getFirst().render().startsWith("@" + session.variables().get("with"))); + assertTrue(result.trace().orElseThrow().plan().startsWith("TypeScan(Review)")); + assertEquals(2, session.execute("RANK EDGE e:Review LIKE ($alice, ANY Document AS target OR observer REQUIRED) TOP 10").rows().size()); + } + + @Test + void invalidRankStatementsFailCleanly() { + session.execute("INSERT NODE Person 'alice' AS $alice"); + List statements = List.of( + "RANK EDGE e:Review LIKE ($alice) TOP 0", + "RANK EDGE e:Review LIKE ($alice) TOP 10001", + "RANK EDGE e:Review LIKE ($alice) TOP 99999999999", + "RANK EDGE e:Review LIKE ($alice) VALID AT 5 TAU -1 TOP 3", + "RANK EDGE e:Review LIKE ($alice) VALID DURING [5, 5) TOP 3", + "RANK EDGE e:Review LIKE ($alice) VALID DURING [*, 5) TOP 3", + "RANK EDGE e:Review LIKE ($alice) WEIGHTS (bogus 1) TOP 3", + "RANK EDGE e:Review LIKE ($alice) WEIGHTS (members 0, roles 0) TOP 3", + "RANK EDGE e:Review LIKE ($alice) WEIGHTS (members -1) TOP 3", + "RANK EDGE e:Review LIKE ($alice WEIGHT 0) TOP 3", + "RANK EDGE e:Review LIKE ($alice) PREFER x.status = 'a' TOP 3", + "RANK EDGE e:Review LIKE ($alice) PREFER e.amount NEAR 5 SCALE 0 TOP 3", + "RANK EDGE e:Review LIKE ($alice) SIZE 0 TOP 3", + "RANK EDGE e:Person LIKE ($alice) TOP 3", + "RANK EDGE e:Review LIKE (@999999) TOP 3", + "RANK EDGE e:Review LIKE (ANY Missing) TOP 3", + "RANK EDGE e:Review LIKE (" + "$alice, ".repeat(16) + "$alice) TOP 3", + "RANK EDGE e:Review LIKE ($alice) PREFER " + "e.amount NEAR 1 SCALE 1, ".repeat(16) + "e.amount NEAR 1 SCALE 1 TOP 3", + "RANK EDGE e:Review LIKE ($alice) VALID DURING [5, *) TOP 3", + "RANK EDGE e:Review LIKE () TOP 3", + "RANK EDGE e:Review ($alice) TOP 3", + "RANK EDGE e:Review LIKE ($alice)", + "RANK NODE e:Review LIKE ($alice) TOP 3", + "RANK EDGE e:Review LIKE ($alice) WEIGHTS (members 1, members 2) TOP 3"); + for (String statement : statements) { + assertThrows(HStoreException.class, () -> session.execute(statement), statement); + } + } + + @Test + void untypedRequiredAndRepeatedSlots() { + session.executeAll(""" + CREATE SET EDGE TYPE Note ROLES (author); + INSERT NODE Person 'alice' AS $alice; + INSERT NODE Person 'bob' AS $bob; + INSERT EDGE Review MEMBERS ($alice AS actor, $bob AS reviewer) AS $review; + INSERT EDGE Note MEMBERS ($alice AS author) AS $note; + INSERT EDGE Note MEMBERS ($bob AS author) AS $other; + """); + QueryResult untyped = session.execute("RANK EDGE e LIKE ($alice) TOP 10"); + assertTrue(untyped.trace().orElseThrow().plan().startsWith("CatalogScan(edges)")); + assertEquals(List.of("1", "1", "0"), column(untyped, 1)); + QueryResult required = session.execute("RANK EDGE e LIKE ($alice REQUIRED, $bob) TOP 10"); + assertEquals(2, required.rows().size()); + assertTrue(required.rows().getFirst().getFirst().render().startsWith("@" + session.variables().get("review"))); + QueryResult twice = session.execute("RANK EDGE e:Review LIKE ($alice AS actor, $alice AS reviewer) TOP 10"); + assertEquals(List.of("1"), column(twice, 2)); + assertEquals(List.of("0.5"), column(twice, 3)); + } + + @Test + void budgetsStopRankingInsteadOfTruncatingIt() { + database.write(writer -> { + long alice = writer.node("Person", "alice"); + for (int i = 0; i < 20; i++) { + writer.add(writer.edge("Review"), alice, "actor"); + } + return null; + }); + database.read(reader -> { + RankQuery query = new RankQuery(Optional.of(reader.requireType("Review")), + List.of(new RankQuery.Slot(new RankQuery.Target.OfType(reader.requireType("Person")), Set.of("actor"), 1, false)), + Optional.empty(), List.of(), OptionalLong.empty(), RankQuery.Weights.DEFAULT, 5); + HStoreException failure = assertThrows(HStoreException.class, () -> Ranking.of(reader, query).top(Budget.DEFAULT.withRows(10).start())); + assertTrue(failure.getMessage().contains("budget"), failure.getMessage()); + assertEquals(5, Ranking.of(reader, query).top(Budget.DEFAULT.start()).size()); + return null; + }); + } +} diff --git a/database/src/test/java/io/hstore/db/retrieve/RoleTimeScorerTest.java b/database/src/test/java/io/hstore/db/retrieve/RoleTimeScorerTest.java new file mode 100644 index 0000000..e28efed --- /dev/null +++ b/database/src/test/java/io/hstore/db/retrieve/RoleTimeScorerTest.java @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.retrieve; + +import io.hstore.db.value.Value; +import io.hstore.engine.topology.Incidence; +import io.hstore.engine.topology.Weight; + +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class RoleTimeScorerTest { + + private static Incidence valid(long from, long to) { + return new Incidence(1, 0, Weight.ONE, from, to, 0, 0); + } + + @Test + void timeScoresAreExactAtTheEdgesOfTheLongRange() { + long big = 1L << 60; + assertEquals(1.0, RoleTimeScorer.time(new RankQuery.TimeSpec.At(big, 10), valid(big, big + 1))); + assertEquals(Math.exp(-0.1), RoleTimeScorer.time(new RankQuery.TimeSpec.At(big + 1, 10), valid(big, big + 1)), 1e-12); + assertEquals(0.0, RoleTimeScorer.time(new RankQuery.TimeSpec.At(big + 1, 0), valid(big, big + 1))); + assertEquals(1.0, RoleTimeScorer.time(new RankQuery.TimeSpec.At(0, 5), valid(Long.MIN_VALUE, Long.MAX_VALUE))); + assertEquals(0.0, RoleTimeScorer.time(new RankQuery.TimeSpec.At(Long.MAX_VALUE - 1, 1), valid(Long.MIN_VALUE, Long.MIN_VALUE + 1))); + assertEquals(1.0, RoleTimeScorer.time(new RankQuery.TimeSpec.During(-big, big), valid(Long.MIN_VALUE, Long.MAX_VALUE))); + assertEquals(0.25, RoleTimeScorer.time(new RankQuery.TimeSpec.During(0, 400), valid(100, 200))); + assertEquals(0.0, RoleTimeScorer.time(new RankQuery.TimeSpec.During(0, 400), valid(400, 500))); + } + + @Test + void nonFiniteValuesScoreZero() { + RankQuery.Preference near = new RankQuery.Preference.Near("amount", 1, 1, 1); + RankQuery.Preference equals = new RankQuery.Preference.Equals("amount", new Value.Real(1), 1); + for (double broken : new double[]{Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}) { + assertEquals(0.0, RoleTimeScorer.similarity(near, new Value.Real(broken))); + assertEquals(0.0, RoleTimeScorer.similarity(equals, new Value.Real(broken))); + } + assertEquals(1.0, RoleTimeScorer.similarity(equals, new Value.Decimal(BigDecimal.ONE))); + assertEquals(0.5, RoleTimeScorer.similarity(near, new Value.Real(1.5)), 1e-12); + assertEquals(0.0, RoleTimeScorer.similarity(near, new Value.Int(2))); + assertEquals(0.0, RoleTimeScorer.similarity(near, new Value.Text("2"))); + } +} diff --git a/docs/README.md b/docs/README.md index 296c397..c1efb7a 100644 --- a/docs/README.md +++ b/docs/README.md @@ -25,6 +25,7 @@ Start here * [database/data-model.md](database/data-model.md): logical data model (types, atoms, properties, roles, documents, payloads) and its mapping onto engine slots. * [database/hql.md](database/hql.md): HQL reference: every statement, its grammar and examples. * [database/planner.md](database/planner.md): how `MATCH` is planned (access paths, cost estimates, `EXPLAIN`, `TRACE`) and executed. +* [database/ranking.md](database/ranking.md): how `RANK` scores hyperedges against a query hyperedge and finds the best ones without scoring all of them. * [database/hora.md](database/hora.md): higher-order relational algebra: gather, scatter, propagate, overlap join, closure, pattern matching. * [database/semantic.md](database/semantic.md): the semantic plane: embeddings, encoders, the persistent HNSW index, freshness. * [database/evidence-and-temporal.md](database/evidence-and-temporal.md): assertions, evidence and provenance; valid time and transaction time. diff --git a/docs/database/hql.md b/docs/database/hql.md index cadcc08..fde4876 100644 --- a/docs/database/hql.md +++ b/docs/database/hql.md @@ -80,7 +80,7 @@ Values and their storage are described in [data model](data-model.md#values). |---|---| | [Schema](#schema-ddl) | `CREATE NODE TYPE`, `CREATE SET EDGE TYPE`, `CREATE ORDERED EDGE TYPE`, `CREATE INDEX`, `CREATE JSON INDEX`, `SHOW TYPES` | | [Data](#data-manipulation) | `INSERT NODE`, `INSERT EDGE`, `ADD`, `REMOVE`, `REMOVE AT`, `SET`, `UNSET`, `DOCUMENT`, `DELETE` | -| [Queries](#queries) | `MATCH`, `EXPLAIN`, `MEMBERS OF`, `INCIDENT TO`, `DESCRIBE`, set algebra | +| [Queries](#queries) | `MATCH`, `EXPLAIN`, `RANK`, `MEMBERS OF`, `INCIDENT TO`, `DESCRIBE`, set algebra | | [HORA](#hora-operators) | `GATHER`, `REDUCE`, `SCATTER`, `PROPAGATE`, `NEIGHBORS`, `OVERLAP JOIN`, `CLOSURE`, `EXPAND`, `PATTERN`, `SAMPLE` | | [Semantic](#semantic) | `EMBED … TEXT`, `EMBED … VECTOR`, `SIMILAR TO` | | [Evidence and state](#evidence-state-and-signals) | `EVIDENCE`, `QUALIFY`, `TRACE`, `STATE`, `SIGNAL`, `SIGNALS FOR` | @@ -372,6 +372,59 @@ IncidentIntersection[@9, @25] rows≈1 cost≈12 rejected IndexRange(Claim.amount (100, +inf]) rows≈28 cost≈17 ``` +### RANK + +```ebnf +rank = "RANK" ( "EDGE" | "EDGES" ) identifier [ ":" identifier ] + "LIKE" "(" slot { "," slot } ")" + [ "VALID" ( "AT" additive [ "TAU" number ] | "DURING" interval ) ] + [ "PREFER" preference { "," preference } ] + [ "SIZE" integer ] + [ "WEIGHTS" "(" component number { "," component number } ")" ] + "TOP" integer ; +slot = ( ref | "ANY" identifier ) [ "AS" identifier { "OR" identifier } ] [ "WEIGHT" number ] [ "REQUIRED" ] ; +preference = identifier "." identifier ( "=" literal | "NEAR" number "SCALE" number ) [ "WEIGHT" number ] ; +component = "members" | "roles" | "attributes" | "time" | "topology" ; +``` + +Returns the `TOP` hyperedges that best fit a query hyperedge, best first. `MATCH` answers which hyperedges satisfy +a condition; `RANK` scores how well each one fits and never changes what `MATCH` returns. + +A slot asks for a named atom or for any atom of a type, optionally in one of several roles (`AS actor OR reviewer` +means either role). `REQUIRED` makes the slot a hard condition: the hyperedge must have that member in an accepted +role. Every other part of the query only changes the score. The score is a weighted sum of five parts, each between +0 and 1: + +| Part | Meaning of 1 | +|---|---| +| `members` | every slot is filled | +| `roles` | every slot is filled in an accepted role | +| `attributes` | every `PREFER` holds (`=` exactly; `NEAR t SCALE s` scores `1 - |x - t| / s`, at least 0) | +| `time` | every filled member is valid at the `VALID AT` instant, or covers the whole `VALID DURING` window | +| `topology` | the hyperedge has exactly `SIZE` members | + +The default weights are `members 0.3, roles 0.2, attributes 0.2, time 0.2, topology 0.1`. A part the query does not +use (no `PREFER`, no `VALID`, no `SIZE`) gets weight 0 and the rest are scaled to add up to 1. `VALID AT t TAU d` +scores a member outside its validity `exp(-distance / d)`; without `TAU` it scores 0. `VALID DURING` needs a finite +interval (no `*`). Ties are broken by the lower hyperedge id. + +Output columns: `edge, score, members, roles, attributes, time, topology`; unused parts are `NULL`. `TOP` is at most +10,000, a query has at most 16 slots and 16 preferences. The algorithm, its bounds and its costs are in +[ranking](ranking.md). + +``` +RANK EDGE e:Review + LIKE ($alice AS actor REQUIRED, $bob AS reviewer, ANY Document AS target) + VALID AT 100 TAU 100 + PREFER e.status = 'approved' + SIZE 3 + TOP 3; +-- | edge | score | members | roles | attributes | time | topology | +-- | @14 Review | 0.9734 | 1 | 1 | 1 | 0.8668 | 1 | +-- | @16 Review | 0.8083 | 0.6667 | 0.6667 | 1 | 1 | 0.75 | +-- | @15 Review | 0.7333 | 1 | 0.6667 | 0 | 1 | 1 | +``` + ### MEMBERS OF ```ebnf diff --git a/docs/database/ranking.md b/docs/database/ranking.md new file mode 100644 index 0000000..53f1e4b --- /dev/null +++ b/docs/database/ranking.md @@ -0,0 +1,108 @@ +# Ranking hyperedges (`RANK`) + +This page describes how `RANK` ([HQL](hql.md#rank)) scores hyperedges against a query hyperedge and returns the best +`k`: `retrieve/RankQuery.java`, `retrieve/RoleTimeScorer.java`, `retrieve/Ranking.java` and `Executor.rank`. + +## Query and score + +A query is a list of slots, each a named atom or "any atom of type `T`", with accepted roles, a weight `w_i > 0` and +a `REQUIRED` flag; optional `VALID AT` or `VALID DURING`, `PREFER` predicates, a target `SIZE`, component weights +and `TOP k`. + +For a hyperedge `e`, every slot picks its best incidence of `e`: a named atom has at most one; a typed slot looks at +every member of that type. "Best" means filled in an accepted role over filled in another role over empty, and among +equals the one with the higher time score. Two slots may pick the same member. With `W = Σ w_i`: + +| Part | Value | +|---|---| +| `members` | `Σ w_i` over filled slots, divided by `W` | +| `roles` | `Σ w_i` over slots filled in an accepted role, divided by `W` | +| `time` | weighted mean of `τ` over filled slots, 0 if none is filled | +| `attributes` | weighted mean of the `PREFER` similarities; a missing property scores 0 | +| `topology` | `min(|M(e)|, SIZE) / max(|M(e)|, SIZE)` | + +`τ` for an incidence valid on `[a, b)`: under `VALID AT t TAU d`, 1 if `a ≤ t < b`, otherwise `exp(-dist / d)` where +`dist` is the distance from `t` to `[a, b - 1]` (0 without `TAU`); under `VALID DURING [f, u)`, the fraction of +`[f, u)` that `[a, b)` covers. + +The score is `Σ α_c · part_c` with the effective weights `α`: the given or default weights, with the parts the query +does not use set to 0, scaled to add up to 1. So every score is between 0 and 1 and means the same thing in every +query. Parts are never normalised over the candidates, which would need every candidate scored before any score is +known. + +## Candidates + +```mermaid +flowchart LR + Q["RANK statement"] --> R{"REQUIRED named slots?"} + R -- yes --> I["leapfrog over their postings,
role-filtered (PostingCursor)"] + R -- no --> T["TypeScan, or every hyperedge
when no type is given"] + I --> A["admit: tenant, kind, type"] + T --> A + A --> B["bound U(e)"] + B --> S["sort by U desc, id asc"] + S --> P{"heap full and
U(e) < k-th score?"} + P -- yes --> D["stop"] + P -- no --> F["full score S(e);
drop if a REQUIRED typed slot is not met"] + F --> H["top-k heap"] --> P +``` + +Required named slots drive the candidates: their reverse-index postings are intersected with the same cursors as +`MATCH` ([planner](planner.md#incidence-intersection-contains-has)), a slot with roles accepting any role set that +contains one of them. Without a required named slot every hyperedge of the type is a candidate, because an edge +that shares no named member can still score above zero. That is exact but reads the whole type, so a query over a +large type should mark its main participant `REQUIRED`. Every candidate passes the tenant, kind and type check of +`Reader.atom` before anything else reads it. + +## Bounds and early stop + +Each candidate first gets an upper bound `U(e) ≥ S(e)`: + +| Part | In the bound | +|---|---| +| named slots | exact: one `View.incidence(e, a)` lookup per slot | +| typed slots | assumed filled in an accepted role with `τ = 1` | +| `attributes` | 1 | +| `time` | exact over named slots, typed slots counted at `τ = 1`; a weighted mean only grows when terms of 1 are added | +| `topology` | exact, from the membership count | + +Candidates are then scored in descending bound order, keeping the best `k` in a heap. The scan stops at the first +candidate whose bound is below the `k`-th score: no later candidate can enter. A candidate whose bound equals that +score is still scored, so ties are broken by id exactly as an exhaustive scorer would. This is the threshold idea of +Fagin, Lotem and Naor applied to one list of per-candidate bounds. + +Named slots are exact in the bound, so queries built from known participants usually stop after a few full scores. +Typed slots and preferences have a bound of 1, and a query made only of those scores every candidate. + +## Costs + +`C` candidates, `s` slots, `p` preferences, `|M(e)|` the size of a scored hyperedge. + +| Step | Time | +|---|---| +| candidates from required slots | the leapfrog cost of the planner page | +| bound | `O(s · log |M(e)|)` per candidate | +| order | `O(C log C)` | +| full score | `O(s · log |M(e)| + p)`, plus `O(|M(e)|)` member reads when a slot is typed | + +Each candidate, each full score and each member read for a typed slot takes one row of a `Budget.DEFAULT` meter +(1,000,000 rows, 30 s), and page reads are charged to the statement's page budget. Member types are cached for up +to 65,536 members per query. A query that exceeds either fails with +`ABORTED_RESOURCE_LIMIT`; it never returns a partial ranking as if it were complete. + +## Tenancy + +Named atoms are resolved with `Reader.require`, so naming another tenant's atom fails as if it did not exist. Typed +slots only count members whose catalog record `Reader.atom` returns, which are the reader's own. Candidates from +postings are checked with `Reader.atom` before their incidences or properties are read. `topology` uses the +hyperedge's member count, like `card()`: in a database written before memberships were checked, a hyperedge may +still count a member of another tenant. + +## Limits + +| Not supported | Reason | +|---|---| +| distinct members per slot (`DISTINCT`) | needs an assignment problem per hyperedge; slots may share a member | +| graded role similarity | the role part is 0 or 1 per slot | +| ordered positions in slots | position is not part of a slot | +| a tight bound for typed slots | needs the edge-signature index proposed for structural matching | diff --git a/server/src/main/java/io/hstore/server/StatementLogger.java b/server/src/main/java/io/hstore/server/StatementLogger.java index 0b4ef04..d659e73 100644 --- a/server/src/main/java/io/hstore/server/StatementLogger.java +++ b/server/src/main/java/io/hstore/server/StatementLogger.java @@ -51,7 +51,7 @@ private static Category category(Statement statement) { case Statement.CreateType _, Statement.CreateIndex _, Statement.CreateJsonIndex _, Statement.CreateView _, Statement.CreateBranch _, Statement.DropBranch _, Statement.CreateTenant _, Statement.AlterTenant _, Statement.CreateUser _, Statement.AlterUser _, Statement.DropUser _, Statement.CheckpointAt _ -> Category.DDL; - case Statement.Query _, Statement.Explain _, Statement.Members _, Statement.IncidentTo _, Statement.Describe _, + case Statement.Query _, Statement.Explain _, Statement.Rank _, Statement.Members _, Statement.IncidentTo _, Statement.Describe _, Statement.Gather _, Statement.Reduce _, Statement.Scatter _, Statement.Propagate _, Statement.Neighbors _, Statement.OverlapJoin _, Statement.Closure _, Statement.PatternQuery _, Statement.Expand _, Statement.History _, Statement.Stats _, Statement.ShowTypes _, Statement.ShowBranches _, Statement.ShowViews _, Statement.ShowView _,