From 79552d2905c616684f09c6cdfc5e355f6a8bdef8 Mon Sep 17 00:00:00 2001 From: venkat1701 Date: Sun, 25 Oct 2026 19:52:48 +0530 Subject: [PATCH 1/2] feat(query): add RANK for scoring hyperedges against a query hyperedge RANK EDGE e:T LIKE (slots) [VALID AT t [TAU d] | VALID DURING [f, u)] [PREFER ...] [SIZE n] [WEIGHTS (...)] TOP k returns the k hyperedges that best fit a query hyperedge. The score is a weighted sum of five parts in [0, 1]: members, roles, attributes, time and topology. MATCH is unchanged. REQUIRED named slots drive the candidates through role-filtered posting cursors; otherwise every hyperedge of the type is a candidate. Each candidate gets an upper bound from cheap reads, and candidates are fully scored in bound order until no remaining bound can enter the top k, which returns exactly what scoring every candidate would. Candidates, full scores and member reads are charged to a row budget, slots and preferences are capped while parsing, and an empty interval in any statement is now a syntax error instead of an internal one. --- .../src/main/java/io/hstore/db/query/Ast.java | 31 ++ .../java/io/hstore/db/query/Executor.java | 67 +++++ .../main/java/io/hstore/db/query/Parser.java | 89 ++++++ .../java/io/hstore/db/retrieve/RankQuery.java | 128 +++++++++ .../java/io/hstore/db/retrieve/Ranking.java | 121 ++++++++ .../io/hstore/db/retrieve/RoleTimeScorer.java | 230 +++++++++++++++ database/src/main/java/module-info.java | 1 + .../test/java/io/hstore/db/SecurityTest.java | 30 ++ .../java/io/hstore/db/query/RankTest.java | 269 ++++++++++++++++++ .../db/retrieve/RoleTimeScorerTest.java | 48 ++++ docs/README.md | 1 + docs/database/hql.md | 55 +++- docs/database/ranking.md | 108 +++++++ .../io/hstore/server/StatementLogger.java | 2 +- 14 files changed, 1178 insertions(+), 2 deletions(-) create mode 100644 database/src/main/java/io/hstore/db/retrieve/RankQuery.java create mode 100644 database/src/main/java/io/hstore/db/retrieve/Ranking.java create mode 100644 database/src/main/java/io/hstore/db/retrieve/RoleTimeScorer.java create mode 100644 database/src/test/java/io/hstore/db/query/RankTest.java create mode 100644 database/src/test/java/io/hstore/db/retrieve/RoleTimeScorerTest.java create mode 100644 docs/database/ranking.md diff --git a/database/src/main/java/io/hstore/db/query/Ast.java b/database/src/main/java/io/hstore/db/query/Ast.java index ec695d6f..774c704e 100644 --- a/database/src/main/java/io/hstore/db/query/Ast.java +++ b/database/src/main/java/io/hstore/db/query/Ast.java @@ -110,6 +110,33 @@ public record Match(boolean edges, String variable, Optional type, Optio List projections, Optional orderBy, boolean descending, OptionalLong limit) { } + public sealed interface RankTarget { + record Atom(Ref ref) implements RankTarget { + } + + record OfType(String type) implements RankTarget { + } + } + + public record RankSlot(RankTarget target, List roles, OptionalDouble weight, boolean required) { + } + + public sealed interface RankTime { + record At(Expr instant, OptionalDouble tau) implements RankTime { + } + + record During(long from, long to) implements RankTime { + } + } + + public sealed interface RankPreference { + record Equals(String variable, String property, Value target, OptionalDouble weight) implements RankPreference { + } + + record Near(String variable, String property, double target, double scale, OptionalDouble weight) implements RankPreference { + } + } + public sealed interface Statement { record Begin(boolean serializable) implements Statement { } @@ -265,6 +292,10 @@ record SetAlgebra(SetOp op, Ref left, Ref right, Optional into) implemen record Expand(Ref atom, int depth, OptionalLong limit) implements Statement { } + record Rank(String variable, Optional type, List slots, Optional time, + List preferences, OptionalLong size, Map weights, long top) implements Statement { + } + record Gather(Field field, Ref edge) implements Statement { } diff --git a/database/src/main/java/io/hstore/db/query/Executor.java b/database/src/main/java/io/hstore/db/query/Executor.java index 1c75e05a..8bdeea43 100644 --- a/database/src/main/java/io/hstore/db/query/Executor.java +++ b/database/src/main/java/io/hstore/db/query/Executor.java @@ -23,6 +23,9 @@ import io.hstore.db.query.Ast.MemberClause; import io.hstore.db.query.Ast.Statement; import io.hstore.db.query.QueryResult.Cell; +import io.hstore.db.retrieve.RankQuery; +import io.hstore.db.retrieve.Ranking; +import io.hstore.db.retrieve.RoleTimeScorer; import io.hstore.db.schema.TypeDef; import io.hstore.db.security.Principal; import io.hstore.db.security.Quota; @@ -61,6 +64,7 @@ import java.util.Optional; import java.util.OptionalLong; import java.util.Set; +import java.util.TreeSet; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; import java.util.stream.Collectors; @@ -268,6 +272,7 @@ QueryResult run(Statement statement) { case Statement.IncidentTo(Ast.Ref atom, Optional validAt) -> read(reader -> incident(reader, session.resolve(reader, atom), validAt)); case Statement.Describe(Ast.Ref atom) -> read(reader -> describe(reader, session.resolve(reader, atom))); case Statement.SetAlgebra algebra -> algebra.into().isPresent() ? write(writer -> setAlgebra(writer, algebra)) : read(reader -> setAlgebra(reader, algebra)); + case Statement.Rank rank -> read(reader -> rank(reader, rank)); case Statement.Expand(Ast.Ref atom, int depth, OptionalLong limit) -> read(reader -> table(List.of("path", "atom", "status"), Hora.on(reader).expand(session.resolve(reader, atom), depth(depth), budget(limit).withDepth(depth(depth))) .map(expansion -> List.of(new Cell.Items(expansion.path().stream().map(id -> (Cell) new Cell.AtomCell(id, "")).toList()), @@ -466,6 +471,68 @@ private QueryResult query(Reader reader, Ast.Match match) { rows.size(), trace.pagesRead(), trace.cacheHits(), System.nanoTime() - started)); } + private QueryResult rank(Reader reader, Statement.Rank statement) { + long started = System.nanoTime(); + IoTrace trace = IoTrace.withBudget(database().queryPageBudget(session.principal())); + Ranking ranking = Ranking.of(reader, rankQuery(reader, statement)); + List scores = trace.call(() -> ranking.top(Budget.DEFAULT.start())); + List> rows = scores.stream().map(scored -> List.of(atomCell(reader, scored.edge()), QueryResult.number(scored.score()), + component(scored.components().members()), component(scored.components().roles()), component(scored.components().attributes()), + component(scored.components().time()), component(scored.components().topology()))).toList(); + return QueryResult.table(List.of("edge", "score", "members", "roles", "attributes", "time", "topology"), rows) + .withTrace(new QueryResult.Trace(0, reader.generation(), ranking.access(), Double.NaN, rows.size(), + trace.pagesRead(), trace.cacheHits(), System.nanoTime() - started)); + } + + private static Cell component(double value) { + return Double.isNaN(value) ? QueryResult.scalar(Value.NULL) : QueryResult.number(value); + } + + private RankQuery rankQuery(Reader reader, Statement.Rank statement) { + Optional type = statement.type().map(reader::requireType); + List slots = statement.slots().stream().map(slot -> new RankQuery.Slot(switch (slot.target()) { + case Ast.RankTarget.Atom(Ast.Ref ref) -> new RankQuery.Target.Atom(reader.require(session.resolve(reader, ref)).id()); + case Ast.RankTarget.OfType(String name) -> new RankQuery.Target.OfType(reader.requireType(name)); + }, Set.copyOf(slot.roles()), slot.weight().orElse(1), slot.required())).toList(); + Evaluator evaluator = evaluator(reader); + Optional time = statement.time().map(spec -> switch (spec) { + case Ast.RankTime.At(Expr instant, var tau) -> new RankQuery.TimeSpec.At( + (long) evaluator.value(instant, Map.of()).number().orElseThrow(() -> HStoreException.invalid("VALID AT needs a number or a timestamp")), + tau.orElse(0)); + case Ast.RankTime.During(long from, long to) -> new RankQuery.TimeSpec.During(from, to); + }); + List preferences = statement.preferences().stream().map(preference -> switch (preference) { + case Ast.RankPreference.Equals(String variable, String property, Value target, var weight) -> { + requireRankVariable(statement, variable); + yield (RankQuery.Preference) new RankQuery.Preference.Equals(property, target, weight.orElse(1)); + } + case Ast.RankPreference.Near(String variable, String property, double target, double scale, var weight) -> { + requireRankVariable(statement, variable); + yield new RankQuery.Preference.Near(property, target, scale, weight.orElse(1)); + } + }).toList(); + RankQuery.Weights defaults = RankQuery.Weights.DEFAULT; + Map given = statement.weights(); + Set unknown = new TreeSet<>(given.keySet()); + unknown.removeAll(Set.of("members", "roles", "attributes", "time", "topology")); + if (!unknown.isEmpty()) { + throw HStoreException.invalid("unknown component " + unknown + "; expected members, roles, attributes, time or topology"); + } + RankQuery.Weights weights = new RankQuery.Weights(given.getOrDefault("members", defaults.members()), + given.getOrDefault("roles", defaults.roles()), given.getOrDefault("attributes", defaults.attributes()), + given.getOrDefault("time", defaults.time()), given.getOrDefault("topology", defaults.topology())); + if (statement.top() > RankQuery.MAX_TOP) { + throw HStoreException.invalid("TOP must be between 1 and " + RankQuery.MAX_TOP); + } + return new RankQuery(type, slots, time, preferences, statement.size(), weights, (int) Math.max(0, statement.top())); + } + + private static void requireRankVariable(Statement.Rank statement, String variable) { + if (!variable.equals(statement.variable())) { + throw HStoreException.invalid("unknown variable " + variable + "; RANK binds only " + statement.variable()); + } + } + private static boolean admits(Reader reader, Planner.Plan plan, long atom) { if (plan.chosen().path() instanceof AccessPath.TypeScan) { return true; diff --git a/database/src/main/java/io/hstore/db/query/Parser.java b/database/src/main/java/io/hstore/db/query/Parser.java index 45e0950d..a710b511 100644 --- a/database/src/main/java/io/hstore/db/query/Parser.java +++ b/database/src/main/java/io/hstore/db/query/Parser.java @@ -15,6 +15,7 @@ import io.hstore.db.query.Ast.Ref; import io.hstore.db.query.Ast.SetOp; import io.hstore.db.query.Ast.Statement; +import io.hstore.db.retrieve.RankQuery; import io.hstore.db.schema.AtomKind; import io.hstore.db.schema.TypeDef.PropertyDef; import io.hstore.db.security.Role; @@ -314,6 +315,9 @@ private Statement statement() { return new Statement.SetAlgebra(op, left, right, accept("INTO") ? Optional.of(variable()) : Optional.empty()); } } + if (accept("RANK")) { + return rank(); + } if (accept("EXPAND")) { Ref atom = ref(); int depth = accept("DEPTH") ? intValue() : 4; @@ -651,6 +655,88 @@ private Bounds bounds() { return new Bounds(exact, exact); } + private Statement.Rank rank() { + if (!accept("EDGE") && !accept("EDGES")) { + throw error("expected EDGE"); + } + String variable = identifier(); + Optional type = acceptSymbol(":") ? Optional.of(identifier()) : Optional.empty(); + expect("LIKE"); + expectSymbol("("); + List slots = new ArrayList<>(); + do { + if (slots.size() == RankQuery.MAX_SLOTS) { + throw error("RANK accepts at most " + RankQuery.MAX_SLOTS + " slots"); + } + slots.add(rankSlot()); + } while (acceptSymbol(",")); + expectSymbol(")"); + Optional time = Optional.empty(); + if (accept("VALID")) { + if (accept("DURING")) { + Validity interval = interval(); + time = Optional.of(new Ast.RankTime.During(interval.from(), interval.to())); + } else { + expect("AT"); + Expr instant = additive(); + time = Optional.of(new Ast.RankTime.At(instant, accept("TAU") ? OptionalDouble.of(number().doubleValue()) : OptionalDouble.empty())); + } + } + List preferences = new ArrayList<>(); + if (accept("PREFER")) { + do { + if (preferences.size() == RankQuery.MAX_PREFERENCES) { + throw error("RANK accepts at most " + RankQuery.MAX_PREFERENCES + " preferences"); + } + preferences.add(preference()); + } while (acceptSymbol(",")); + } + OptionalLong size = accept("SIZE") ? OptionalLong.of(longValue()) : OptionalLong.empty(); + Map weights = new LinkedHashMap<>(); + if (accept("WEIGHTS")) { + expectSymbol("("); + do { + String component = identifier().toLowerCase(Locale.ROOT); + if (weights.put(component, number().doubleValue()) != null) { + throw error("the weight of " + component + " is given twice"); + } + } while (acceptSymbol(",")); + expectSymbol(")"); + } + expect("TOP"); + return new Statement.Rank(variable, type, slots, time, preferences, size, weights, longValue()); + } + + private Ast.RankSlot rankSlot() { + Ast.RankTarget target = accept("ANY") ? new Ast.RankTarget.OfType(identifier()) : new Ast.RankTarget.Atom(ref()); + List roles = new ArrayList<>(); + if (accept("AS")) { + do { + roles.add(identifier()); + } while (accept("OR")); + } + OptionalDouble weight = accept("WEIGHT") ? OptionalDouble.of(number().doubleValue()) : OptionalDouble.empty(); + return new Ast.RankSlot(target, roles, weight, accept("REQUIRED")); + } + + private Ast.RankPreference preference() { + String variable = identifier(); + expectSymbol("."); + String property = identifier(); + if (accept("NEAR")) { + double target = number().doubleValue(); + expect("SCALE"); + double scale = number().doubleValue(); + return new Ast.RankPreference.Near(variable, property, target, scale, preferenceWeight()); + } + expectSymbol("="); + return new Ast.RankPreference.Equals(variable, property, literal(), preferenceWeight()); + } + + private OptionalDouble preferenceWeight() { + return accept("WEIGHT") ? OptionalDouble.of(number().doubleValue()) : OptionalDouble.empty(); + } + private Ast.Match match() { boolean edges; if (accept("EDGE") || accept("EDGES")) { @@ -867,6 +953,9 @@ private Validity interval() { expectSymbol(","); long to = acceptSymbol("*") ? Long.MAX_VALUE : instant(); expectSymbol(")"); + if (from >= to) { + throw error("the interval [" + from + ", " + to + ") is empty"); + } return new Validity(from, to); } diff --git a/database/src/main/java/io/hstore/db/retrieve/RankQuery.java b/database/src/main/java/io/hstore/db/retrieve/RankQuery.java new file mode 100644 index 00000000..35b7d0ec --- /dev/null +++ b/database/src/main/java/io/hstore/db/retrieve/RankQuery.java @@ -0,0 +1,128 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.retrieve; + +import io.hstore.db.schema.TypeDef; +import io.hstore.db.value.Value; +import io.hstore.engine.HStoreException; + +import java.util.List; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; + +public record RankQuery(Optional type, List slots, Optional time, List preferences, + OptionalLong size, Weights weights, int top) { + + public static final int MAX_SLOTS = 16; + public static final int MAX_PREFERENCES = 16; + public static final int MAX_TOP = 10_000; + + public RankQuery { + slots = List.copyOf(slots); + preferences = List.copyOf(preferences); + if (slots.isEmpty() || slots.size() > MAX_SLOTS) { + throw HStoreException.invalid("RANK needs between 1 and " + MAX_SLOTS + " slots"); + } + if (preferences.size() > MAX_PREFERENCES) { + throw HStoreException.invalid("RANK accepts at most " + MAX_PREFERENCES + " preferences"); + } + if (top < 1 || top > MAX_TOP) { + throw HStoreException.invalid("TOP must be between 1 and " + MAX_TOP); + } + if (size.isPresent() && size.getAsLong() < 1) { + throw HStoreException.invalid("SIZE must be at least 1"); + } + type.ifPresent(found -> { + if (!found.kind().isEdge()) { + throw HStoreException.invalid(found.name() + " is not a hyperedge type"); + } + }); + } + + public Weights effectiveWeights() { + return weights.restrictedTo(!preferences.isEmpty(), time.isPresent(), size.isPresent()); + } + + public sealed interface Target { + record Atom(long id) implements Target { + } + + record OfType(TypeDef type) implements Target { + } + } + + public record Slot(Target target, Set roles, double weight, boolean required) { + public Slot { + roles = Set.copyOf(roles); + requireWeight("slot", weight); + } + } + + public sealed interface TimeSpec { + record At(long instant, double tau) implements TimeSpec { + public At { + if (!Double.isFinite(tau) || tau < 0) { + throw HStoreException.invalid("TAU must be a number of at least 0"); + } + } + } + + record During(long from, long to) implements TimeSpec { + public During { + if (from >= to || from == Long.MIN_VALUE || to == Long.MAX_VALUE) { + throw HStoreException.invalid("VALID DURING needs a finite, non-empty interval"); + } + } + } + } + + public sealed interface Preference { + String property(); + + double weight(); + + record Equals(String property, Value target, double weight) implements Preference { + public Equals { + requireWeight("preference", weight); + } + } + + record Near(String property, double target, double scale, double weight) implements Preference { + public Near { + requireWeight("preference", weight); + if (!Double.isFinite(target) || !Double.isFinite(scale) || scale <= 0) { + throw HStoreException.invalid("NEAR needs a finite target and a positive SCALE"); + } + } + } + } + + public record Weights(double members, double roles, double attributes, double time, double topology) { + public static final Weights DEFAULT = new Weights(0.3, 0.2, 0.2, 0.2, 0.1); + + public Weights { + for (double weight : new double[]{members, roles, attributes, time, topology}) { + if (!Double.isFinite(weight) || weight < 0) { + throw HStoreException.invalid("component weights must be finite and not negative"); + } + } + } + + Weights restrictedTo(boolean useAttributes, boolean useTime, boolean useTopology) { + Weights used = new Weights(members, roles, useAttributes ? attributes : 0, useTime ? time : 0, useTopology ? topology : 0); + double total = used.members + used.roles + used.attributes + used.time + used.topology; + if (total <= 0) { + throw HStoreException.invalid("the weights of the components this query uses add up to zero"); + } + return new Weights(used.members / total, used.roles / total, used.attributes / total, used.time / total, used.topology / total); + } + } + + private static void requireWeight(String kind, double weight) { + if (!Double.isFinite(weight) || weight <= 0 || weight > 1_000_000) { + throw HStoreException.invalid("a " + kind + " WEIGHT must be a positive number up to 1000000"); + } + } +} diff --git a/database/src/main/java/io/hstore/db/retrieve/Ranking.java b/database/src/main/java/io/hstore/db/retrieve/Ranking.java new file mode 100644 index 00000000..aa2e2fff --- /dev/null +++ b/database/src/main/java/io/hstore/db/retrieve/Ranking.java @@ -0,0 +1,121 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.retrieve; + +import io.hstore.db.Atom; +import io.hstore.db.Reader; +import io.hstore.db.hora.Budget; +import io.hstore.db.retrieve.RankQuery.Slot; +import io.hstore.db.retrieve.RankQuery.Target; +import io.hstore.db.retrieve.RoleTimeScorer.EdgeScore; +import io.hstore.engine.index.IndexValues; +import io.hstore.engine.index.PostingCursor; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.BitSet; +import java.util.Comparator; +import java.util.List; +import java.util.Optional; +import java.util.PriorityQueue; +import java.util.stream.IntStream; +import java.util.stream.LongStream; + +public final class Ranking { + + private static final double TOLERANCE = 1e-9; + private static final Comparator BEST_FIRST = + Comparator.comparingDouble(EdgeScore::score).reversed().thenComparingLong(EdgeScore::edge); + + private final Reader reader; + private final RankQuery query; + + private Ranking(Reader reader, RankQuery query) { + this.reader = reader; + this.query = query; + } + + public static Ranking of(Reader reader, RankQuery query) { + return new Ranking(reader, query); + } + + public String access() { + List anchors = requiredAnchors(); + if (!anchors.isEmpty()) { + return "IncidentIntersection" + anchors.stream().map(i -> describe(query.slots().get(i))).toList(); + } + return query.type().map(type -> "TypeScan(" + type.name() + ")").orElse("CatalogScan(edges)"); + } + + public List top(Budget.Meter meter) { + RoleTimeScorer scorer = new RoleTimeScorer(reader, query, meter); + long[] edges = new long[64]; + double[] bounds = new double[64]; + int count = 0; + for (var candidates = candidates(scorer).iterator(); candidates.hasNext(); ) { + long edge = candidates.nextLong(); + meter.consume(1); + Optional admitted = RoleTimeScorer.admitted(reader, edge, query); + if (admitted.isEmpty()) { + continue; + } + if (count == edges.length) { + edges = Arrays.copyOf(edges, count * 2); + bounds = Arrays.copyOf(bounds, count * 2); + } + edges[count] = edge; + bounds[count] = scorer.bound(edge); + count++; + } + long[] ids = edges; + double[] limits = bounds; + Integer[] order = IntStream.range(0, count).boxed().toArray(Integer[]::new); + Arrays.sort(order, Comparator.comparingDouble(i -> limits[i]).reversed().thenComparingLong(i -> ids[i])); + PriorityQueue best = new PriorityQueue<>(BEST_FIRST.reversed()); + for (int i : order) { + if (best.size() == query.top() && limits[i] + TOLERANCE < best.peek().score()) { + break; + } + meter.consume(1); + scorer.score(ids[i]).ifPresent(scored -> { + best.add(scored); + if (best.size() > query.top()) { + best.poll(); + } + }); + } + List result = new ArrayList<>(best); + result.sort(BEST_FIRST); + return result; + } + + private List requiredAnchors() { + return IntStream.range(0, query.slots().size()) + .filter(i -> query.slots().get(i).required() && query.slots().get(i).target() instanceof Target.Atom) + .boxed().toList(); + } + + private LongStream candidates(RoleTimeScorer scorer) { + List anchors = requiredAnchors(); + if (!anchors.isEmpty()) { + List> cursors = anchors.stream().map(i -> { + Slot slot = query.slots().get(i); + long atom = ((Target.Atom) slot.target()).id(); + if (slot.roles().isEmpty()) { + return reader.view().incidentCursor(atom); + } + BitSet accepted = scorer.accepted(i); + return reader.view().incidentCursor(atom, incident -> accepted.get(incident.roleSet())); + }).toList(); + return PostingCursor.intersect(cursors); + } + return query.type().map(type -> reader.atomsOfType(type.id())) + .orElseGet(() -> reader.allAtoms(true).mapToLong(Atom::id)); + } + + private static String describe(Slot slot) { + long atom = ((Target.Atom) slot.target()).id(); + return "@" + atom + (slot.roles().isEmpty() ? "" : " AS " + String.join(" OR ", slot.roles().stream().sorted().toList())); + } +} diff --git a/database/src/main/java/io/hstore/db/retrieve/RoleTimeScorer.java b/database/src/main/java/io/hstore/db/retrieve/RoleTimeScorer.java new file mode 100644 index 00000000..0d70ab00 --- /dev/null +++ b/database/src/main/java/io/hstore/db/retrieve/RoleTimeScorer.java @@ -0,0 +1,230 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.retrieve; + +import io.hstore.db.Atom; +import io.hstore.db.Reader; +import io.hstore.db.hora.Budget; +import io.hstore.db.retrieve.RankQuery.Preference; +import io.hstore.db.retrieve.RankQuery.Slot; +import io.hstore.db.retrieve.RankQuery.Target; +import io.hstore.db.retrieve.RankQuery.TimeSpec; +import io.hstore.db.retrieve.RankQuery.Weights; +import io.hstore.db.value.Value; +import io.hstore.engine.Dictionary; +import io.hstore.engine.topology.Incidence; + +import java.util.BitSet; +import java.util.HashMap; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalInt; + +public final class RoleTimeScorer { + + private static final int MEMBER_TYPE_CACHE = 65_536; + + public record Components(double members, double roles, double attributes, double time, double topology) { + } + + public record EdgeScore(long edge, double score, Components components, double[] slotCover) { + } + + private record Fill(int value, double time) { + static final Fill EMPTY = new Fill(0, 0); + + boolean better(Fill other) { + return value > other.value || value == other.value && time > other.time; + } + } + + private final Reader reader; + private final RankQuery query; + private final Weights alpha; + private final List accepted; + private final double slotWeights; + private final double preferenceWeights; + private final Map memberTypes = new HashMap<>(); + private final Budget.Meter meter; + + public RoleTimeScorer(Reader reader, RankQuery query, Budget.Meter meter) { + this.reader = reader; + this.query = query; + this.meter = meter; + this.alpha = query.effectiveWeights(); + Dictionary dictionary = reader.database().engine().dictionary(); + this.accepted = query.slots().stream().map(slot -> anyOf(dictionary, slot)).toList(); + this.slotWeights = query.slots().stream().mapToDouble(Slot::weight).sum(); + this.preferenceWeights = query.preferences().stream().mapToDouble(Preference::weight).sum(); + } + + static BitSet anyOf(Dictionary dictionary, Slot slot) { + BitSet sets = new BitSet(); + slot.roles().forEach(role -> sets.or(dictionary.roleSetsWithAll(List.of(role)))); + return sets; + } + + BitSet accepted(int slot) { + return accepted.get(slot); + } + + public double bound(long edge) { + Fill[] fills = new Fill[query.slots().size()]; + for (int i = 0; i < fills.length; i++) { + fills[i] = query.slots().get(i).target() instanceof Target.Atom(long atom) + ? fixed(edge, i, atom) + : new Fill(2, 1); + } + return combine(edge, fills, 1).score(); + } + + public Optional score(long edge) { + Fill[] fills = new Fill[query.slots().size()]; + boolean typed = false; + for (int i = 0; i < fills.length; i++) { + if (query.slots().get(i).target() instanceof Target.Atom(long atom)) { + fills[i] = fixed(edge, i, atom); + } else { + fills[i] = Fill.EMPTY; + typed = true; + } + } + if (typed) { + reader.edge(edge).stream().forEach(incidence -> { + meter.consume(1); + OptionalInt type = memberType(incidence.member()); + if (type.isEmpty()) { + return; + } + for (int i = 0; i < fills.length; i++) { + if (query.slots().get(i).target() instanceof Target.OfType(var slotType) && slotType.id() == type.getAsInt()) { + Fill candidate = fill(i, incidence); + if (candidate.better(fills[i])) { + fills[i] = candidate; + } + } + } + }); + } + for (int i = 0; i < fills.length; i++) { + if (query.slots().get(i).required() && fills[i].value() < 2) { + return Optional.empty(); + } + } + return Optional.of(combine(edge, fills, attributes(edge))); + } + + private OptionalInt memberType(long member) { + OptionalInt cached = memberTypes.get(member); + if (cached != null) { + return cached; + } + OptionalInt type = reader.atom(member).map(found -> OptionalInt.of(found.type().id())).orElse(OptionalInt.empty()); + if (memberTypes.size() < MEMBER_TYPE_CACHE) { + memberTypes.put(member, type); + } + return type; + } + + private Fill fixed(long edge, int slot, long atom) { + return reader.view().incidence(edge, atom).map(incidence -> fill(slot, incidence)).orElse(Fill.EMPTY); + } + + private Fill fill(int slot, Incidence incidence) { + boolean inRole = query.slots().get(slot).roles().isEmpty() || accepted.get(slot).get(incidence.roleSet()); + return new Fill(inRole ? 2 : 1, query.time().map(spec -> time(spec, incidence)).orElse(1.0)); + } + + static double time(TimeSpec spec, Incidence incidence) { + return switch (spec) { + case TimeSpec.At(long instant, double tau) -> { + if (incidence.validAt(instant)) { + yield 1; + } + double distance = instant < incidence.validFrom() + ? gap(incidence.validFrom(), instant) + : gap(instant, incidence.validTo() - 1); + yield tau == 0 ? 0 : Math.exp(-distance / tau); + } + case TimeSpec.During(long start, long end) -> { + long low = Math.max(incidence.validFrom(), start); + long high = Math.min(incidence.validTo(), end); + yield high <= low ? 0 : Math.min(1, gap(high, low) / gap(end, start)); + } + }; + } + + private static double gap(long high, long low) { + long difference = high - low; + return ((high ^ low) & (high ^ difference)) < 0 ? (double) high - (double) low : difference; + } + + private double attributes(long edge) { + if (query.preferences().isEmpty()) { + return 0; + } + double total = 0; + for (Preference preference : query.preferences()) { + Optional value = reader.property(edge, preference.property()); + total += preference.weight() * value.map(found -> similarity(preference, found)).orElse(0.0); + } + return total / preferenceWeights; + } + + static double similarity(Preference preference, Value value) { + return switch (preference) { + case Preference.Equals(String _, Value target, double _) -> + comparable(value) && comparable(target) && value.compareTo(target) == 0 ? 1 : 0; + case Preference.Near(String _, double target, double scale, double _) -> { + double number = value.tag().numeric() ? value.number().orElse(Double.NaN) : Double.NaN; + yield Double.isFinite(number) ? Math.max(0, 1 - Math.abs(number - target) / scale) : 0; + } + }; + } + + private static boolean comparable(Value value) { + return !(value instanceof Value.Null) && !(value instanceof Value.Real(double number) && !Double.isFinite(number)); + } + + private EdgeScore combine(long edge, Fill[] fills, double attributes) { + double members = 0; + double roles = 0; + double filledWeight = 0; + double time = 0; + double[] cover = new double[fills.length]; + for (int i = 0; i < fills.length; i++) { + double weight = query.slots().get(i).weight(); + Fill fill = fills[i]; + if (fill.value() > 0) { + members += weight; + filledWeight += weight; + time += weight * fill.time(); + } + if (fill.value() == 2) { + roles += weight; + } + cover[i] = fill.value() / 2.0; + } + Components components = new Components(members / slotWeights, roles / slotWeights, + query.preferences().isEmpty() ? Double.NaN : attributes, + query.time().isEmpty() ? Double.NaN : filledWeight == 0 ? 0 : time / filledWeight, + query.size().isEmpty() ? Double.NaN : topology(edge)); + double score = alpha.members() * components.members() + alpha.roles() * components.roles() + + (query.preferences().isEmpty() ? 0 : alpha.attributes() * components.attributes()) + + (query.time().isEmpty() ? 0 : alpha.time() * components.time()) + + (query.size().isEmpty() ? 0 : alpha.topology() * components.topology()); + return new EdgeScore(edge, score, components, cover); + } + + private double topology(long edge) { + double size = reader.cardinality(edge); + double target = query.size().orElseThrow(); + return size == 0 ? 0 : Math.min(size, target) / Math.max(size, target); + } + + static Optional admitted(Reader reader, long edge, RankQuery query) { + return reader.atom(edge).filter(Atom::isEdge).filter(atom -> query.type().map(type -> type.id() == atom.type().id()).orElse(true)); + } +} diff --git a/database/src/main/java/module-info.java b/database/src/main/java/module-info.java index 3396d011..cbd0c231 100644 --- a/database/src/main/java/module-info.java +++ b/database/src/main/java/module-info.java @@ -11,6 +11,7 @@ exports io.hstore.db.payload; exports io.hstore.db.property; exports io.hstore.db.query; + exports io.hstore.db.retrieve; exports io.hstore.db.schema; exports io.hstore.db.security; exports io.hstore.db.semantic; diff --git a/database/src/test/java/io/hstore/db/SecurityTest.java b/database/src/test/java/io/hstore/db/SecurityTest.java index 16844a3f..654cf094 100644 --- a/database/src/test/java/io/hstore/db/SecurityTest.java +++ b/database/src/test/java/io/hstore/db/SecurityTest.java @@ -326,6 +326,36 @@ void roleFilteredMatchesStayInsideTheTenant() { } } + @Test + void rankStaysInsideTheTenant() { + try (Session alice = login("alice", "s3cret"); Session bob = login("bob", "hunter2")) { + alice.executeAll("INSERT NODE Patient 'p1' {mrn: 'A-1'} AS $p; INSERT EDGE Encounter MEMBERS ($p AS patient) AS $e;"); + bob.executeAll(""" + INSERT NODE Patient 'b1' {mrn: 'B-1'} AS $p; + INSERT NODE Patient 'b2' {mrn: 'B-2'} AS $q; + INSERT EDGE Encounter MEMBERS ($q AS clinician) AS $e; + """); + long acmePatient = alice.variables().get("p"); + long acmeEdge = alice.variables().get("e"); + long globexEdge = bob.variables().get("e"); + int patient = database.engine().dictionary().roleSet(List.of("patient")); + database.write(writer -> { + writer.transaction().upsert(globexEdge, acmePatient, _ -> new Incidence(acmePatient, patient, Weight.ONE, + Long.MIN_VALUE, Long.MAX_VALUE, 0, 0)); + return null; + }); + String failure = assertThrows(HStoreException.InvalidSchema.class, + () -> bob.execute("RANK EDGE e:Encounter LIKE (@" + acmePatient + " AS patient) TOP 5")).getMessage(); + assertEquals("atom " + acmePatient + " does not exist", failure); + QueryResult ranked = bob.execute("RANK EDGE e:Encounter LIKE (ANY Patient AS patient) TOP 5"); + assertEquals(List.of("@" + globexEdge), ranked.rows().stream().map(row -> row.getFirst().render().split(" ")[0]).toList()); + assertEquals("0.6", ranked.rows().getFirst().get(1).render()); + assertEquals("0", ranked.rows().getFirst().get(3).render()); + QueryResult own = alice.execute("RANK EDGE e:Encounter LIKE (@" + acmePatient + " AS patient REQUIRED) TOP 5"); + assertEquals(List.of("@" + acmeEdge), own.rows().stream().map(row -> row.getFirst().render().split(" ")[0]).toList()); + } + } + @Test void qualifyOnlyAcceptsVisibleMembers() { try (Session alice = login("alice", "s3cret"); Session bob = login("bob", "hunter2")) { diff --git a/database/src/test/java/io/hstore/db/query/RankTest.java b/database/src/test/java/io/hstore/db/query/RankTest.java new file mode 100644 index 00000000..995dcc2d --- /dev/null +++ b/database/src/test/java/io/hstore/db/query/RankTest.java @@ -0,0 +1,269 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.query; + +import io.hstore.db.DatabaseOptions; +import io.hstore.db.HypergraphDatabase; +import io.hstore.db.MemberSpec; +import io.hstore.db.hora.Budget; +import io.hstore.db.retrieve.RankQuery; +import io.hstore.db.retrieve.Ranking; +import io.hstore.db.retrieve.RoleTimeScorer; +import io.hstore.db.schema.TypeDef; +import io.hstore.db.value.Value; +import io.hstore.engine.EngineOptions; +import io.hstore.engine.HStoreException; +import io.hstore.engine.topology.Validity; + +import org.junit.jupiter.api.AfterEach; +import org.junit.jupiter.api.BeforeEach; +import org.junit.jupiter.api.Test; +import org.junit.jupiter.api.io.TempDir; + +import java.nio.file.Path; +import java.util.ArrayList; +import java.util.Comparator; +import java.util.List; +import java.util.Map; +import java.util.Optional; +import java.util.OptionalLong; +import java.util.Set; +import java.util.random.RandomGenerator; +import java.util.random.RandomGeneratorFactory; +import java.util.stream.IntStream; + +import static org.junit.jupiter.api.Assertions.assertEquals; +import static org.junit.jupiter.api.Assertions.assertThrows; +import static org.junit.jupiter.api.Assertions.assertTrue; + +class RankTest { + + private static final List ROLES = List.of("actor", "reviewer", "target", "observer"); + + @TempDir + Path directory; + + HypergraphDatabase database; + Session session; + + @BeforeEach + void open() { + database = HypergraphDatabase.open(directory, DatabaseOptions.defaults() + .withEngine(EngineOptions.defaults().withPageSize(4096))); + session = new Session(database); + session.executeAll(""" + CREATE NODE TYPE Person; + CREATE NODE TYPE Document; + CREATE SET EDGE TYPE Review (status STRING, amount FLOAT) ROLES (actor, reviewer, target, observer); + """); + } + + @AfterEach + void close() { + session.close(); + database.close(); + } + + private static List column(QueryResult result, int index) { + return result.rows().stream().map(row -> row.get(index).render()).toList(); + } + + @Test + void workedExampleRanksByTheFiveComponents() { + session.executeAll(""" + INSERT NODE Person 'alice' AS $alice; + INSERT NODE Person 'bob' AS $bob; + INSERT NODE Person 'carol' AS $carol; + INSERT NODE Person 'dave' AS $dave; + INSERT NODE Document 'd1' AS $d1; + INSERT NODE Document 'd2' AS $d2; + INSERT NODE Document 'd3' AS $d3; + INSERT EDGE Review {status: 'approved'} MEMBERS ($alice AS actor, $bob AS reviewer VALID [0, 50), $d1 AS target) AS $e1; + INSERT EDGE Review {status: 'draft'} MEMBERS ($alice AS actor, $bob AS actor, $d2 AS target) AS $e2; + INSERT EDGE Review {status: 'approved'} MEMBERS ($alice AS actor, $carol AS reviewer, $d3 AS target, $dave AS observer) AS $e3; + """); + String query = """ + RANK EDGE e:Review LIKE ($alice AS actor REQUIRED, $bob AS reviewer, ANY Document AS target) + VALID AT 100 TAU 100 PREFER e.status = 'approved' SIZE 3 TOP %d"""; + QueryResult all = session.execute(query.formatted(3)); + List order = all.rows().stream().map(row -> Long.parseLong(row.getFirst().render().substring(1).split(" ")[0])).toList(); + assertEquals(List.of(session.variables().get("e1"), session.variables().get("e3"), session.variables().get("e2")), order); + List scores = all.rows().stream().map(row -> Double.parseDouble(row.get(1).render())).toList(); + double tau = Math.exp(-51.0 / 100); + assertEquals(0.3 + 0.2 + 0.2 + 0.2 * (2 + tau) / 3 + 0.1, scores.get(0), 1e-9); + assertEquals(0.3 * 2 / 3 + 0.2 * 2 / 3 + 0.2 + 0.2 + 0.1 * 0.75, scores.get(1), 1e-9); + assertEquals(0.3 + 0.2 * 2 / 3 + 0.2 + 0.1, scores.get(2), 1e-9); + assertEquals(List.of(session.variables().get("e1")), session.execute(query.formatted(1)).rows().stream() + .map(row -> Long.parseLong(row.getFirst().render().substring(1).split(" ")[0])).toList()); + QueryResult plain = session.execute("RANK EDGE e:Review LIKE ($bob AS reviewer) TOP 5"); + assertEquals(List.of("NULL", "NULL", "NULL"), List.of(plain.rows().getFirst().get(4).render().toUpperCase(), + plain.rows().getFirst().get(5).render().toUpperCase(), plain.rows().getFirst().get(6).render().toUpperCase())); + assertEquals("1", plain.rows().getFirst().get(1).render()); + assertEquals(3, plain.rows().size()); + assertTrue(all.trace().orElseThrow().plan().startsWith("IncidentIntersection[@" + session.variables().get("alice") + " AS actor]")); + } + + @Test + void earlyStoppingAgreesWithScoringEverything() { + RandomGenerator random = RandomGeneratorFactory.of("L64X128MixRandom").create(5); + List people = new ArrayList<>(); + List documents = new ArrayList<>(); + database.write(writer -> { + for (int i = 0; i < 25; i++) { + people.add(writer.node("Person", "p" + i)); + documents.add(writer.node("Document", "d" + i)); + } + for (int e = 0; e < 500; e++) { + long edge = writer.edge("Review", Map.of("status", random.nextBoolean() ? "approved" : "draft", + "amount", random.nextInt(200) * 1.0)); + for (int k = 0, size = 1 + random.nextInt(5); k < size; k++) { + long member = random.nextInt(3) == 0 ? documents.get(random.nextInt(documents.size())) + : people.get(random.nextInt(k == 0 ? 3 : people.size())); + long from = random.nextInt(100); + Validity validity = random.nextBoolean() ? Validity.ALWAYS : new Validity(from, from + 1 + random.nextInt(60)); + writer.add(edge, member, MemberSpec.roles(ROLES.stream().filter(_ -> random.nextInt(3) == 0).toArray(String[]::new)) + .withValidity(validity)); + } + } + return null; + }); + database.read(reader -> { + TypeDef review = reader.requireType("Review"); + TypeDef document = reader.requireType("Document"); + for (int round = 0; round < 300; round++) { + List slots = new ArrayList<>(); + for (int i = 0, n = 1 + random.nextInt(4); i < n; i++) { + RankQuery.Target target = random.nextInt(4) == 0 + ? new RankQuery.Target.OfType(document) + : new RankQuery.Target.Atom(people.get(random.nextInt(i == 0 ? 3 : people.size()))); + Set roles = Set.copyOf(ROLES.stream().filter(_ -> random.nextInt(3) == 0).toList()); + slots.add(new RankQuery.Slot(target, roles, 0.5 + random.nextInt(3), random.nextInt(4) == 0)); + } + Optional time = switch (random.nextInt(3)) { + case 0 -> Optional.empty(); + case 1 -> Optional.of(new RankQuery.TimeSpec.At(random.nextInt(150), random.nextInt(3) * 25.0)); + default -> { + long from = random.nextInt(120); + yield Optional.of(new RankQuery.TimeSpec.During(from, from + 1 + random.nextInt(80))); + } + }; + List preferences = random.nextBoolean() ? List.of() + : List.of(new RankQuery.Preference.Equals("status", new Value.Text("approved"), 1), + new RankQuery.Preference.Near("amount", 100, 50, 2)); + OptionalLong size = random.nextBoolean() ? OptionalLong.empty() : OptionalLong.of(1 + random.nextInt(4)); + int top = 1 + random.nextInt(12); + RankQuery.Weights weights = random.nextBoolean() ? RankQuery.Weights.DEFAULT : new RankQuery.Weights(0.05 + random.nextDouble(), + random.nextDouble(), random.nextDouble(), random.nextDouble(), random.nextDouble()); + RankQuery query = new RankQuery(Optional.of(review), slots, time, preferences, size, weights, top); + RoleTimeScorer scorer = new RoleTimeScorer(reader, query, Budget.UNBOUNDED.start()); + List everything = reader.atomsOfType(review.id()).boxed() + .flatMap(edge -> { + Optional scored = scorer.score(edge); + scored.ifPresent(found -> assertTrue(scorer.bound(edge) + 1e-9 >= found.score(), + "bound below score for " + edge + " in " + query)); + return scored.stream(); + }) + .sorted(Comparator.comparingDouble(RoleTimeScorer.EdgeScore::score).reversed() + .thenComparingLong(RoleTimeScorer.EdgeScore::edge)) + .limit(top) + .toList(); + List ranked = Ranking.of(reader, query).top(Budget.DEFAULT.start()); + assertEquals(everything.stream().map(RoleTimeScorer.EdgeScore::edge).toList(), + ranked.stream().map(RoleTimeScorer.EdgeScore::edge).toList(), query.toString()); + IntStream.range(0, ranked.size()).forEach(i -> assertEquals(everything.get(i).score(), ranked.get(i).score(), 1e-12)); + } + return null; + }); + } + + @Test + void requiredTypedSlotsAreHardConstraints() { + session.executeAll(""" + INSERT NODE Person 'alice' AS $alice; + INSERT NODE Document 'd1' AS $d1; + INSERT EDGE Review MEMBERS ($alice AS actor, $d1 AS target) AS $with; + INSERT EDGE Review MEMBERS ($alice AS actor, $d1 AS observer) AS $wrongRole; + INSERT EDGE Review MEMBERS ($alice AS actor) AS $without; + """); + QueryResult result = session.execute("RANK EDGE e:Review LIKE ($alice, ANY Document AS target REQUIRED) TOP 10"); + assertEquals(1, result.rows().size()); + assertTrue(result.rows().getFirst().getFirst().render().startsWith("@" + session.variables().get("with"))); + assertTrue(result.trace().orElseThrow().plan().startsWith("TypeScan(Review)")); + assertEquals(2, session.execute("RANK EDGE e:Review LIKE ($alice, ANY Document AS target OR observer REQUIRED) TOP 10").rows().size()); + } + + @Test + void invalidRankStatementsFailCleanly() { + session.execute("INSERT NODE Person 'alice' AS $alice"); + List statements = List.of( + "RANK EDGE e:Review LIKE ($alice) TOP 0", + "RANK EDGE e:Review LIKE ($alice) TOP 10001", + "RANK EDGE e:Review LIKE ($alice) TOP 99999999999", + "RANK EDGE e:Review LIKE ($alice) VALID AT 5 TAU -1 TOP 3", + "RANK EDGE e:Review LIKE ($alice) VALID DURING [5, 5) TOP 3", + "RANK EDGE e:Review LIKE ($alice) VALID DURING [*, 5) TOP 3", + "RANK EDGE e:Review LIKE ($alice) WEIGHTS (bogus 1) TOP 3", + "RANK EDGE e:Review LIKE ($alice) WEIGHTS (members 0, roles 0) TOP 3", + "RANK EDGE e:Review LIKE ($alice) WEIGHTS (members -1) TOP 3", + "RANK EDGE e:Review LIKE ($alice WEIGHT 0) TOP 3", + "RANK EDGE e:Review LIKE ($alice) PREFER x.status = 'a' TOP 3", + "RANK EDGE e:Review LIKE ($alice) PREFER e.amount NEAR 5 SCALE 0 TOP 3", + "RANK EDGE e:Review LIKE ($alice) SIZE 0 TOP 3", + "RANK EDGE e:Person LIKE ($alice) TOP 3", + "RANK EDGE e:Review LIKE (@999999) TOP 3", + "RANK EDGE e:Review LIKE (ANY Missing) TOP 3", + "RANK EDGE e:Review LIKE (" + "$alice, ".repeat(16) + "$alice) TOP 3", + "RANK EDGE e:Review LIKE ($alice) PREFER " + "e.amount NEAR 1 SCALE 1, ".repeat(16) + "e.amount NEAR 1 SCALE 1 TOP 3", + "RANK EDGE e:Review LIKE ($alice) VALID DURING [5, *) TOP 3", + "RANK EDGE e:Review LIKE () TOP 3", + "RANK EDGE e:Review ($alice) TOP 3", + "RANK EDGE e:Review LIKE ($alice)", + "RANK NODE e:Review LIKE ($alice) TOP 3", + "RANK EDGE e:Review LIKE ($alice) WEIGHTS (members 1, members 2) TOP 3"); + for (String statement : statements) { + assertThrows(HStoreException.class, () -> session.execute(statement), statement); + } + } + + @Test + void untypedRequiredAndRepeatedSlots() { + session.executeAll(""" + CREATE SET EDGE TYPE Note ROLES (author); + INSERT NODE Person 'alice' AS $alice; + INSERT NODE Person 'bob' AS $bob; + INSERT EDGE Review MEMBERS ($alice AS actor, $bob AS reviewer) AS $review; + INSERT EDGE Note MEMBERS ($alice AS author) AS $note; + INSERT EDGE Note MEMBERS ($bob AS author) AS $other; + """); + QueryResult untyped = session.execute("RANK EDGE e LIKE ($alice) TOP 10"); + assertTrue(untyped.trace().orElseThrow().plan().startsWith("CatalogScan(edges)")); + assertEquals(List.of("1", "1", "0"), column(untyped, 1)); + QueryResult required = session.execute("RANK EDGE e LIKE ($alice REQUIRED, $bob) TOP 10"); + assertEquals(2, required.rows().size()); + assertTrue(required.rows().getFirst().getFirst().render().startsWith("@" + session.variables().get("review"))); + QueryResult twice = session.execute("RANK EDGE e:Review LIKE ($alice AS actor, $alice AS reviewer) TOP 10"); + assertEquals(List.of("1"), column(twice, 2)); + assertEquals(List.of("0.5"), column(twice, 3)); + } + + @Test + void budgetsStopRankingInsteadOfTruncatingIt() { + database.write(writer -> { + long alice = writer.node("Person", "alice"); + for (int i = 0; i < 20; i++) { + writer.add(writer.edge("Review"), alice, "actor"); + } + return null; + }); + database.read(reader -> { + RankQuery query = new RankQuery(Optional.of(reader.requireType("Review")), + List.of(new RankQuery.Slot(new RankQuery.Target.OfType(reader.requireType("Person")), Set.of("actor"), 1, false)), + Optional.empty(), List.of(), OptionalLong.empty(), RankQuery.Weights.DEFAULT, 5); + HStoreException failure = assertThrows(HStoreException.class, () -> Ranking.of(reader, query).top(Budget.DEFAULT.withRows(10).start())); + assertTrue(failure.getMessage().contains("budget"), failure.getMessage()); + assertEquals(5, Ranking.of(reader, query).top(Budget.DEFAULT.start()).size()); + return null; + }); + } +} diff --git a/database/src/test/java/io/hstore/db/retrieve/RoleTimeScorerTest.java b/database/src/test/java/io/hstore/db/retrieve/RoleTimeScorerTest.java new file mode 100644 index 00000000..e28efed8 --- /dev/null +++ b/database/src/test/java/io/hstore/db/retrieve/RoleTimeScorerTest.java @@ -0,0 +1,48 @@ +// SPDX-FileCopyrightText: MetaCognition India Pvt Limited +// SPDX-License-Identifier: PolyForm-Noncommercial-1.0.0 + +package io.hstore.db.retrieve; + +import io.hstore.db.value.Value; +import io.hstore.engine.topology.Incidence; +import io.hstore.engine.topology.Weight; + +import org.junit.jupiter.api.Test; + +import java.math.BigDecimal; + +import static org.junit.jupiter.api.Assertions.assertEquals; + +class RoleTimeScorerTest { + + private static Incidence valid(long from, long to) { + return new Incidence(1, 0, Weight.ONE, from, to, 0, 0); + } + + @Test + void timeScoresAreExactAtTheEdgesOfTheLongRange() { + long big = 1L << 60; + assertEquals(1.0, RoleTimeScorer.time(new RankQuery.TimeSpec.At(big, 10), valid(big, big + 1))); + assertEquals(Math.exp(-0.1), RoleTimeScorer.time(new RankQuery.TimeSpec.At(big + 1, 10), valid(big, big + 1)), 1e-12); + assertEquals(0.0, RoleTimeScorer.time(new RankQuery.TimeSpec.At(big + 1, 0), valid(big, big + 1))); + assertEquals(1.0, RoleTimeScorer.time(new RankQuery.TimeSpec.At(0, 5), valid(Long.MIN_VALUE, Long.MAX_VALUE))); + assertEquals(0.0, RoleTimeScorer.time(new RankQuery.TimeSpec.At(Long.MAX_VALUE - 1, 1), valid(Long.MIN_VALUE, Long.MIN_VALUE + 1))); + assertEquals(1.0, RoleTimeScorer.time(new RankQuery.TimeSpec.During(-big, big), valid(Long.MIN_VALUE, Long.MAX_VALUE))); + assertEquals(0.25, RoleTimeScorer.time(new RankQuery.TimeSpec.During(0, 400), valid(100, 200))); + assertEquals(0.0, RoleTimeScorer.time(new RankQuery.TimeSpec.During(0, 400), valid(400, 500))); + } + + @Test + void nonFiniteValuesScoreZero() { + RankQuery.Preference near = new RankQuery.Preference.Near("amount", 1, 1, 1); + RankQuery.Preference equals = new RankQuery.Preference.Equals("amount", new Value.Real(1), 1); + for (double broken : new double[]{Double.NaN, Double.POSITIVE_INFINITY, Double.NEGATIVE_INFINITY}) { + assertEquals(0.0, RoleTimeScorer.similarity(near, new Value.Real(broken))); + assertEquals(0.0, RoleTimeScorer.similarity(equals, new Value.Real(broken))); + } + assertEquals(1.0, RoleTimeScorer.similarity(equals, new Value.Decimal(BigDecimal.ONE))); + assertEquals(0.5, RoleTimeScorer.similarity(near, new Value.Real(1.5)), 1e-12); + assertEquals(0.0, RoleTimeScorer.similarity(near, new Value.Int(2))); + assertEquals(0.0, RoleTimeScorer.similarity(near, new Value.Text("2"))); + } +} diff --git a/docs/README.md b/docs/README.md index 296c3976..c1efb7a3 100644 --- a/docs/README.md +++ b/docs/README.md @@ -25,6 +25,7 @@ Start here * [database/data-model.md](database/data-model.md): logical data model (types, atoms, properties, roles, documents, payloads) and its mapping onto engine slots. * [database/hql.md](database/hql.md): HQL reference: every statement, its grammar and examples. * [database/planner.md](database/planner.md): how `MATCH` is planned (access paths, cost estimates, `EXPLAIN`, `TRACE`) and executed. +* [database/ranking.md](database/ranking.md): how `RANK` scores hyperedges against a query hyperedge and finds the best ones without scoring all of them. * [database/hora.md](database/hora.md): higher-order relational algebra: gather, scatter, propagate, overlap join, closure, pattern matching. * [database/semantic.md](database/semantic.md): the semantic plane: embeddings, encoders, the persistent HNSW index, freshness. * [database/evidence-and-temporal.md](database/evidence-and-temporal.md): assertions, evidence and provenance; valid time and transaction time. diff --git a/docs/database/hql.md b/docs/database/hql.md index d94f971c..86abb2f4 100644 --- a/docs/database/hql.md +++ b/docs/database/hql.md @@ -80,7 +80,7 @@ Values and their storage are described in [data model](data-model.md#values). |---|---| | [Schema](#schema-ddl) | `CREATE NODE TYPE`, `CREATE SET EDGE TYPE`, `CREATE ORDERED EDGE TYPE`, `CREATE INDEX`, `CREATE JSON INDEX`, `SHOW TYPES` | | [Data](#data-manipulation) | `INSERT NODE`, `INSERT EDGE`, `ADD`, `REMOVE`, `REMOVE AT`, `SET`, `UNSET`, `DOCUMENT`, `DELETE` | -| [Queries](#queries) | `MATCH`, `EXPLAIN`, `MEMBERS OF`, `INCIDENT TO`, `DESCRIBE`, set algebra | +| [Queries](#queries) | `MATCH`, `EXPLAIN`, `RANK`, `MEMBERS OF`, `INCIDENT TO`, `DESCRIBE`, set algebra | | [HORA](#hora-operators) | `GATHER`, `REDUCE`, `SCATTER`, `PROPAGATE`, `NEIGHBORS`, `OVERLAP JOIN`, `CLOSURE`, `EXPAND`, `PATTERN`, `SAMPLE` | | [Semantic](#semantic) | `EMBED … TEXT`, `EMBED … VECTOR`, `SIMILAR TO` | | [Evidence and state](#evidence-state-and-signals) | `EVIDENCE`, `QUALIFY`, `TRACE`, `STATE`, `SIGNAL`, `SIGNALS FOR` | @@ -372,6 +372,59 @@ IncidentIntersection[@9, @25] rows≈1 cost≈12 rejected IndexRange(Claim.amount (100, +inf]) rows≈28 cost≈17 ``` +### RANK + +```ebnf +rank = "RANK" ( "EDGE" | "EDGES" ) identifier [ ":" identifier ] + "LIKE" "(" slot { "," slot } ")" + [ "VALID" ( "AT" additive [ "TAU" number ] | "DURING" interval ) ] + [ "PREFER" preference { "," preference } ] + [ "SIZE" integer ] + [ "WEIGHTS" "(" component number { "," component number } ")" ] + "TOP" integer ; +slot = ( ref | "ANY" identifier ) [ "AS" identifier { "OR" identifier } ] [ "WEIGHT" number ] [ "REQUIRED" ] ; +preference = identifier "." identifier ( "=" literal | "NEAR" number "SCALE" number ) [ "WEIGHT" number ] ; +component = "members" | "roles" | "attributes" | "time" | "topology" ; +``` + +Returns the `TOP` hyperedges that best fit a query hyperedge, best first. `MATCH` answers which hyperedges satisfy +a condition; `RANK` scores how well each one fits and never changes what `MATCH` returns. + +A slot asks for a named atom or for any atom of a type, optionally in one of several roles (`AS actor OR reviewer` +means either role). `REQUIRED` makes the slot a hard condition: the hyperedge must have that member in an accepted +role. Every other part of the query only changes the score. The score is a weighted sum of five parts, each between +0 and 1: + +| Part | Meaning of 1 | +|---|---| +| `members` | every slot is filled | +| `roles` | every slot is filled in an accepted role | +| `attributes` | every `PREFER` holds (`=` exactly; `NEAR t SCALE s` scores `1 - |x - t| / s`, at least 0) | +| `time` | every filled member is valid at the `VALID AT` instant, or covers the whole `VALID DURING` window | +| `topology` | the hyperedge has exactly `SIZE` members | + +The default weights are `members 0.3, roles 0.2, attributes 0.2, time 0.2, topology 0.1`. A part the query does not +use (no `PREFER`, no `VALID`, no `SIZE`) gets weight 0 and the rest are scaled to add up to 1. `VALID AT t TAU d` +scores a member outside its validity `exp(-distance / d)`; without `TAU` it scores 0. `VALID DURING` needs a finite +interval (no `*`). Ties are broken by the lower hyperedge id. + +Output columns: `edge, score, members, roles, attributes, time, topology`; unused parts are `NULL`. `TOP` is at most +10,000, a query has at most 16 slots and 16 preferences. The algorithm, its bounds and its costs are in +[ranking](ranking.md). + +``` +RANK EDGE e:Review + LIKE ($alice AS actor REQUIRED, $bob AS reviewer, ANY Document AS target) + VALID AT 100 TAU 100 + PREFER e.status = 'approved' + SIZE 3 + TOP 3; +-- | edge | score | members | roles | attributes | time | topology | +-- | @14 Review | 0.9734 | 1 | 1 | 1 | 0.8668 | 1 | +-- | @16 Review | 0.8083 | 0.6667 | 0.6667 | 1 | 1 | 0.75 | +-- | @15 Review | 0.7333 | 1 | 0.6667 | 0 | 1 | 1 | +``` + ### MEMBERS OF ```ebnf diff --git a/docs/database/ranking.md b/docs/database/ranking.md new file mode 100644 index 00000000..53f1e4b6 --- /dev/null +++ b/docs/database/ranking.md @@ -0,0 +1,108 @@ +# Ranking hyperedges (`RANK`) + +This page describes how `RANK` ([HQL](hql.md#rank)) scores hyperedges against a query hyperedge and returns the best +`k`: `retrieve/RankQuery.java`, `retrieve/RoleTimeScorer.java`, `retrieve/Ranking.java` and `Executor.rank`. + +## Query and score + +A query is a list of slots, each a named atom or "any atom of type `T`", with accepted roles, a weight `w_i > 0` and +a `REQUIRED` flag; optional `VALID AT` or `VALID DURING`, `PREFER` predicates, a target `SIZE`, component weights +and `TOP k`. + +For a hyperedge `e`, every slot picks its best incidence of `e`: a named atom has at most one; a typed slot looks at +every member of that type. "Best" means filled in an accepted role over filled in another role over empty, and among +equals the one with the higher time score. Two slots may pick the same member. With `W = Σ w_i`: + +| Part | Value | +|---|---| +| `members` | `Σ w_i` over filled slots, divided by `W` | +| `roles` | `Σ w_i` over slots filled in an accepted role, divided by `W` | +| `time` | weighted mean of `τ` over filled slots, 0 if none is filled | +| `attributes` | weighted mean of the `PREFER` similarities; a missing property scores 0 | +| `topology` | `min(|M(e)|, SIZE) / max(|M(e)|, SIZE)` | + +`τ` for an incidence valid on `[a, b)`: under `VALID AT t TAU d`, 1 if `a ≤ t < b`, otherwise `exp(-dist / d)` where +`dist` is the distance from `t` to `[a, b - 1]` (0 without `TAU`); under `VALID DURING [f, u)`, the fraction of +`[f, u)` that `[a, b)` covers. + +The score is `Σ α_c · part_c` with the effective weights `α`: the given or default weights, with the parts the query +does not use set to 0, scaled to add up to 1. So every score is between 0 and 1 and means the same thing in every +query. Parts are never normalised over the candidates, which would need every candidate scored before any score is +known. + +## Candidates + +```mermaid +flowchart LR + Q["RANK statement"] --> R{"REQUIRED named slots?"} + R -- yes --> I["leapfrog over their postings,
role-filtered (PostingCursor)"] + R -- no --> T["TypeScan, or every hyperedge
when no type is given"] + I --> A["admit: tenant, kind, type"] + T --> A + A --> B["bound U(e)"] + B --> S["sort by U desc, id asc"] + S --> P{"heap full and
U(e) < k-th score?"} + P -- yes --> D["stop"] + P -- no --> F["full score S(e);
drop if a REQUIRED typed slot is not met"] + F --> H["top-k heap"] --> P +``` + +Required named slots drive the candidates: their reverse-index postings are intersected with the same cursors as +`MATCH` ([planner](planner.md#incidence-intersection-contains-has)), a slot with roles accepting any role set that +contains one of them. Without a required named slot every hyperedge of the type is a candidate, because an edge +that shares no named member can still score above zero. That is exact but reads the whole type, so a query over a +large type should mark its main participant `REQUIRED`. Every candidate passes the tenant, kind and type check of +`Reader.atom` before anything else reads it. + +## Bounds and early stop + +Each candidate first gets an upper bound `U(e) ≥ S(e)`: + +| Part | In the bound | +|---|---| +| named slots | exact: one `View.incidence(e, a)` lookup per slot | +| typed slots | assumed filled in an accepted role with `τ = 1` | +| `attributes` | 1 | +| `time` | exact over named slots, typed slots counted at `τ = 1`; a weighted mean only grows when terms of 1 are added | +| `topology` | exact, from the membership count | + +Candidates are then scored in descending bound order, keeping the best `k` in a heap. The scan stops at the first +candidate whose bound is below the `k`-th score: no later candidate can enter. A candidate whose bound equals that +score is still scored, so ties are broken by id exactly as an exhaustive scorer would. This is the threshold idea of +Fagin, Lotem and Naor applied to one list of per-candidate bounds. + +Named slots are exact in the bound, so queries built from known participants usually stop after a few full scores. +Typed slots and preferences have a bound of 1, and a query made only of those scores every candidate. + +## Costs + +`C` candidates, `s` slots, `p` preferences, `|M(e)|` the size of a scored hyperedge. + +| Step | Time | +|---|---| +| candidates from required slots | the leapfrog cost of the planner page | +| bound | `O(s · log |M(e)|)` per candidate | +| order | `O(C log C)` | +| full score | `O(s · log |M(e)| + p)`, plus `O(|M(e)|)` member reads when a slot is typed | + +Each candidate, each full score and each member read for a typed slot takes one row of a `Budget.DEFAULT` meter +(1,000,000 rows, 30 s), and page reads are charged to the statement's page budget. Member types are cached for up +to 65,536 members per query. A query that exceeds either fails with +`ABORTED_RESOURCE_LIMIT`; it never returns a partial ranking as if it were complete. + +## Tenancy + +Named atoms are resolved with `Reader.require`, so naming another tenant's atom fails as if it did not exist. Typed +slots only count members whose catalog record `Reader.atom` returns, which are the reader's own. Candidates from +postings are checked with `Reader.atom` before their incidences or properties are read. `topology` uses the +hyperedge's member count, like `card()`: in a database written before memberships were checked, a hyperedge may +still count a member of another tenant. + +## Limits + +| Not supported | Reason | +|---|---| +| distinct members per slot (`DISTINCT`) | needs an assignment problem per hyperedge; slots may share a member | +| graded role similarity | the role part is 0 or 1 per slot | +| ordered positions in slots | position is not part of a slot | +| a tight bound for typed slots | needs the edge-signature index proposed for structural matching | diff --git a/server/src/main/java/io/hstore/server/StatementLogger.java b/server/src/main/java/io/hstore/server/StatementLogger.java index 0b4ef04a..d659e73d 100644 --- a/server/src/main/java/io/hstore/server/StatementLogger.java +++ b/server/src/main/java/io/hstore/server/StatementLogger.java @@ -51,7 +51,7 @@ private static Category category(Statement statement) { case Statement.CreateType _, Statement.CreateIndex _, Statement.CreateJsonIndex _, Statement.CreateView _, Statement.CreateBranch _, Statement.DropBranch _, Statement.CreateTenant _, Statement.AlterTenant _, Statement.CreateUser _, Statement.AlterUser _, Statement.DropUser _, Statement.CheckpointAt _ -> Category.DDL; - case Statement.Query _, Statement.Explain _, Statement.Members _, Statement.IncidentTo _, Statement.Describe _, + case Statement.Query _, Statement.Explain _, Statement.Rank _, Statement.Members _, Statement.IncidentTo _, Statement.Describe _, Statement.Gather _, Statement.Reduce _, Statement.Scatter _, Statement.Propagate _, Statement.Neighbors _, Statement.OverlapJoin _, Statement.Closure _, Statement.PatternQuery _, Statement.Expand _, Statement.History _, Statement.Stats _, Statement.ShowTypes _, Statement.ShowBranches _, Statement.ShowViews _, Statement.ShowView _, From 6999366500f9d2031ba51c18e122e9c7820a83da Mon Sep 17 00:00:00 2001 From: venkat1701 Date: Sun, 25 Oct 2026 21:51:14 +0530 Subject: [PATCH 2/2] Use passwords that meet the new length rule in the tenant tests --- database/src/test/java/io/hstore/db/SecurityTest.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/database/src/test/java/io/hstore/db/SecurityTest.java b/database/src/test/java/io/hstore/db/SecurityTest.java index 46e2e994..81b07f76 100644 --- a/database/src/test/java/io/hstore/db/SecurityTest.java +++ b/database/src/test/java/io/hstore/db/SecurityTest.java @@ -303,7 +303,7 @@ void membershipsAcrossTenantsAreNotFollowed() { @Test void roleFilteredMatchesStayInsideTheTenant() { - try (Session alice = login("alice", "s3cret"); Session bob = login("bob", "hunter2")) { + try (Session alice = login("alice", "s3cret-pass"); Session bob = login("bob", "hunter2-pass")) { alice.executeAll("INSERT NODE Patient 'p1' {mrn: 'A-1'} AS $p; INSERT EDGE Encounter MEMBERS ($p AS patient) AS $e;"); bob.executeAll("INSERT NODE Patient 'b1' {mrn: 'B-1'} AS $p; INSERT EDGE Encounter MEMBERS ($p AS patient) AS $e;"); long acmePatient = alice.variables().get("p"); @@ -328,7 +328,7 @@ void roleFilteredMatchesStayInsideTheTenant() { @Test void rankStaysInsideTheTenant() { - try (Session alice = login("alice", "s3cret"); Session bob = login("bob", "hunter2")) { + try (Session alice = login("alice", "s3cret-pass"); Session bob = login("bob", "hunter2-pass")) { alice.executeAll("INSERT NODE Patient 'p1' {mrn: 'A-1'} AS $p; INSERT EDGE Encounter MEMBERS ($p AS patient) AS $e;"); bob.executeAll(""" INSERT NODE Patient 'b1' {mrn: 'B-1'} AS $p;