diff --git a/server/src/instant/db/datalog.clj b/server/src/instant/db/datalog.clj index 70e0f83335..53380bd59e 100644 --- a/server/src/instant/db/datalog.clj +++ b/server/src/instant/db/datalog.clj @@ -86,7 +86,7 @@ :variable symbol? :function (s/keys :req-un [(or ::$not ::$isNull ::$comparator ::$entityIdStartsWith)]))) -(s/def ::idx-key #{:ea :eav :av :ave :vae}) +(s/def ::idx-key #{:ea :eav :av :ave :vae :mutated}) (s/def ::data-type #{:string :number :boolean :date}) (s/def ::index-map (s/keys :req-un [::idx-key ::data-type])) (s/def ::index (s/or :keyword ::idx-key @@ -2775,6 +2775,21 @@ (let [{:keys [cte-cols symbol-fields pattern page-info]} pattern-meta {:keys [symbol-values symbol-values-for-topics]} acc topics (named-pattern->topics pattern symbol-values-for-topics) + ;; The page-info pattern (order/limit) re-sorts the result, so its + ;; topic exists to catch a row whose sort key *changes* (an update), + ;; including a row reordering into the window. A *new* row entering + ;; the window is already caught by the where/enumeration topic, so + ;; the page topic doesn't need to fire on inserts -- and firing on + ;; them is what made it app-wide spam (e.g. for + ;; `messages where conversation.id = X order by createdAt`, the + ;; topic is `[:ave _ createdAt _]`, which every new message anywhere + ;; matched). Rewrite the page topic's index to `:mutated`, which + ;; only value-changed updates emit (see + ;; topics/topics-for-triple-update), so it matches reorders but not + ;; inserts. + topics (if (and page-info (flags/order-topics-match-updates-only?)) + (mapv (fn [topic] (assoc topic 0 #{:mutated})) topics) + topics) {:keys [join-rows page-info-rows symbol-values symbol-values-for-topics]} (reduce (fn [acc row] (let [join-row (sql-row->triple row cte-cols coerce-uuids?) diff --git a/server/src/instant/db/model/triple.clj b/server/src/instant/db/model/triple.clj index e89abb38fe..85ccd67c14 100644 --- a/server/src/instant/db/model/triple.clj +++ b/server/src/instant/db/model/triple.clj @@ -44,7 +44,9 @@ #(s/gen #{"foo" (UUID/randomUUID) 25 nil true}))) (s/def ::triple (s/cat :e ::lookup :a ::attr-id :v ::value)) -(s/def ::index #{:ea :eav :av :ave :vae}) +;; `:ea`..`:vae` are the triple index permutations; `:mutated` is a topic-only +;; marker meaning "this attr's value changed" (see topics/topics-for-triple-update). +(s/def ::index #{:ea :eav :av :ave :vae :mutated}) (s/def ::md5 ::uspec/non-blank-string) (s/def ::enhanced-triple diff --git a/server/src/instant/flags.clj b/server/src/instant/flags.clj index 0e990dc88f..cb26803ddc 100644 --- a/server/src/instant/flags.clj +++ b/server/src/instant/flags.clj @@ -367,6 +367,20 @@ (defn use-coarse-topics? [app-id] (contains? (flag :coarse-topics-apps) app-id)) +(defn order-topics-match-updates-only? + "When true (the default), an ordered+limited query's page-info topic only + matches sort-key changes on rows that *already exist* -- value updates, plus + first setting or clearing the value (see topics/lifecycle-entities) -- not the + writes that *create* a brand-new row. New rows are caught by the + where/enumeration topic, so the page topic doesn't need to fire on them, and + not firing keeps a new row in some *other* parent (e.g. a new message in + another conversation for `messages where conversation.id = X order by + createdAt`) from re-running the query, while still catching a row reordering + into the window. Set the `skip-order-topic-updates-only` toggle to revert to + the old app-wide behavior." + [] + (not (toggled? :skip-order-topic-updates-only))) + (defn use-get-datalog-queries-for-topics-v3? [] (toggled? :use-get-datalog-queries-for-topics-v3? true)) diff --git a/server/src/instant/reactive/topics.clj b/server/src/instant/reactive/topics.clj index b55a721774..fd8e1f8bd9 100644 --- a/server/src/instant/reactive/topics.clj +++ b/server/src/instant/reactive/topics.clj @@ -79,14 +79,23 @@ :else v-parsed))) -(defn- topics-for-triple-insert [change] +(defn- topics-for-triple-insert [change created-entities] (let [m (columns->map (:columns change) true) e (UUID/fromString (:entity_id m)) a (UUID/fromString (:attr_id m)) v (parse-v m) ks (->> #{:ea :eav :av :ave :vae} (filter m) - set)] + set) + ;; A value appearing on an entity that *already existed* (e.g. setting an + ;; order attr for the first time) is a re-sort an ordered+limited query + ;; must catch, so we add the `:mutated` marker. A value written while + ;; *creating* the entity is already caught by the where/enumeration topic + ;; (the new row's id/link triples), so we leave it off -- that's what + ;; keeps a new row in some other parent from re-running the query. + ks (if (contains? created-entities e) + ks + (conj ks :mutated))] [[ks #{e} #{a} #{v}]])) (defn- topics-for-triple-update @@ -114,34 +123,83 @@ (and (= e old-e) (= a old-a)) - [[ks #{e} #{a} (set [v old-v])]] + ;; `:mutated` marks that this attr's value changed (vs was just + ;; inserted). Ordered+limited queries subscribe to it so they catch a + ;; row reordering into the window without also re-running on every + ;; insert (new rows are caught by the where/enumeration topic). + [[(conj ks :mutated) #{e} #{a} (set [v old-v])]] ;; We shouldn't hit this, but just in case :else - [[ks #{e} #{a} #{v}] - [ks #{e} #{a} #{old-v}]]))) + [[(conj ks :mutated) #{e} #{a} #{v}] + [(conj ks :mutated) #{e} #{a} #{old-v}]]))) -(defn- topics-for-triple-delete [change] +(defn- topics-for-triple-delete [change deleted-entities] (let [m (columns->map (:identity change) true) e (UUID/fromString (:entity_id m)) a (UUID/fromString (:attr_id m)) v (parse-v m) ks (->> #{:ea :eav :av :ave :vae} (filter m) - set)] + set) + ;; Retracting one of an entity's values (without deleting the whole + ;; entity) can drop a row out of an ordered window, so we mark it + ;; `:mutated` like an update. Deleting the *entity* instead removes its + ;; id triple and is caught by the where/enumeration topic, so its value + ;; retractions don't need the marker (see lifecycle-entities). + ks (if (contains? deleted-entities e) + ks + (conj ks :mutated))] [[ks #{e} #{a} #{v}]])) -(defn topics-for-change [{:keys [action] :as change}] - (case action - :insert (topics-for-triple-insert change) - :update (topics-for-triple-update change) - :delete (topics-for-triple-delete change) - [])) +(defn- id-self-triple? + "An entity's id triple is its self-triple `[e id-attr e]`: the value equals the + entity and it's an object (non-ref) attr. Its insert/delete is our marker that + the *entity itself* was created/destroyed in this batch (the client always + writes the id triple on create, and we skip no-op id updates otherwise)." + [m] + (boolean + (and (not (:eav m)) + (:value m) + (= (:entity_id m) (<-json (:value m)))))) + +(defn- lifecycle-entities + "Scans a batch of triple changes for entities whose existence changed in it, + keyed off the id self-triple. Returns `{:created #{..} :deleted #{..}}`. A + value insert/delete on an entity *outside* these sets is a value appearing or + disappearing on a surviving row -- a re-sort -- so it earns `:mutated`; one + *inside* them is part of creating/destroying the row and is already caught by + the where/enumeration topic." + [changes] + (reduce (fn [acc {:keys [action columns identity]}] + (case action + :insert (let [m (columns->map columns true)] + (cond-> acc + (id-self-triple? m) + (update :created conj (UUID/fromString (:entity_id m))))) + :delete (let [m (columns->map identity true)] + (cond-> acc + (id-self-triple? m) + (update :deleted conj (UUID/fromString (:entity_id m))))) + acc)) + {:created #{} :deleted #{}} + changes)) + +(defn topics-for-change + ([change] + (topics-for-change change nil)) + ([{:keys [action] :as change} {:keys [created deleted]}] + (case action + :insert (topics-for-triple-insert change (or created #{})) + :update (topics-for-triple-update change) + :delete (topics-for-triple-delete change (or deleted #{})) + []))) (defn topics-for-triple-changes [changes] - (->> changes - (mapcat topics-for-change) - set)) + (let [lifecycle (lifecycle-entities changes)] + (->> changes + (mapcat #(topics-for-change % lifecycle)) + set))) (defn- topics-for-ident-upsert [{:keys [columns]}] (let [indexes #{:ea :eav :av :ave :vae} diff --git a/server/test/instant/db/instaql_test.clj b/server/test/instant/db/instaql_test.clj index 4c0f62c398..95614161ad 100644 --- a/server/test/instant/db/instaql_test.clj +++ b/server/test/instant/db/instaql_test.clj @@ -10,6 +10,7 @@ [instant.db.model.attr :as attr-model] [instant.db.model.triple :as triple-model] [instant.db.transaction :as tx] + [instant.flags :as flags] [instant.fixtures :refer [with-empty-app with-zeneca-app with-zeneca-byop @@ -29,6 +30,7 @@ [instant.util.test :refer [instant-ex-data pretty-perm-q with-sketches stuid]] [instant.reactive.ephemeral :as eph] [instant.reactive.store :as rs] + [instant.reactive.topics :as topics] [next.jdbc :as next-jdbc] [rewrite-clj.zip :as z] [zprint.core :as zprint]) @@ -94,6 +96,14 @@ (resolvers/walk-friendly r) (map ->pretty-node))) +(defn- query-topics + "Returns the raw (unresolved) set of topics a query subscribes to." + [ctx q] + (->> (iq/query ctx q) + (mapcat (fn [node] + (mapcat :topics (map :datalog-result (iq/data-seq node))))) + set)) + (defn- validation-err [ctx q] (try (iq/query ctx @@ -524,7 +534,7 @@ (is-pretty-eq? (query-pretty {:users {:$ {:limit 2 :order {:serverCreatedAt :desc}}}}) '({:topics ([#{:ea} _ #{:users/id} _] - [#{:ea} #{"eid-alex" "eid-nicole"} #{:users/id} _] + [#{:mutated} #{"eid-alex" "eid-nicole"} #{:users/id} _] -- [#{:ea} #{"eid-alex"} #{:users/createdAt :users/email :users/id :users/fullName @@ -534,28 +544,28 @@ #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _]) :triples (("eid-alex" :users/id "eid-alex") - ("eid-alex" :users/id "eid-alex") - ("eid-nicole" :users/id "eid-nicole") - ("eid-nicole" :users/id "eid-nicole") - -- - ("eid-alex" :users/id "eid-alex") - ("eid-alex" :users/fullName "Alex") - ("eid-alex" :users/email "alex@instantdb.com") - ("eid-alex" :users/handle "alex") - ("eid-alex" :users/createdAt "2021-01-09 18:53:07.993689") - -- - ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264") - ("eid-nicole" :users/email "nicole@instantdb.com") - ("eid-nicole" :users/handle "nicolegf") - ("eid-nicole" :users/id "eid-nicole") - ("eid-nicole" :users/fullName "Nicole"))}))) + ("eid-alex" :users/id "eid-alex") + ("eid-nicole" :users/id "eid-nicole") + ("eid-nicole" :users/id "eid-nicole") + -- + ("eid-alex" :users/id "eid-alex") + ("eid-alex" :users/fullName "Alex") + ("eid-alex" :users/email "alex@instantdb.com") + ("eid-alex" :users/handle "alex") + ("eid-alex" :users/createdAt "2021-01-09 18:53:07.993689") + -- + ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264") + ("eid-nicole" :users/email "nicole@instantdb.com") + ("eid-nicole" :users/handle "nicolegf") + ("eid-nicole" :users/id "eid-nicole") + ("eid-nicole" :users/fullName "Nicole"))}))) (testing "limit with where" (is-pretty-eq? (query-pretty {:users {:$ {:where {:handle {:in ["joe" "stopa" "nicolegf"]}} :limit 2 :order {:serverCreatedAt :desc}}}}) '({:topics ([#{:av} _ #{:users/handle} #{"stopa" "joe" "nicolegf"}] - [#{:ea} #{"eid-joe-averbukh" "eid-nicole"} #{:users/id} _] + [#{:mutated} #{"eid-joe-averbukh" "eid-nicole"} #{:users/id} _] -- [#{:ea} #{"eid-joe-averbukh"} #{:users/createdAt :users/email :users/id :users/fullName @@ -565,21 +575,21 @@ #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _]) :triples (("eid-nicole" :users/handle "nicolegf") - ("eid-nicole" :users/id "eid-nicole") - ("eid-joe-averbukh" :users/id "eid-joe-averbukh") - ("eid-joe-averbukh" :users/handle "joe") - -- - ("eid-joe-averbukh" :users/id "eid-joe-averbukh") - ("eid-joe-averbukh" :users/email "joe@instantdb.com") - ("eid-joe-averbukh" :users/handle "joe") - ("eid-joe-averbukh" :users/fullName "Joe Averbukh") - ("eid-joe-averbukh" :users/createdAt "2021-01-07 18:51:23.742637") - -- - ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264") - ("eid-nicole" :users/email "nicole@instantdb.com") - ("eid-nicole" :users/handle "nicolegf") - ("eid-nicole" :users/id "eid-nicole") - ("eid-nicole" :users/fullName "Nicole"))}))) + ("eid-nicole" :users/id "eid-nicole") + ("eid-joe-averbukh" :users/id "eid-joe-averbukh") + ("eid-joe-averbukh" :users/handle "joe") + -- + ("eid-joe-averbukh" :users/id "eid-joe-averbukh") + ("eid-joe-averbukh" :users/email "joe@instantdb.com") + ("eid-joe-averbukh" :users/handle "joe") + ("eid-joe-averbukh" :users/fullName "Joe Averbukh") + ("eid-joe-averbukh" :users/createdAt "2021-01-07 18:51:23.742637") + -- + ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264") + ("eid-nicole" :users/email "nicole@instantdb.com") + ("eid-nicole" :users/handle "nicolegf") + ("eid-nicole" :users/id "eid-nicole") + ("eid-nicole" :users/fullName "Nicole"))}))) (testing "makes sure we use distinct" (is (= (-> (iq/query ctx {:users {:$ {:where {:bookshelves {:in @@ -605,8 +615,8 @@ (is-pretty-eq? (query-pretty {:users {:$ {:offset 2 :order {:serverCreatedAt :desc}}}}) '({:topics ([#{:ea} _ #{:users/id} _] - [#{:ea} #{"eid-joe-averbukh" "eid-stepan-parunashvili"} #{:users/id} - _] + [#{:mutated} #{"eid-joe-averbukh" "eid-stepan-parunashvili"} + #{:users/id} _] -- [#{:ea} #{"eid-joe-averbukh"} #{:users/createdAt :users/email :users/id :users/fullName @@ -616,22 +626,22 @@ #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _]) :triples - (("eid-joe-averbukh" :users/id "eid-joe-averbukh") - ("eid-joe-averbukh" :users/id "eid-joe-averbukh") - ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") - ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") - -- - ("eid-joe-averbukh" :users/id "eid-joe-averbukh") - ("eid-joe-averbukh" :users/email "joe@instantdb.com") - ("eid-joe-averbukh" :users/handle "joe") - ("eid-joe-averbukh" :users/fullName "Joe Averbukh") - ("eid-joe-averbukh" :users/createdAt "2021-01-07 18:51:23.742637") - -- - ("eid-stepan-parunashvili" :users/email "stopa@instantdb.com") - ("eid-stepan-parunashvili" :users/createdAt "2021-01-07 18:50:43.447955") - ("eid-stepan-parunashvili" :users/fullName "Stepan Parunashvili") - ("eid-stepan-parunashvili" :users/handle "stopa") - ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili"))}))) + (("eid-joe-averbukh" :users/id "eid-joe-averbukh") + ("eid-joe-averbukh" :users/id "eid-joe-averbukh") + ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") + ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") + -- + ("eid-joe-averbukh" :users/id "eid-joe-averbukh") + ("eid-joe-averbukh" :users/email "joe@instantdb.com") + ("eid-joe-averbukh" :users/handle "joe") + ("eid-joe-averbukh" :users/fullName "Joe Averbukh") + ("eid-joe-averbukh" :users/createdAt "2021-01-07 18:51:23.742637") + -- + ("eid-stepan-parunashvili" :users/email "stopa@instantdb.com") + ("eid-stepan-parunashvili" :users/createdAt "2021-01-07 18:50:43.447955") + ("eid-stepan-parunashvili" :users/fullName "Stepan Parunashvili") + ("eid-stepan-parunashvili" :users/handle "stopa") + ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili"))}))) (testing "cursors" (let [{:keys [start-cursor end-cursor]} @@ -646,108 +656,109 @@ :after end-cursor :order {:serverCreatedAt :desc}}}}) '({:topics ([#{:ea} _ #{:users/id} _] - [#{:ea} #{"eid-alex"} #{:users/id} _] + [#{:mutated} #{"eid-alex"} #{:users/id} _] -- [#{:ea} #{"eid-alex"} #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _]) :triples (("eid-alex" :users/id "eid-alex") - ("eid-alex" :users/id "eid-alex") - -- - ("eid-alex" :users/id "eid-alex") - ("eid-alex" :users/fullName "Alex") - ("eid-alex" :users/email "alex@instantdb.com") - ("eid-alex" :users/handle "alex") - ("eid-alex" :users/createdAt "2021-01-09 18:53:07.993689"))})) + ("eid-alex" :users/id "eid-alex") + -- + ("eid-alex" :users/id "eid-alex") + ("eid-alex" :users/fullName "Alex") + ("eid-alex" :users/email "alex@instantdb.com") + ("eid-alex" :users/handle "alex") + ("eid-alex" :users/createdAt "2021-01-09 18:53:07.993689"))})) (testing "afterInclusive" (is-pretty-eq? (query-pretty {:users {:$ {:limit 1 :after end-cursor :afterInclusive true :order {:serverCreatedAt :desc}}}}) '({:topics ([#{:ea} _ #{:users/id} _] - [#{:ea} #{"eid-nicole"} #{:users/id} _] + [#{:mutated} #{"eid-nicole"} #{:users/id} _] -- [#{:ea} #{"eid-nicole"} #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _]) - :triples #{("eid-nicole" :users/fullName "Nicole") + :triples (("eid-nicole" :users/id "eid-nicole") ("eid-nicole" :users/id "eid-nicole") - ("eid-nicole" :users/handle "nicolegf") -- + ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264") ("eid-nicole" :users/email "nicole@instantdb.com") - ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264")}})))) + ("eid-nicole" :users/handle "nicolegf") + ("eid-nicole" :users/id "eid-nicole") + ("eid-nicole" :users/fullName "Nicole"))})))) (testing "before" (is-pretty-eq? (query-pretty {:users {:$ {:limit 1 :before start-cursor :order {:serverCreatedAt :desc}}}}) - '({:topics ([#{:ea} _ #{:users/id} _] [#{:ea} #{} #{:users/id} _]) :triples ()})) + '({:topics ([#{:ea} _ #{:users/id} _] [#{:mutated} #{} #{:users/id} _]) + :triples ()})) (is-pretty-eq? (query-pretty {:users {:$ {:limit 1 :before start-cursor :order {:serverCreatedAt "asc"}}}}) '({:topics ([#{:ea} _ #{:users/id} _] - [#{:ea} #{"eid-stepan-parunashvili"} #{:users/id} _] + [#{:mutated} #{"eid-stepan-parunashvili"} #{:users/id} _] -- [#{:ea} #{"eid-stepan-parunashvili"} #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _]) :triples - (("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") - ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") - -- - ("eid-stepan-parunashvili" :users/email "stopa@instantdb.com") - ("eid-stepan-parunashvili" :users/createdAt "2021-01-07 18:50:43.447955") - ("eid-stepan-parunashvili" :users/fullName "Stepan Parunashvili") - ("eid-stepan-parunashvili" :users/handle "stopa") - ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili"))})) + (("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") + ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") + -- + ("eid-stepan-parunashvili" :users/email "stopa@instantdb.com") + ("eid-stepan-parunashvili" :users/createdAt "2021-01-07 18:50:43.447955") + ("eid-stepan-parunashvili" :users/fullName "Stepan Parunashvili") + ("eid-stepan-parunashvili" :users/handle "stopa") + ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili"))})) (testing "beforeInclusive" (is-pretty-eq? (query-pretty {:users {:$ {:limit 1 :before start-cursor :beforeInclusive true :order {:serverCreatedAt :desc}}}}) - '({:topics #{[#{:ea} #{"eid-nicole"} #{:users/id} _] - [#{:ea} _ #{:users/id} _] - [#{:ea} - #{"eid-nicole"} - #{:users/createdAt - :users/email - :users/id - :users/fullName - :users/handle} - _] - --} - :triples #{("eid-nicole" :users/fullName "Nicole") + '({:topics ([#{:ea} _ #{:users/id} _] + [#{:mutated} #{"eid-nicole"} #{:users/id} _] + -- + [#{:ea} #{"eid-nicole"} + #{:users/createdAt :users/email :users/id :users/fullName + :users/handle} _]) + :triples (("eid-nicole" :users/id "eid-nicole") ("eid-nicole" :users/id "eid-nicole") - ("eid-nicole" :users/handle "nicolegf") -- + ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264") ("eid-nicole" :users/email "nicole@instantdb.com") - ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264")}})))) + ("eid-nicole" :users/handle "nicolegf") + ("eid-nicole" :users/id "eid-nicole") + ("eid-nicole" :users/fullName "Nicole"))})))) (testing "last" (is-pretty-eq? (query-pretty {:users {:$ {:limit 1 :before start-cursor :order {:serverCreatedAt :desc}}}}) - '({:topics ([#{:ea} _ #{:users/id} _] [#{:ea} #{} #{:users/id} _]) :triples ()})) + '({:topics ([#{:ea} _ #{:users/id} _] [#{:mutated} #{} #{:users/id} _]) + :triples ()})) (is-pretty-eq? (query-pretty {:users {:$ {:last 1 :before start-cursor :order {:serverCreatedAt "asc"}}}}) '({:topics ([#{:ea} _ #{:users/id} _] - [#{:ea} #{"eid-alex"} #{:users/id} _] + [#{:mutated} #{"eid-alex"} #{:users/id} _] -- [#{:ea} #{"eid-alex"} #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _]) :triples (("eid-alex" :users/id "eid-alex") - ("eid-alex" :users/id "eid-alex") - -- - ("eid-alex" :users/id "eid-alex") - ("eid-alex" :users/fullName "Alex") - ("eid-alex" :users/email "alex@instantdb.com") - ("eid-alex" :users/handle "alex") - ("eid-alex" :users/createdAt "2021-01-09 18:53:07.993689"))}))) + ("eid-alex" :users/id "eid-alex") + -- + ("eid-alex" :users/id "eid-alex") + ("eid-alex" :users/fullName "Alex") + ("eid-alex" :users/email "alex@instantdb.com") + ("eid-alex" :users/handle "alex") + ("eid-alex" :users/createdAt "2021-01-09 18:53:07.993689"))}))) (let [alex-cursor (-> (iq/query ctx {:users {:$ {:limit 1 :where {:handle "alex"}}}}) @@ -880,8 +891,8 @@ (is-pretty-eq? (query-pretty ctx r {:users {:$ {:limit 2 :order {:handle :desc}}}}) '({:topics ([#{:ea} _ #{:users/id} _] - [#{:ave} #{"eid-nicole" "eid-stepan-parunashvili"} #{:users/handle} - _] + [#{:mutated} #{"eid-nicole" "eid-stepan-parunashvili"} + #{:users/handle} _] -- [#{:ea} #{"eid-stepan-parunashvili"} #{:users/createdAt :users/email :users/id :users/fullName @@ -891,29 +902,29 @@ #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _]) :triples - (("eid-nicole" :users/handle "nicolegf") - ("eid-nicole" :users/id "eid-nicole") - ("eid-stepan-parunashvili" :users/handle "stopa") - ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") - -- - ("eid-stepan-parunashvili" :users/email "stopa@instantdb.com") - ("eid-stepan-parunashvili" :users/createdAt "2021-01-07 18:50:43.447955") - ("eid-stepan-parunashvili" :users/fullName "Stepan Parunashvili") - ("eid-stepan-parunashvili" :users/handle "stopa") - ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") - -- - ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264") - ("eid-nicole" :users/email "nicole@instantdb.com") - ("eid-nicole" :users/handle "nicolegf") - ("eid-nicole" :users/id "eid-nicole") - ("eid-nicole" :users/fullName "Nicole"))}))) + (("eid-nicole" :users/handle "nicolegf") + ("eid-nicole" :users/id "eid-nicole") + ("eid-stepan-parunashvili" :users/handle "stopa") + ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") + -- + ("eid-stepan-parunashvili" :users/email "stopa@instantdb.com") + ("eid-stepan-parunashvili" :users/createdAt "2021-01-07 18:50:43.447955") + ("eid-stepan-parunashvili" :users/fullName "Stepan Parunashvili") + ("eid-stepan-parunashvili" :users/handle "stopa") + ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") + -- + ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264") + ("eid-nicole" :users/email "nicole@instantdb.com") + ("eid-nicole" :users/handle "nicolegf") + ("eid-nicole" :users/id "eid-nicole") + ("eid-nicole" :users/fullName "Nicole"))}))) (testing "limit with where" (is-pretty-eq? (query-pretty ctx r {:users {:$ {:where {:handle {:in ["joe" "stopa" "alex"]}} :limit 2 :order {:handle :desc}}}}) '({:topics ([#{:ave} _ #{:users/handle} #{"alex" "stopa" "joe"}] - [#{:ave} #{"eid-joe-averbukh" "eid-stepan-parunashvili"} + [#{:mutated} #{"eid-joe-averbukh" "eid-stepan-parunashvili"} #{:users/handle} _] -- [#{:ea} #{"eid-stepan-parunashvili"} @@ -924,28 +935,28 @@ #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _]) :triples - (("eid-joe-averbukh" :users/handle "joe") - ("eid-joe-averbukh" :users/handle "joe") - ("eid-stepan-parunashvili" :users/handle "stopa") - ("eid-stepan-parunashvili" :users/handle "stopa") - -- - ("eid-stepan-parunashvili" :users/email "stopa@instantdb.com") - ("eid-stepan-parunashvili" :users/createdAt "2021-01-07 18:50:43.447955") - ("eid-stepan-parunashvili" :users/fullName "Stepan Parunashvili") - ("eid-stepan-parunashvili" :users/handle "stopa") - ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") - -- - ("eid-joe-averbukh" :users/id "eid-joe-averbukh") - ("eid-joe-averbukh" :users/email "joe@instantdb.com") - ("eid-joe-averbukh" :users/handle "joe") - ("eid-joe-averbukh" :users/fullName "Joe Averbukh") - ("eid-joe-averbukh" :users/createdAt "2021-01-07 18:51:23.742637"))}))) + (("eid-joe-averbukh" :users/handle "joe") + ("eid-joe-averbukh" :users/handle "joe") + ("eid-stepan-parunashvili" :users/handle "stopa") + ("eid-stepan-parunashvili" :users/handle "stopa") + -- + ("eid-stepan-parunashvili" :users/email "stopa@instantdb.com") + ("eid-stepan-parunashvili" :users/createdAt "2021-01-07 18:50:43.447955") + ("eid-stepan-parunashvili" :users/fullName "Stepan Parunashvili") + ("eid-stepan-parunashvili" :users/handle "stopa") + ("eid-stepan-parunashvili" :users/id "eid-stepan-parunashvili") + -- + ("eid-joe-averbukh" :users/id "eid-joe-averbukh") + ("eid-joe-averbukh" :users/email "joe@instantdb.com") + ("eid-joe-averbukh" :users/handle "joe") + ("eid-joe-averbukh" :users/fullName "Joe Averbukh") + ("eid-joe-averbukh" :users/createdAt "2021-01-07 18:51:23.742637"))}))) (testing "offset" (is-pretty-eq? (query-pretty ctx r {:users {:$ {:offset 2 :order {:handle :desc}}}}) '({:topics ([#{:ea} _ #{:users/id} _] - [#{:ave} #{"eid-joe-averbukh" "eid-alex"} #{:users/handle} _] + [#{:mutated} #{"eid-joe-averbukh" "eid-alex"} #{:users/handle} _] -- [#{:ea} #{"eid-joe-averbukh"} #{:users/createdAt :users/email :users/id :users/fullName @@ -955,21 +966,21 @@ #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _]) :triples (("eid-alex" :users/id "eid-alex") - ("eid-alex" :users/handle "alex") - ("eid-joe-averbukh" :users/id "eid-joe-averbukh") - ("eid-joe-averbukh" :users/handle "joe") - -- - ("eid-joe-averbukh" :users/id "eid-joe-averbukh") - ("eid-joe-averbukh" :users/email "joe@instantdb.com") - ("eid-joe-averbukh" :users/handle "joe") - ("eid-joe-averbukh" :users/fullName "Joe Averbukh") - ("eid-joe-averbukh" :users/createdAt "2021-01-07 18:51:23.742637") - -- - ("eid-alex" :users/id "eid-alex") - ("eid-alex" :users/fullName "Alex") - ("eid-alex" :users/email "alex@instantdb.com") - ("eid-alex" :users/handle "alex") - ("eid-alex" :users/createdAt "2021-01-09 18:53:07.993689"))}))) + ("eid-alex" :users/handle "alex") + ("eid-joe-averbukh" :users/id "eid-joe-averbukh") + ("eid-joe-averbukh" :users/handle "joe") + -- + ("eid-joe-averbukh" :users/id "eid-joe-averbukh") + ("eid-joe-averbukh" :users/email "joe@instantdb.com") + ("eid-joe-averbukh" :users/handle "joe") + ("eid-joe-averbukh" :users/fullName "Joe Averbukh") + ("eid-joe-averbukh" :users/createdAt "2021-01-07 18:51:23.742637") + -- + ("eid-alex" :users/id "eid-alex") + ("eid-alex" :users/fullName "Alex") + ("eid-alex" :users/email "alex@instantdb.com") + ("eid-alex" :users/handle "alex") + ("eid-alex" :users/createdAt "2021-01-09 18:53:07.993689"))}))) (testing "cursors" (let [{:keys [start-cursor end-cursor]} @@ -985,69 +996,69 @@ :after end-cursor :order {:handle :desc}}}}) '({:topics ([#{:ea} _ #{:users/id} _] - [#{:ave} #{"eid-nicole"} #{:users/handle} _] + [#{:mutated} #{"eid-nicole"} #{:users/handle} _] -- [#{:ea} #{"eid-nicole"} #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _]) :triples (("eid-nicole" :users/handle "nicolegf") - ("eid-nicole" :users/id "eid-nicole") - -- - ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264") - ("eid-nicole" :users/email "nicole@instantdb.com") - ("eid-nicole" :users/handle "nicolegf") - ("eid-nicole" :users/id "eid-nicole") - ("eid-nicole" :users/fullName "Nicole"))}))) + ("eid-nicole" :users/id "eid-nicole") + -- + ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264") + ("eid-nicole" :users/email "nicole@instantdb.com") + ("eid-nicole" :users/handle "nicolegf") + ("eid-nicole" :users/id "eid-nicole") + ("eid-nicole" :users/fullName "Nicole"))}))) (testing "before" (is-pretty-eq? (query-pretty ctx r {:users {:$ {:limit 1 :before start-cursor :order {:handle :desc}}}}) - '({:topics ([#{:ea} _ #{:users/id} _] [#{:ave} #{} #{:users/handle} _]) + '({:topics ([#{:ea} _ #{:users/id} _] [#{:mutated} #{} #{:users/handle} _]) :triples ()})) (is-pretty-eq? (query-pretty ctx r {:users {:$ {:limit 1 :before start-cursor :order {:handle "asc"}}}}) '({:topics ([#{:ea} _ #{:users/id} _] - [#{:ave} #{"eid-alex"} #{:users/handle} _] + [#{:mutated} #{"eid-alex"} #{:users/handle} _] -- [#{:ea} #{"eid-alex"} #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _]) :triples (("eid-alex" :users/id "eid-alex") - ("eid-alex" :users/handle "alex") - -- - ("eid-alex" :users/id "eid-alex") - ("eid-alex" :users/fullName "Alex") - ("eid-alex" :users/email "alex@instantdb.com") - ("eid-alex" :users/handle "alex") - ("eid-alex" :users/createdAt "2021-01-09 18:53:07.993689"))}))) + ("eid-alex" :users/handle "alex") + -- + ("eid-alex" :users/id "eid-alex") + ("eid-alex" :users/fullName "Alex") + ("eid-alex" :users/email "alex@instantdb.com") + ("eid-alex" :users/handle "alex") + ("eid-alex" :users/createdAt "2021-01-09 18:53:07.993689"))}))) (testing "last" (is-pretty-eq? (query-pretty ctx r {:users {:$ {:limit 1 :before start-cursor :order {:handle :desc}}}}) - '({:topics ([#{:ea} _ #{:users/id} _] [#{:ave} #{} #{:users/handle} _]) + '({:topics ([#{:ea} _ #{:users/id} _] [#{:mutated} #{} #{:users/handle} _]) :triples ()})) (is-pretty-eq? (query-pretty ctx r {:users {:$ {:last 1 :before start-cursor :order {:handle "asc"}}}}) '({:topics ([#{:ea} _ #{:users/id} _] - [#{:ave} #{"eid-nicole"} #{:users/handle} _] + [#{:mutated} #{"eid-nicole"} #{:users/handle} _] -- [#{:ea} #{"eid-nicole"} #{:users/createdAt :users/email :users/id :users/fullName :users/handle} _]) :triples (("eid-nicole" :users/handle "nicolegf") - ("eid-nicole" :users/id "eid-nicole") - -- - ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264") - ("eid-nicole" :users/email "nicole@instantdb.com") - ("eid-nicole" :users/handle "nicolegf") - ("eid-nicole" :users/id "eid-nicole") - ("eid-nicole" :users/fullName "Nicole"))}))) + ("eid-nicole" :users/id "eid-nicole") + -- + ("eid-nicole" :users/createdAt "2021-02-05 22:35:23.754264") + ("eid-nicole" :users/email "nicole@instantdb.com") + ("eid-nicole" :users/handle "nicolegf") + ("eid-nicole" :users/id "eid-nicole") + ("eid-nicole" :users/fullName "Nicole"))}))) (let [nicole-cursor (-> (iq/query ctx {:users {:$ {:limit 1 :where {:handle "nicolegf"} @@ -3194,6 +3205,105 @@ (is (= #{true false} (run-query :boolean {:etype {:$ {:where {:boolean {:$in [true false]}}}}}))) (is (= #{false} (run-query :boolean {:etype {:$ {:where {:boolean false}}}})))))))) +(defn- wal-col [n v] {:name n :value v}) +(defn- wal-index-cols [] + [(wal-col "ea" true) (wal-col "eav" false) (wal-col "av" true) + (wal-col "ave" true) (wal-col "vae" false)]) +(defn- wal-update-change [eid attr old new] + {:action :update + :columns (into [(wal-col "entity_id" (str eid)) (wal-col "attr_id" (str attr)) + (wal-col "value" (str new))] (wal-index-cols)) + :identity (into [(wal-col "entity_id" (str eid)) (wal-col "attr_id" (str attr)) + (wal-col "value" (str old))] (wal-index-cols))}) +(defn- wal-insert-change [eid attr v] + {:action :insert + :columns (into [(wal-col "entity_id" (str eid)) (wal-col "attr_id" (str attr)) + (wal-col "value" (str v))] (wal-index-cols))}) +(defn- wal-id-insert + "An entity's id self-triple insert ([e id-attr e]) -- marks the entity as + created in the batch, the way the client always writes it on create." + [eid id-attr] + {:action :insert + :columns (into [(wal-col "entity_id" (str eid)) (wal-col "attr_id" (str id-attr)) + (wal-col "value" (str "\"" eid "\""))] (wal-index-cols))}) + +(deftest order-topic-matches-updates-not-inserts + ;; An ordered+limited query whose entities are matched through a parent->child + ;; many-link (e.g. `messages where conversation.id = X order by createdAt`) + ;; binds the matched rows as `ref-value`s, so the page-info (order) topic is + ;; app-wide (`[:ave _ createdAt _]`) and every write to the order attr anywhere + ;; re-runs the query. The order topic only needs to catch *re-sorts*; brand-new + ;; rows are caught by the where/link topic. So the page topic subscribes to the + ;; `:mutated` marker, which a value change -- or a value appearing/disappearing + ;; on a row that *already existed* -- emits, but a brand-new row (which writes + ;; its own id triple) does not. That catches a re-sort, including first setting + ;; the order key on an existing row, while still ignoring inserts elsewhere. + (with-empty-app + (fn [app] + (let [conn (aurora/conn-pool :write) + convos-id (random-uuid) + msgs-id (random-uuid) + priority (random-uuid) + convo-msgs (random-uuid) + convo-eid (random-uuid) + msg-eids (vec (repeatedly 15 random-uuid))] + (tx/transact! conn (attr-model/get-by-app-id (:id app)) (:id app) + [[:add-attr {:id convos-id :forward-identity [(random-uuid) "conversations" "id"] + :unique? true :index? false :value-type :blob :cardinality :one}] + [:add-attr {:id msgs-id :forward-identity [(random-uuid) "messages" "id"] + :unique? true :index? false :value-type :blob :cardinality :one}] + [:add-attr {:id priority :forward-identity [(random-uuid) "messages" "priority"] + :unique? false :index? true :value-type :blob + :checked-data-type :number :cardinality :one}] + [:add-attr {:id convo-msgs + :forward-identity [(random-uuid) "conversations" "messages"] + :reverse-identity [(random-uuid) "messages" "conversation"] + :unique? false :index? false :value-type :ref :cardinality :many}]]) + (tx/transact! conn (attr-model/get-by-app-id (:id app)) (:id app) + (into [[:add-triple convo-eid convos-id (str convo-eid)]] + (mapcat (fn [i eid] + [[:add-triple eid msgs-id (str eid)] + [:add-triple eid priority (inc i)] + [:add-triple convo-eid convo-msgs eid]]) + (range 15) msg-eids))) + (let [ctx {:db {:conn-pool conn} :app-id (:id app) + :attrs (attr-model/get-by-app-id (:id app))} + link-q {:messages {:$ {:where {:conversation.id (str convo-eid)} + :order {:priority "desc"} :limit 5}}} + off-window (first msg-eids) ;; priority 1, outside a desc/limit-5 window + ;; an existing off-window row reordered into the window (sort-key update) + reorder-ivs (topics/topics-for-changes + {:triple-changes [(wal-update-change off-window priority 1 100)]}) + ;; an existing off-window row gets its sort key set for the first + ;; time (an insert, but on a row that already exists) + first-set-ivs (topics/topics-for-changes + {:triple-changes [(wal-insert-change off-window priority 100)]}) + ;; a brand-new message elsewhere: it writes its own id triple, so + ;; its sort-key insert must not re-run the query + new-msg (random-uuid) + insert-ivs (topics/topics-for-changes + {:triple-changes [(wal-id-insert new-msg msgs-id) + (wal-insert-change new-msg priority 100)]}) + caught? (fn [ivs toggle?] + (let [run #(boolean (rs/matching-topic-intersection? + ivs (query-topics ctx link-q)))] + (if toggle? + (binding [flags/*toggle-overrides* {:skip-order-topic-updates-only true}] + (run)) + (run))))] + (testing "default: catches re-sorts of existing rows, ignores new rows elsewhere" + (is (caught? reorder-ivs false) + "an off-window row reordered into the window still invalidates") + (is (caught? first-set-ivs false) + "first setting the sort key on an existing row still invalidates") + (is (not (caught? insert-ivs false)) + "a brand-new message elsewhere does NOT invalidate (no spam)")) + + (testing "skip toggle reverts to the old app-wide behavior" + (is (caught? reorder-ivs true)) + (is (caught? insert-ivs true) + "old behavior re-runs on every insert (the spam)"))))))) + (deftest $not-with-refs (with-zeneca-checked-data-app (fn [app r] @@ -3225,20 +3335,20 @@ [#{:vae} {:$not "eid-joe-averbukh"} #{:users/bookshelves} #{}] [#{:ea} #{} #{:bookshelves/id} _] [#{:ea} _ #{:users/bookshelves} _] - [#{:ave} #{} #{:bookshelves/order} _] + [#{:mutated} #{} #{:bookshelves/order} _] -- [#{:ea} #{"eid-nonfiction"} #{:bookshelves/desc :bookshelves/name :bookshelves/order :bookshelves/id} _]) :triples (("eid-alex" :users/bookshelves "eid-nonfiction") - ("eid-alex" :users/bookshelves "eid-nonfiction") - ("eid-alex" :users/bookshelves "eid-nonfiction") - ("eid-nonfiction" :bookshelves/order 1) - -- - ("eid-nonfiction" :bookshelves/id "eid-nonfiction") - ("eid-nonfiction" :bookshelves/name "Nonfiction") - ("eid-nonfiction" :bookshelves/desc "") - ("eid-nonfiction" :bookshelves/order 1))})))))) + ("eid-alex" :users/bookshelves "eid-nonfiction") + ("eid-alex" :users/bookshelves "eid-nonfiction") + ("eid-nonfiction" :bookshelves/order 1) + -- + ("eid-nonfiction" :bookshelves/id "eid-nonfiction") + ("eid-nonfiction" :bookshelves/name "Nonfiction") + ("eid-nonfiction" :bookshelves/desc "") + ("eid-nonfiction" :bookshelves/order 1))})))))) (deftest lookup-unique-uses-the-av-index (binding [d/*enable-pg-hints* true] diff --git a/server/test/instant/reactive/invalidator_test.clj b/server/test/instant/reactive/invalidator_test.clj index 8c1e94c0fd..028378e3b8 100644 --- a/server/test/instant/reactive/invalidator_test.clj +++ b/server/test/instant/reactive/invalidator_test.clj @@ -405,15 +405,48 @@ (deftest changes-produce-correct-topics (testing "insert triples" - (is (= #{[#{:ea} #{#uuid "7c6b379b-d841-46e1-8970-2da7e0cbc490"} + ;; These inserts carry no id self-triple in the batch, so they read as values + ;; set on an already-existing entity (e.g. backfilling a field) and get the + ;; `:mutated` marker so an ordered query catches the re-sort. A real entity + ;; *creation* also writes the id triple and does not -- see "create entity". + (is (= #{[#{:mutated :ea} #{#uuid "7c6b379b-d841-46e1-8970-2da7e0cbc490"} #{#uuid "a2f7b8b7-5c6f-4b8c-a7aa-2ba400336acb"} #{"New Movie"}] - [#{:ea} #{#uuid "7c6b379b-d841-46e1-8970-2da7e0cbc490"} + [#{:mutated :ea} #{#uuid "7c6b379b-d841-46e1-8970-2da7e0cbc490"} #{#uuid "6a631008-d315-4bbd-8665-c92aed9abc9c"} #{1987}]} (topics/topics-for-changes {:triple-changes create-triple-changes})))) + (testing "create entity (id triple in batch) does not mark its values :mutated" + (let [eid (random-uuid) + id-attr (random-uuid) + title-attr (random-uuid) + cols (fn [e a v] [{:name "entity_id" :value (str e)} + {:name "attr_id" :value (str a)} + {:name "value" :value v} + {:name "ea" :value true} {:name "eav" :value false} + {:name "av" :value false} {:name "ave" :value false} + {:name "vae" :value false}])] + (is (= #{[#{:ea} #{eid} #{id-attr} #{(str eid)}] + [#{:ea} #{eid} #{title-attr} #{"Hi"}]} + (topics/topics-for-changes + {:triple-changes [{:action :insert :columns (cols eid id-attr (str "\"" eid "\""))} + {:action :insert :columns (cols eid title-attr "\"Hi\"")}]}))))) + (testing "retracting a single value (entity survives) is marked :mutated" + (let [eid (random-uuid) + attr (random-uuid)] + (is (= #{[#{:mutated :ea} #{eid} #{attr} #{"Hi"}]} + (topics/topics-for-changes + {:triple-changes [{:action :delete + :identity [{:name "entity_id" :value (str eid)} + {:name "attr_id" :value (str attr)} + {:name "value" :value "\"Hi\""} + {:name "ea" :value true} {:name "eav" :value false} + {:name "av" :value false} {:name "ave" :value false} + {:name "vae" :value false}]}]}))))) (testing "update triples" - (is (= '#{[#{:ea} + ;; `:mutated` marks a value-changed update so ordered queries can catch + ;; reorders without firing on inserts (see topics/topics-for-triple-update). + (is (= '#{[#{:mutated :ea} #{#uuid "7c6b379b-d841-46e1-8970-2da7e0cbc490"} #{#uuid "a2f7b8b7-5c6f-4b8c-a7aa-2ba400336acb"} #{"Updated Movie3" "Old Movie"}]}