diff --git a/server/src/instant/db/scoped_query_plans.clj b/server/src/instant/db/scoped_query_plans.clj index 033314e773..102302f0c9 100644 --- a/server/src/instant/db/scoped_query_plans.clj +++ b/server/src/instant/db/scoped_query_plans.clj @@ -1,5 +1,6 @@ (ns instant.db.scoped-query-plans (:require [clojure.set :as set] + [instant.flags :as flags] [instant.util.pg-hint-plan :as pg-hint])) ;; These overrides are scoped to measured query shapes. A different layout or @@ -257,12 +258,102 @@ :pg-hints (conj (vec pg-hints) (pg-hint/index-scan :t-upper-bound-match :triples_number_type_idx))}))) +(defn- reassign-calendar-child-ids [app-id normalized ctes pg-hints result-tables] + (when (and (not (flags/toggled? :disable-scoped-query-plans)) + (true? (get-in (flags/flag :scoped-query-plans) + [(str app-id) "reassign-calendar-child-ids"])) + (= normalized + {:sections + {:$ {:where {:or [{:and [{:date {:$gte :string}} {:date {:$lte :string}}]} + {:and [{:recurrenceRule {:$isNull :boolean}} {:date {:$lte :string}}]} + {:and [{:exceptionDate {:$gte :string}} {:exceptionDate {:$lte :string}}]}]}} + :area {} :activityType {} :parent {} :eventLinks {}}})) + (let [ctes (vec ctes) + [[_ q0] [_ q1] [_ q2] [_ q3] [_ q4] [_ q5]] ctes + where-vec (fn [query] (when (sequential? (:where query)) (vec (:where query)))) + [w0 w1 w2 w3 w4 w5] (mapv where-vec [q0 q1 q2 q3 q4 q5]) + date-id (get-in w0 [3 2]) + id-attr (get-in w2 [3 2]) + recurrence-id (get-in (where-vec (get-in w2 [4 2])) [3 2]) + exception-id (get-in w4 [3 2]) + child-attrs (get-in (where-vec (get-in ctes [8 1])) [3 2 1]) + bounds (mapv #(get-in % [4 2 1]) [w0 w1 w3 w4 w5]) + [date-lower date-upper recurrence-upper exception-lower exception-upper] bounds + base (fn [index attr-id] + [[:= :app-id app-id] [:= index :true] [:= :attr-id attr-id]]) + scan (fn [idx previous where] + (let [table (str "m-" idx)] + {:select (concat (when previous [(keyword (str (name previous) ".*"))]) + (mapv (fn [[column suffix]] + [column (keyword (str table suffix))]) + [[:entity-id "-entity-id"] [:attr-id "-attr-id"] + [:value "-value"] [:eav "-is-ref-val"] + [:created-at "-created-at"]])) + :from (cond-> [[:triples (keyword (str "t" idx))]] previous (conj previous)) + :where (into [:and] where)})) + range-scan (fn [idx previous attr-id op bound] + (scan idx previous + (cond-> (conj (base :ave attr-id) [op :value [:cast bound :jsonb]]) + previous (conj [:= :entity-id (keyword (str (name previous) "-entity-id"))])))) + expected + [(range-scan 0 nil date-id :>= date-lower) + (range-scan 1 :m-0 date-id :<= date-upper) + (scan 2 nil + (conj (base :ea id-attr) + [:in :entity-id + {:select :t2-subquery.entity-id + :from [[:triples :t2-subquery]] + :where [:and [:= :t2-subquery.app-id app-id] + [:= :t2-subquery.entity-id :t2.entity-id] + [:= :t2-subquery.attr-id recurrence-id] + nil [:not= :t2-subquery.value [:cast "null" :jsonb]]]}])) + (range-scan 3 :m-2 date-id :<= recurrence-upper) + (range-scan 4 nil exception-id :>= exception-lower) + (range-scan 5 :m-4 exception-id :<= exception-upper) + {:select [:m-1.* :m-3.* :m-5.* + [[:coalesce :m-1.m-1-entity-id :m-3.m-3-entity-id :m-5.m-5-entity-id] + :m-6-entity-id]] + :from :m-1 + :full-join [:m-3 [:= :0 :1] :m-5 [:= :0 :1]]} + {:select [[[:distinct :entity-id] :m-7-entity-id]] + :from [:triples :m-6] + :where [:and [:= :app-id app-id] + [:or [:and [:= :entity-id :m-4-entity-id] [:= :entity-id :m-5-entity-id]] + [:and [:or [:and [:= :entity-id :m-0-entity-id] [:= :entity-id :m-1-entity-id]] + [:and [:= :entity-id :m-2-entity-id] [:= :entity-id :m-3-entity-id]]]]]]} + (assoc (scan 8 nil + (conj (base :ea [:any child-attrs]) [:= :entity-id :m-7-entity-id])) + :from [[:triples :t8] :m-7])]] + (when (and (every? uuid? [date-id id-attr recurrence-id exception-id]) + (set? child-attrs) (seq child-attrs) (every? uuid? child-attrs) + (every? string? bounds) + (scans? pg-hints [(pg-hint/index-scan :t8 :ea_index)]) + (= (mapv #(keyword (str "m-" %)) (range 22)) (mapv first ctes)) + (= (mapv #(vector (keyword (str "m-" %1)) %2 :materialized) + (range 9) expected) + (subvec ctes 0 (min 9 (count ctes)))) + (= #{:m-6 :m-8 :m-10 :m-12 :m-13 :m-15 :m-16 :m-18 :m-19 :m-21} + result-tables) + (= 2 (count (filter #{:m-7} (tree-seq coll? seq ctes))))) + ;; Each false full join row contains one branch's same-app entity. + ;; Its two bindings are equal and already came from triples, so the + ;; collector's extra existence scan only multiplies rows before DISTINCT. + {:ctes (seq (assoc-in ctes [7 1] + {:select [[[:distinct :m-6-entity-id] :m-7-entity-id]] + :from :m-6})) + ;; Keep the selected entities outside the parameterized ea_index lookup. + :pg-hints (into (vec pg-hints) [(pg-hint/leading [:m-7 :t8]) + (pg-hint/nest-loop :m-7 :t8)])})))) + (defn apply-plan [app-id normalized ctes pg-hints result-tables attrs] (or (when (not-any? #(contains? #{:'Leading :'HashJoin :'NestLoop :'MergeJoin :'NoHashJoin :'NoNestLoop :'NoMergeJoin} (first %)) pg-hints) (case app-id + #uuid "19bde4a4-559c-4274-9bba-5e4bff9fcffe" + (reassign-calendar-child-ids app-id normalized ctes pg-hints result-tables) + #uuid "299e756d-b3e6-41d3-a0ba-d097bfe7e04d" (status-filter app-id normalized ctes pg-hints) diff --git a/server/src/instant/util/hsql.clj b/server/src/instant/util/hsql.clj index e16c7f1f17..7696e7b5d4 100644 --- a/server/src/instant/util/hsql.clj +++ b/server/src/instant/util/hsql.clj @@ -14,6 +14,17 @@ (.append (hsql/format-entity (first more)))) (next more)))))) +(defn- add-leading-pair! [^StringBuilder s pair] + (if (vector? pair) + (do + (assert (= 2 (count pair)) "Leading join pairs must have two members") + (.append s \() + (add-leading-pair! s (first pair)) + (.append s " ") + (add-leading-pair! s (second pair)) + (.append s \))) + (.append s (hsql/format-entity pair)))) + ;; pg-hints expects a list of hints, e.g. ;; {:select :* ;; :pg-hints [(index-scan :t2 :ea_index)] @@ -31,6 +42,11 @@ (.append s (hsql/sql-kw op)) (.append s \() (case op + :'Leading (if (some vector? args) + (do (assert (= 1 (count args)) "Leading requires one outer join pair") + (add-leading-pair! s (first args))) + (add-args! s args)) + :'Rows (do (add-args! s (butlast args)) (.append s " #") (assert (number? (last args))) diff --git a/server/test/instant/db/reassign_calendar_child_ids_integration_test.clj b/server/test/instant/db/reassign_calendar_child_ids_integration_test.clj new file mode 100644 index 0000000000..35112cddca --- /dev/null +++ b/server/test/instant/db/reassign_calendar_child_ids_integration_test.clj @@ -0,0 +1,272 @@ +(ns instant.db.reassign-calendar-child-ids-integration-test + (:require [clojure.test :refer [deftest is testing]] + [instant.db.attr-sketch :as cms] + [instant.db.instaql :as iq] + [instant.db.model.attr :as attr-model] + [instant.db.reassign-calendar-child-ids-test :as fixtures] + [instant.flags :as flags] + [instant.jdbc.aurora :as aurora] + [instant.jdbc.sql :as sql] + [instant.model.rule :as rule-model] + [instant.util.instaql :as instaql-util] + [instant.util.json :as json] + [next.jdbc :as jdbc])) + +(defn entity [n] (java.util.UUID. 10 n)) + +(defn attr [etype label] + (attr-model/seek-by-fwd-ident-name [etype label] fixtures/measured-attrs)) + +(defn create-triples! [conn] + (jdbc/execute! conn + ["CREATE TEMP TABLE triples ( + app_id uuid, entity_id uuid, attr_id uuid, value jsonb, + checked_data_type checked_data_type, created_at bigint, + ea boolean, eav boolean, av boolean, ave boolean, vae boolean) + ON COMMIT DROP"]) + (doseq [query ["CREATE INDEX triples_pkey ON triples (app_id, entity_id, attr_id)" + "CREATE INDEX ea_index ON triples (app_id, entity_id, attr_id) WHERE ea" + "CREATE INDEX av_index ON triples (app_id, attr_id, value) WHERE av" + "CREATE INDEX ave_with_e_index ON triples (app_id, attr_id, value, entity_id) WHERE ave" + "CREATE INDEX triples_created_at_idx ON triples (app_id, attr_id, created_at, entity_id)" + "CREATE INDEX triples_string_type_idx ON triples + (app_id, attr_id, (value #>> '{}'), entity_id) + WHERE ave AND checked_data_type = 'string'" + "CREATE INDEX vae_uuid_index ON triples + (app_id, attr_id, json_uuid_to_uuid(value), entity_id) WHERE vae" + "CREATE INDEX eav_uuid_index ON triples + (app_id, entity_id, attr_id, json_uuid_to_uuid(value)) WHERE eav"]] + (jdbc/execute! conn [query]))) + +(defn insert-fields! [conn app-id etype eid fields] + (doseq [[label value] (assoc fields "id" eid)] + (let [{aid :id :keys [value-type cardinality unique? index? checked-data-type]} + (attr etype label) + ref? (= :ref value-type)] + (jdbc/execute! conn + ["INSERT INTO triples VALUES (?, ?, ?, ?::jsonb, ?::checked_data_type, + 100, ?, ?, ?, ?, ?)" + app-id eid aid + (json/->json value) (some-> checked-data-type name) + (= :one cardinality) ref? (boolean unique?) (boolean index?) ref?])))) + +(defn- seed! [conn] + (let [app fixtures/measured-app] + (doseq [[n fields] + [[1 {"date" "2026-09-01" "recurrenceRule" "daily" "exceptionDate" "2026-09-15" + "area" (entity 101) "activityType" (entity 201) "parent" (entity 3)}] + [2 {"date" "2026-09-30" "recurrenceRule" nil "exceptionDate" "2026-09-30" + "area" (entity 101) "activityType" (entity 999) "parent" (entity 998)}] + [3 {"date" "2026-08-15" "recurrenceRule" "daily" + "area" (entity 102) "parent" (entity 5)}] + [4 {"date" "2026-10-01" "exceptionDate" "2026-09-01" "area" nil}] + [5 {"date" "2026-08-31" "recurrenceRule" nil}] + [6 {"date" "2026-10-01" "recurrenceRule" "daily" "exceptionDate" "2026-10-01"}] + [7 {"exceptionDate" "2026-09-15"}] + [8 {"recurrenceRule" "daily"}] + [9 {"date" "2026-09-15"}] + [10 {"date" nil "recurrenceRule" nil "exceptionDate" nil}]]] + (insert-fields! conn app "sections" (entity n) fields)) + (insert-fields! conn app "areas" (entity 101) {"name" "shared area"}) + (insert-fields! conn app "areas" (entity 102) {"name" "private area"}) + (insert-fields! conn app "activityTypes" (entity 201) {"name" "activity"}) + (doseq [[n parent] [[301 1] [302 1] [303 2] [304 7] [305 5] [306 997]]] + (insert-fields! conn app "sourceLinks" (entity n) + {"name" (str "link " n) "section" (entity parent)})) + ;; Same entity IDs and relationships in a second tenant cannot enter results. + (let [other-app (random-uuid)] + (insert-fields! conn other-app "sections" (entity 1) {"date" "2026-09-01" "area" (entity 103)}) + (insert-fields! conn other-app "areas" (entity 103) {"name" "other tenant"})) + (jdbc/execute! conn ["ANALYZE triples"]))) + +(defn range-query [lower upper] + (-> fixtures/measured-query + (assoc-in [:sections :$ :where :or 0 :and 0 :date :$gte] lower) + (assoc-in [:sections :$ :where :or 0 :and 1 :date :$lte] upper) + (assoc-in [:sections :$ :where :or 1 :and 1 :date :$lte] upper) + (assoc-in [:sections :$ :where :or 2 :and 0 :exceptionDate :$gte] lower) + (assoc-in [:sections :$ :where :or 2 :and 1 :exceptionDate :$lte] upper))) + +(defn- query-flags [enabled?] + {:scoped-query-plans {(str fixtures/measured-app) {"reassign-calendar-child-ids" enabled?}}}) + +(defn- synthetic-sketches [_conn keys] + (into {} (map (fn [k] [k {:sketch (assoc (cms/make-sketch) :total 2000)}])) keys)) + +(defn- query-result + ([conn query enabled? rules] + (query-result conn query enabled? rules fixtures/measured-attrs)) + ([conn query enabled? rules attrs] + (binding [flags/*flag-overrides* (query-flags enabled?) + flags/*toggle-overrides* {:disable-scoped-query-plans false + :disable-pg-hints false :pg-hints-by-default true}] + (let [ctx {:app-id fixtures/measured-app :attrs attrs + :db {:conn-pool conn} :skip-app-status-read-check? true + :use-rule-wheres? false :testing-rule-wheres true} + statements (atom []) + raw-results (atom []) + select-arrays sql/select-arrays + nodes (with-redefs [cms/lookup synthetic-sketches + rule-model/get-by-app-id (constantly {:code rules}) + sql/select-arrays + (fn [op conn q & args] + (let [result (apply select-arrays op conn q args)] + (when (= :instant.db.datalog/send-query-nested op) + (swap! statements conj q) + (swap! raw-results conj (apply merge (-> result second first)))) + result))] + (if rules + (iq/permissioned-query ctx query) + (iq/query-normal ctx query)))] + {:nodes nodes + :objects (instaql-util/instaql-nodes->object-tree ctx nodes) + :datalog (mapv #(select-keys (:datalog-result %) [:join-rows :topics :page-info :aggregate]) + (mapcat iq/data-seq nodes)) + :raw (into {} (map (fn [[table rows]] [table (frequencies rows)])) (first @raw-results)) + :sql (first @statements)})))) + +(defn- projected? [result] + (boolean (re-find #"SELECT DISTINCT m_6_entity_id AS m_7_entity_id FROM m_6" (first (:sql result))))) + +(defn plan-nodes [plan] + (tree-seq #(seq (get % "Plans")) #(get % "Plans") (get plan "Plan"))) + +(defn m8-plan [plan] + (some #(when (= "CTE m_8" (get % "Subplan Name")) %) (plan-nodes plan))) + +(defn bound-m8-lookup? [plan] + (let [join (m8-plan plan) + [outer inner] (get join "Plans") + index-cond (get inner "Index Cond" "")] + (and (= "Nested Loop" (get join "Node Type")) + (= "CTE Scan" (get outer "Node Type")) + (= "m_7" (get outer "CTE Name")) + (= "Outer" (get outer "Parent Relationship")) + (= "Index Scan" (get inner "Node Type")) + (= "ea_index" (get inner "Index Name")) + (= "t8" (get inner "Alias")) + (= "Inner" (get inner "Parent Relationship")) + (boolean (re-find #"app_id = '[0-9a-f-]+'::uuid" index-cond)) + (boolean (re-find #"entity_id = m_7.m_7_entity_id" index-cond)) + (boolean (re-find #"attr_id = ANY \('[{][0-9a-f,-]+[}]'::uuid\[\]\)" index-cond))))) + +(deftest calendar-projection-keeps-child-lookups-bound-to-matched-ids + (jdbc/with-transaction [conn (aurora/conn-pool :write) {:rollback-only true}] + (create-triples! conn) + (seed! conn) + (doseq [[label lower upper] + [[:sparse "2026-09-30" "2026-09-30"] + [:empty "2025-01-01" "2025-01-02"] + [:broad "2020-01-01" "2030-01-01"]]] + (testing (name label) + (let [query (range-query lower upper) + baseline (query-result conn query false nil) + candidate (query-result conn query true nil) + explain (update (:sql candidate) 0 #(str "EXPLAIN (FORMAT JSON) " %)) + plan (-> (jdbc/execute! conn explain {:builder-fn sql/as-string-maps}) + first (get "QUERY PLAN") first)] + (is (projected? candidate)) + (is (= (dissoc baseline :sql) (dissoc candidate :sql))) + (is (bound-m8-lookup? plan) (pr-str (m8-plan plan)))))))) + +(deftest calendar-projection-preserves-full-results-and-topics + (jdbc/with-transaction [conn (aurora/conn-pool :write) {:rollback-only true}] + (create-triples! conn) + (seed! conn) + (doseq [[lower upper expected] + [["2026-09-01" "2026-09-30" #{1 2 3 4 7 9}] + ["2026-09-01" "2026-09-01" #{1 3 4}] + ["2026-09-30" "2026-09-30" #{1 2 3}] + ["2026-09-30" "2026-09-01" #{1 3}] + ["2027-01-01" "2027-01-01" #{1 3 6}] + ["2025-01-01" "2025-01-02" #{}]]] + (testing (str lower " to " upper) + (let [query (range-query lower upper) + baseline (query-result conn query false nil) + candidate (query-result conn query true nil) + objects (get-in candidate [:objects "sections"])] + (is (not (projected? baseline))) + (is (projected? candidate)) + (is (= (dissoc baseline :sql) (dissoc candidate :sql))) + (is (= (set (map #(str (entity %)) expected)) (set (map #(get % "id") objects))))))) + (let [result (query-result conn fixtures/measured-query true nil) + objects (into {} (map (juxt #(get % "id") identity)) (get-in result [:objects "sections"]))] + (is (= #{(str (entity 301)) (str (entity 302))} + (set (map #(get % "id") (get-in objects [(str (entity 1)) "eventLinks"]))))) + (is (= [(str (entity 101))] (map #(get % "id") (get-in objects [(str (entity 1)) "area"])))) + (is (= [(str (entity 5))] (map #(get % "id") (get-in objects [(str (entity 3)) "parent"])))) + (is (= [{}] (get-in objects [(str (entity 2)) "parent"]))) + (is (= [{}] (get-in objects [(str (entity 2)) "activityType"]))) + (is (empty? (get-in objects [(str (entity 4)) "area"]))) + (is (empty? (get-in objects [(str (entity 9)) "eventLinks"])))) + (doseq [[label removed expected] + [[:range-only ["recurrenceRule" "exceptionDate"] #{1 2 9}] + [:exception-only ["date"] #{1 2 4 7}]]] + (testing (name label) + (jdbc/execute! conn ["SAVEPOINT empty_branches"]) + (doseq [field removed] + (jdbc/execute! conn ["DELETE FROM triples WHERE app_id = ? AND attr_id = ?" + fixtures/measured-app (:id (attr "sections" field))])) + (let [baseline (query-result conn fixtures/measured-query false nil) + candidate (query-result conn fixtures/measured-query true nil)] + (is (projected? candidate)) + (is (= (dissoc baseline :sql) (dissoc candidate :sql))) + (is (= (set (map #(str (entity %)) expected)) + (set (map #(get % "id") (get-in candidate [:objects "sections"])))))) + (jdbc/execute! conn ["ROLLBACK TO SAVEPOINT empty_branches"]))))) + +(deftest calendar-projection-retains-permission-filtering + (jdbc/with-transaction [conn (aurora/conn-pool :write) {:rollback-only true}] + (create-triples! conn) + (seed! conn) + (let [rules {"$default" {"allow" {"view" "true"}} + "sections" {"allow" {"view" (str "data.id != '" (entity 2) "'")}} + "areas" {"allow" {"view" (str "data.id != '" (entity 102) "'")}} + "sourceLinks" {"allow" {"view" (str "data.id != '" (entity 302) "'")}}} + baseline (query-result conn fixtures/measured-query false rules) + candidate (query-result conn fixtures/measured-query true rules) + objects (into {} (map (juxt #(get % "id") identity)) (get-in candidate [:objects "sections"]))] + (is (projected? candidate)) + (is (= (dissoc baseline :sql) (dissoc candidate :sql))) + (is (= (set (map #(str (entity %)) [1 3 4 7 9])) (set (keys objects)))) + (is (empty? (get-in objects [(str (entity 3)) "area"]))) + (is (= [(str (entity 301))] (map #(get % "id") (get-in objects [(str (entity 1)) "eventLinks"]))))))) + +(deftest many-valued-date-joins-retain-all-combinations + (jdbc/with-transaction [conn (aurora/conn-pool :write) {:rollback-only true}] + (create-triples! conn) + (seed! conn) + (let [date-ids #{(:id (attr "sections" "date")) (:id (attr "sections" "exceptionDate"))} + attrs (attr-model/wrap-attrs (mapv #(if (date-ids (:id %)) (assoc % :cardinality :many) %) + fixtures/measured-attrs)) + rules {"$default" {"allow" {"view" "true"}} + "sections" {"allow" {"view" (str "data.id != '" (entity 2) "'")}} + "sourceLinks" {"allow" {"view" (str "data.id != '" (entity 302) "'")}}}] + (doseq [aid date-ids] + (jdbc/execute! conn ["UPDATE triples SET ea = false WHERE app_id = ? AND attr_id = ?" + fixtures/measured-app aid])) + (doseq [[label value] [["date" "2026-08-10"] ["date" "2026-09-15"] + ["date" "2026-09-30"] ["date" "2026-10-15"] + ["exceptionDate" "2026-09-01"] ["exceptionDate" "2026-09-30"]]] + (jdbc/execute! conn + ["INSERT INTO triples + SELECT app_id, entity_id, attr_id, ?::jsonb, checked_data_type, + created_at, false, eav, av, ave, vae + FROM triples WHERE app_id = ? AND entity_id = ? AND attr_id = ? LIMIT 1" + (json/->json value) fixtures/measured-app (entity 1) (:id (attr "sections" label))])) + (doseq [policy [nil rules]] + (let [baseline (query-result conn fixtures/measured-query false policy attrs) + candidate (query-result conn fixtures/measured-query true policy attrs) + objects (into {} (map (juxt #(get % "id") identity)) (get-in candidate [:objects "sections"])) + root-rows (get-in candidate [:raw "m-6"])] + (is (projected? candidate)) + (is (= (dissoc baseline :sql) (dissoc candidate :sql))) + ;; Four lower-bound values pair with four upper-bound values, while + ;; the recurring branch has four matches and the exception branch 3x3. + (is (= 29 (reduce + 0 (for [[row n] root-rows + :when (= (str (entity 1)) (get row "m_6_entity_id"))] + n)))) + (is (= (set (map #(str (entity %)) (if policy [1 3 4 7 9] [1 2 3 4 7 9]))) + (set (keys objects)))) + (is (= (set (map #(str (entity %)) (if policy [301] [301 302]))) + (set (map #(get % "id") (get-in objects [(str (entity 1)) "eventLinks"])))))))))) diff --git a/server/test/instant/db/reassign_calendar_child_ids_test.clj b/server/test/instant/db/reassign_calendar_child_ids_test.clj new file mode 100644 index 0000000000..998c76e9a2 --- /dev/null +++ b/server/test/instant/db/reassign_calendar_child_ids_test.clj @@ -0,0 +1,271 @@ +(ns instant.db.reassign-calendar-child-ids-test + (:require [clojure.test :refer [deftest is testing]] + [honey.sql :as hsql] + [instant.db.attr-sketch :as cms] + [instant.db.datalog :as d] + [instant.db.instaql :as iq] + [instant.db.model.attr :as attr-model] + [instant.db.scoped-query-plans :as plans] + [instant.flags :as flags] + [instant.util.instaql :as instaql-util] + [instant.util.pg-hint-plan :as pg-hint] + [instant.util.tracer :as tracer])) + +(def measured-app #uuid "19bde4a4-559c-4274-9bba-5e4bff9fcffe") +(def measured-query + {:sections + {:$ {:where {:or [{:and [{:date {:$gte "2026-09-01"}} + {:date {:$lte "2026-09-30"}}]} + {:and [{:recurrenceRule {:$isNull false}} + {:date {:$lte "2026-09-30"}}]} + {:and [{:exceptionDate {:$gte "2026-09-01"}} + {:exceptionDate {:$lte "2026-09-30"}}]}]}} + :area {} :activityType {} :parent {} :eventLinks {}}}) +(def measured-normalized (instaql-util/normalized-forms measured-query)) + +(def measured-attrs + (attr-model/wrap-attrs + (mapv (fn [[n etype label opts]] + (merge {:id (java.util.UUID. 0 n) + :forward-identity [(java.util.UUID. 1 n) etype label] + :value-type :blob :cardinality :one + :unique? (= label "id") :index? false} + opts)) + [[1 "sections" "id" {}] + [2 "sections" "date" {:index? true :checked-data-type :string}] + [3 "sections" "recurrenceRule" {:checked-data-type :string}] + [4 "sections" "exceptionDate" {:index? true :checked-data-type :string}] + [5 "sections" "area" {:value-type :ref + :reverse-identity [(java.util.UUID. 2 5) "areas" "sections"]}] + [6 "sections" "activityType" {:value-type :ref + :reverse-identity [(java.util.UUID. 2 6) "activityTypes" "sections"]}] + [7 "sections" "parent" {:value-type :ref + :reverse-identity [(java.util.UUID. 2 7) "sections" "children"]}] + [8 "sourceLinks" "section" {:value-type :ref + :reverse-identity [(java.util.UUID. 2 8) "sections" "eventLinks"]}] + [9 "areas" "id" {}] + [10 "areas" "name" {}] + [11 "activityTypes" "id" {}] + [12 "activityTypes" "name" {}] + [13 "sourceLinks" "id" {}] + [14 "sourceLinks" "name" {}]]))) + +(defn compile-query + ([app-id q] (compile-query app-id q measured-attrs)) + ([app-id q attrs] + (let [ctx {:app-id app-id :attrs attrs + :query-normalized (instaql-util/normalized-forms q) + :sketches (into {} (map (fn [a] + [{:app-id app-id :attr-id (:id a)} + {:sketch (assoc (cms/make-sketch) :total 2000)}]) + attrs))} + patterns (:patterns (iq/instaql-query->patterns ctx q)) + named (d/annotate-with-hints-impl ctx {} (d/nested->named-patterns patterns))] + (:query (d/nested-match-query ctx :m- app-id named))))) + +(defn baseline [app-id q] + (with-redefs [plans/apply-plan (fn [_ _ ctes pg-hints _ _] + {:ctes ctes :pg-hints pg-hints})] + (compile-query app-id q))) + +(def result-tables #{:m-6 :m-8 :m-10 :m-12 :m-13 :m-15 :m-16 :m-18 :m-19 :m-21}) +(def projected-collector + {:select [[[:distinct :m-6-entity-id] :m-7-entity-id]] :from :m-6}) +(def child-lookup-hints [(pg-hint/leading [:m-7 :t8]) (pg-hint/nest-loop :m-7 :t8)]) + +(defn with-plan [f] + (with-redefs [flags/flag (fn [flag & [default]] + (if (= flag :scoped-query-plans) + {(str measured-app) {"reassign-calendar-child-ids" true}} + default)) + flags/toggled? (fn [_ & [default]] default)] + (binding [d/*enable-pg-hints* true] (f)))) + +(deftest calendar-projection-changes-only-the-intermediate-collector + (with-plan + (fn [] + (let [original (baseline measured-app measured-query) + before (update original :with vec) + after (compile-query measured-app measured-query) + qid [:qid {:select [[[:inline measured-app]] [[:inline -843103532]]]}]] + (is (= projected-collector (get-in (vec (:with after)) [7 1]))) + (is (= (-> before + (assoc-in [:with 7 1] projected-collector) + (update :pg-hints into child-lookup-hints)) + after)) + (is (= :materialized (get-in (vec (:with after)) [7 2]))) + (is (= (into (:pg-hints before) child-lookup-hints) (:pg-hints after))) + (is (= [[:'Leading [:m-7 :t8]] [:'NestLoop :m-7 :t8]] child-lookup-hints)) + (is (re-find #"Leading\(\(m_7 t8\)\)\nNestLoop\(m_7 t8\)" + (first (hsql/format after)))) + (is (some #{(pg-hint/index-scan :t8 :ea_index)} (:pg-hints after))) + ;; send-query-nested adds qid with conj. Keep its position unchanged. + (is (= :qid (ffirst (:with (update after :with conj qid))))) + (is (= (hsql/format (update original :with conj qid)) + (hsql/format (-> after + (update :with #(vec (conj % qid))) + (assoc :pg-hints (:pg-hints before)) + (assoc-in [:with 8 1] (get-in before [:with 7 1])))))))))) + +(deftest calendar-projection-requires-exact-opt-in + (binding [d/*enable-pg-hints* true] + (let [before (baseline measured-app measured-query)] + (doseq [setting [nil {} {(str measured-app) {}} + {(str measured-app) {"reassign-calendar-child-ids" false}} + {(str measured-app) {"reassign-calendar-child-ids" "true"}} + {measured-app {"reassign-calendar-child-ids" true}}]] + (with-redefs [flags/flag (fn [flag & [default]] + (if (= flag :scoped-query-plans) setting default)) + flags/toggled? (fn [_ & [default]] default)] + (is (= before (compile-query measured-app measured-query))))))) + (with-plan + (fn [] + (with-redefs [flags/toggled? (fn [flag & [default]] + (if (= flag :disable-scoped-query-plans) true default))] + (is (= (baseline measured-app measured-query) (compile-query measured-app measured-query))))))) + +(deftest calendar-projection-respects-existing-hint-switches + (with-plan + (fn [] + (binding [d/*enable-pg-hints* false] + (is (= (baseline measured-app measured-query) (compile-query measured-app measured-query)))) + (with-redefs [flags/toggled? (fn [flag & [default]] + (if (= flag :disable-pg-hints) true default))] + (is (= (baseline measured-app measured-query) (compile-query measured-app measured-query)))) + (let [original-flag flags/flag] + (with-redefs [flags/flag (fn [flag & [default]] + (if (= flag :disable-hint-query-hashes) + #{-843103532} + (original-flag flag default))) + tracer/add-data! (fn [& _]) + d/send-query-nested (fn [& _] (compile-query measured-app measured-query))] + (is (= (binding [d/*enable-pg-hints* false] (baseline measured-app measured-query)) + (d/query-nested {:app-id measured-app :query-hash -843103532} + '{:children {:pattern-groups + [{:patterns [[:ea ?e #uuid "00000000-0000-0000-0000-000000000001" _]]}]}})))))))) + +(deftest calendar-projection-requires-the-whole-query-and-app + (with-plan + (fn [] + (doseq [[label app-id q] + [[:other-app (random-uuid) measured-query] + [:missing-child measured-app (update measured-query :sections dissoc :area)] + [:limit measured-app (assoc-in measured-query [:sections :$ :limit] 10)] + [:order measured-app (assoc-in measured-query [:sections :$ :order] {:date "asc"})] + [:fields measured-app (assoc-in measured-query [:sections :$ :fields] ["id"])] + [:extra-filter measured-app (assoc-in measured-query [:sections :$ :where :date] "2026-09-10")] + [:is-null measured-app (assoc-in measured-query [:sections :$ :where :or 1 :and 0 :recurrenceRule :$isNull] true)]]] + (testing (name label) + (is (= (baseline app-id q) (compile-query app-id q))))) + (let [q (-> measured-query + (assoc-in [:sections :$ :where :or 0 :and 0 :date :$gte] "2030-01-01") + (assoc-in [:sections :$ :where :or 1 :and 1 :date :$lte] "2020-01-01"))] + (is (= projected-collector (get-in (vec (:with (compile-query measured-app q))) [7 1]))))))) + +(deftest calendar-projection-falls-back-on-compiler-drift + (with-plan + (fn [] + (let [{:keys [with pg-hints]} (baseline measured-app measured-query) + ctes (vec with) + apply-plan (fn [ctes hints tables] + (plans/apply-plan measured-app measured-normalized ctes hints tables measured-attrs)) + where-change (fn [ctes idx f] + (update-in ctes [idx 1 :where] #(f (vec %))))] + (doseq [[label changed] + (concat + [[:missing-cte (pop ctes)] + [:duplicate-cte (conj ctes (first ctes))] + [:reordered-ctes (assoc ctes 0 (ctes 1) 1 (ctes 0))] + [:materialization (assoc-in ctes [0 2] :not-materialized)] + [:foreign-app (where-change ctes 0 #(assoc % 1 [:= :app-id (random-uuid)]))] + [:foreign-subquery-app (where-change ctes 2 #(assoc-in % [4 2 :where] [:and [:= :app-id (random-uuid)]]))] + [:attribute-drift (where-change ctes 1 #(assoc-in % [3 2] (random-uuid)))] + [:subquery-null-value (where-change ctes 2 #(update-in % [4 2 :where] (fn [w] (assoc (vec w) 5 [:not= :t2-subquery.value [:cast "false" :jsonb]]))))] + [:malformed-where (assoc-in ctes [0 1 :where] true)] + [:full-join-not-false (update-in ctes [6 1 :full-join] #(assoc (vec %) 1 [:= :0 :0]))] + [:left-join (update-in ctes [6 1] #(-> % (assoc :left-join (:full-join %)) (dissoc :full-join)))] + [:gather-filter (assoc-in ctes [6 1 :where] [:= :m-1-entity-id :m-3-entity-id])] + [:coalesce-binding (update-in ctes [6 1 :select] #(assoc (vec %) 3 [[:coalesce :m-0-value] :m-6-entity-id]))] + [:collector-app (where-change ctes 7 #(assoc % 1 [:= :app-id (random-uuid)]))] + [:collector-extra-filter (where-change ctes 7 #(conj % [:= :entity-id (random-uuid)]))] + [:collector-limit (assoc-in ctes [7 1 :limit] 10)] + [:collector-distinct (assoc-in ctes [7 1 :select] [[:entity-id :m-7-entity-id]])] + [:different-consumer (assoc-in ctes [8 1 :from] [[:triples :t8] :m-6])] + [:child-select (assoc-in ctes [8 1 :select] [[:entity-id :m-8-entity-id]])] + [:child-app (where-change ctes 8 #(assoc-in % [1 2] (random-uuid)))] + [:child-index (where-change ctes 8 #(assoc-in % [2 1] :eav))] + [:child-attribute-op (where-change ctes 8 #(assoc-in % [3 2 0] :all))] + [:child-attribute-vector (where-change ctes 8 #(update-in % [3 2 1] vec))] + [:child-attribute-empty (where-change ctes 8 #(assoc-in % [3 2 1] #{}))] + [:child-attribute-non-uuid (where-change ctes 8 #(update-in % [3 2 1] conj "invalid"))] + [:child-attribute-missing (where-change ctes 8 #(assoc-in % [3 2 1] nil))] + [:child-entity-binding (where-change ctes 8 #(assoc-in % [4 2] :m-6-entity-id))] + [:child-extra-filter (where-change ctes 8 #(conj % [:= :value [:cast "null" :jsonb]]))] + [:child-limit (assoc-in ctes [8 1 :limit] 10)] + [:child-order (assoc-in ctes [8 1 :order-by] [:entity-id])] + [:child-materialization (assoc-in ctes [8 2] :not-materialized)] + [:extra-consumer (assoc-in ctes [9 1 :from] :m-7)]] + (for [idx (range 6)] + [(keyword (str "branch-select-" idx)) + (assoc-in ctes [idx 1 :select] [[:value (keyword (str "m-" idx "-entity-id"))]])]) + (for [idx [1 3 5]] + [(keyword (str "branch-join-" idx)) + (where-change ctes idx #(assoc % 5 [:= :value :m-0-value]))]))] + (testing (name label) + (is (= {:ctes changed :pg-hints pg-hints} + (apply-plan changed pg-hints result-tables))))) + (doseq [tables [(conj result-tables :m-7) (disj result-tables :m-6)]] + (is (= {:ctes ctes :pg-hints pg-hints} (apply-plan ctes pg-hints tables)))) + (let [hints (conj (vec pg-hints) [:'HashJoin :t0 :t1])] + (is (= {:ctes ctes :pg-hints hints} (apply-plan ctes hints result-tables)))) + (doseq [hints [(vec (remove #{(pg-hint/index-scan :t8 :ea_index)} pg-hints)) + (conj (vec pg-hints) (pg-hint/index-scan :t8 :ea_index)) + (mapv #(if (= % (pg-hint/index-scan :t8 :ea_index)) + (pg-hint/index-scan :t8 :triples_pkey) %) pg-hints)]] + (is (= {:ctes ctes :pg-hints hints} (apply-plan ctes hints result-tables)))))))) + +(deftest calendar-projection-preserves-current-child-attributes + (with-plan + (fn [] + (let [{:keys [with pg-hints]} (baseline measured-app measured-query) + attrs (with-meta (set [(random-uuid) (random-uuid)]) {:pgtype "uuid[]"}) + ctes (update-in (vec with) [8 1 :where] + #(assoc-in (vec %) [3 2 1] attrs)) + after (plans/apply-plan measured-app measured-normalized ctes pg-hints result-tables measured-attrs) + after-ctes (vec (:ctes after))] + (is (= projected-collector (get-in after-ctes [7 1]))) + (is (= (ctes 8) (after-ctes 8))) + (is (identical? attrs (get-in (vec (get-in after-ctes [8 1 :where])) [3 2 1]))) + (is (= (into pg-hints child-lookup-hints) (:pg-hints after))))))) + +(deftest calendar-projection-preserves-schema-dependent-compilation + (with-plan + (fn [] + (let [compile (fn [attrs enabled?] + (if enabled? + (compile-query measured-app measured-query attrs) + (with-redefs [plans/apply-plan (fn [_ _ ctes pg-hints _ _] + {:ctes ctes :pg-hints pg-hints})] + (compile-query measured-app measured-query attrs))))] + (doseq [[label attrs] + [[:typed-date (attr-model/wrap-attrs + (mapv #(if (= "date" (nth (:forward-identity %) 2)) + (assoc % :checked-data-type :date) %) + measured-attrs))] + [:indexed-recurrence (attr-model/wrap-attrs + (mapv #(if (= "recurrenceRule" (nth (:forward-identity %) 2)) + (assoc % :index? true) %) + measured-attrs))]]] + (testing (name label) + (is (= (compile attrs false) (compile attrs true))))) + (let [attrs (attr-model/wrap-attrs + (mapv #(if (= "date" (nth (:forward-identity %) 2)) + (assoc % :cardinality :many) %) + measured-attrs)) + before (update (compile attrs false) :with vec) + after (compile attrs true)] + (is (= projected-collector (get-in (vec (:with after)) [7 1]))) + (is (= (-> before + (assoc-in [:with 7 1] projected-collector) + (update :pg-hints into child-lookup-hints)) + after))))))) diff --git a/server/test/instant/util/hsql_test.clj b/server/test/instant/util/hsql_test.clj index 437550bcfb..f3ab1423da 100644 --- a/server/test/instant/util/hsql_test.clj +++ b/server/test/instant/util/hsql_test.clj @@ -2,7 +2,28 @@ (:require [clojure.test :refer [deftest is]] [honey.sql :as hsql] - [instant.util.hsql :as uhsql])) + [instant.util.hsql :as uhsql] + [instant.util.pg-hint-plan :as pg-hint])) + +(deftest join-order-hints-preserve-pair-direction + (doseq [[hint expected] + [[(pg-hint/leading :m-7 :t8) "Leading(m_7 t8)"] + [(pg-hint/leading [:m-7 :t8]) "Leading((m_7 t8))"] + [(pg-hint/leading [[:m-7 :t8] :t9]) "Leading(((m_7 t8) t9))"] + [(pg-hint/leading [:m-7 [:t8 :t9]]) "Leading((m_7 (t8 t9)))"] + [(pg-hint/nest-loop :m-7 :t8) "NestLoop(m_7 t8)"] + [(pg-hint/rows :m-7 :t8 1000) "Rows(m_7 t8 #1000)"] + [(pg-hint/parallel :t8 2 :hard) "Parallel(t8 2 hard)"]]] + (is (= [(str "/*+\n" expected "\n*/ SELECT * FROM triples")] + (hsql/format {:select :* :from :triples :pg-hints [hint]}))))) + +(deftest join-order-hints-reject-malformed-pairs + (doseq [hint [(pg-hint/leading [:m-7]) + (pg-hint/leading [:m-7 :t8 :t9]) + (pg-hint/leading :m-7 [:t8 :t9]) + (pg-hint/leading [[:m-7] :t8])]] + (is (thrown? AssertionError + (hsql/format {:select :* :from :triples :pg-hints [hint]}))))) (deftest formatp-works-the-same-as-format (doseq [{:keys [query params]}