diff --git a/.clj-kondo/README.md b/.clj-kondo/README.md index 67638c3..b2c41c7 100644 --- a/.clj-kondo/README.md +++ b/.clj-kondo/README.md @@ -1,5 +1,5 @@ Update `clj-kondo` configs for libraries using ```sh -clj-kondo --copy-configs --dependencies --lint "$(clojure -Spath)" +clojure -M:kondo --copy-configs --dependencies --lint "$(clojure -Spath -A:dev)" ``` diff --git a/.clj-kondo/config.edn b/.clj-kondo/config.edn index 7623f08..3ea5a77 100644 --- a/.clj-kondo/config.edn +++ b/.clj-kondo/config.edn @@ -2,17 +2,61 @@ ["../resources/clj-kondo.exports/com.github.metabase/hawk"] :linters - {:docstring-leading-trailing-whitespace {:level :warning} + {:aliased-namespace-symbol {:level :warning} + :case-symbol-test {:level :warning} + :condition-always-true {:level :warning} + :def-fn {:level :warning} + :docstring-leading-trailing-whitespace {:level :warning} + :dynamic-var-not-earmuffed {:level :warning} + :equals-expected-position {:level :warning, :only-in-test-assertion true} + :equals-true {:level :warning} + :keyword-binding {:level :warning} + :main-without-gen-class {:level :warning} + :minus-one {:level :warning} + :misplaced-docstring {:level :warning} + :missing-body-in-when {:level :warning} :missing-docstring {:level :warning} + :missing-else-branch {:level :warning} + :namespace-name-mismatch {:level :warning} + :non-arg-vec-return-type-hint {:level :warning} + :plus-one {:level :warning} + :reduce-without-init {:level :warning} + :redundant-call {:level :warning} + :redundant-fn-wrapper {:level :warning} + :redundant-str-call {:level :warning} + :self-requiring-namespace {:level :warning} + :shadowed-var {:level :warning} + :single-key-in {:level :warning} + :unsorted-imports {:level :warning} :unsorted-required-namespaces {:level :warning} + :unused-alias {:level :warning} + :use {:level :warning} + :used-underscored-binding {:level :warning} + :warn-on-reflection {:level :warning} :refer-all {:level :warning :exclude [clojure.test]} + :unresolved-var + {:exclude [io.aviso.ansi]} + :consistent-alias {:aliases - {methodical.core methodical}}} + {methodical.core methodical}} + +;;; disabled linters + + :redundant-ignore {:level :off}} ; false positives :config-in-comment - {:linters {:unresolved-symbol {:level :off}}}} + {:linters {:unresolved-symbol {:level :off}}} + + :ns-groups + [{:pattern "^mb\\.eftest\\..*$" + :name eftest-namespaces}] + + :config-in-ns + {eftest-namespaces + {:linters + {:missing-docstring {:level :off}}}}} diff --git a/README.md b/README.md index c766a99..ca20340 100644 --- a/README.md +++ b/README.md @@ -277,3 +277,9 @@ for more information. ``` clj -X:test '{:fail-fast? true}' ``` + +## License + +Copyright © 2019-2026 James Reeves, 2023-2026 Metabase, Inc. + +Distributed under the Eclipse Public License either version 2.0 or (at your option) any later version. diff --git a/deps.edn b/deps.edn index 570da2f..41b5087 100644 --- a/deps.edn +++ b/deps.edn @@ -3,14 +3,16 @@ :deps {commons-io/commons-io {:mvn/version "2.18.0"} - eftest/eftest {:mvn/version "0.6.0"} environ/environ {:mvn/version "1.2.0"} + io.aviso/pretty {:mvn/version "1.3"} methodical/methodical {:mvn/version "1.0.124"} - pjstadig/humane-test-output {:mvn/version "0.11.0"} - prismatic/schema {:mvn/version "1.4.1"} metosin/malli {:mvn/version "0.17.0"} + mvxcvi/puget {:mvn/version "1.3.4"} org.clojure/java.classpath {:mvn/version "1.1.0"} - org.clojure/tools.namespace {:mvn/version "1.5.0"}} + org.clojure/tools.namespace {:mvn/version "1.5.0"} + pjstadig/humane-test-output {:mvn/version "0.11.0"} + prismatic/schema {:mvn/version "1.4.1"} + progrock/progrock {:mvn/version "0.1.2"}} :aliases {:dev @@ -19,7 +21,8 @@ ;; clj -X:dev:test :test {:exec-fn mb.hawk.core/find-and-run-tests-cli - :exec-args {:exclude-directories ["src" "resources"]}} + :exec-args {:exclude-directories ["src" "resources"] + :exclude-tags [:hawk.tests/skip]}} ;; clojure -T:build :build @@ -37,7 +40,7 @@ ;; version here which may be different from the version installed on your computer. :kondo {:replace-deps - {clj-kondo/clj-kondo {:mvn/version "2024.08.29"}} + {clj-kondo/clj-kondo {:mvn/version "2026.05.25"}} :main-opts ["-m" "clj-kondo.main"]} @@ -47,5 +50,5 @@ ;; Find outdated versions of dependencies. Run with `clojure -M:outdated` :outdated {;; Note that it is `:deps`, not `:extra-deps` - :deps {com.github.liquidz/antq {:mvn/version "RELEASE"}} + :deps {com.github.liquidz/antq {:mvn/version "RELEASE"}} :main-opts ["-m" "antq.core" "--skip=github-action"]}}} diff --git a/src/mb/eftest/output_capture.clj b/src/mb/eftest/output_capture.clj new file mode 100644 index 0000000..7a12393 --- /dev/null +++ b/src/mb/eftest/output_capture.clj @@ -0,0 +1,61 @@ +(ns mb.eftest.output-capture + (:import + (java.io ByteArrayOutputStream OutputStream PrintStream PrintWriter))) + +(set! *warn-on-reflection* true) + +(def ^:dynamic ^ByteArrayOutputStream *test-buffer* nil) + +(defn read-test-buffer [] + (some-> *test-buffer* (.toByteArray) (String.))) + +(def active-buffers (atom #{})) + +(defmacro with-test-buffer [& body] + `(let [buffer# (ByteArrayOutputStream.)] + (try + (swap! active-buffers conj buffer#) + (binding [*test-buffer* buffer#] ~@body) + (finally (swap! active-buffers disj buffer#))))) + +(defn- doto-capture-buffer [f] + (if *test-buffer* + (f *test-buffer*) + (doseq [buffer @active-buffers] + (f buffer)))) + +(defn- create-proxy-output-stream ^OutputStream [] + (proxy [OutputStream] [] + (write + ([data] + (if (instance? Integer data) + (doto-capture-buffer #(.write ^OutputStream % ^int data)) + (doto-capture-buffer #(.write ^OutputStream % ^bytes data 0 (alength ^bytes data))))) + ([data off len] + (doto-capture-buffer #(.write ^OutputStream % data off len)))))) + +(defn init-capture [] + (let [old-out System/out + old-err System/err + proxy-output-stream (create-proxy-output-stream) + new-stream (PrintStream. proxy-output-stream) + new-writer (PrintWriter. proxy-output-stream)] + (System/setOut new-stream) + (System/setErr new-stream) + {:captured-writer new-writer + :old-system-out old-out + :old-system-err old-err})) + +(defn restore-capture [{:keys [old-system-out old-system-err]}] + (System/setOut old-system-out) + (System/setErr old-system-err)) + +(defmacro with-capture [& body] + `(let [context# (init-capture) + writer# (:captured-writer context#)] + (try + (binding [*out* writer#, *err* writer#] + (with-redefs [*out* writer#, *err* writer#] + ~@body)) + (finally + (restore-capture context#))))) diff --git a/src/mb/eftest/report.clj b/src/mb/eftest/report.clj new file mode 100644 index 0000000..9a9d6d3 --- /dev/null +++ b/src/mb/eftest/report.clj @@ -0,0 +1,33 @@ +(ns mb.eftest.report + (:require + [clojure.java.io :as io] + [clojure.test :refer [*test-out*]])) + +(set! *warn-on-reflection* true) + +(def ^:dynamic *context* + "Used by eftest.runner/run-tests to hold a mutable atom that persists for the + duration of the test run. This atom can be used by reporters to hold + additional statistics and information during the tests." + nil) + +(def ^:dynamic *testing-path* + "2-element vector [ns scope] where scope is either :clojure.test/once-fixtures, + :clojure.test/each-fixtures or var under test" + nil) + +(defn report-to-file + "Wrap a report function so that its output is directed to a file. output-file + should be something that can be coerced into a Writer." + [report output-file] + (let [output-key [::output-files output-file]] + (fn [m] + (when (= (:type m) :begin-test-run) + (io/make-parents (io/file output-file)) + (swap! *context* assoc-in output-key (io/writer output-file))) + (let [^java.io.Writer writer (get-in @*context* output-key)] + (binding [*test-out* writer] + (report m)) + (when (= (:type m) :summary) + (swap! *context* update ::output-files dissoc output-file) + (.close writer)))))) diff --git a/src/mb/eftest/report/pretty.clj b/src/mb/eftest/report/pretty.clj new file mode 100644 index 0000000..b1e0967 --- /dev/null +++ b/src/mb/eftest/report/pretty.clj @@ -0,0 +1,161 @@ +(ns mb.eftest.report.pretty + "A test reporter with an emphasis on pretty formatting." + (:refer-clojure :exclude [test]) + (:require + [clojure.data :as data] + [clojure.string :as str] + [clojure.test :as test] + [fipp.engine :as fipp] + [io.aviso.ansi :as ansi] + [io.aviso.exception :as exception] + [io.aviso.repl :as repl] + [mb.eftest.output-capture :as capture] + [mb.eftest.report :as report] + [puget.printer :as puget])) + +(def ^:dynamic *fonts* + "The ANSI codes to use for reporting on tests." + {:exception ansi/red-font + :reset ansi/reset-font + :message ansi/italic-font + :property ansi/bold-font + :source ansi/italic-font + :function-name ansi/blue-font + :clojure-frame ansi/white-font + :java-frame ansi/reset-font + :omitted-frame ansi/reset-font + :pass ansi/green-font + :fail ansi/red-font + :error ansi/red-font + :divider ansi/yellow-font}) + +(def ^:dynamic *divider* + "The divider to use between test failure and error reports." + "\n") + +(defn- testing-scope-str [{:keys [file line]}] + (let [[test-ns scope] report/*testing-path*] + (str + (cond + (keyword? scope) + (str (:clojure-frame *fonts*) (ns-name test-ns) (:reset *fonts*) " during " + (:function-name *fonts*) scope (:reset *fonts*)) + + (var? scope) + (str (:clojure-frame *fonts*) (ns-name test-ns) "/" + (:function-name *fonts*) (:name (meta scope)) (:reset *fonts*))) + (when (or file line) + (str " (" (:source *fonts*) file ":" line (:reset *fonts*) ")"))))) + +(defn- diff-all [expected actuals] + (map vector actuals (map #(take 2 (data/diff expected %)) actuals))) + +(defn- pretty-printer [] + (puget/pretty-printer {:print-color true + :print-meta false})) + +(defn- pprint-document [doc] + (fipp/pprint-document doc {:width 80})) + +(defn- equals-fail-report [{:keys [actual]}] + (let [[_ [_ expected & actuals]] actual + p (pretty-printer)] + (doseq [[actual [a b]] (diff-all expected actuals)] + (pprint-document + [:group + [:span "expected: " (puget/format-doc p expected) :break] + [:span " actual: " (puget/format-doc p actual) :break] + (when (and (not= expected a) (not= actual b)) + [:span " diff: " + (when a + [:span "- " (puget/format-doc p a) :break]) + (when b + [:span + (if a " + " "+ ") + (puget/format-doc p b)])])])))) + +(defn- predicate-fail-report [{:keys [expected actual]}] + (let [p (pretty-printer)] + (pprint-document + [:group + [:span "expected: " (puget/format-doc p expected) :break] + [:span " actual: " (puget/format-doc p actual)]]))) + +(defn- print-stacktrace [t] + (binding [exception/*traditional* true + exception/*fonts* *fonts*] + (repl/pretty-print-stack-trace t test/*stack-trace-depth*))) + +(defn- error-report [{:keys [expected actual]}] + (if expected + (let [p (pretty-printer)] + (pprint-document + [:group + [:span "expected: " (puget/format-doc p expected) :break] + [:span " actual: " (with-out-str (print-stacktrace actual))]])) + (print-stacktrace actual))) + +(defn- print-output [output] + (let [c (:divider *fonts*) + r (:reset *fonts*)] + (when-not (str/blank? output) + (println (str c "---" r " Test output " c "---" r)) + (println (str/trim-newline output)) + (println (str c "-------------------" r))))) + +(defmulti report + "A reporting function compatible with clojure.test. Uses ANSI colors and + terminal formatting to produce readable and 'pretty' reports." + :type) + +(defmethod report :default [_m]) + +(defmethod report :pass [_m] + (test/with-test-out (test/inc-report-counter :pass))) + +(defmethod report :fail [{:keys [message expected] :as m}] + (test/with-test-out + (test/inc-report-counter :fail) + (print *divider*) + (println (str (:fail *fonts*) "FAIL" (:reset *fonts*) " in") (testing-scope-str m)) + (when (seq test/*testing-contexts*) (println (test/testing-contexts-str))) + (when message (println message)) + (if (and (sequential? expected) + (= (first expected) '=)) + (equals-fail-report m) + (predicate-fail-report m)) + (print-output (capture/read-test-buffer)))) + +(defmethod report :error [{:keys [message _expected _actual] :as m}] + (test/with-test-out + (test/inc-report-counter :error) + (print *divider*) + (println (str (:error *fonts*) "ERROR" (:reset *fonts*) " in") (testing-scope-str m)) + (when (seq test/*testing-contexts*) (println (test/testing-contexts-str))) + (when message (println message)) + (error-report m) + (some-> (capture/read-test-buffer) (print-output)))) + +(defn- pluralize [word n] + (if (= n 1) word (str word "s"))) + +(defn- format-interval [duration] + (format "%.3f seconds" (double (/ duration 1e3)))) + +(defmethod report :long-test [{:keys [duration] :as m}] + (test/with-test-out + (print *divider*) + (println (str (:fail *fonts*) "LONG TEST" (:reset *fonts*) " in") (testing-scope-str m)) + (when duration (println "Test took" (format-interval duration) "seconds to run")))) + +(defmethod report :summary [{:keys [test pass fail error duration]}] + (let [total (+ pass fail error) + color (if (= pass total) (:pass *fonts*) (:error *fonts*))] + (test/with-test-out + (print *divider*) + (println "Ran" test "tests in" (format-interval duration)) + (println (str color + total " " (pluralize "assertion" total) ", " + fail " " (pluralize "failure" fail) ", " + error " " (pluralize "error" error) "." + (:reset *fonts*)))))) diff --git a/src/mb/eftest/report/progress.clj b/src/mb/eftest/report/progress.clj new file mode 100644 index 0000000..2288870 --- /dev/null +++ b/src/mb/eftest/report/progress.clj @@ -0,0 +1,73 @@ +(ns mb.eftest.report.progress + "A test reporter with a progress bar." + (:require + [clojure.test :as test] + [mb.eftest.report :as report] + [mb.eftest.report.pretty :as pretty] + [progrock.core :as prog])) + +(def ^:private clear-line (apply str "\r" (repeat 80 " "))) + +(defn- colored-format [state] + (str ":progress/:total :percent% [" + (if state + (str (pretty/*fonts* state) ":bar" (pretty/*fonts* :reset)) + ":bar") + "] ETA: :remaining")) + +(defn- print-progress [{:keys [bar state]}] + (prog/print bar {:format (colored-format state)})) + +(defn- set-state [old-state new-state] + (case [old-state new-state] + [nil :pass] :pass + [nil :fail] :fail + [:pass :fail] :fail + [nil :error] :error + [:pass :error] :error + [:fail :error] :error + old-state)) + +(defmulti report :type) + +(defmethod report :default [_m]) + +(defmethod report :begin-test-run [m] + (test/with-test-out + (newline) + (print-progress (reset! report/*context* {:bar (prog/progress-bar (:count m))})))) + +(defmethod report :pass [m] + (test/with-test-out + (pretty/report m) + (print-progress (swap! report/*context* update-in [:state] set-state :pass)))) + +(defmethod report :fail [m] + (test/with-test-out + (print clear-line) + (binding [pretty/*divider* "\r"] (pretty/report m)) + (newline) + (print-progress (swap! report/*context* update-in [:state] set-state :fail)))) + +(defmethod report :error [m] + (test/with-test-out + (print clear-line) + (binding [pretty/*divider* "\r"] (pretty/report m)) + (newline) + (print-progress (swap! report/*context* update-in [:state] set-state :error)))) + +(defmethod report :end-test-var [_m] + (test/with-test-out + (print-progress (swap! report/*context* update-in [:bar] prog/tick)))) + +(defmethod report :summary [m] + (test/with-test-out + (print-progress (swap! report/*context* update-in [:bar] prog/done)) + (pretty/report m))) + +(defmethod report :long-test [m] + (test/with-test-out + (print clear-line) + (binding [pretty/*divider* "\r"] (pretty/report m)) + (newline) + (print-progress @report/*context*))) diff --git a/src/mb/eftest/runner.clj b/src/mb/eftest/runner.clj new file mode 100644 index 0000000..3119f2e --- /dev/null +++ b/src/mb/eftest/runner.clj @@ -0,0 +1,195 @@ +(ns mb.eftest.runner + "Functions to run tests written with clojure.test or compatible libraries." + (:require + [clojure.test :as test] + [mb.eftest.output-capture :as capture] + [mb.eftest.report :as report] + [mb.eftest.report.progress :as progress] + [mb.hawk.parallel :as hawk.parallel]) + (:import + (java.util.concurrent Executors ExecutorService))) + +(set! *warn-on-reflection* true) + +(defmethod test/report :begin-test-run [_]) + +;; deterministic shuffle stuff is disabled for now since it breaks too much stuff in Metabase. + +#_(defn- deterministic-shuffle [seed ^java.util.Collection coll] + (let [al (java.util.ArrayList. coll) + rng (java.util.Random. seed)] + (java.util.Collections/shuffle al rng) + (vec al))) + +(defn- synchronize [f] + (let [lock (Object.)] (fn [x] (locking lock (f x))))) + +(defn- synchronized? [test-var] + (not (hawk.parallel/parallel? test-var))) + +(defn- known-slow? [v] + (or (-> v meta :eftest/slow true?) + (-> v meta :ns meta :eftest/slow true?))) + +(defn- failed-test? [] + (or (not= :pass (get @report/*context* :state :pass)) + (< 0 (:error @test/*report-counters* 0)) + (< 0 (:fail @test/*report-counters* 0)))) + +(defn- wrap-test-with-timer [test-fn test-ns test-warn-time] + (fn [v] + (let [start-time (System/nanoTime) + result (test-fn v) + end-time (System/nanoTime) + duration (/ (- end-time start-time) 1e6)] + (when (and (not (known-slow? v)) + (number? test-warn-time) + (<= test-warn-time duration)) + (binding [test/*testing-vars* (conj test/*testing-vars* v) + report/*testing-path* [test-ns v]] + (test/report {:type :long-test + :duration duration + :var v}))) + result))) + +(defn- bound-callback ^Callable [f] + (let [bindings (get-thread-bindings)] + (reify Callable + (call [_] + (with-bindings* bindings f))))) + +(defn- default-thread-count [] + (+ 2 (.availableProcessors (Runtime/getRuntime)))) + +(defn- threadpool-executor + ^ExecutorService [{:keys [thread-count] :or {thread-count (default-thread-count)}}] + (Executors/newFixedThreadPool thread-count)) + +(defn- pcalls* [^ExecutorService executor fs] + (->> fs + (map #(.submit executor (bound-callback %))) + (doall) + (map #(.get ^java.util.concurrent.Future %)) + (doall))) + +(defn- pmap* [executor f xs] + (pcalls* executor (map (fn [x] #(f x)) xs))) + +(defn- multithread-vars? [{:keys [multithread?] :or {multithread? true}}] + (or (true? multithread?) (= multithread? :vars))) + +(defn- multithread-namespaces? [{:keys [multithread?] :or {multithread? true}}] + (or (true? multithread?) (= multithread? :namespaces))) + +(defn- multithread? [opts] + (or (multithread-vars? opts) (multithread-namespaces? opts))) + +(defn- fixture-exception [throwable] + {:type :error + :message "Uncaught exception during fixture initialization." + :actual throwable}) + +(defn- test-vars + [test-ns vars report + {:as opts :keys [executor fail-fast? capture-output? test-warn-time] + :or {capture-output? true}}] + (let [once-fixtures (-> test-ns meta ::test/once-fixtures test/join-fixtures) + each-fixtures (-> test-ns meta ::test/each-fixtures test/join-fixtures) + test-var (-> (fn [v] + (when-not (and fail-fast? (failed-test?)) + (binding [report/*testing-path* [test-ns ::test/each-fixtures] + hawk.parallel/*parallel?* (hawk.parallel/parallel? v)] + (try + (each-fixtures + (if capture-output? + #(binding [test/report report + report/*testing-path* [test-ns v]] + (capture/with-test-buffer + (test/test-var v))) + #(binding [test/report report + report/*testing-path* [test-ns v]] + (test/test-var v)))) + (catch Throwable t + (test/do-report (fixture-exception t))))))) + (wrap-test-with-timer test-ns test-warn-time))] + (binding [report/*testing-path* [test-ns ::test/once-fixtures]] + (try + (once-fixtures + (fn [] + (if (multithread-vars? opts) + (do (->> vars (filter synchronized?) (map test-var) (dorun)) + (->> vars (remove synchronized?) (pmap* executor test-var) (dorun))) + (doseq [v vars] (test-var v))))) + (catch Throwable t + (test/do-report (fixture-exception t))))))) + +(defn- test-ns [namespac vars report opts] + (let [namespac (the-ns namespac)] + (binding [test/*report-counters* (ref test/*initial-report-counters*)] + (test/do-report {:type :begin-test-ns, :ns namespac}) + (test-vars namespac vars report opts) + (test/do-report {:type :end-test-ns, :ns namespac}) + @test/*report-counters*))) + +(defn- test-all [vars {:as opts + :keys [capture-output? #_randomize-seed] + :or {capture-output? true #_randomize-seed #_0}}] + (let [report (synchronize test/report) + executor (delay (Executors/newCachedThreadPool)) + mapf (if (multithread-namespaces? opts) + (partial pmap* @executor) + map) + f #(->> (group-by (comp :ns meta) vars) + (sort-by (comp str key)) + #_(deterministic-shuffle randomize-seed) + (mapf (fn [[namespac vars]] (test-ns namespac vars report opts))) + (apply merge-with +))] + (try (if capture-output? + (capture/with-capture (f)) + (f)) + (finally (when (realized? executor) + (.shutdownNow ^ExecutorService @executor)))))) + +(defn find-tests-in-namespace [namespac] + (->> namespac ns-interns vals (filter (comp :test meta)))) + +(defn run-tests + "Run the supplied test vars. Accepts the following options: + + :fail-fast? - if true, stop after first failure or error + :capture-output? - if true, catch test output and print it only if + the test fails (defaults to true) + :multithread? - one of: true, false, :namespaces or :vars (defaults to + true). If set to true, namespaces and vars are run in + parallel; if false, they are run in serial. If set to + :namespaces, namespaces are run in parallel but the vars + in those namespaces are run serially. If set to :vars, + the namespaces are run serially, but the vars inside run + in parallel. + :thread-count - the number of threads used to run the tests in parallel + (as per :multithread?). If not specified, the number + reported by java.lang.Runtime.availableProcessors (which + is not always accurate) *plus two* will be used. + :randomize-seed - the random seed used to deterministically shuffle + test namespaces before running tests (defaults to 0). + :report - the test reporting function to use + (defaults to eftest.report.progress/report) + :test-warn-time - print a warning for any test that exceeds this time + (measured in milliseconds)" + ([vars] (run-tests vars {})) + ([vars opts] + (let [start-time (System/nanoTime)] + (if (empty? vars) + (do (println "No tests found.") + test/*initial-report-counters*) + (binding [report/*context* (atom {}) + test/report (:report opts progress/report)] + (test/do-report {:type :begin-test-run, :count (count vars)}) + (let [executor (when (multithread? opts) (threadpool-executor opts)) + opts (assoc opts :executor executor) + counters (try (test-all vars opts) + (finally (when executor (.shutdownNow executor)))) + duration (/ (- (System/nanoTime) start-time) 1e6) + summary (assoc counters :type :summary, :duration duration)] + (test/do-report summary) + summary)))))) diff --git a/src/mb/hawk/assert_exprs/approximately_equal.clj b/src/mb/hawk/assert_exprs/approximately_equal.clj index dffb792..e45c3cd 100644 --- a/src/mb/hawk/assert_exprs/approximately_equal.clj +++ b/src/mb/hawk/assert_exprs/approximately_equal.clj @@ -145,8 +145,8 @@ (defn schema "Used inside a =? expression. Compares things to a schema.core schema." - [schema] - (->Schema schema)) + [a-schema] + (->Schema a-schema)) (defmethod print-method Schema [this writer] @@ -166,12 +166,12 @@ [^Schema this actual] (s/check (.schema this) actual)) -(deftype Malli [schema]) +(deftype Malli [#_{:clj-kondo/ignore [:shadowed-var]} schema]) (defn malli "Used inside a =? expression. Compares things to a malli schema." - [schema] - (->Malli schema)) + [malli-schema] + (->Malli malli-schema)) (defmethod print-dup Malli [^Malli this ^java.io.Writer writer] diff --git a/src/mb/hawk/core.clj b/src/mb/hawk/core.clj index b73ca1c..fa79fa6 100644 --- a/src/mb/hawk/core.clj +++ b/src/mb/hawk/core.clj @@ -8,10 +8,10 @@ [clojure.test :as t] [clojure.tools.namespace.file :as ns.file] [clojure.tools.namespace.find :as ns.find] - [eftest.report.pretty] - [eftest.report.progress] - [eftest.runner] [environ.core :as env] + [mb.eftest.report.pretty] + [mb.eftest.report.progress] + [mb.eftest.runner] [mb.hawk.assert-exprs] [mb.hawk.hooks :as hawk.hooks] [mb.hawk.init :as hawk.init] @@ -116,7 +116,7 @@ (when-not (skip-by-tags? (find-ns ns-symb) options) (remove (some-fn #(skip-by-tags? % options) #(ignored-var? % options)) - (eftest.runner/find-tests ns-symb)))) + (mb.eftest.runner/find-tests-in-namespace ns-symb)))) ;; a test namespace or individual test (defmethod find-tests clojure.lang.Symbol @@ -170,8 +170,8 @@ for every [[clojure.test]] event, including stuff like `:begin-test-run`, `:end-test-var`, and `:fail`." [options] (let [stdout-reporter (case (:mode options) - (:cli/ci :repl) eftest.report.pretty/report - :cli/local eftest.report.progress/report)] + (:cli/ci :repl) mb.eftest.report.pretty/report + :cli/local mb.eftest.report.progress/report)] (fn handle-event [event] (hawk.junit/handle-event! event) (hawk.speak/handle-event! event) @@ -186,7 +186,7 @@ :cli/ci)) (defn run-tests - "Run `test-vars` with `options`, which are passed directly to [[eftest.runner/run-tests]]. + "Run `test-vars` with `options`, which are passed directly to [[mb.eftest.runner/run-tests]]. To run tests from the REPL, use this function. @@ -203,18 +203,16 @@ options)] (when-not (every? var? test-vars) (throw (ex-info "Invalid test vars" {:test-vars test-vars, :options options}))) - ;; don't randomize test order for now please, thanks anyway - (with-redefs [eftest.runner/deterministic-shuffle (fn [_ test-vars] test-vars)] - (binding [*parallel-test-counter* (atom {})] + (binding [*parallel-test-counter* (atom {})] + (merge + (mb.eftest.runner/run-tests + test-vars (merge - (eftest.runner/run-tests - test-vars - (merge - {:capture-output? false - :multithread? :vars - :report (reporter options)} - options)) - @*parallel-test-counter*)))))) + {:capture-output? false + :multithread? :vars + :report (reporter options)} + options)) + @*parallel-test-counter*))))) (defn- run-tests-n-times "[[run-tests]] but repeat `n` times. @@ -227,6 +225,7 @@ %2) acc test-result)) + {} (for [i (range 1 (inc n))] (do (println "----------------------------") diff --git a/src/mb/hawk/junit.clj b/src/mb/hawk/junit.clj index bd36341..5bc888f 100644 --- a/src/mb/hawk/junit.clj +++ b/src/mb/hawk/junit.clj @@ -3,6 +3,8 @@ [clojure.test :as t] [mb.hawk.junit.write :as write])) +(set! *warn-on-reflection* true) + (defmulti ^:private handle-event!* {:arglists '([event])} :type) @@ -87,9 +89,9 @@ (let [event (assoc event :testing-contexts (vec t/*testing-contexts*))] (alter-meta! test-var update ::context (fn [context] - (-> context - (update :assertion-count inc) - (update :results conj event)))))) + (some-> context + (update :assertion-count inc) + (update :results conj event)))))) (defmethod handle-event!* :pass [event] diff --git a/src/mb/hawk/junit/write.clj b/src/mb/hawk/junit/write.clj index 919e6c8..decff45 100644 --- a/src/mb/hawk/junit/write.clj +++ b/src/mb/hawk/junit/write.clj @@ -11,6 +11,8 @@ (javax.xml.stream XMLOutputFactory XMLStreamWriter) (org.apache.commons.io FileUtils))) +(set! *warn-on-reflection* true) + (def ^String ^:private output-dir "target/junit") (defn clean-output-dir! @@ -189,13 +191,15 @@ "Submit a background thread task to write the JUnit output for the tests in a namespace when an `:end-test-ns` event is encountered." [result] - (let [^Callable thunk (fn [] - (write-ns-result!* result))] - (.submit ^ThreadPoolExecutor @thread-pool thunk))) + (when @thread-pool + (let [^Callable thunk (fn [] + (write-ns-result!* result))] + (.submit ^ThreadPoolExecutor @thread-pool thunk)))) (defn wait-for-writes-to-finish "Wait up to 10 seconds for the thread pool that writes results to finish." [] - (.shutdown ^ThreadPoolExecutor @thread-pool) - (.awaitTermination ^ThreadPoolExecutor @thread-pool 10 TimeUnit/SECONDS) - (reset! thread-pool nil)) + (when @thread-pool + (.shutdown ^ThreadPoolExecutor @thread-pool) + (.awaitTermination ^ThreadPoolExecutor @thread-pool 10 TimeUnit/SECONDS) + (reset! thread-pool nil))) diff --git a/src/mb/hawk/parallel.clj b/src/mb/hawk/parallel.clj index 120d7ab..0007e4e 100644 --- a/src/mb/hawk/parallel.clj +++ b/src/mb/hawk/parallel.clj @@ -1,8 +1,7 @@ (ns mb.hawk.parallel "Code related to running parallel tests, and utilities for disallowing dangerous stuff inside them." (:require - [clojure.test :as t] - [eftest.runner])) + [clojure.test :as t])) (defn parallel? "Whether `test-var` can be ran in parallel with other parallel tests." @@ -12,10 +11,6 @@ var-parallel (:parallel (-> metta :ns meta))))) -(def ^:private synchronized? (complement parallel?)) - -(alter-var-root #'eftest.runner/synchronized? (constantly synchronized?)) - (def ^:dynamic *parallel?* "Whether test currently being ran is being ran in parallel." nil) diff --git a/src/mb/hawk/partition.clj b/src/mb/hawk/partition.clj index 9c10723..bf8d19a 100644 --- a/src/mb/hawk/partition.clj +++ b/src/mb/hawk/partition.clj @@ -57,10 +57,10 @@ For most namespaces there should only be one possible partition but for some the ideal split happens in the middle of the namespace which means we have two possible candidate partitions to put it into." [num-partitions test-vars] - (let [test-var->ideal-partition (test-var->ideal-partition num-partitions test-vars)] + (let [var->ideal-partition (test-var->ideal-partition num-partitions test-vars)] (reduce (fn [m test-var] - (update m (namespace* test-var) #(conj (set %) (test-var->ideal-partition test-var)))) + (update m (namespace* test-var) #(conj (set %) (var->ideal-partition test-var)))) {} test-vars))) @@ -71,26 +71,26 @@ If there are multiple possible candidate partitions for a namespace, choose the one that has the least tests in it." [num-partitions test-vars] - (let [namespace->num-tests (namespace->num-tests test-vars) - namespace->possible-partitions (namespace->possible-partitions num-partitions test-vars) + (let [ns->num-tests (namespace->num-tests test-vars) + ns->possible-partitions (namespace->possible-partitions num-partitions test-vars) ;; process all the namespaces that have no question about what partition they should go into first so we have as ;; accurate a picture of the size of each partition as possible before dealing with the ambiguous ones namespaces (distinct (map namespace* test-vars)) multiple-possible-partitions? (fn [nmspace] - (> (count (namespace->possible-partitions nmspace)) + (> (count (ns->possible-partitions nmspace)) 1)) - namespaces (concat (remove multiple-possible-partitions? namespaces) - (filter multiple-possible-partitions? namespaces))] + namespaces (concat (remove multiple-possible-partitions? namespaces) + (filter multiple-possible-partitions? namespaces))] ;; Keep track of how many tests are in each partition so far (:namespace->partition (reduce (fn [m nmspace] - (let [partition (first (sort-by (fn [partition] - (get-in m [:partition->size partition])) - (namespace->possible-partitions nmspace)))] + (let [part (first (sort-by (fn [part] + (get-in m [:partition->size part])) + (ns->possible-partitions nmspace)))] (-> m - (update-in [:partition->size partition] (fnil + 0) (namespace->num-tests nmspace)) - (assoc-in [:namespace->partition nmspace] partition)))) + (update-in [:partition->size part] (fnil + 0) (ns->num-tests nmspace)) + (assoc-in [:namespace->partition nmspace] part)))) {} namespaces)))) @@ -99,9 +99,9 @@ (f test-var) => partititon-number" [num-partitions test-vars] - (let [namespace->partition (namespace->partition num-partitions test-vars)] + (let [ns->partition (namespace->partition num-partitions test-vars)] (fn test-var->partition [test-var] - (get namespace->partition (namespace* test-var))))) + (get ns->partition (namespace* test-var))))) (defn- partition-tests-into-n-partitions "Split a sequence of `test-vars` into `num-partitions`, returning a map of @@ -140,11 +140,11 @@ (do (validate-partition-options tests options) (let [partition-index->tests (partition-tests-into-n-partitions num-partitions tests) - partition (get partition-index->tests partition-index)] + part (get partition-index->tests partition-index)] (printf "Running tests in partition %d of %d (%d tests of %d)...\n" (inc partition-index) num-partitions - (count partition) + (count part) (count tests)) - partition)) + part)) tests)) diff --git a/src/mb/hawk/speak.clj b/src/mb/hawk/speak.clj index 520cb76..4da20d0 100644 --- a/src/mb/hawk/speak.clj +++ b/src/mb/hawk/speak.clj @@ -1,6 +1,8 @@ (ns mb.hawk.speak (:require [clojure.java.shell :as sh])) +(set! *warn-on-reflection* true) + (defmulti handle-event! "Handles a test event by speaking(!?) it if appropriate" :type) diff --git a/test/hawk/test_ns/parallel_test.clj b/test/hawk/test_ns/parallel_test.clj new file mode 100644 index 0000000..34926dd --- /dev/null +++ b/test/hawk/test_ns/parallel_test.clj @@ -0,0 +1,17 @@ +(ns ^:hawk.tests/skip hawk.test-ns.parallel-test + (:require + [clojure.test :refer :all] + [mb.hawk.parallel :as parallel])) + +(defn- disallowed-parallel-function [] + (parallel/assert-test-is-not-parallel "disallowed-parallel-function")) + +(use-fixtures :each (fn [thunk] + (disallowed-parallel-function) + (thunk))) + +(deftest synchronized-test + (is (= 1 1))) + +(deftest ^:parallel parallel-test + (is (= 1 1))) diff --git a/test/mb/eftest/report_test.clj b/test/mb/eftest/report_test.clj new file mode 100644 index 0000000..ae3345c --- /dev/null +++ b/test/mb/eftest/report_test.clj @@ -0,0 +1,32 @@ +(ns mb.eftest.report-test + (:require + [clojure.test :refer :all] + [mb.eftest.output-capture :as output-capture] + [mb.eftest.report :as report] + [mb.eftest.report.pretty :as pretty] + [puget.printer :as puget])) + +(def ^:private this-ns *ns*) + +(deftest file-and-line-in-pretty-fail-report + (let [pretty-nil (puget/pprint-str nil {:print-color true + :print-meta false}) + result (with-out-str + (binding [*test-out* *out* + pretty/*fonts* {} + report/*testing-path* [this-ns #'file-and-line-in-pretty-fail-report] + *report-counters* (ref *initial-report-counters*)] + (output-capture/with-test-buffer + (pretty/report {:type :fail + :file "report_test.clj" + :line 999 + :message "foo"}))))] + (is (= (str "\nFAIL in mb.eftest.report-test/file-and-line-in-pretty-fail-report" + " (report_test.clj:999)\n" + "foo\n" + "expected: " + pretty-nil + "\n actual: " + pretty-nil + "\n") + result)))) diff --git a/test/mb/eftest/runner_test.clj b/test/mb/eftest/runner_test.clj new file mode 100644 index 0000000..2334632 --- /dev/null +++ b/test/mb/eftest/runner_test.clj @@ -0,0 +1,56 @@ +(ns mb.eftest.runner-test + (:require + [clojure.test :refer :all] + [mb.eftest.runner :as sut] + [mb.hawk.core :as hawk.core])) + +(set! *warn-on-reflection* true) + +(defn- with-test-out-str* [f] + (let [s (java.io.StringWriter.)] + (binding [clojure.test/*test-out* s] + (f)) + (-> (str s) + (.replace "\u001B" "") + (.replaceAll "\\[([0-9]{1};)?[0-9]{0,2}m" "")))) + +(defmacro ^:private with-test-out-str [& body] + `(with-test-out-str* (fn [] ~@body))) + +(defn- test-run-tests + ([test-locs] + (test-run-tests test-locs {})) + ([test-locs opts] + (let [vars (hawk.core/find-tests test-locs {}) + ret (promise) + out (with-test-out-str (deliver ret (sut/run-tests vars opts)))] + {:output out + :return @ret}))) + +(deftest test-reporting + (let [out (:output (test-run-tests 'mb.eftest.test-ns.single-failing-test))] + (is (re-find #"FAIL in mb\.eftest\.test-ns\.single-failing-test/single-failing-test" out)) + (is (not (re-find #"IllegalArgumentException" out))))) + +(deftest test-fail-fast + (let [result (:return + (test-run-tests + '[mb.eftest.test-ns.single-failing-test + mb.eftest.test-ns.another-failing-test] + {:fail-fast? true, :multithread? false}))] + (is (= {:test 1 :fail 1} (select-keys result [:test :fail]))))) + +(deftest test-fail-multi + (let [out (:output + (test-run-tests + '[mb.eftest.test-ns.single-failing-test + mb.eftest.test-ns.another-failing-test]))] + (println out) + (is (re-find #"(?m)expected: 1\n actual: 2" out)) + (is (re-find #"(?m)expected: 3\n actual: 4" out)))) + +(deftest test-slow-test-report + (testing "should fail with an accurate var location" + (let [out (:output + (test-run-tests ['mb.eftest.test-ns.slow-test] {:test-warn-time 5}))] + (is (re-find #"LONG TEST in mb\.eftest\.test-ns\.slow-test/a-slow-test\n" out))))) diff --git a/test/mb/eftest/test_ns/another_failing_test.clj b/test/mb/eftest/test_ns/another_failing_test.clj new file mode 100644 index 0000000..6d63058 --- /dev/null +++ b/test/mb/eftest/test_ns/another_failing_test.clj @@ -0,0 +1,6 @@ +(ns ^:hawk.tests/skip mb.eftest.test-ns.another-failing-test + (:require + [clojure.test :refer :all])) + +(deftest another-failing-test + (is (= 3 4))) diff --git a/test/mb/eftest/test_ns/single_failing_test.clj b/test/mb/eftest/test_ns/single_failing_test.clj new file mode 100644 index 0000000..1d39b03 --- /dev/null +++ b/test/mb/eftest/test_ns/single_failing_test.clj @@ -0,0 +1,6 @@ +(ns ^:hawk.tests/skip mb.eftest.test-ns.single-failing-test + (:require + [clojure.test :refer :all])) + +(deftest single-failing-test + (is (= 1 2))) diff --git a/test/mb/eftest/test_ns/slow_test.clj b/test/mb/eftest/test_ns/slow_test.clj new file mode 100644 index 0000000..abc8476 --- /dev/null +++ b/test/mb/eftest/test_ns/slow_test.clj @@ -0,0 +1,8 @@ +(ns ^:hawk.tests/skip mb.eftest.test-ns.slow-test + (:require + [clojure.test :refer :all])) + +(set! *warn-on-reflection* true) + +(deftest a-slow-test + (is (true? (do (Thread/sleep 10) true)))) diff --git a/test/mb/hawk/assert_exprs/approximately_equal_test.clj b/test/mb/hawk/assert_exprs/approximately_equal_test.clj index 19fd407..3a1e62c 100644 --- a/test/mb/hawk/assert_exprs/approximately_equal_test.clj +++ b/test/mb/hawk/assert_exprs/approximately_equal_test.clj @@ -5,6 +5,8 @@ [mb.hawk.assert-exprs.approximately-equal :as =?] [schema.core :as s])) +(set! *warn-on-reflection* true) + (comment test-runner.assert-exprs/keep-me) (deftest ^:parallel passing-tests @@ -96,7 +98,7 @@ (is (=? (=?/exactly 2) 2)) (testing "should evaluate args" - (is (=? (=?/exactly (+ 1 1)) + (is (=? (=?/exactly (inc 1)) 2))) (testing "Inside a map" (is (=? {:a 1, :b (=?/exactly 2)} diff --git a/test/mb/hawk/parallel_fixtures_test.clj b/test/mb/hawk/parallel_fixtures_test.clj new file mode 100644 index 0000000..6b3b3d7 --- /dev/null +++ b/test/mb/hawk/parallel_fixtures_test.clj @@ -0,0 +1,17 @@ +(ns mb.hawk.parallel-fixtures-test + (:require + [clojure.test :refer :all] + [mb.hawk.core :as hawk])) + +(deftest assert-test-is-not-parallel-works-inside-fixtures-test + (testing "asssert-test-is-not-parallel should work inside :each fixtures (#26)" + (letfn [(run-test [symb] + (with-open [w (java.io.StringWriter.)] + (binding [*test-out* w] + (hawk/run-tests [(requiring-resolve symb)]))))] + (testing "single-threaded test should be ok" + (is (=? {:test 1, :pass 1, :fail 0, :error 0, :single-threaded 1} + (run-test 'hawk.test-ns.parallel-test/synchronized-test)))) + (testing "parallel test should fail because of parallel-unsafe fixture" + (is (=? {:test 1, :pass 1, :fail 0, :error 1, :parallel 1} + (run-test 'hawk.test-ns.parallel-test/parallel-test)))))))