Add eval-many and tests

README.md

@@ -88,3 +88,64 @@ nil
 
 [1]: https://git.launchpad.net/ubuntu/+source/sqlite3/tree/debian/rules?h=debian/sid#n41
 [2]: https://github.com/gentoo/gentoo/blob/653b190ffe5f4433112ad6786d1bfd2e26143711/dev-db/sqlite/sqlite-3.34.0.ebuild
+
+## Dataframe Example
+
+2x faster than `eval` for reads, `eval-to-dataframe` returns a dataframe (e.g. `{:name ["Bob" "Janet" "Jack"] :age [25 22 31]}`). For bulk writes, `eval-many` runs a single SQL statement in a transaction, (4x faster than `eval` with a transaction.)
+
+```janet
+(sqlite3/eval-many db "INSERT INTO f VALUES (?, ?, ?);"
+  @[["a.c" "c" 120]
+    ["b.h" "h" 40]
+    ["Makefile" "" 300]])
+
+(sqlite3/eval-many db "INSERT INTO f VALUES (:path, :ext, :size);"
+  @[{:path "a.c" :ext "c" :size 120}
+    {:path "b.h" :ext "h" :size 40}
+    {:path "Makefile" :ext "" :size 300}])
+```
+Here is a cute, self-contained program printing number of files ordered by size (use like `janet a.janet my-dir`):
+
+```janet
+(import sqlite3)
+
+(defn walk-dir
+  "Return [path ext size] for every regular file under dir, recursing."
+  [dir]
+  (mapcat
+    (fn [name]
+      (let [path (string dir "/" name)
+            st   (os/lstat path)]
+        (case (get st :mode)
+          :file      [[path
+                       (if-let [i (string/find-all "." name)
+                                j (last i)]
+                         (string/slice name (inc j)) "")
+                       (in st :size)]]
+          :directory (walk-dir path)
+          [])))
+    (os/dir dir)))
+
+(let [root  (get (dyn :args) 1 ".")
+      path  "usage.db"
+      files (walk-dir root)]
+  (when   (os/stat path) (os/rm path))
+  (os/execute ["sqlite3" path "CREATE TABLE f (path TEXT, ext TEXT, size INTEGER);"] :px)
+  (def db (sqlite3/open path))
+  (defer   (os/rm path)
+    (defer (sqlite3/close db)
+      (sqlite3/eval-many db "INSERT INTO f VALUES (?, ?, ?);" files)
+      (let [agg (sqlite3/eval-to-dataframe db
+                  "SELECT ext, count(*) AS n, sum(size) AS bytes
+                   FROM f GROUP BY ext ORDER BY bytes DESC;")
+            # sqlite struggles, but columns excel at cumulative share
+            total (sum (in agg :bytes))
+            cum (accumulate + 0 (map |(/ $ total) (in agg :bytes)))]
+        (printf "%d files, %.1f MB under %s" (length files) (/ total 1e6) root)
+        (eachp [i ext] (in agg :ext)
+          (printf "%10s %7d files %10.2f MB  %5.1f%% cum"
+                  (if (empty? ext) "<none>" ext)
+                  (in (in agg :n) i)
+                  (/ (in (in agg :bytes) i) 1e6)
+                  (* 100 (in cum i))))))))
+```

main.c

@@ -384,6 +384,72 @@ static Janet sql_eval_to_dataframe(int32_t argc, Janet *argv) {
     return sql_eval_impl(argc, argv, COLLECT_TO_DF);
 }
 
+/* Execute statement repeatedly against parameter set */
+static Janet eql_eval_many(int32_t argc, Janet *argv) {
+    janet_fixarity(argc, 3);
+    const char *err;
+    sqlite3_stmt *stmt = NULL, *stmt_extra = NULL;
+    Db *db = janet_getabstract(argv, 0, &sql_conn_type);
+    if (db->flags & FLAG_CLOSED) janet_panic(MSG_DB_CLOSED);
+    const uint8_t *query = janet_getstring(argv, 1);
+    if (has_null(query, janet_string_length(query))) {
+        janet_panic("cannot have embedded NULL in sql statements");
+    }
+    const Janet *sets;
+    int32_t nsets;
+    if (!janet_indexed_view(argv[2], &sets, &nsets)) {
+        janet_panic("expected array or tuple of parameter sets");
+    }
+
+    const char *c = (const char *)query;
+    if (sqlite3_prepare_v2(db->handle, c, -1, &stmt, &c) != SQLITE_OK) {
+        janet_panic(sqlite3_errmsg(db->handle));
+    }
+    if (NULL == stmt) janet_panic("expected a sql statement");
+    /* Ignore trailing whitespace and comments, err on anything else*/
+    /* (treated like a 2nd statement which won't compile) */
+    if (sqlite3_prepare_v2(db->handle, c, -1, &stmt_extra, &c) != SQLITE_OK) {
+        /* because this executes many things, I think we need to copy the messages in such spots*/
+        err = janet_cstring(sqlite3_errmsg(db->handle));
+        goto error;
+    }
+    if (NULL != stmt_extra) {
+        err = janet_cstring("expected only one sql statement");
+        goto error;
+    }
+
+    /* Wrap in a transaction (if not already in one) */
+    int own_txn = sqlite3_get_autocommit(db->handle);
+    if (own_txn && sqlite3_exec(db->handle, "BEGIN;", NULL, NULL, NULL) != SQLITE_OK) {
+        err = janet_cstring(sqlite3_errmsg(db->handle));
+        goto error;
+    }
+
+    for (int32_t i = 0; i < nsets; i++) {
+        const char *berr = bindmany(stmt, sets[i]);
+        if (berr) { err = janet_cstring(berr); goto rollback; }
+        berr = execute(stmt);
+        if (berr) { err = janet_cstring(berr); goto rollback; }
+        sqlite3_reset(stmt);
+        sqlite3_clear_bindings(stmt);
+    }
+
+    if (own_txn && sqlite3_exec(db->handle, "COMMIT;", NULL, NULL, NULL) != SQLITE_OK) {
+        err = janet_cstring(sqlite3_errmsg(db->handle));
+        goto rollback;
+    }
+    sqlite3_finalize(stmt);
+    return janet_wrap_nil();
+
+rollback:
+    if (own_txn) sqlite3_exec(db->handle, "ROLLBACK;", NULL, NULL, NULL);
+error:
+    if (stmt) sqlite3_finalize(stmt);
+    if (stmt_extra) sqlite3_finalize(stmt_extra);
+    janet_panics(err);
+    return janet_wrap_nil();
+}
+
 /* Gets the last inserted row id */
 static Janet sql_last_insert_rowid(int32_t argc, Janet *argv) {
     janet_fixarity(argc, 1);
@@ -443,6 +509,7 @@ static JanetMethod conn_methods[] = {
     {"last-insert-rowid", sql_last_insert_rowid},
     {"allow-loading-extensions", sql_allow_loading_extensions},
     {"load-extension", sql_load_extension},
+    {"eval-many", eql_eval_many},
     {NULL, NULL}
 };
 
@@ -517,6 +584,16 @@ static const JanetReg cfuns[] = {
         "the library-entrypoint. Extension loading must be enabled prior to calling this function. "
         "Returns library-file-path."
     },
+    {"eval-many", eql_eval_many,
+        "(sqlite3/eval-many db sql param-sets)\n\n"
+        "Evaluates an sql statement once per element of param-sets (like map) "
+        "only preparing statement once, binding arguments like the params argument of "
+        "(sqlite3/eval ...); both indexed and named parameters work. All executions "
+        "run inside one transaction unless the connection is already in a transaction, "
+        "and any error rolls it back. The purpose is bulk writes, so it returns nil.\n\n"
+        "  * (sqlite3/eval-many db \"INSERT INTO tracks VALUES (?, ?, ?);\"\n"
+        "        (map tuple (tracks :title) (tracks :bpm) (tracks :gain_db)))\n"
+    },
     {NULL, NULL, NULL}
 };
 

test/benchmark.janet

@@ -0,0 +1,38 @@
+(import ../build/sqlite3 :as sql)
+
+(defmacro timed `Return [seconds result]`
+  [& body]
+  (with-syms [t0 res]
+    ~(let [,t0 (os/clock :monotonic)
+           ,res (do ,;body)]
+       [(- (os/clock :monotonic) ,t0) ,res])))
+
+(let [path   "bench.db"
+      _      (when (os/stat path) (os/rm path))
+      n      20000
+      ids    (seq [i :range [0 n]] i)
+      names  (seq [i :range [0 n]] (string "user-" i))
+      scores (seq [i :range [0 n]] (* 0.25 (- i (/ n 2))))
+      flags  (seq [i :range [0 n]] (mod i 2))
+      src   @{:id ids :name names :score scores :flag flags}] 
+  (defer (os/rm path)
+         (def db (sql/open path))
+    (defer (sql/close db)
+           (sql/eval  db   "CREATE TABLE bulk (id INTEGER, name TEXT, score REAL, flag INTEGER);
+                            CREATE TABLE rows (id INTEGER, name TEXT, score REAL, flag INTEGER);")
+      (let [[insert-a _]    (timed (sql/eval-many db "INSERT INTO bulk VALUES (?, ?, ?, ?);"
+                                                  (map tuple ids names scores flags)))
+            [read-a df-a]   (timed (sql/eval-to-dataframe db "SELECT * FROM bulk ORDER BY id;"))
+            [insert-b _]    (timed
+                             (sql/eval db "BEGIN;") # extremely slow without this
+                             (loop [i :range [0 n]]
+                               (sql/eval db "INSERT INTO rows VALUES (:id, :name, :score, :flag);"
+                                         {:id    (in ids    i) :name (in names i)
+                                          :score (in scores i) :flag (in flags i)}))
+                             (sql/eval db "COMMIT;"))
+            [read-b rows-b] (timed (sql/eval db "SELECT * FROM rows ORDER BY id;"))] 
+        (assert (= n (length rows-b) (length (df-a :name))))
+        (printf "a eval-many insert: %.4f s" insert-a)
+        (printf "b per-row insert:   %.4f s  (%.1fx)" insert-b (/ insert-b insert-a))
+        (printf "a dataframe read:   %.4f s" read-a)
+        (printf "b row-seq read:     %.4f s  (%.1fx)" read-b   (/ read-b read-a))))))

test/dataframe-roundtrip.janet

@@ -0,0 +1,29 @@
+(import ../build/sqlite3 :as sql)
+
+(let [path    "roundtrip.db"
+      _       (when (os/stat path) (os/rm path))
+      tracks @{:title @["axiom" "briar" "cinder" "dune" "ember" "fjord"]
+               :bpm @[122 98 140 87 133 104]
+               :gain_db @[-6.5 -3.25 0.0 -12.75 2.5 -0.125]}
+      db      (sql/open path)]
+
+  (defer (os/rm path)
+         (defer (sql/close db)
+                (sql/eval db "CREATE TABLE tracks (title TEXT, bpm INTEGER, gain_db REAL);
+              CREATE TABLE mirror (title TEXT, bpm INTEGER, gain_db REAL);")
+
+                # to and from tracks
+                (sql/eval-many db "INSERT INTO tracks VALUES (?, ?, ?);"
+                               (map tuple (in tracks :title) (in tracks :bpm) (in tracks :gain_db)))
+                (def once (sql/eval-to-dataframe db "SELECT * FROM tracks ORDER BY title;"))
+                # to and from mirroir
+                (sql/eval-many db "INSERT INTO mirror VALUES (?, ?, ?);"
+                               (map tuple (in once :title) (in once :bpm) (in once :gain_db)))
+                (def twice (sql/eval-to-dataframe db "SELECT * FROM mirror ORDER BY title;"))
+
+                (assert (deep= once twice)) # both are identical
+                (assert (deep= twice @{:title @["axiom" "briar" "cinder" "dune" "ember" "fjord"]
+                                       :bpm @[122 98 140 87 133 104]
+                                       :gain_db @[-6.5 -3.25 0.0 -12.75 2.5 -0.125]}))
+                (assert (deep= once tracks)) # the actual round trip, tracks defined in top let
+                )))

test/test.janet

@@ -1,27 +1,21 @@
-(import /build/sqlite3 :as sql)
+(import ../build/sqlite3 :as sql)
 
-#
-# Testing
-#
-
-(defn assert [c]
-  (if (not c) (error "failed assertion")))
-(def db (sql/open "build/test.db"))
-(try (sql/eval db `DROP TABLE people`) ([_]))
-(sql/eval db `CREATE TABLE people(name TEXT, age INTEGER, bool INTEGER);`)
-(sql/eval db `INSERT INTO people values(:name, :age, :bool)` {:name "John" :age 20 :bool false})
-(sql/eval db `INSERT INTO people values(:name, :age, :bool)` {:name "Paul" :age 30 :bool true})
-(sql/eval db `INSERT INTO people values(:name, :age, :bool)` {:name "Bob" :age 40 :bool false})
-(sql/eval db `INSERT INTO people values(:name, :age, :bool)` {:name "Joe" :age 50 :bool true})
-(def results (sql/eval db `SELECT * FROM people`))
-(assert (= (length results) 4))
-
-(def update-result
-  (-> (sql/eval db
-                `UPDATE people set name = :new_name where name = :old_name RETURNING name, age, bool`
-                {:new_name "Harry" :old_name "Paul"})
-      (first)))
-(assert (= update-result {:name "Harry" :age 30 :bool 1}))
-
-
-(sql/close db)
+(let [path   "test.db" 
+      _      (when (os/stat path) (os/rm path))
+      db     (sql/open path)]
+  (defer (os/rm path)
+    (defer (sql/close db) 
+           (sql/eval db `CREATE TABLE people(name TEXT, age INTEGER, bool INTEGER);`)
+           (sql/eval db `INSERT INTO people values(:name, :age, :bool)` {:name "John" :age 20 :bool false})
+           (sql/eval db `INSERT INTO people values(:name, :age, :bool)` {:name "Paul" :age 30 :bool true})
+           (sql/eval db `INSERT INTO people values(:name, :age, :bool)` {:name "Bob" :age 40 :bool false})
+           (sql/eval db `INSERT INTO people values(:name, :age, :bool)` {:name "Joe" :age 50 :bool true})
+           (def results (sql/eval db `SELECT * FROM people`))
+           (assert (= (length results) 4))
+
+           (def update-result
+             (-> (sql/eval db
+                           `UPDATE people set name = :new_name where name = :old_name RETURNING name, age, bool`
+                           {:new_name "Harry" :old_name "Paul"})
+                 (first)))
+           (assert (deep= update-result {:name "Harry" :age 30 :bool 1})))))