From 0d9dcd7b6d227362259bec593db21c3e65ad9806 Mon Sep 17 00:00:00 2001 From: Devin AI <158243242+devin-ai-integration[bot]@users.noreply.github.com> Date: Tue, 3 Nov 2026 17:20:15 +0000 Subject: [PATCH] Fix bugs found by the final pre-release sweep - MySQL :copyfrom into JSON columns (LOAD DATA CHARACTER SET binary): convert JSON columns via user variables - PostgreSQL :copyfrom with booleans: text boolean encoder - PostgreSQL nested transaction rescuing a failed statement: roll back only the savepoint - PostgreSQL array parameters (bytea[], jsonb[], [] / leading nil, preparedStatements: false) as array literals; floats as exact text - :copyfrom keeps INSERT OR REPLACE / OR IGNORE / ON CONFLICT (SQLite), INSERT IGNORE / REPLACE (MySQL); rejects clauses COPY / LOAD DATA cannot express - acronym-aware snake_case (MIN_ID -> min_id); RuboCop exemptions only where needed (Naming/VariableNumber) - mysql2 batches prepare per row; nested isolation:/mode: raise; dead connections reset (pg) / discarded from the pool (mysql2) - E2E tests for every case Co-Authored-By: Doron Eli Rachman --- README.md | 20 +- examples/authors/mysql/query.sql | 21 ++ examples/authors/mysql/schema.sql | 8 + examples/authors/postgresql/query.sql | 15 ++ examples/authors/postgresql/schema.sql | 11 + examples/authors/sqlite/query.sql | 15 ++ examples/authors/sqlite/schema.sql | 5 + examples/mysql2/models.rb | 9 + examples/mysql2/models.rbs | 15 ++ examples/mysql2/query_sql.rb | 215 +++++++++++++++-- examples/mysql2/query_sql.rbs | 85 ++++++- examples/pg/models.rb | 12 + examples/pg/models.rbs | 21 ++ examples/pg/query_sql.rb | 198 +++++++++++++++- examples/pg/query_sql.rbs | 94 ++++++++ examples/sqlite3/models.rb | 3 + examples/sqlite3/models.rbs | 6 + examples/sqlite3/query_sql.rb | 84 +++++++ examples/sqlite3/query_sql.rbs | 38 +++ lib/sqlc_gen_ruby/code.rb | 4 +- lib/sqlc_gen_ruby/drivers/base.rb | 11 +- lib/sqlc_gen_ruby/drivers/mysql2.rb | 51 ++++- lib/sqlc_gen_ruby/drivers/pg.rb | 78 +++++-- lib/sqlc_gen_ruby/drivers/sqlite3.rb | 10 +- lib/sqlc_gen_ruby/method_gen.rb | 51 ++++- steep_expectations.yml | 194 ++++++++++++---- test/features_test.rb | 50 +++- test/fixtures/mysql2.bin | 71 +++++- test/fixtures/pg.bin | 55 ++++- test/fixtures/sqlite3.bin | Bin 9062 -> 10069 bytes test/golden/mysql2/default/models.rb | 9 + test/golden/mysql2/default/models.rbs | 15 ++ test/golden/mysql2/default/my_queries_sql.rb | 10 +- test/golden/mysql2/default/other_sql.rb | 10 +- test/golden/mysql2/default/query_sql.rb | 215 +++++++++++++++-- test/golden/mysql2/default/query_sql.rbs | 85 ++++++- test/golden/mysql2/ruby31_gemfile/models.rb | 9 + test/golden/mysql2/ruby31_gemfile/models.rbs | 22 ++ .../mysql2/ruby31_gemfile/my_queries_sql.rb | 10 +- .../golden/mysql2/ruby31_gemfile/other_sql.rb | 10 +- .../golden/mysql2/ruby31_gemfile/query_sql.rb | 216 ++++++++++++++++-- .../mysql2/ruby31_gemfile/query_sql.rbs | 104 ++++++++- test/golden/mysql2/ruby32_notypes/models.rb | 9 + .../mysql2/ruby32_notypes/my_queries_sql.rb | 10 +- .../golden/mysql2/ruby32_notypes/other_sql.rb | 10 +- .../golden/mysql2/ruby32_notypes/query_sql.rb | 215 +++++++++++++++-- test/golden/pg/default/models.rb | 12 + test/golden/pg/default/models.rbs | 21 ++ test/golden/pg/default/my_queries_sql.rb | 17 +- test/golden/pg/default/other_sql.rb | 24 +- test/golden/pg/default/other_sql.rbs | 2 + test/golden/pg/default/query_sql.rb | 200 ++++++++++++++-- test/golden/pg/default/query_sql.rbs | 94 ++++++++ test/golden/pg/ruby31_gemfile/models.rb | 12 + test/golden/pg/ruby31_gemfile/models.rbs | 31 +++ .../pg/ruby31_gemfile/my_queries_sql.rb | 17 +- test/golden/pg/ruby31_gemfile/other_sql.rb | 24 +- test/golden/pg/ruby31_gemfile/other_sql.rbs | 2 + test/golden/pg/ruby31_gemfile/query_sql.rb | 199 ++++++++++++++-- test/golden/pg/ruby31_gemfile/query_sql.rbs | 125 ++++++++++ test/golden/pg/ruby32_notypes/models.rb | 12 + .../pg/ruby32_notypes/my_queries_sql.rb | 17 +- test/golden/pg/ruby32_notypes/other_sql.rb | 24 +- test/golden/pg/ruby32_notypes/query_sql.rb | 200 ++++++++++++++-- test/golden/sqlite3/default/models.rb | 3 + test/golden/sqlite3/default/models.rbs | 6 + test/golden/sqlite3/default/my_queries_sql.rb | 2 + test/golden/sqlite3/default/other_sql.rb | 2 + test/golden/sqlite3/default/query_sql.rb | 84 +++++++ test/golden/sqlite3/default/query_sql.rbs | 38 +++ test/golden/sqlite3/ruby31_gemfile/models.rb | 2 + test/golden/sqlite3/ruby31_gemfile/models.rbs | 7 + .../sqlite3/ruby31_gemfile/my_queries_sql.rb | 2 + .../sqlite3/ruby31_gemfile/other_sql.rb | 2 + .../sqlite3/ruby31_gemfile/query_sql.rb | 80 +++++++ .../sqlite3/ruby31_gemfile/query_sql.rbs | 42 ++++ test/golden/sqlite3/ruby32_notypes/models.rb | 3 + .../sqlite3/ruby32_notypes/my_queries_sql.rb | 2 + .../sqlite3/ruby32_notypes/other_sql.rb | 2 + .../sqlite3/ruby32_notypes/query_sql.rb | 84 +++++++ tests/end2end_mysql2.rb | 43 ++++ tests/end2end_pg.rb | 79 +++++++ tests/end2end_sqlite3.rb | 17 ++ 83 files changed, 3660 insertions(+), 282 deletions(-) diff --git a/README.md b/README.md index 9cd6b8c..a256366 100644 --- a/README.md +++ b/README.md @@ -117,9 +117,13 @@ end * Everything in the block runs on one connection. It commits when the block ends (also on `break`, `return`, `throw`) and rolls back when it raises or the thread is killed. A nested `transaction` rolls back only its own part. * `isolation:` (`pg`, `mysql2`) takes `:read_uncommitted`, `:read_committed`, `:repeatable_read`, `:serializable`; - `mode:` (`sqlite3`) takes `:deferred`, `:immediate`, `:exclusive`. Other values raise before anything runs. + `mode:` (`sqlite3`) takes `:deferred`, `:immediate`, `:exclusive`. Other values raise before anything runs, and so + do `isolation:` / `mode:` on a nested `transaction` (a savepoint cannot change them). * PostgreSQL: if a statement fails and the block rescues the error, PostgreSQL still rolls the whole transaction - back, so `transaction` raises `PG::InFailedSqlTransaction` instead of pretending it committed. + back, so `transaction` raises `PG::InFailedSqlTransaction` instead of pretending it committed. In a nested + `transaction` only the savepoint is rolled back: the outer block can rescue that error and go on. +* A connection the server closed (restart, failover, killed backend) is reconnected (`pg`) or replaced in the pool + (`mysql2`) the next time it is checked out; the call that hit the dead connection still raises. * SQLite: threads and fibers sharing one `SQLite3::Database` take turns; a transaction holds the database until it ends. ## Supported features @@ -137,7 +141,12 @@ end `:copyfrom` uses `COPY ... FROM STDIN` on PostgreSQL and batched multi-row `INSERT`s on SQLite. On MySQL it uses `LOAD DATA LOCAL INFILE`, which needs `local_infile: true` in the `Mysql2::Client` params and `local_infile=1` on the -server; duplicate keys and invalid values raise `Mysql2::Error` instead of being skipped. +server; duplicate keys and invalid values raise `Mysql2::Error` instead of being skipped. Conflict clauses are kept: +`INSERT OR REPLACE` / `OR IGNORE` / `ON CONFLICT ... DO ...` on SQLite, `INSERT IGNORE` (duplicates skipped) and +`REPLACE` on MySQL. Clauses `COPY` / `LOAD DATA` cannot express (`ON CONFLICT` / `RETURNING` on PostgreSQL, +`ON DUPLICATE KEY UPDATE` on MySQL) are generation errors; use `:batchexec` for those. +* MySQL: `LOAD DATA` commits an open transaction that was started with raw SQL (`BEGIN`); call `:copyfrom` inside + `transaction` instead. ### Macros | Macro | PostgreSQL | MySQL | SQLite | Notes | @@ -161,7 +170,7 @@ server; duplicate keys and invalid values raise `Mysql2::Error` instead of being The generated SQLite code uses `Monitor`: if you run Steep, add `library "monitor"` to your `Steepfile`. ### Names -* Names are snake_cased. Ruby keywords and names a row or method already uses get a trailing `_` (`class_`, `hash_`). +* Names are snake_cased, acronyms kept together (`MIN_ID` → `min_id`, `HTTPServer` → `http_server`). Ruby keywords and names a row or method already uses get a trailing `_` (`class_`, `hash_`). * A parameter used twice is one keyword when named with `sqlc.arg`; repeated `?` / `$n` of one column become `id:`, `id_2:`. * A table `authors` next to a table `author` keeps its plural model name (`Authors`). Query names that would replace a generated method (`Transaction`, `Initialize`, ...) are errors. @@ -170,6 +179,9 @@ server; duplicate keys and invalid values raise `Mysql2::Error` instead of being * Two result columns with the same name (e.g. `a.id, b.id`) are an error: alias one (`b.id AS book_id`) or use `sqlc.embed`. * `rubyVersion` takes `3.4` but not a patch version of it (`3.4.1`). * SQLite's `NUMERIC` affinity can return an `Integer` where `Float` is declared. +* `:batchexec` / `:batchone` / `:batchmany` are not atomic on their own: wrap them in `transaction` for all-or-nothing. +* SQLite: a transaction belongs to the thread that opened it; a Fiber / `Enumerator#next` inside it that uses the same + database waits for it (with `async`, run the work in the transaction's own fiber). * Not applicable to Ruby (they exist in the C# plugin): `withAsyncSuffix` / `withCancellationToken` (use the Fiber scheduler), `useDapper`, target framework / csproj options (`moduleName` replaces the namespace). diff --git a/examples/authors/mysql/query.sql b/examples/authors/mysql/query.sql index 6ee7430..9ebbb5e 100644 --- a/examples/authors/mysql/query.sql +++ b/examples/authors/mysql/query.sql @@ -118,3 +118,24 @@ SELECT id, payload FROM regressions ORDER BY id; -- name: TruncateRegressions :exec TRUNCATE TABLE regressions; + +-- name: CopyCopyTypes :copyfrom +INSERT INTO copy_types (id, code, doc, flags, amount) VALUES (?, ?, ?, ?, ?); + +-- name: CopyCopyTypesIgnore :copyfrom +INSERT IGNORE INTO copy_types (id, code) VALUES (?, ?); + +-- name: CopyCopyTypesReplace :copyfrom +REPLACE INTO copy_types (id, code) VALUES (?, ?); + +-- name: CreateCopyTypesBatch :batchexec +INSERT INTO copy_types (id, code, flags, amount) VALUES (?, ?, ?, ?); + +-- name: CountCopyTypesByCodes :batchone +SELECT count(*) AS matches FROM copy_types WHERE code = ? OR code = ?; + +-- name: ListCopyTypes :many +SELECT id, code, doc, flags, amount FROM copy_types ORDER BY id; + +-- name: TruncateCopyTypes :exec +TRUNCATE TABLE copy_types; diff --git a/examples/authors/mysql/schema.sql b/examples/authors/mysql/schema.sql index 99a4904..e9985c9 100644 --- a/examples/authors/mysql/schema.sql +++ b/examples/authors/mysql/schema.sql @@ -70,3 +70,11 @@ CREATE TABLE regressions ( note text, payload varbinary(16) ); + +CREATE TABLE copy_types ( + id bigint NOT NULL PRIMARY KEY, + code varchar(32) NOT NULL UNIQUE, + doc json, + flags bit(8), + amount int unsigned +); diff --git a/examples/authors/postgresql/query.sql b/examples/authors/postgresql/query.sql index e348817..cbffa4c 100644 --- a/examples/authors/postgresql/query.sql +++ b/examples/authors/postgresql/query.sql @@ -163,3 +163,18 @@ SELECT * FROM regressions WHERE (sqlc.narg(order_name)::text IS NULL OR "order" -- name: TruncateRegressions :exec TRUNCATE TABLE regressions; + +-- name: CreateBatchTypesBatch :batchexec +INSERT INTO batch_types (id, ok, ratio, blobs, docs, counts, amounts, stamps) VALUES ($1, $2, $3, $4, $5, $6, $7, $8); + +-- name: CopyBatchTypes :copyfrom +INSERT INTO batch_types (id, ok, ratio, blobs, docs, counts, amounts, stamps) VALUES ($1, $2, $3, $4, $5, $6, $7, $8); + +-- name: ListBatchTypes :many +SELECT * FROM batch_types ORDER BY id; + +-- name: CountBatchTypesAbove :one +SELECT count(*) FROM batch_types WHERE id > sqlc.arg(MIN_ID); + +-- name: TruncateBatchTypes :exec +TRUNCATE TABLE batch_types; diff --git a/examples/authors/postgresql/schema.sql b/examples/authors/postgresql/schema.sql index 8905d68..573e1dc 100644 --- a/examples/authors/postgresql/schema.sql +++ b/examples/authors/postgresql/schema.sql @@ -91,3 +91,14 @@ CREATE TABLE regressions ( bio_types extended.bio_type[], uids uuid[] ); + +CREATE TABLE batch_types ( + id bigint PRIMARY KEY, + ok boolean, + ratio float8, + blobs bytea[], + docs jsonb[], + counts int[], + amounts numeric[], + stamps timestamptz[] +); diff --git a/examples/authors/sqlite/query.sql b/examples/authors/sqlite/query.sql index 9455e03..faf2146 100644 --- a/examples/authors/sqlite/query.sql +++ b/examples/authors/sqlite/query.sql @@ -112,3 +112,18 @@ SELECT coalesce(max(price), 0) AS max_price FROM regressions; -- name: TruncateRegressions :exec DELETE FROM regressions; + +-- name: CopyTagsOrReplace :copyfrom +INSERT OR REPLACE INTO tags (name, uses) VALUES (?, ?); + +-- name: CopyTagsOrIgnore :copyfrom +INSERT OR IGNORE INTO tags (name, uses) VALUES (?, ?); + +-- name: CopyTagsAddingUses :copyfrom +INSERT INTO tags (name, uses) VALUES (?, ?) ON CONFLICT (name) DO UPDATE SET uses = tags.uses + excluded.uses; + +-- name: ListTags :many +SELECT name, uses FROM tags ORDER BY name; + +-- name: TruncateTags :exec +DELETE FROM tags; diff --git a/examples/authors/sqlite/schema.sql b/examples/authors/sqlite/schema.sql index 59e653c..5b78e26 100644 --- a/examples/authors/sqlite/schema.sql +++ b/examples/authors/sqlite/schema.sql @@ -28,3 +28,8 @@ CREATE TABLE regressions ( "order" VARCHAR(255) NOT NULL, price DECIMAL(10,2) ); + +CREATE TABLE tags ( + name TEXT NOT NULL PRIMARY KEY, + uses INTEGER NOT NULL DEFAULT 0 +); diff --git a/examples/mysql2/models.rb b/examples/mysql2/models.rb index 95ad226..07b421a 100644 --- a/examples/mysql2/models.rb +++ b/examples/mysql2/models.rb @@ -69,4 +69,13 @@ class ExtendedBio < Data.define(:author_name, :name, :bio_type, :author_type) class Regression < Data.define(:id, :order, :note, :payload) end + + class CopyType < Data.define( + :id, + :code, + :doc, + :flags, + :amount + ) + end end diff --git a/examples/mysql2/models.rbs b/examples/mysql2/models.rbs index b97f70b..15dee64 100644 --- a/examples/mysql2/models.rbs +++ b/examples/mysql2/models.rbs @@ -123,4 +123,19 @@ module Mysql2Codegen attr_reader payload: String? def initialize: ( Integer id, String order, String? note, String? payload) -> void end + + class CopyType + attr_reader id: Integer + attr_reader code: String + attr_reader doc: String? + attr_reader flags: String? + attr_reader amount: Integer? + def initialize: ( + Integer id, + String code, + String? doc, + String? flags, + Integer? amount + ) -> void + end end diff --git a/examples/mysql2/query_sql.rb b/examples/mysql2/query_sql.rb index 9ce3b60..298ad97 100644 --- a/examples/mysql2/query_sql.rb +++ b/examples/mysql2/query_sql.rb @@ -249,6 +249,57 @@ class ListRegressionPayloadsRow < Data.define(:id, :payload) TRUNCATE TABLE regressions SQL + class CopyCopyTypesArgs < Data.define( + :id, + :code, + :doc, + :flags, + :amount + ) + end + + class CopyCopyTypesIgnoreArgs < Data.define(:id, :code) + end + + class CopyCopyTypesReplaceArgs < Data.define(:id, :code) + end + + CREATE_COPY_TYPES_BATCH_SQL = <<~'SQL' + INSERT INTO copy_types (id, code, flags, amount) VALUES (?, ?, ?, ?) + SQL + + class CreateCopyTypesBatchArgs < Data.define(:id, :code, :flags, :amount) + end + + COUNT_COPY_TYPES_BY_CODES_SQL = <<~'SQL' + SELECT count(*) AS matches FROM copy_types WHERE code = ? OR code = ? + SQL + + class CountCopyTypesByCodesRow < Data.define(:matches) + end + + # rubocop:disable Naming/VariableNumber + class CountCopyTypesByCodesArgs < Data.define(:code, :code_2) + end + # rubocop:enable Naming/VariableNumber + + LIST_COPY_TYPES_SQL = <<~'SQL' + SELECT id, code, doc, flags, amount FROM copy_types ORDER BY id + SQL + + class ListCopyTypesRow < Data.define( + :id, + :code, + :doc, + :flags, + :amount + ) + end + + TRUNCATE_COPY_TYPES_SQL = <<~'SQL' + TRUNCATE TABLE copy_types + SQL + class QuerySql # Raised by the bang variants (get_author!) when the query returns no row. class NotFound < StandardError; end @@ -272,6 +323,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = !client.in_transaction + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.query("SET TRANSACTION ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}") if isolation @@ -669,48 +722,48 @@ def create_author_result(name:, bio:) def create_authors_batch(rows) with_client('create_authors_batch') do |client| - stmt = client.prepare(CREATE_AUTHORS_BATCH_SQL) - begin - rows.each do |args| - query_params = [args.name, args.bio] + rows.each do |args| + query_params = [args.name, args.bio] + stmt = client.prepare(CREATE_AUTHORS_BATCH_SQL) + begin stmt.execute(*query_params) + ensure + stmt.close end - nil - ensure - stmt.close end + nil end end def get_author_batch(rows) with_client('get_author_batch') do |client| - stmt = client.prepare(GET_AUTHOR_BATCH_SQL) - begin - rows.map do |args| - query_params = [args.id] + rows.map do |args| + query_params = [args.id] + stmt = client.prepare(GET_AUTHOR_BATCH_SQL) + begin result = stmt.execute(*query_params, as: :array) row = result.first row && GetAuthorBatchRow.new(row[0], row[1], row[2]) + ensure + stmt.close end - ensure - stmt.close end end end def list_authors_by_name_batch(rows) with_client('list_authors_by_name_batch') do |client| - stmt = client.prepare(LIST_AUTHORS_BY_NAME_BATCH_SQL) - begin - rows.map do |args| - query_params = [args.name] + rows.map do |args| + query_params = [args.name] + stmt = client.prepare(LIST_AUTHORS_BY_NAME_BATCH_SQL) + begin result = stmt.execute(*query_params, as: :array) result.map do |row| ListAuthorsByNameBatchRow.new(row[0], row[1], row[2]) end + ensure + stmt.close end - ensure - stmt.close end end end @@ -805,6 +858,114 @@ def truncate_regressions end end + def copy_copy_types(rows) + with_client('copy_copy_types') do |client| + return if rows.empty? + + Tempfile.create(['copy_copy_types', '.csv']) do |file| + rows.each do |row| + file.write(csv_line([row.id, row.code, row.doc, row.flags, row.amount])) + end + file.flush + load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' INTO TABLE `copy_types` CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (`id`, `code`, @sqlc_v2, `flags`, `amount`) SET `doc` = CONVERT(@sqlc_v2 USING utf8mb4)") + end + nil + end + end + + def copy_copy_types_ignore(rows) + with_client('copy_copy_types_ignore') do |client| + return if rows.empty? + + Tempfile.create(['copy_copy_types_ignore', '.csv']) do |file| + rows.each do |row| + file.write(csv_line([row.id, row.code])) + end + file.flush + load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' IGNORE INTO TABLE `copy_types` CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (`id`, `code`)", strict: false) + end + nil + end + end + + def copy_copy_types_replace(rows) + with_client('copy_copy_types_replace') do |client| + return if rows.empty? + + Tempfile.create(['copy_copy_types_replace', '.csv']) do |file| + rows.each do |row| + file.write(csv_line([row.id, row.code])) + end + file.flush + load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' REPLACE INTO TABLE `copy_types` CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (`id`, `code`)") + end + nil + end + end + + def create_copy_types_batch(rows) + with_client('create_copy_types_batch') do |client| + rows.each do |args| + query_params = [args.id, args.code, args.flags, args.amount] + stmt = client.prepare(CREATE_COPY_TYPES_BATCH_SQL) + begin + stmt.execute(*query_params) + ensure + stmt.close + end + end + nil + end + end + + def count_copy_types_by_codes(rows) + with_client('count_copy_types_by_codes') do |client| + rows.map do |args| + query_params = [args.code, args.code_2] + stmt = client.prepare(COUNT_COPY_TYPES_BY_CODES_SQL) + begin + result = stmt.execute(*query_params, as: :array) + row = result.first + row && CountCopyTypesByCodesRow.new(row[0]) + ensure + stmt.close + end + end + end + end + + def list_copy_types + with_client('list_copy_types') do |client| + stmt = client.prepare(LIST_COPY_TYPES_SQL) + begin + result = stmt.execute(as: :array) + result.map do |row| + ListCopyTypesRow.new( + row[0], + row[1], + row[2], + row[3], + row[4] + ) + end + ensure + stmt.close + end + end + end + + def truncate_copy_types + with_client('truncate_copy_types') do |client| + stmt = client.prepare(TRUNCATE_COPY_TYPES_SQL) + begin + stmt.execute + nil + ensure + stmt.close + end + end + end + private def instrument(name) @@ -823,7 +984,13 @@ def with_client(name) instrument(name) do @db.with do |client| client.extend(ConnectionState) unless client.is_a?(ConnectionState) - yield client + begin + yield client + rescue Mysql2::Error + # a connection closed by the server (restart, KILL, wait_timeout) is replaced by the pool + @db.discard_current_connection if client.closed? && @db.respond_to?(:discard_current_connection) + raise + end end end end @@ -842,15 +1009,15 @@ def csv_line(values) "#{fields.join(',')}\n" end - # Runs LOAD DATA in a transaction (a savepoint inside #transaction): all rows or none, and an error instead - # of the warnings LOAD DATA LOCAL gives for duplicate keys and invalid values. - def load_data(client, sql) + # Runs LOAD DATA in a transaction (a savepoint inside #transaction): all rows or none, and (strict) an error + # instead of the warnings LOAD DATA LOCAL gives for duplicate keys and invalid values. + def load_data(client, sql, strict: true) outermost = !client.in_transaction client.query(outermost ? 'BEGIN' : 'SAVEPOINT sqlc_copy') done = false begin client.query(sql) - unless client.warning_count.zero? + if strict && !client.warning_count.zero? warnings = client.query('SHOW WARNINGS', as: :array).reject { |level, *| level == 'Note' } raise Mysql2::Error, "LOAD DATA: #{warnings.map { |w| w[2] }.join('; ')}" unless warnings.empty? end diff --git a/examples/mysql2/query_sql.rbs b/examples/mysql2/query_sql.rbs index fc0d044..31d4b61 100644 --- a/examples/mysql2/query_sql.rbs +++ b/examples/mysql2/query_sql.rbs @@ -273,6 +273,75 @@ module Mysql2Codegen TRUNCATE_REGRESSIONS_SQL: String + class CopyCopyTypesArgs + attr_reader id: Integer + attr_reader code: String + attr_reader doc: String? + attr_reader flags: String? + attr_reader amount: Integer? + def initialize: ( + Integer id, + String code, + String? doc, + String? flags, + Integer? amount + ) -> void + end + + class CopyCopyTypesIgnoreArgs + attr_reader id: Integer + attr_reader code: String + def initialize: ( Integer id, String code) -> void + end + + class CopyCopyTypesReplaceArgs + attr_reader id: Integer + attr_reader code: String + def initialize: ( Integer id, String code) -> void + end + + CREATE_COPY_TYPES_BATCH_SQL: String + + class CreateCopyTypesBatchArgs + attr_reader id: Integer + attr_reader code: String + attr_reader flags: String? + attr_reader amount: Integer? + def initialize: ( Integer id, String code, String? flags, Integer? amount) -> void + end + + COUNT_COPY_TYPES_BY_CODES_SQL: String + + class CountCopyTypesByCodesRow + attr_reader matches: Integer + def initialize: ( Integer matches) -> void + end + + class CountCopyTypesByCodesArgs + attr_reader code: String + attr_reader code_2: String + def initialize: ( String code, String code_2) -> void + end + + LIST_COPY_TYPES_SQL: String + + class ListCopyTypesRow + attr_reader id: Integer + attr_reader code: String + attr_reader doc: String? + attr_reader flags: String? + attr_reader amount: Integer? + def initialize: ( + Integer id, + String code, + String? doc, + String? flags, + Integer? amount + ) -> void + end + + TRUNCATE_COPY_TYPES_SQL: String + class QuerySql class NotFound < StandardError end @@ -359,6 +428,20 @@ module Mysql2Codegen def truncate_regressions: -> void + def copy_copy_types: (Array[CopyCopyTypesArgs] rows) -> void + + def copy_copy_types_ignore: (Array[CopyCopyTypesIgnoreArgs] rows) -> void + + def copy_copy_types_replace: (Array[CopyCopyTypesReplaceArgs] rows) -> void + + def create_copy_types_batch: (Array[CreateCopyTypesBatchArgs] rows) -> void + + def count_copy_types_by_codes: (Array[CountCopyTypesByCodesArgs] rows) -> Array[CountCopyTypesByCodesRow?] + + def list_copy_types: -> Array[ListCopyTypesRow] + + def truncate_copy_types: -> void + private def instrument: [T] (String name) { () -> T } -> T @@ -367,7 +450,7 @@ module Mysql2Codegen def csv_line: (Array[untyped] values) -> String - def load_data: (untyped client, String sql) -> void + def load_data: (untyped client, String sql, ?strict: bool) -> void def slice_placeholders: (Array[untyped] values) -> String end diff --git a/examples/pg/models.rb b/examples/pg/models.rb index 02758eb..b9a3073 100644 --- a/examples/pg/models.rb +++ b/examples/pg/models.rb @@ -76,6 +76,18 @@ class Regression < Data.define( ) end + class BatchType < Data.define( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps + ) + end + class ExtendedBio < Data.define(:author_name, :name, :bio_type) end end diff --git a/examples/pg/models.rbs b/examples/pg/models.rbs index 69c5313..edc0520 100644 --- a/examples/pg/models.rbs +++ b/examples/pg/models.rbs @@ -130,6 +130,27 @@ module PgCodegen ) -> void end + class BatchType + attr_reader id: Integer + attr_reader ok: bool? + attr_reader ratio: Float? + attr_reader blobs: Array[String]? + attr_reader docs: Array[untyped]? + attr_reader counts: Array[Integer]? + attr_reader amounts: Array[BigDecimal]? + attr_reader stamps: Array[Time]? + def initialize: ( + Integer id, + bool? ok, + Float? ratio, + Array[String]? blobs, + Array[untyped]? docs, + Array[Integer]? counts, + Array[BigDecimal]? amounts, + Array[Time]? stamps + ) -> void + end + class ExtendedBio attr_reader author_name: String attr_reader name: String diff --git a/examples/pg/query_sql.rb b/examples/pg/query_sql.rb index 3bdf842..e8bb3ad 100644 --- a/examples/pg/query_sql.rb +++ b/examples/pg/query_sql.rb @@ -343,6 +343,61 @@ class ListRegressionsRow < Data.define( TRUNCATE TABLE regressions SQL + CREATE_BATCH_TYPES_BATCH_SQL = <<~'SQL' + INSERT INTO batch_types (id, ok, ratio, blobs, docs, counts, amounts, stamps) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + SQL + + class CreateBatchTypesBatchArgs < Data.define( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps + ) + end + + class CopyBatchTypesArgs < Data.define( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps + ) + end + + LIST_BATCH_TYPES_SQL = <<~'SQL' + SELECT id, ok, ratio, blobs, docs, counts, amounts, stamps FROM batch_types ORDER BY id + SQL + + class ListBatchTypesRow < Data.define( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps + ) + end + + COUNT_BATCH_TYPES_ABOVE_SQL = <<~'SQL' + SELECT count(*) FROM batch_types WHERE id > $1 + SQL + + class CountBatchTypesAboveRow < Data.define(:count) + end + + TRUNCATE_BATCH_TYPES_SQL = <<~'SQL' + TRUNCATE TABLE batch_types + SQL + class QuerySql # Raised by the bang variants (get_author!) when the query returns no row. class NotFound < StandardError; end @@ -366,6 +421,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = client.transaction_status == PG::PQTRANS_IDLE + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.exec(isolation ? "BEGIN ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}" : 'BEGIN') @@ -382,11 +439,12 @@ def transaction(isolation: nil) raise ensure unless failed + # a statement failed and the block rescued the error: PostgreSQL can only roll back + rescued = client.transaction_status == PG::PQTRANS_INERROR commit = outermost ? 'COMMIT' : "RELEASE SAVEPOINT #{savepoint}" - result = client.exec(Thread.current.status == 'aborting' ? rollback : commit) - # COMMIT of a transaction aborted by an error the block rescued is a ROLLBACK - rolled_back = outermost && result.cmd_status == 'ROLLBACK' && Thread.current.status != 'aborting' - raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rolled_back + client.exec(rescued || Thread.current.status == 'aborting' ? rollback : commit) + client.exec("RELEASE SAVEPOINT #{savepoint}") if rescued && savepoint + raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rescued && Thread.current.status != 'aborting' end end end @@ -549,7 +607,7 @@ def update_author_bios(bio:) def get_authors_by_ids(param1:) with_client('get_authors_by_ids') do |client| - query_params = [param1] + query_params = [encode_array(param1)] unless client.prepared_statements.include?('query_sql_get_authors_by_ids') client.prepare('query_sql_get_authors_by_ids', GET_AUTHORS_BY_IDS_SQL) client.prepared_statements.add('query_sql_get_authors_by_ids') @@ -563,7 +621,7 @@ def get_authors_by_ids(param1:) def get_authors_by_ids_and_names(param1:, param2:) with_client('get_authors_by_ids_and_names') do |client| - query_params = [param1, param2] + query_params = [encode_array(param1), encode_array(param2)] unless client.prepared_statements.include?('query_sql_get_authors_by_ids_and_names') client.prepare('query_sql_get_authors_by_ids_and_names', GET_AUTHORS_BY_IDS_AND_NAMES_SQL) client.prepared_statements.add('query_sql_get_authors_by_ids_and_names') @@ -701,8 +759,8 @@ def insert_extra_types(c_uuid:, c_jsonb:, c_text_array:, c_integer_array:, c_tim query_params = [ c_uuid, encode_json(c_jsonb), - c_text_array, - c_integer_array, + encode_array(c_text_array), + encode_array(c_integer_array), c_timestamptz, c_inet, c_money, @@ -756,7 +814,7 @@ def truncate_extra_types def insert_array_types(c_uuid_array:, c_bio_type_array:, c_integer_matrix:) with_client('insert_array_types') do |client| - query_params = [c_uuid_array, c_bio_type_array, c_integer_matrix] + query_params = [encode_array(c_uuid_array), encode_array(c_bio_type_array), encode_array(c_integer_matrix)] unless client.prepared_statements.include?('query_sql_insert_array_types') client.prepare('query_sql_insert_array_types', INSERT_ARRAY_TYPES_SQL) client.prepared_statements.add('query_sql_insert_array_types') @@ -944,8 +1002,8 @@ def create_regression(id:, order:, payload:, doc:, bio_types:, uids:) order, encode_bytea(payload), encode_json(doc), - bio_types, - uids + encode_array(bio_types), + encode_array(uids) ] unless client.prepared_statements.include?('query_sql_create_regression') client.prepare('query_sql_create_regression', CREATE_REGRESSION_SQL) @@ -1040,6 +1098,92 @@ def truncate_regressions end end + def create_batch_types_batch(rows) + with_client('create_batch_types_batch') do |client| + unless client.prepared_statements.include?('query_sql_create_batch_types_batch') + client.prepare('query_sql_create_batch_types_batch', CREATE_BATCH_TYPES_BATCH_SQL) + client.prepared_statements.add('query_sql_create_batch_types_batch') + end + rows.each do |args| + query_params = [ + args.id, + args.ok, + encode_float(args.ratio), + encode_array(args.blobs, :bytea), + encode_array(args.docs, :json), + encode_array(args.counts), + encode_array(args.amounts), + encode_array(args.stamps) + ] + client.exec_prepared('query_sql_create_batch_types_batch', query_params) + end + nil + end + end + + def copy_batch_types(rows) + with_client('copy_batch_types') do |client| + encoder = PG::TextEncoder::CopyRow.new(type_map: client.type_map_for_queries) + client.copy_data('COPY "batch_types" ("id", "ok", "ratio", "blobs", "docs", "counts", "amounts", "stamps") FROM STDIN', encoder) do + rows.each do |row| + client.put_copy_data([row.id, row.ok, encode_float(row.ratio), encode_array(row.blobs, :bytea), encode_array(row.docs, :json), encode_array(row.counts), encode_array(row.amounts), encode_array(row.stamps)]) + end + end + nil + end + end + + def list_batch_types + with_client('list_batch_types') do |client| + unless client.prepared_statements.include?('query_sql_list_batch_types') + client.prepare('query_sql_list_batch_types', LIST_BATCH_TYPES_SQL) + client.prepared_statements.add('query_sql_list_batch_types') + end + result = client.exec_prepared('query_sql_list_batch_types') + result.map do |row| + ListBatchTypesRow.new( + row['id'], + row['ok'], + row['ratio'], + row['blobs'], + row['docs'], + row['counts'], + row['amounts'], + row['stamps'] + ) + end + end + end + + def count_batch_types_above(min_id:) + with_client('count_batch_types_above') do |client| + query_params = [min_id] + unless client.prepared_statements.include?('query_sql_count_batch_types_above') + client.prepare('query_sql_count_batch_types_above', COUNT_BATCH_TYPES_ABOVE_SQL) + client.prepared_statements.add('query_sql_count_batch_types_above') + end + result = client.exec_prepared('query_sql_count_batch_types_above', query_params) + row = result.first + row && CountBatchTypesAboveRow.new(row['count']) + end + end + + # Like #count_batch_types_above, but raises NotFound when there is no row. + def count_batch_types_above!(...) + count_batch_types_above(...) || raise(NotFound, 'CountBatchTypesAbove returned no row') + end + + def truncate_batch_types + with_client('truncate_batch_types') do |client| + unless client.prepared_statements.include?('query_sql_truncate_batch_types') + client.prepare('query_sql_truncate_batch_types', TRUNCATE_BATCH_TYPES_SQL) + client.prepared_statements.add('query_sql_truncate_batch_types') + end + client.exec_prepared('query_sql_truncate_batch_types') + nil + end + end + private def instrument(name) @@ -1057,6 +1201,7 @@ def instrument(name) def with_client(name) instrument(name) do @db.with do |client| + client.reset if client.status == PG::CONNECTION_BAD # closed by the server (restart, failover, killed) configure(client) unless client.is_a?(ConnectionState) yield client end @@ -1068,7 +1213,10 @@ def configure(client) type_map = PG::BasicTypeMapForResults.new(client) type_map.default_type_map = string_arrays(client) # uuid, enums, money, ... come back as Strings client.type_map_for_results = type_map - client.type_map_for_queries = PG::BasicTypeMapForQueries.new(client) + queries = PG::BasicTypeMapForQueries.new(client) + # true / false as text ('t' / 'f'), also valid in COPY (the default binary encoder is not) + queries[TrueClass] = queries[FalseClass] = PG::TextEncoder::Boolean.new(oid: 16) + client.type_map_for_queries = queries client.extend(ConnectionState) end @@ -1093,5 +1241,31 @@ def encode_bytea(value) def encode_json(value) value.is_a?(Array) ? PG::TextEncoder::JSON.new.encode(value) : value end + + # Float#to_s is the shortest text that reads back as the same Float (the pg gem's encoder drops digits). + def encode_float(value) + value.is_a?(Float) ? value.to_s : value + end + + # Array parameters are sent as array literals, which the server converts to the parameter's type (the pg gem + # would type them from their first element: wrong for [], [nil, ...], bytea[], json[]). + def encode_array(value, kind = :value) + return value unless value.is_a?(Array) + + PG::TextEncoder::Array.new(elements_type: PG::TextEncoder::String.new).encode(array_elements(value, kind)) + end + + def array_elements(values, kind) + values.map do |value| + case value + when nil then nil + when Array then kind == :json ? PG::TextEncoder::JSON.new.encode(value) : array_elements(value, kind) + when Hash then PG::TextEncoder::JSON.new.encode(value) + when String then kind == :bytea ? PG::TextEncoder::Bytea.new.encode(value) : value + when Time then PG::TextEncoder::TimestampWithTimeZone.new.encode(value) + else defined?(IPAddr) && value.is_a?(IPAddr) ? "#{value}/#{value.prefix}" : value.to_s + end + end + end end end diff --git a/examples/pg/query_sql.rbs b/examples/pg/query_sql.rbs index bb90ab1..dad457d 100644 --- a/examples/pg/query_sql.rbs +++ b/examples/pg/query_sql.rbs @@ -356,6 +356,82 @@ module PgCodegen TRUNCATE_REGRESSIONS_SQL: String + CREATE_BATCH_TYPES_BATCH_SQL: String + + class CreateBatchTypesBatchArgs + attr_reader id: Integer + attr_reader ok: bool? + attr_reader ratio: Float? + attr_reader blobs: Array[String]? + attr_reader docs: Array[untyped]? + attr_reader counts: Array[Integer]? + attr_reader amounts: Array[BigDecimal]? + attr_reader stamps: Array[Time]? + def initialize: ( + Integer id, + bool? ok, + Float? ratio, + Array[String]? blobs, + Array[untyped]? docs, + Array[Integer]? counts, + Array[BigDecimal]? amounts, + Array[Time]? stamps + ) -> void + end + + class CopyBatchTypesArgs + attr_reader id: Integer + attr_reader ok: bool? + attr_reader ratio: Float? + attr_reader blobs: Array[String]? + attr_reader docs: Array[untyped]? + attr_reader counts: Array[Integer]? + attr_reader amounts: Array[BigDecimal]? + attr_reader stamps: Array[Time]? + def initialize: ( + Integer id, + bool? ok, + Float? ratio, + Array[String]? blobs, + Array[untyped]? docs, + Array[Integer]? counts, + Array[BigDecimal]? amounts, + Array[Time]? stamps + ) -> void + end + + LIST_BATCH_TYPES_SQL: String + + class ListBatchTypesRow + attr_reader id: Integer + attr_reader ok: bool? + attr_reader ratio: Float? + attr_reader blobs: Array[String]? + attr_reader docs: Array[untyped]? + attr_reader counts: Array[Integer]? + attr_reader amounts: Array[BigDecimal]? + attr_reader stamps: Array[Time]? + def initialize: ( + Integer id, + bool? ok, + Float? ratio, + Array[String]? blobs, + Array[untyped]? docs, + Array[Integer]? counts, + Array[BigDecimal]? amounts, + Array[Time]? stamps + ) -> void + end + + COUNT_BATCH_TYPES_ABOVE_SQL: String + + class CountBatchTypesAboveRow + attr_reader count: Integer + def initialize: ( Integer count) -> void + end + + TRUNCATE_BATCH_TYPES_SQL: String + class QuerySql class NotFound < StandardError end @@ -470,6 +546,18 @@ module PgCodegen def truncate_regressions: -> void + def create_batch_types_batch: (Array[CreateBatchTypesBatchArgs] rows) -> void + + def copy_batch_types: (Array[CopyBatchTypesArgs] rows) -> void + + def list_batch_types: -> Array[ListBatchTypesRow] + + def count_batch_types_above: (min_id: Integer) -> CountBatchTypesAboveRow? + + def count_batch_types_above!: (min_id: Integer) -> CountBatchTypesAboveRow + + def truncate_batch_types: -> void + private def instrument: [T] (String name) { () -> T } -> T @@ -483,5 +571,11 @@ module PgCodegen def encode_bytea: (String? value) -> String? def encode_json: (untyped value) -> untyped + + def encode_float: (untyped value) -> untyped + + def encode_array: (Array[untyped]? value, ?Symbol kind) -> untyped + + def array_elements: (Array[untyped] values, Symbol kind) -> Array[untyped] end end diff --git a/examples/sqlite3/models.rb b/examples/sqlite3/models.rb index a880702..498e262 100644 --- a/examples/sqlite3/models.rb +++ b/examples/sqlite3/models.rb @@ -22,4 +22,7 @@ class DeclaredType < Data.define( class Regression < Data.define(:id, :order, :price) end + + class Tag < Data.define(:name, :uses) + end end diff --git a/examples/sqlite3/models.rbs b/examples/sqlite3/models.rbs index 7b76c71..60532b3 100644 --- a/examples/sqlite3/models.rbs +++ b/examples/sqlite3/models.rbs @@ -40,4 +40,10 @@ module Sqlite3Codegen attr_reader price: Float? def initialize: ( Integer id, String order, Float? price) -> void end + + class Tag + attr_reader name: String + attr_reader uses: Integer + def initialize: ( String name, Integer uses) -> void + end end diff --git a/examples/sqlite3/query_sql.rb b/examples/sqlite3/query_sql.rb index 18a779d..38b041b 100644 --- a/examples/sqlite3/query_sql.rb +++ b/examples/sqlite3/query_sql.rb @@ -218,6 +218,26 @@ class GetMaxRegressionPriceRow < Data.define(:max_price) DELETE FROM regressions SQL + class CopyTagsOrReplaceArgs < Data.define(:name, :uses) + end + + class CopyTagsOrIgnoreArgs < Data.define(:name, :uses) + end + + class CopyTagsAddingUsesArgs < Data.define(:name, :uses) + end + + LIST_TAGS_SQL = <<~'SQL' + SELECT name, uses FROM tags ORDER BY name + SQL + + class ListTagsRow < Data.define(:name, :uses) + end + + TRUNCATE_TAGS_SQL = <<~'SQL' + DELETE FROM tags + SQL + class QuerySql # Raised by the bang variants (get_author!) when the query returns no row. class NotFound < StandardError; end @@ -240,6 +260,8 @@ def transaction(mode: nil) instrument('transaction') do @lock.synchronize do outermost = !@db.transaction_active? + raise ArgumentError, 'mode: cannot be set on a nested transaction' if mode && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" @db.execute(outermost ? "BEGIN#{" #{mode.to_s.upcase}" if mode}" : "SAVEPOINT #{savepoint}") rollback = outermost ? 'ROLLBACK' : "ROLLBACK TO SAVEPOINT #{savepoint}" @@ -701,6 +723,68 @@ def truncate_regressions end end + def copy_tags_or_replace(rows) + with_client('copy_tags_or_replace') do |client| + atomically(client) do + rows.each_slice(499) do |batch| + sql = %(INSERT OR REPLACE INTO "tags" ("name", "uses") VALUES #{Array.new(batch.size, '(?, ?)').join(', ')}) + client.execute(sql, batch.flat_map { |row| [row.name, row.uses] }) + end + end + nil + end + end + + def copy_tags_or_ignore(rows) + with_client('copy_tags_or_ignore') do |client| + atomically(client) do + rows.each_slice(499) do |batch| + sql = %(INSERT OR IGNORE INTO "tags" ("name", "uses") VALUES #{Array.new(batch.size, '(?, ?)').join(', ')}) + client.execute(sql, batch.flat_map { |row| [row.name, row.uses] }) + end + end + nil + end + end + + def copy_tags_adding_uses(rows) + with_client('copy_tags_adding_uses') do |client| + atomically(client) do + rows.each_slice(499) do |batch| + sql = %(INSERT INTO "tags" ("name", "uses") VALUES #{Array.new(batch.size, '(?, ?)').join(', ')} ON CONFLICT (name) DO UPDATE SET uses = tags.uses + excluded.uses) + client.execute(sql, batch.flat_map { |row| [row.name, row.uses] }) + end + end + nil + end + end + + def list_tags + with_client('list_tags') do |client| + stmt = client.prepare(LIST_TAGS_SQL) + begin + result = stmt.execute + result.map do |row| + ListTagsRow.new(row[0], row[1]) + end + ensure + stmt.close + end + end + end + + def truncate_tags + with_client('truncate_tags') do |client| + stmt = client.prepare(TRUNCATE_TAGS_SQL) + begin + stmt.execute + nil + ensure + stmt.close + end + end + end + private def instrument(name) diff --git a/examples/sqlite3/query_sql.rbs b/examples/sqlite3/query_sql.rbs index 1276e0a..0f3f568 100644 --- a/examples/sqlite3/query_sql.rbs +++ b/examples/sqlite3/query_sql.rbs @@ -209,6 +209,34 @@ module Sqlite3Codegen TRUNCATE_REGRESSIONS_SQL: String + class CopyTagsOrReplaceArgs + attr_reader name: String + attr_reader uses: Integer + def initialize: ( String name, Integer uses) -> void + end + + class CopyTagsOrIgnoreArgs + attr_reader name: String + attr_reader uses: Integer + def initialize: ( String name, Integer uses) -> void + end + + class CopyTagsAddingUsesArgs + attr_reader name: String + attr_reader uses: Integer + def initialize: ( String name, Integer uses) -> void + end + + LIST_TAGS_SQL: String + + class ListTagsRow + attr_reader name: String + attr_reader uses: Integer + def initialize: ( String name, Integer uses) -> void + end + + TRUNCATE_TAGS_SQL: String + class QuerySql class NotFound < StandardError end @@ -287,6 +315,16 @@ module Sqlite3Codegen def truncate_regressions: -> void + def copy_tags_or_replace: (Array[CopyTagsOrReplaceArgs] rows) -> void + + def copy_tags_or_ignore: (Array[CopyTagsOrIgnoreArgs] rows) -> void + + def copy_tags_adding_uses: (Array[CopyTagsAddingUsesArgs] rows) -> void + + def list_tags: -> Array[ListTagsRow] + + def truncate_tags: -> void + private def instrument: [T] (String name) { () -> T } -> T diff --git a/lib/sqlc_gen_ruby/code.rb b/lib/sqlc_gen_ruby/code.rb index 41787ba..ac40578 100644 --- a/lib/sqlc_gen_ruby/code.rb +++ b/lib/sqlc_gen_ruby/code.rb @@ -20,8 +20,9 @@ def comma_join(items, wrap: false) "\n#{indent(items.join(",\n"))}\n" end + # GetAuthor => get_author, userId => user_id, GetByID => get_by_id, HTTPCode => http_code, ABC => abc def snake_case(str) - str.gsub(/([A-Z])/, '_\1').sub(/\A_+/, '').downcase + str.gsub(/([A-Z]+)([A-Z][a-z])/, '\1_\2').gsub(/([[:lower:]\d])([A-Z])/, '\1_\2').sub(/\A_+/, '').downcase end def upcase_first(str) @@ -101,6 +102,7 @@ def naming_cops(attributes: [], params: []) cops << 'Naming/MethodName' unless attributes.all? { |n| n.match?(/\A[\d[[:lower:]]_]+\z/) } cops << 'Naming/MethodParameterName' if params.any? { |n| n.length < 3 && !n.match?(/\d\z/) && !SHORT_NAMES.include?(n) } cops << 'Naming/AsciiIdentifiers' unless params.all?(&:ascii_only?) + cops << 'Naming/VariableNumber' if attributes.any? { |n| n.match?(/_\d+\z/) } # code_2 (repeated parameters) cops end diff --git a/lib/sqlc_gen_ruby/drivers/base.rb b/lib/sqlc_gen_ruby/drivers/base.rb index 1170d5a..4ed7447 100644 --- a/lib/sqlc_gen_ruby/drivers/base.rb +++ b/lib/sqlc_gen_ruby/drivers/base.rb @@ -66,8 +66,15 @@ def exec_rows_lines(_exec, _query) = raise(NotImplementedError) def exec_last_id_lines(_exec, _query) = raise(NotImplementedError) - # table: quoted SQL name; columns: quoted SQL column names; values: Ruby expressions of the row values - def copy_from_lines(_rows_var, _table, _columns, _values, _label) = raise(NotImplementedError) + # table: quoted SQL name; columns: quoted SQL column names; values: Ruby expressions of the row values; + # sources: the sqlc columns of the values; verb: the SQL before INTO (`INSERT OR IGNORE`); tail: the SQL after + # the VALUES tuple (`ON CONFLICT ...`) + def copy_from_lines(_rows_var, _table, _columns, _values, _label, query_name:, sources:, verb:, tail:) + raise(NotImplementedError) + end + + # :batch* prepare the statement for every row instead of once (mysql2 corrupts reused statements' parameters). + def prepares_batch_rows? = false # SQL identifier quoting for the statements the plugin writes itself (:copyfrom). def quote_identifier(name) = %("#{name.gsub('"', '""')}") diff --git a/lib/sqlc_gen_ruby/drivers/mysql2.rb b/lib/sqlc_gen_ruby/drivers/mysql2.rb index 7e7467c..dd43338 100644 --- a/lib/sqlc_gen_ruby/drivers/mysql2.rb +++ b/lib/sqlc_gen_ruby/drivers/mysql2.rb @@ -54,9 +54,15 @@ def exec_result_lines(exec) = [exec, 'stmt.affected_rows', 'client.last_id'] # LOAD DATA LOCAL INFILE from a temporary CSV (the client needs `local_infile: true`, the server # `local_infile=1`). - def copy_from_lines(rows_var, table, columns, values, label) + def copy_from_lines(rows_var, table, columns, values, label, query_name:, sources:, verb:, tail:) + unless tail.empty? && verb.match?(/\A(INSERT|REPLACE)\b/) + raise ArgumentError, "#{query_name}: :copyfrom (LOAD DATA) takes INSERT [IGNORE] / REPLACE ... VALUES (...), " \ + 'without ON DUPLICATE KEY UPDATE' + end + write = Code.block("#{rows_var}.each do |row|", "file.write(csv_line([#{values.join(', ')}]))") - load_data = format(LOAD_DATA, table: table, columns: columns.join(', ')) + load_data = format(LOAD_DATA, table: table, **load_columns(columns, sources), + mode: load_mode(verb), strict: verb.split.include?('IGNORE') ? ', strict: false' : '') [ "return if #{rows_var}.empty?", '', @@ -65,13 +71,32 @@ def copy_from_lines(rows_var, table, columns, values, label) ] end + # INSERT IGNORE skips duplicates and adjusts invalid values (with warnings); REPLACE replaces duplicates. + def load_mode(verb) + if verb.start_with?('REPLACE') then 'REPLACE ' + elsif verb.split.include?('IGNORE') then 'IGNORE ' + else '' + end + end + + # The file is read as binary (binary columns keep every byte); JSON columns need text: they are loaded through + # a user variable converted to utf8mb4. + def load_columns(columns, sources) + json = columns.each_index.select { |i| normalize_type(sources[i].type.name) == 'json' } + list = columns.each_with_index.map { |c, i| json.include?(i) ? "@sqlc_v#{i}" : c } + set = json.map { |i| "#{columns[i]} = CONVERT(@sqlc_v#{i} USING utf8mb4)" } + { columns: list.join(', '), set: set.empty? ? '' : " SET #{set.join(', ')}" } + end + + def prepares_batch_rows? = true + def extra_requires(queries, _ruby_version) = copy_from?(queries) ? %w[tempfile] : [] def helper_methods(queries) return [] unless copy_from?(queries) [[CSV_LINE, 'def csv_line: (Array[untyped] values) -> String'], - [LOAD_DATA_HELPER, 'def load_data: (untyped client, String sql) -> void']] + [LOAD_DATA_HELPER, 'def load_data: (untyped client, String sql, ?strict: bool) -> void']] end CONNECTION_STATE = <<~'RUBY'.chomp @@ -105,6 +130,8 @@ def self.connect(mysql2_params, logger: nil, **pool_options) def transaction(isolation: nil) with_client('transaction') do |client| outermost = !client.in_transaction + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.query("SET TRANSACTION ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}") if isolation @@ -140,7 +167,13 @@ def with_client(name) instrument(name) do @db.with do |client| client.extend(ConnectionState) unless client.is_a?(ConnectionState) - yield client + begin + yield client + rescue Mysql2::Error + # a connection closed by the server (restart, KILL, wait_timeout) is replaced by the pool + @db.discard_current_connection if client.closed? && @db.respond_to?(:discard_current_connection) + raise + end end end end @@ -148,20 +181,20 @@ def with_client(name) # Template of the generated statement (single-quoted heredoc: backslashes are copied verbatim). LOAD_DATA = <<~'RUBY'.chomp - load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' INTO TABLE %s CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (%s)") + load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' %sINTO TABLE %
s CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (%s)%s"%s) RUBY # LOAD DATA LOCAL skips duplicate keys and converts invalid values with a warning instead of failing. LOAD_DATA_HELPER = <<~'RUBY'.chomp - # Runs LOAD DATA in a transaction (a savepoint inside #transaction): all rows or none, and an error instead - # of the warnings LOAD DATA LOCAL gives for duplicate keys and invalid values. - def load_data(client, sql) + # Runs LOAD DATA in a transaction (a savepoint inside #transaction): all rows or none, and (strict) an error + # instead of the warnings LOAD DATA LOCAL gives for duplicate keys and invalid values. + def load_data(client, sql, strict: true) outermost = !client.in_transaction client.query(outermost ? 'BEGIN' : 'SAVEPOINT sqlc_copy') done = false begin client.query(sql) - unless client.warning_count.zero? + if strict && !client.warning_count.zero? warnings = client.query('SHOW WARNINGS', as: :array).reject { |level, *| level == 'Note' } raise Mysql2::Error, "LOAD DATA: #{warnings.map { |w| w[2] }.join('; ')}" unless warnings.empty? end diff --git a/lib/sqlc_gen_ruby/drivers/pg.rb b/lib/sqlc_gen_ruby/drivers/pg.rb index da6e89b..1cc9eb3 100644 --- a/lib/sqlc_gen_ruby/drivers/pg.rb +++ b/lib/sqlc_gen_ruby/drivers/pg.rb @@ -73,7 +73,12 @@ def exec_last_id_lines(exec, query) end # COPY ... FROM STDIN, one row at a time through the pg gem's CopyRow encoder. - def copy_from_lines(rows_var, table, columns, values, _label) + def copy_from_lines(rows_var, table, columns, values, _label, query_name:, sources:, verb:, tail:) + unless verb == 'INSERT' && tail.empty? + raise ArgumentError, "#{query_name}: :copyfrom on PostgreSQL (COPY) takes a plain INSERT ... VALUES (...), " \ + 'without ON CONFLICT / RETURNING' + end + put = Code.block("#{rows_var}.each do |row|", "client.put_copy_data([#{values.join(', ')}])") [ 'encoder = PG::TextEncoder::CopyRow.new(type_map: client.type_map_for_queries)', @@ -82,15 +87,17 @@ def copy_from_lines(rows_var, table, columns, values, _label) ] end - # bytea, json / jsonb (BasicTypeMapForQueries sends a Ruby Array as a PG array) and, without prepared - # statements (no server-side parameter types), arrays of String-typed values such as uuid[] / enum[] + # bytea, json / jsonb (BasicTypeMapForQueries sends a Ruby Array as a PG array), float (the pg gem's encoder + # drops digits) and arrays (the pg gem types an Array from its first element) def encode_param(expr, column) - helper = encoder_for(column) - helper ? "#{helper}(#{expr})" : expr + helper, kind = encoder_for(column) + return expr unless helper + + kind ? "#{helper}(#{expr}, :#{kind})" : "#{helper}(#{expr})" end def helper_methods(queries) - used = queries.flat_map { |q| q.params.map { |p| encoder_for(p.column) } }.compact.uniq + used = queries.flat_map { |q| q.params.map { |p| encoder_for(p.column)&.first } }.compact.uniq ENCODERS.select { |name, _| used.include?(name) }.values end @@ -107,13 +114,38 @@ def encode_json(value) value.is_a?(Array) ? PG::TextEncoder::JSON.new.encode(value) : value end RUBY - 'encode_text_array' => [<<~'RUBY'.chomp, 'def encode_text_array: (Array[untyped]? value) -> String?'] - # Without prepared statements an Array of Strings would be typed text[]; as an array literal the server - # converts it to the column type (uuid[], enum[], ...). - def encode_text_array(value) - value && PG::TextEncoder::Array.new(elements_type: PG::TextEncoder::String.new).encode(value) + 'encode_float' => [<<~'RUBY'.chomp, 'def encode_float: (untyped value) -> untyped'], + # Float#to_s is the shortest text that reads back as the same Float (the pg gem's encoder drops digits). + def encode_float(value) + value.is_a?(Float) ? value.to_s : value + end + RUBY + 'encode_array' => [<<~'RUBY'.chomp, <<~'RBS'.chomp] + # Array parameters are sent as array literals, which the server converts to the parameter's type (the pg gem + # would type them from their first element: wrong for [], [nil, ...], bytea[], json[]). + def encode_array(value, kind = :value) + return value unless value.is_a?(Array) + + PG::TextEncoder::Array.new(elements_type: PG::TextEncoder::String.new).encode(array_elements(value, kind)) + end + + def array_elements(values, kind) + values.map do |value| + case value + when nil then nil + when Array then kind == :json ? PG::TextEncoder::JSON.new.encode(value) : array_elements(value, kind) + when Hash then PG::TextEncoder::JSON.new.encode(value) + when String then kind == :bytea ? PG::TextEncoder::Bytea.new.encode(value) : value + when Time then PG::TextEncoder::TimestampWithTimeZone.new.encode(value) + else defined?(IPAddr) && value.is_a?(IPAddr) ? "#{value}/#{value.prefix}" : value.to_s + end + end end RUBY + def encode_array: (Array[untyped]? value, ?Symbol kind) -> untyped + + def array_elements: (Array[untyped] values, Symbol kind) -> Array[untyped] + RBS }.freeze CONNECTION_STATE = <<~'RUBY'.chomp @@ -168,6 +200,8 @@ def self.connect(pg_params, logger: nil, **pool_options) def transaction(isolation: nil) with_client('transaction') do |client| outermost = client.transaction_status == PG::PQTRANS_IDLE + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.exec(isolation ? "BEGIN ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}" : 'BEGIN') @@ -184,11 +218,12 @@ def transaction(isolation: nil) raise ensure unless failed + # a statement failed and the block rescued the error: PostgreSQL can only roll back + rescued = client.transaction_status == PG::PQTRANS_INERROR commit = outermost ? 'COMMIT' : "RELEASE SAVEPOINT #{savepoint}" - result = client.exec(Thread.current.status == 'aborting' ? rollback : commit) - # COMMIT of a transaction aborted by an error the block rescued is a ROLLBACK - rolled_back = outermost && result.cmd_status == 'ROLLBACK' && Thread.current.status != 'aborting' - raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rolled_back + client.exec(rescued || Thread.current.status == 'aborting' ? rollback : commit) + client.exec("RELEASE SAVEPOINT #{savepoint}") if rescued && savepoint + raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rescued && Thread.current.status != 'aborting' end end end @@ -199,6 +234,7 @@ def transaction(isolation: nil) def with_client(name) instrument(name) do @db.with do |client| + client.reset if client.status == PG::CONNECTION_BAD # closed by the server (restart, failover, killed) configure(client) unless client.is_a?(ConnectionState) yield client end @@ -212,7 +248,10 @@ def configure(client) type_map = PG::BasicTypeMapForResults.new(client) type_map.default_type_map = string_arrays(client) # uuid, enums, money, ... come back as Strings client.type_map_for_results = type_map - client.type_map_for_queries = PG::BasicTypeMapForQueries.new(client) + queries = PG::BasicTypeMapForQueries.new(client) + # true / false as text ('t' / 'f'), also valid in COPY (the default binary encoder is not) + queries[TrueClass] = queries[FalseClass] = PG::TextEncoder::Boolean.new(oid: 16) + client.type_map_for_queries = queries client.extend(ConnectionState) end @@ -238,9 +277,10 @@ def encoder_for(column) type = normalize_type(column.type.name) if column.is_array - 'encode_text_array' if !@prepared && !%w[bytea json jsonb].include?(type) && column_type(column) == 'String' - elsif type == 'bytea' then 'encode_bytea' - elsif %w[json jsonb].include?(type) then 'encode_json' + ['encode_array', ({ 'bytea' => 'bytea', 'json' => 'json', 'jsonb' => 'json' }[type])] + elsif type == 'bytea' then ['encode_bytea'] + elsif %w[json jsonb].include?(type) then ['encode_json'] + elsif column_type(column, fallback: nil) == 'Float' then ['encode_float'] end end diff --git a/lib/sqlc_gen_ruby/drivers/sqlite3.rb b/lib/sqlc_gen_ruby/drivers/sqlite3.rb index 5482496..9a963b3 100644 --- a/lib/sqlc_gen_ruby/drivers/sqlite3.rb +++ b/lib/sqlc_gen_ruby/drivers/sqlite3.rb @@ -67,11 +67,15 @@ def exec_last_id_lines(exec, query) end # One multi-row `INSERT ... VALUES (?, ?), (?, ?), ...` per batch. - def copy_from_lines(rows_var, table, columns, values, _label) + # The conflict clause is kept: INSERT OR REPLACE / OR IGNORE, REPLACE, ON CONFLICT ... DO ... (without parameters). + def copy_from_lines(rows_var, table, columns, values, _label, query_name:, sources:, verb:, tail:) + raise ArgumentError, "#{query_name}: :copyfrom takes no parameters after VALUES (...)" if tail.include?('?') + rows_per_batch = [MAX_BOUND_PARAMETERS / columns.size, 1].max placeholders = "(#{(['?'] * columns.size).join(', ')})" + tail = tail.empty? ? '' : " #{tail.gsub('\\') { '\\\\' }.gsub('#', '\\#')}" body = [ - "sql = %(INSERT INTO #{table} (#{columns.join(', ')}) VALUES \#{Array.new(batch.size, '#{placeholders}').join(', ')})", + "sql = %(#{verb} INTO #{table} (#{columns.join(', ')}) VALUES \#{Array.new(batch.size, '#{placeholders}').join(', ')}#{tail})", "client.execute(sql, batch.flat_map { |row| [#{values.join(', ')}] })" ].join("\n") [Code.block('atomically(client) do', Code.block("#{rows_var}.each_slice(#{rows_per_batch}) do |batch|", body)), 'nil'] @@ -98,6 +102,8 @@ def transaction(mode: nil) instrument('transaction') do @lock.synchronize do outermost = !@db.transaction_active? + raise ArgumentError, 'mode: cannot be set on a nested transaction' if mode && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" @db.execute(outermost ? "BEGIN#{" #{mode.to_s.upcase}" if mode}" : "SAVEPOINT #{savepoint}") rollback = outermost ? 'ROLLBACK' : "ROLLBACK TO SAVEPOINT #{savepoint}" diff --git a/lib/sqlc_gen_ruby/method_gen.rb b/lib/sqlc_gen_ruby/method_gen.rb index 3a5dddf..968447d 100644 --- a/lib/sqlc_gen_ruby/method_gen.rb +++ b/lib/sqlc_gen_ruby/method_gen.rb @@ -69,7 +69,9 @@ def code(query) end doc = query.comments.map { |c| "# #{c.strip}".rstrip }.join("\n") method = Code.block("def #{func}#{signature}", method_body(func, statements)) - method = Code.without_cops(method, Code.naming_cops(params: params.map(&:name))) + # :copyfrom / :batch* take only `rows`; the Args class gets its own exemptions + method_params = ROWS_COMMANDS.include?(query.cmd) ? [] : params.map(&:name) + method = Code.without_cops(method, Code.naming_cops(params: method_params)) method = "#{doc}\n#{method}" unless doc.empty? query.cmd == ':one' ? "#{method}\n\n#{bang_method(query, func)}" : method end @@ -153,8 +155,38 @@ def copy_statements(query, params) name = table.schema.to_s.empty? ? '' : "#{@driver.quote_identifier(table.schema)}." name += @driver.quote_identifier(table.name) values = params.map { |p| @driver.encode_param("row.#{p.name}", p.column) } + verb, tail = insert_clauses(query.text) @driver.copy_from_lines('rows', name, params.map { |p| @driver.quote_identifier(p.column.name) }, values, - Code.snake_case(query.name)) + Code.snake_case(query.name), + query_name: query.name, sources: params.map(&:column), verb: verb, tail: tail) + end + + # `INSERT OR REPLACE INTO t (a) VALUES (?) ON CONFLICT (a) DO NOTHING;` => ['INSERT OR REPLACE', 'ON CONFLICT (a) DO NOTHING'] + def insert_clauses(sql) + sql = sql.gsub(%r{--[^\n]*|/\*.*?\*/}m, ' ') + verb = sql[/\A\s*(.*?)\s+INTO\b/im, 1].to_s.split.join(' ').upcase + values = sql =~ /\bVALUES\s*\(/i + tail = values ? after_parenthesis(sql, sql.index('(', values)) : '' + [verb, tail.strip.sub(/;\z/, '').strip] + end + + # The text after the parenthesis opened at `start` is closed (quotes skipped). + def after_parenthesis(sql, start) + depth = 0 + quote = nil + (start...sql.size).each do |i| + char = sql[i] + if quote + quote = nil if char == quote + elsif %w[' " `].include?(char) + quote = char + elsif char == '(' + depth += 1 + elsif char == ')' && (depth -= 1).zero? + return sql[(i + 1)..] + end + end + '' end def query_statements(query, func, params) @@ -187,15 +219,24 @@ def batch_statements(query, func, params) const = Code.sql_constant(query.name) stmt_name = "#{@statement_prefix}_#{func}" - statements = [@driver.prepare_stmt(stmt_name, const)].compact + prepare = @driver.prepare_stmt(stmt_name, const) cmd = BATCH_COMMANDS.fetch(query.cmd) body = [] body << "query_params = [#{Code.comma_join(params.map { |p| @driver.encode_param("args.#{p.name}", p.column) })}]" unless params.empty? - body.concat(command_statements(query, stmt_name, const, !params.empty?, cmd: cmd)) - body.pop if cmd == ':exec' # the trailing nil + commands = command_statements(query, stmt_name, const, !params.empty?, cmd: cmd) + commands.pop if cmd == ':exec' # the trailing nil + if @driver.prepares_batch_rows? + body << prepare + body << Code.block('begin', commands.join("\n")).sub(/\nend\z/, "\nensure\n stmt.close\nend") + else + body.concat(commands) + end block_arg = params.empty? ? '_args' : 'args' loop = Code.block("rows.#{cmd == ':exec' ? 'each' : 'map'} do |#{block_arg}|", body.join("\n")) commands = cmd == ':exec' ? [loop, 'nil'] : [loop] + return commands if @driver.prepares_batch_rows? + + statements = [prepare].compact if @driver.closes_statements? statements << Code.block('begin', commands.join("\n")).sub(/\nend\z/, "\nensure\n stmt.close\nend") else diff --git a/steep_expectations.yml b/steep_expectations.yml index e6d8dc6..3aca857 100644 --- a/steep_expectations.yml +++ b/steep_expectations.yml @@ -3,40 +3,50 @@ diagnostics: - range: start: - line: 259 + line: 310 character: 10 end: - line: 259 + line: 310 character: 24 severity: WARNING message: 'Cannot find the declaration of constant: `ConnectionPool`' code: Ruby::UnknownConstant - range: start: - line: 259 + line: 310 character: 47 end: - line: 259 + line: 310 character: 53 severity: WARNING message: 'Cannot find the declaration of constant: `Mysql2`' code: Ruby::UnknownConstant - range: start: - line: 259 + line: 310 character: 66 end: - line: 259 + line: 310 character: 81 severity: WARNING message: Empty kwargs doesn't have type annotation code: Ruby::UnannotatedEmptyCollection - range: start: - line: 855 + line: 989 + character: 17 + end: + line: 989 + character: 23 + severity: WARNING + message: 'Cannot find the declaration of constant: `Mysql2`' + code: Ruby::UnknownConstant + - range: + start: + line: 1022 character: 16 end: - line: 855 + line: 1022 character: 22 severity: WARNING message: 'Cannot find the declaration of constant: `Mysql2`' @@ -45,161 +55,261 @@ diagnostics: - range: start: - line: 353 + line: 408 character: 10 end: - line: 353 + line: 408 character: 24 severity: WARNING message: 'Cannot find the declaration of constant: `ConnectionPool`' code: Ruby::UnknownConstant - range: start: - line: 353 + line: 408 character: 47 end: - line: 353 + line: 408 character: 49 severity: WARNING message: 'Cannot find the declaration of constant: `PG`' code: Ruby::UnknownConstant - range: start: - line: 353 + line: 408 character: 58 end: - line: 353 + line: 408 character: 69 severity: WARNING message: Empty kwargs doesn't have type annotation code: Ruby::UnannotatedEmptyCollection - range: start: - line: 368 + line: 423 character: 49 end: - line: 368 + line: 423 + character: 51 + severity: WARNING + message: 'Cannot find the declaration of constant: `PG`' + code: Ruby::UnknownConstant + - range: + start: + line: 443 character: 51 + end: + line: 443 + character: 53 severity: WARNING message: 'Cannot find the declaration of constant: `PG`' code: Ruby::UnknownConstant - range: start: - line: 389 + line: 447 character: 18 end: - line: 389 + line: 447 character: 20 severity: WARNING message: 'Cannot find the declaration of constant: `PG`' code: Ruby::UnknownConstant - range: start: - line: 581 + line: 639 character: 18 end: - line: 581 + line: 639 character: 20 severity: WARNING message: 'Cannot find the declaration of constant: `PG`' code: Ruby::UnknownConstant - range: start: - line: 987 + line: 1045 character: 18 end: - line: 987 + line: 1045 character: 20 severity: WARNING message: 'Cannot find the declaration of constant: `PG`' code: Ruby::UnknownConstant - range: start: - line: 1068 + line: 1126 + character: 18 + end: + line: 1126 + character: 20 + severity: WARNING + message: 'Cannot find the declaration of constant: `PG`' + code: Ruby::UnknownConstant + - range: + start: + line: 1204 + character: 43 + end: + line: 1204 + character: 45 + severity: WARNING + message: 'Cannot find the declaration of constant: `PG`' + code: Ruby::UnknownConstant + - range: + start: + line: 1213 character: 17 end: - line: 1068 + line: 1213 character: 19 severity: WARNING message: 'Cannot find the declaration of constant: `PG`' code: Ruby::UnknownConstant - range: start: - line: 1071 - character: 36 + line: 1216 + character: 16 end: - line: 1071 - character: 38 + line: 1216 + character: 18 + severity: WARNING + message: 'Cannot find the declaration of constant: `PG`' + code: Ruby::UnknownConstant + - range: + start: + line: 1218 + character: 49 + end: + line: 1218 + character: 51 severity: WARNING message: 'Cannot find the declaration of constant: `PG`' code: Ruby::UnknownConstant - range: start: - line: 1077 + line: 1225 character: 16 end: - line: 1077 + line: 1225 character: 18 severity: WARNING message: 'Cannot find the declaration of constant: `PG`' code: Ruby::UnknownConstant - range: start: - line: 1078 + line: 1226 character: 15 end: - line: 1078 + line: 1226 character: 17 severity: WARNING message: 'Cannot find the declaration of constant: `PG`' code: Ruby::UnknownConstant - range: start: - line: 1079 + line: 1227 character: 79 end: - line: 1079 + line: 1227 character: 81 severity: WARNING message: Empty array doesn't have type annotation code: Ruby::UnannotatedEmptyCollection - range: start: - line: 1080 + line: 1228 character: 25 end: - line: 1080 + line: 1228 character: 27 severity: WARNING message: 'Cannot find the declaration of constant: `PG`' code: Ruby::UnknownConstant - range: start: - line: 1081 + line: 1229 character: 67 end: - line: 1081 + line: 1229 character: 69 severity: WARNING message: 'Cannot find the declaration of constant: `PG`' code: Ruby::UnknownConstant - range: start: - line: 1089 + line: 1237 character: 15 end: - line: 1089 + line: 1237 character: 17 severity: WARNING message: 'Cannot find the declaration of constant: `PG`' code: Ruby::UnknownConstant - range: start: - line: 1094 + line: 1242 character: 27 end: - line: 1094 + line: 1242 character: 29 severity: WARNING message: 'Cannot find the declaration of constant: `PG`' code: Ruby::UnknownConstant + - range: + start: + line: 1255 + character: 6 + end: + line: 1255 + character: 8 + severity: WARNING + message: 'Cannot find the declaration of constant: `PG`' + code: Ruby::UnknownConstant + - range: + start: + line: 1255 + character: 48 + end: + line: 1255 + character: 50 + severity: WARNING + message: 'Cannot find the declaration of constant: `PG`' + code: Ruby::UnknownConstant + - range: + start: + line: 1262 + character: 40 + end: + line: 1262 + character: 42 + severity: WARNING + message: 'Cannot find the declaration of constant: `PG`' + code: Ruby::UnknownConstant + - range: + start: + line: 1263 + character: 23 + end: + line: 1263 + character: 25 + severity: WARNING + message: 'Cannot find the declaration of constant: `PG`' + code: Ruby::UnknownConstant + - range: + start: + line: 1264 + character: 42 + end: + line: 1264 + character: 44 + severity: WARNING + message: 'Cannot find the declaration of constant: `PG`' + code: Ruby::UnknownConstant + - range: + start: + line: 1265 + character: 23 + end: + line: 1265 + character: 25 + severity: WARNING + message: 'Cannot find the declaration of constant: `PG`' + code: Ruby::UnknownConstant diff --git a/test/features_test.rb b/test/features_test.rb index f41eee7..359281f 100644 --- a/test/features_test.rb +++ b/test/features_test.rb @@ -1,6 +1,7 @@ # frozen_string_literal: true require_relative 'test_helper' +require 'sqlc_gen_ruby/code' # Generator behaviour for the sqlc features the example schemas exercise (details of the runtime behaviour of the # generated code are covered by the end-to-end tests under tests/). @@ -424,10 +425,10 @@ def test_pg_encodes_bytea_json_and_string_arrays rb = generate(load_request('pg')).fetch('query_sql.rb') assert_includes rb, 'encode_bytea(payload),' assert_includes rb, 'encode_json(doc),' - refute_includes rb, 'encode_text_array' + assert_includes rb, 'encode_array(bio_types)' + assert_includes rb, 'def encode_array(value, kind = :value)' noprep = generate(load_request('pg', preparedStatements: false)).fetch('query_sql.rb') - assert_includes noprep, 'encode_text_array(bio_types)' - assert_includes noprep, 'def encode_text_array(value)' + assert_includes noprep, 'encode_array(bio_types)' end def test_transactions_finish_on_non_local_exits @@ -462,6 +463,49 @@ def test_sqlite_json_and_uuid_declarations_are_strings generate(request).fetch('query_sql.rbs')) end + def test_snake_case_keeps_acronyms_together + { 'MIN_ID' => 'min_id', 'ABC' => 'abc', 'HTTPServer' => 'http_server', 'UserId' => 'user_id', + 'getHTTPResponse2Code' => 'get_http_response2_code', 'authorID' => 'author_id' }.each do |name, snake| + assert_equal snake, SqlcGenRuby::Code.snake_case(name) + end + end + + def test_copyfrom_keeps_or_rejects_conflict_clauses + sqlite = generate(load_request('sqlite3')).fetch('query_sql.rb') + assert_includes sqlite, 'INSERT OR REPLACE INTO "tags"' + assert_includes sqlite, 'ON CONFLICT (name) DO UPDATE SET uses = tags.uses + excluded.uses' + mysql = generate(load_request('mysql2')).fetch('query_sql.rb') + assert_includes mysql, "IGNORE INTO TABLE `copy_types`" + assert_includes mysql, "REPLACE INTO TABLE `copy_types`" + { 'pg' => ' ON CONFLICT DO NOTHING', 'mysql2' => ' ON DUPLICATE KEY UPDATE bio = bio', + 'sqlite3' => ' ON CONFLICT (name) DO UPDATE SET bio = ?' }.each do |driver, tail| + request = load_request(driver) + query(request, 'CopyAuthors').text += tail + error = assert_raises(StandardError, driver) { generate(request) } + assert_includes error.message, 'CopyAuthors', driver + end + end + + def test_nested_transactions_reject_isolation_and_mode + { 'pg' => 'isolation:', 'mysql2' => 'isolation:', 'sqlite3' => 'mode:' }.each do |driver, option| + assert_includes generate(load_request(driver)).fetch('query_sql.rb'), + "raise ArgumentError, '#{option} cannot be set on a nested transaction'" + end + end + + def test_pg_encodes_float_and_array_parameters_as_text + rb = generate(load_request('pg')).fetch('query_sql.rb') + assert_includes rb, 'encode_float(args.ratio)' + assert_includes rb, 'encode_array(args.blobs, :bytea)' + assert_includes rb, 'encode_array(args.docs, :json)' + assert_includes rb, 'queries[TrueClass] = queries[FalseClass] = PG::TextEncoder::Boolean.new(oid: 16)' + end + + def test_mysql_copyfrom_converts_json_columns_from_binary + rb = generate(load_request('mysql2')).fetch('query_sql.rb') + assert_includes rb, '(`id`, `code`, @sqlc_v2, `flags`, `amount`) SET `doc` = CONVERT(@sqlc_v2 USING utf8mb4)' + end + def test_schema_names_breaking_naming_cops_are_exempted_where_they_occur request = load_request('pg') get_author = query(request, 'GetAuthor') diff --git a/test/fixtures/mysql2.bin b/test/fixtures/mysql2.bin index ad0f6e3..7e15263 100644 --- a/test/fixtures/mysql2.bin +++ b/test/fixtures/mysql2.bin @@ -1,7 +1,7 @@ -2mysqlÍpublic"Âpublic€ +2mysql£public"˜public€ authors& id0ÿÿÿÿÿÿÿÿÿR authorsbbigint& name0ÿÿÿÿÿÿÿÿÿR authorsbtext# @@ -75,7 +75,19 @@ c_longtext0 id0ÿÿÿÿÿÿÿÿÿR  regressionsbbigint& order0ÿR  regressionsb varchar( note0ÿÿÿÿÿÿÿÿÿR  regressionsbtext' -payload0R  regressionsb  varbinary": +payload0R  regressionsb  varbinaryÓ +  +copy_types) +id0ÿÿÿÿÿÿÿÿÿR  +copy_typesbbigint# +code0 R  +copy_typesb varchar& +doc0ÿÿÿÿÿÿÿÿÿR  +copy_typesbjson +flags0R  +copy_typesbbit+ +amount0ÿÿÿÿÿÿÿÿÿR  +copy_typesbint€": extended_bios_bio_type Autobiography BiographyMemoir"7 extended_bios_author_typeAuthorEditor TranslatorË @@ -281,4 +293,57 @@ HINSERT INTO regressions (id, `order`, note, payload) VALUES (?, ?, ?, ?)CopyR /SELECT id, payload FROM regressions ORDER BY idListRegressionPayloads:many". id0ÿÿÿÿÿÿÿÿÿR  regressionsbbigintzid"0 payload0R  regressionsb  varbinaryzpayload: query.sqlC -TRUNCATE TABLE regressionsTruncateRegressions:exec: query.sql"v1.27.0*T{"driver":"mysql2","generateGemfile":false,"generateTypes":true,"rubyVersion":"3.3"} \ No newline at end of file +TRUNCATE TABLE regressionsTruncateRegressions:exec: query.sql  +LINSERT INTO copy_types (id, code, doc, flags, amount) VALUES (?, ?, ?, ?, ?) CopyCopyTypes :copyfrom*95 +id0ÿÿÿÿÿÿÿÿÿRpublic +copy_typesbbigintzid*51 +code0 Rpublic +copy_typesb varcharzcode*73 +doc0ÿÿÿÿÿÿÿÿÿRpublic +copy_typesbjsonzdoc*1- +flags0Rpublic +copy_typesbbitzflags*?; +amount0ÿÿÿÿÿÿÿÿÿRpublic +copy_typesbintzamount€: query.sqlB  +copy_typesã +6INSERT IGNORE INTO copy_types (id, code) VALUES (?, ?)CopyCopyTypesIgnore :copyfrom*95 +id0ÿÿÿÿÿÿÿÿÿRpublic +copy_typesbbigintzid*51 +code0 Rpublic +copy_typesb varcharzcode: query.sqlB  +copy_typesÞ +0REPLACE INTO copy_types (id, code) VALUES (?, ?)CopyCopyTypesReplace :copyfrom*95 +id0ÿÿÿÿÿÿÿÿÿRpublic +copy_typesbbigintzid*51 +code0 Rpublic +copy_typesb varcharzcode: query.sqlB  +copy_typesç +DINSERT INTO copy_types (id, code, flags, amount) VALUES (?, ?, ?, ?)CreateCopyTypesBatch +:batchexec*95 +id0ÿÿÿÿÿÿÿÿÿRpublic +copy_typesbbigintzid*51 +code0 Rpublic +copy_typesb varcharzcode*1- +flags0Rpublic +copy_typesbbitzflags*?; +amount0ÿÿÿÿÿÿÿÿÿRpublic +copy_typesbintzamount€: query.sqlB  +copy_typesö +ESELECT count(*) AS matches FROM copy_types WHERE code = ? OR code = ?CountCopyTypesByCodes :batchone"" +matches0ÿÿÿÿÿÿÿÿÿ@bbigint*-) +code0 R  +copy_typesb varcharzcode*-) +code0 R  +copy_typesb varcharzcode: query.sqlÅ +?SELECT id, code, doc, flags, amount FROM copy_types ORDER BY id ListCopyTypes:many"- +id0ÿÿÿÿÿÿÿÿÿR  +copy_typesbbigintzid") +code0 R  +copy_typesb varcharzcode"+ +doc0ÿÿÿÿÿÿÿÿÿR  +copy_typesbjsonzdoc"% +flags0R  +copy_typesbbitzflags"3 +amount0ÿÿÿÿÿÿÿÿÿR  +copy_typesbintzamount€: query.sql@ +TRUNCATE TABLE copy_typesTruncateCopyTypes:exec: query.sql"v1.27.0*T{"driver":"mysql2","generateGemfile":false,"generateTypes":true,"rubyVersion":"3.3"} \ No newline at end of file diff --git a/test/fixtures/pg.bin b/test/fixtures/pg.bin index 5c89375..762b6bc 100644 --- a/test/fixtures/pg.bin +++ b/test/fixtures/pg.bin @@ -1,7 +1,7 @@  2 -postgresql‹public"ÉpublicÍ +postgresqlËpublic"‰publicÍ authors) id0ÿÿÿÿÿÿÿÿÿR authorsb  bigserial& name0ÿÿÿÿÿÿÿÿÿR authorsbtextH @@ -81,7 +81,20 @@ pg_catalogint8+ payload0ÿÿÿÿÿÿÿÿÿR  regressionsbbytea( doc0ÿÿÿÿÿÿÿÿÿR  regressionsbjsonb@ bio_types 0ÿÿÿÿÿÿÿÿÿR  regressionsbextendedbio_typeˆ- -uids 0ÿÿÿÿÿÿÿÿÿR  regressionsbuuidˆ"´extendedÔ +uids 0ÿÿÿÿÿÿÿÿÿR  regressionsbuuidˆ½ +  batch_types4 +id0ÿÿÿÿÿÿÿÿÿR  batch_typesb +pg_catalogint82 +ok0ÿÿÿÿÿÿÿÿÿR  batch_typesb +pg_catalogbool+ +ratio0ÿÿÿÿÿÿÿÿÿR  batch_typesbfloat8/ +blobs 0ÿÿÿÿÿÿÿÿÿR  batch_typesbbyteaˆ. +docs 0ÿÿÿÿÿÿÿÿÿR  batch_typesbjsonbˆ; +counts 0ÿÿÿÿÿÿÿÿÿR  batch_typesb +pg_catalogint4ˆ? +amounts 0ÿÿÿÿÿÿÿÿÿR  batch_typesb +pg_catalognumericˆ6 +stamps 0ÿÿÿÿÿÿÿÿÿR  batch_typesb  timestamptzˆ"´extendedÔ extendedbiosC author_name0ÿÿÿÿÿÿÿÿÿRextendedbiosb pg_catalogvarchar< @@ -372,4 +385,40 @@ pg_catalogint8zid"2 uids 0ÿÿÿÿÿÿÿÿÿR  regressionsbuuidzuidsˆ*# order_name0ÿÿÿÿÿÿÿÿÿbtext: query.sqlC -TRUNCATE TABLE regressionsTruncateRegressions:exec: query.sql"v1.27.0*P{"driver":"pg","generateGemfile":false,"generateTypes":true,"rubyVersion":"3.3"} \ No newline at end of file +TRUNCATE TABLE regressionsTruncateRegressions:exec: query.sqló +uINSERT INTO batch_types (id, ok, ratio, blobs, docs, counts, amounts, stamps) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)CreateBatchTypesBatch +:batchexec*C? +id0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbpg_catalog.int8zid*A= +ok0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbpg_catalog.boolzok*>: +ratio0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbfloat8zratio*B> +blobs 0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbbyteazblobsˆ*@< +docs 0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbjsonbzdocsˆ*NJ +counts 0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbpg_catalog.int4zcountsˆ*SO +amounts 0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbpg_catalog.numericzamountsˆ*JF +stamps 0ÿÿÿÿÿÿÿÿÿRpublic batch_typesb  timestamptzzstampsˆ: query.sqlB  batch_typesë +uINSERT INTO batch_types (id, ok, ratio, blobs, docs, counts, amounts, stamps) VALUES ($1, $2, $3, $4, $5, $6, $7, $8)CopyBatchTypes :copyfrom*C? +id0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbpg_catalog.int8zid*A= +ok0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbpg_catalog.boolzok*>: +ratio0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbfloat8zratio*B> +blobs 0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbbyteazblobsˆ*@< +docs 0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbjsonbzdocsˆ*NJ +counts 0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbpg_catalog.int4zcountsˆ*SO +amounts 0ÿÿÿÿÿÿÿÿÿRpublic batch_typesbpg_catalog.numericzamountsˆ*JF +stamps 0ÿÿÿÿÿÿÿÿÿRpublic batch_typesb  timestamptzzstampsˆ: query.sqlB  batch_typesÞ +WSELECT id, ok, ratio, blobs, docs, counts, amounts, stamps FROM batch_types ORDER BY idListBatchTypes:many"8 +id0ÿÿÿÿÿÿÿÿÿR  batch_typesb +pg_catalogint8zid"6 +ok0ÿÿÿÿÿÿÿÿÿR  batch_typesb +pg_catalogboolzok"2 +ratio0ÿÿÿÿÿÿÿÿÿR  batch_typesbfloat8zratio"6 +blobs 0ÿÿÿÿÿÿÿÿÿR  batch_typesbbyteazblobsˆ"4 +docs 0ÿÿÿÿÿÿÿÿÿR  batch_typesbjsonbzdocsˆ"C +counts 0ÿÿÿÿÿÿÿÿÿR  batch_typesb +pg_catalogint4zcountsˆ"H +amounts 0ÿÿÿÿÿÿÿÿÿR  batch_typesb +pg_catalognumericzamountsˆ"> +stamps 0ÿÿÿÿÿÿÿÿÿR  batch_typesb  timestamptzzstampsˆ: query.sql¼ +.SELECT count(*) FROM batch_types WHERE id > $1CountBatchTypesAbove:one" +count0ÿÿÿÿÿÿÿÿÿ@bbigint*A= +min_id0ÿÿÿÿÿÿÿÿÿ8R  batch_typesbpg_catalog.int8zid: query.sqlB +TRUNCATE TABLE batch_typesTruncateBatchTypes:exec: query.sql"v1.27.0*P{"driver":"pg","generateGemfile":false,"generateTypes":true,"rubyVersion":"3.3"} \ No newline at end of file diff --git a/test/fixtures/sqlite3.bin b/test/fixtures/sqlite3.bin index e50bb15826c78e727e8ea27ecaaf30ba8efb9c39..c2e55dca9966081216d1d853bab214e233ec8081 100644 GIT binary patch delta 985 zcmaFncGXXUi<^tlNQkYtFekGlRp>3d5KC@iW}ecMjS~IL+@V}-QYBU0IlNDJ+ zY@oa(ARXcw5hA3<#Zp?FS}eh6@E-;kgJ6m{rPw|FLR{TlgQU(fahXl_7gOAPhS^F! zUg{bnmzk$uuxn6=f`5=gkZXXCqqD05P_e%P$T9_uyu{p89R-llnhIf#KB2C`3L5r0 z3ig^pg3kE`l_5Yy{zXBl1v!bysZyL)$w0BRqWoMfV-7|keUMA*c_99PxE|)-Di)Bz zT4o$fLPpqJj;aaa{wlC9tvCxyQ;RC~fI;O1^`q1kMlMr?PdwfI{DZ(AaUtvz0fbLH z)ARC+2G%2eOkBB8kD$auJ&uS_@b^=2_V;u1@pKM>#FwT5(9@v-E{-9t3c;=+;Mh~J z1xKhJC?*xO6;dmbb4pWEQ^0&7Ay@!7rle%%rH2C5V8kWYWF;11dPU{}MlQ`@S07iP zj}Qq^!7WI`-xrcD{exUwgA|-1!O}t;KAFWOprk0pYL%OqSE-~4PI<8856vTxG^eD4 wExo~%;z?;zMqC0eKsNyc)h!6>4k4b9qSCzN#FA8~?Wq;1$*|NpnOQj*0Qpfh)c^nh delta 42 ycmccW_smU#i<^tlNQkYtFekGlRcIQ!5KC@iW}Z^-Mu~pr&HtF=<$ void end + + class CopyType + attr_reader id: Integer + attr_reader code: String + attr_reader doc: String? + attr_reader flags: String? + attr_reader amount: Integer? + def initialize: ( + Integer id, + String code, + String? doc, + String? flags, + Integer? amount + ) -> void + end end diff --git a/test/golden/mysql2/default/my_queries_sql.rb b/test/golden/mysql2/default/my_queries_sql.rb index 2f677b4..7e3b3a8 100644 --- a/test/golden/mysql2/default/my_queries_sql.rb +++ b/test/golden/mysql2/default/my_queries_sql.rb @@ -35,6 +35,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = !client.in_transaction + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.query("SET TRANSACTION ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}") if isolation @@ -95,7 +97,13 @@ def with_client(name) instrument(name) do @db.with do |client| client.extend(ConnectionState) unless client.is_a?(ConnectionState) - yield client + begin + yield client + rescue Mysql2::Error + # a connection closed by the server (restart, KILL, wait_timeout) is replaced by the pool + @db.discard_current_connection if client.closed? && @db.respond_to?(:discard_current_connection) + raise + end end end end diff --git a/test/golden/mysql2/default/other_sql.rb b/test/golden/mysql2/default/other_sql.rb index 3d5d7c8..c067da1 100644 --- a/test/golden/mysql2/default/other_sql.rb +++ b/test/golden/mysql2/default/other_sql.rb @@ -48,6 +48,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = !client.in_transaction + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.query("SET TRANSACTION ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}") if isolation @@ -145,7 +147,13 @@ def with_client(name) instrument(name) do @db.with do |client| client.extend(ConnectionState) unless client.is_a?(ConnectionState) - yield client + begin + yield client + rescue Mysql2::Error + # a connection closed by the server (restart, KILL, wait_timeout) is replaced by the pool + @db.discard_current_connection if client.closed? && @db.respond_to?(:discard_current_connection) + raise + end end end end diff --git a/test/golden/mysql2/default/query_sql.rb b/test/golden/mysql2/default/query_sql.rb index 6ad9117..d49bcfa 100644 --- a/test/golden/mysql2/default/query_sql.rb +++ b/test/golden/mysql2/default/query_sql.rb @@ -249,6 +249,57 @@ class ListRegressionPayloadsRow < Data.define(:id, :payload) TRUNCATE TABLE regressions SQL + class CopyCopyTypesArgs < Data.define( + :id, + :code, + :doc, + :flags, + :amount + ) + end + + class CopyCopyTypesIgnoreArgs < Data.define(:id, :code) + end + + class CopyCopyTypesReplaceArgs < Data.define(:id, :code) + end + + CREATE_COPY_TYPES_BATCH_SQL = <<~'SQL' + INSERT INTO copy_types (id, code, flags, amount) VALUES (?, ?, ?, ?) + SQL + + class CreateCopyTypesBatchArgs < Data.define(:id, :code, :flags, :amount) + end + + COUNT_COPY_TYPES_BY_CODES_SQL = <<~'SQL' + SELECT count(*) AS matches FROM copy_types WHERE code = ? OR code = ? + SQL + + class CountCopyTypesByCodesRow < Data.define(:matches) + end + + # rubocop:disable Naming/VariableNumber + class CountCopyTypesByCodesArgs < Data.define(:code, :code_2) + end + # rubocop:enable Naming/VariableNumber + + LIST_COPY_TYPES_SQL = <<~'SQL' + SELECT id, code, doc, flags, amount FROM copy_types ORDER BY id + SQL + + class ListCopyTypesRow < Data.define( + :id, + :code, + :doc, + :flags, + :amount + ) + end + + TRUNCATE_COPY_TYPES_SQL = <<~'SQL' + TRUNCATE TABLE copy_types + SQL + MANY_PARAMS_SQL = <<~'SQL' SELECT 1 -- ? ? WHERE a = ? @@ -302,6 +353,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = !client.in_transaction + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.query("SET TRANSACTION ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}") if isolation @@ -699,48 +752,48 @@ def create_author_result(name:, bio:) def create_authors_batch(rows) with_client('create_authors_batch') do |client| - stmt = client.prepare(CREATE_AUTHORS_BATCH_SQL) - begin - rows.each do |args| - query_params = [args.name, args.bio] + rows.each do |args| + query_params = [args.name, args.bio] + stmt = client.prepare(CREATE_AUTHORS_BATCH_SQL) + begin stmt.execute(*query_params) + ensure + stmt.close end - nil - ensure - stmt.close end + nil end end def get_author_batch(rows) with_client('get_author_batch') do |client| - stmt = client.prepare(GET_AUTHOR_BATCH_SQL) - begin - rows.map do |args| - query_params = [args.id] + rows.map do |args| + query_params = [args.id] + stmt = client.prepare(GET_AUTHOR_BATCH_SQL) + begin result = stmt.execute(*query_params, as: :array) row = result.first row && GetAuthorBatchRow.new(row[0], row[1], row[2]) + ensure + stmt.close end - ensure - stmt.close end end end def list_authors_by_name_batch(rows) with_client('list_authors_by_name_batch') do |client| - stmt = client.prepare(LIST_AUTHORS_BY_NAME_BATCH_SQL) - begin - rows.map do |args| - query_params = [args.name] + rows.map do |args| + query_params = [args.name] + stmt = client.prepare(LIST_AUTHORS_BY_NAME_BATCH_SQL) + begin result = stmt.execute(*query_params, as: :array) result.map do |row| ListAuthorsByNameBatchRow.new(row[0], row[1], row[2]) end + ensure + stmt.close end - ensure - stmt.close end end end @@ -835,6 +888,114 @@ def truncate_regressions end end + def copy_copy_types(rows) + with_client('copy_copy_types') do |client| + return if rows.empty? + + Tempfile.create(['copy_copy_types', '.csv']) do |file| + rows.each do |row| + file.write(csv_line([row.id, row.code, row.doc, row.flags, row.amount])) + end + file.flush + load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' INTO TABLE `copy_types` CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (`id`, `code`, @sqlc_v2, `flags`, `amount`) SET `doc` = CONVERT(@sqlc_v2 USING utf8mb4)") + end + nil + end + end + + def copy_copy_types_ignore(rows) + with_client('copy_copy_types_ignore') do |client| + return if rows.empty? + + Tempfile.create(['copy_copy_types_ignore', '.csv']) do |file| + rows.each do |row| + file.write(csv_line([row.id, row.code])) + end + file.flush + load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' IGNORE INTO TABLE `copy_types` CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (`id`, `code`)", strict: false) + end + nil + end + end + + def copy_copy_types_replace(rows) + with_client('copy_copy_types_replace') do |client| + return if rows.empty? + + Tempfile.create(['copy_copy_types_replace', '.csv']) do |file| + rows.each do |row| + file.write(csv_line([row.id, row.code])) + end + file.flush + load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' REPLACE INTO TABLE `copy_types` CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (`id`, `code`)") + end + nil + end + end + + def create_copy_types_batch(rows) + with_client('create_copy_types_batch') do |client| + rows.each do |args| + query_params = [args.id, args.code, args.flags, args.amount] + stmt = client.prepare(CREATE_COPY_TYPES_BATCH_SQL) + begin + stmt.execute(*query_params) + ensure + stmt.close + end + end + nil + end + end + + def count_copy_types_by_codes(rows) + with_client('count_copy_types_by_codes') do |client| + rows.map do |args| + query_params = [args.code, args.code_2] + stmt = client.prepare(COUNT_COPY_TYPES_BY_CODES_SQL) + begin + result = stmt.execute(*query_params, as: :array) + row = result.first + row && CountCopyTypesByCodesRow.new(row[0]) + ensure + stmt.close + end + end + end + end + + def list_copy_types + with_client('list_copy_types') do |client| + stmt = client.prepare(LIST_COPY_TYPES_SQL) + begin + result = stmt.execute(as: :array) + result.map do |row| + ListCopyTypesRow.new( + row[0], + row[1], + row[2], + row[3], + row[4] + ) + end + ensure + stmt.close + end + end + end + + def truncate_copy_types + with_client('truncate_copy_types') do |client| + stmt = client.prepare(TRUNCATE_COPY_TYPES_SQL) + begin + stmt.execute + nil + ensure + stmt.close + end + end + end + def many_params(c0:, c1:, c2:, c3:, c4:) with_client('many_params') do |client| query_params = [ @@ -913,7 +1074,13 @@ def with_client(name) instrument(name) do @db.with do |client| client.extend(ConnectionState) unless client.is_a?(ConnectionState) - yield client + begin + yield client + rescue Mysql2::Error + # a connection closed by the server (restart, KILL, wait_timeout) is replaced by the pool + @db.discard_current_connection if client.closed? && @db.respond_to?(:discard_current_connection) + raise + end end end end @@ -932,15 +1099,15 @@ def csv_line(values) "#{fields.join(',')}\n" end - # Runs LOAD DATA in a transaction (a savepoint inside #transaction): all rows or none, and an error instead - # of the warnings LOAD DATA LOCAL gives for duplicate keys and invalid values. - def load_data(client, sql) + # Runs LOAD DATA in a transaction (a savepoint inside #transaction): all rows or none, and (strict) an error + # instead of the warnings LOAD DATA LOCAL gives for duplicate keys and invalid values. + def load_data(client, sql, strict: true) outermost = !client.in_transaction client.query(outermost ? 'BEGIN' : 'SAVEPOINT sqlc_copy') done = false begin client.query(sql) - unless client.warning_count.zero? + if strict && !client.warning_count.zero? warnings = client.query('SHOW WARNINGS', as: :array).reject { |level, *| level == 'Note' } raise Mysql2::Error, "LOAD DATA: #{warnings.map { |w| w[2] }.join('; ')}" unless warnings.empty? end diff --git a/test/golden/mysql2/default/query_sql.rbs b/test/golden/mysql2/default/query_sql.rbs index e38d25c..b734145 100644 --- a/test/golden/mysql2/default/query_sql.rbs +++ b/test/golden/mysql2/default/query_sql.rbs @@ -273,6 +273,75 @@ module Mysql2Codegen TRUNCATE_REGRESSIONS_SQL: String + class CopyCopyTypesArgs + attr_reader id: Integer + attr_reader code: String + attr_reader doc: String? + attr_reader flags: String? + attr_reader amount: Integer? + def initialize: ( + Integer id, + String code, + String? doc, + String? flags, + Integer? amount + ) -> void + end + + class CopyCopyTypesIgnoreArgs + attr_reader id: Integer + attr_reader code: String + def initialize: ( Integer id, String code) -> void + end + + class CopyCopyTypesReplaceArgs + attr_reader id: Integer + attr_reader code: String + def initialize: ( Integer id, String code) -> void + end + + CREATE_COPY_TYPES_BATCH_SQL: String + + class CreateCopyTypesBatchArgs + attr_reader id: Integer + attr_reader code: String + attr_reader flags: String? + attr_reader amount: Integer? + def initialize: ( Integer id, String code, String? flags, Integer? amount) -> void + end + + COUNT_COPY_TYPES_BY_CODES_SQL: String + + class CountCopyTypesByCodesRow + attr_reader matches: Integer + def initialize: ( Integer matches) -> void + end + + class CountCopyTypesByCodesArgs + attr_reader code: String + attr_reader code_2: String + def initialize: ( String code, String code_2) -> void + end + + LIST_COPY_TYPES_SQL: String + + class ListCopyTypesRow + attr_reader id: Integer + attr_reader code: String + attr_reader doc: String? + attr_reader flags: String? + attr_reader amount: Integer? + def initialize: ( + Integer id, + String code, + String? doc, + String? flags, + Integer? amount + ) -> void + end + + TRUNCATE_COPY_TYPES_SQL: String + MANY_PARAMS_SQL: String class ManyParamsRow @@ -394,6 +463,20 @@ module Mysql2Codegen def truncate_regressions: -> void + def copy_copy_types: (Array[CopyCopyTypesArgs] rows) -> void + + def copy_copy_types_ignore: (Array[CopyCopyTypesIgnoreArgs] rows) -> void + + def copy_copy_types_replace: (Array[CopyCopyTypesReplaceArgs] rows) -> void + + def create_copy_types_batch: (Array[CreateCopyTypesBatchArgs] rows) -> void + + def count_copy_types_by_codes: (Array[CountCopyTypesByCodesArgs] rows) -> Array[CountCopyTypesByCodesRow?] + + def list_copy_types: -> Array[ListCopyTypesRow] + + def truncate_copy_types: -> void + def many_params: (c0: Integer, c1: String, c2: Float, c3: String, c4: Integer) -> ManyParamsRow? def many_params!: (c0: Integer, c1: String, c2: Float, c3: String, c4: Integer) -> ManyParamsRow @@ -410,7 +493,7 @@ module Mysql2Codegen def csv_line: (Array[untyped] values) -> String - def load_data: (untyped client, String sql) -> void + def load_data: (untyped client, String sql, ?strict: bool) -> void def slice_placeholders: (Array[untyped] values) -> String end diff --git a/test/golden/mysql2/ruby31_gemfile/models.rb b/test/golden/mysql2/ruby31_gemfile/models.rb index c22168b..8da87bb 100644 --- a/test/golden/mysql2/ruby31_gemfile/models.rb +++ b/test/golden/mysql2/ruby31_gemfile/models.rb @@ -83,4 +83,13 @@ module ExtendedBiosAuthorType :payload, keyword_init: true ) + + CopyType = Struct.new( + :id, + :code, + :doc, + :flags, + :amount, + keyword_init: true + ) end diff --git a/test/golden/mysql2/ruby31_gemfile/models.rbs b/test/golden/mysql2/ruby31_gemfile/models.rbs index 26d60db..e5769a0 100644 --- a/test/golden/mysql2/ruby31_gemfile/models.rbs +++ b/test/golden/mysql2/ruby31_gemfile/models.rbs @@ -166,4 +166,26 @@ module Mysql2Codegen def initialize: ( id: Integer, order: String, note: String?, payload: String?) -> void def self.new: ( id: Integer, order: String, note: String?, payload: String?) -> Regression end + + class CopyType < ::Struct[untyped] + attr_accessor id: Integer + attr_accessor code: String + attr_accessor doc: String? + attr_accessor flags: String? + attr_accessor amount: Integer? + def initialize: ( + id: Integer, + code: String, + doc: String?, + flags: String?, + amount: Integer? + ) -> void + def self.new: ( + id: Integer, + code: String, + doc: String?, + flags: String?, + amount: Integer? + ) -> CopyType + end end diff --git a/test/golden/mysql2/ruby31_gemfile/my_queries_sql.rb b/test/golden/mysql2/ruby31_gemfile/my_queries_sql.rb index 2f677b4..7e3b3a8 100644 --- a/test/golden/mysql2/ruby31_gemfile/my_queries_sql.rb +++ b/test/golden/mysql2/ruby31_gemfile/my_queries_sql.rb @@ -35,6 +35,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = !client.in_transaction + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.query("SET TRANSACTION ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}") if isolation @@ -95,7 +97,13 @@ def with_client(name) instrument(name) do @db.with do |client| client.extend(ConnectionState) unless client.is_a?(ConnectionState) - yield client + begin + yield client + rescue Mysql2::Error + # a connection closed by the server (restart, KILL, wait_timeout) is replaced by the pool + @db.discard_current_connection if client.closed? && @db.respond_to?(:discard_current_connection) + raise + end end end end diff --git a/test/golden/mysql2/ruby31_gemfile/other_sql.rb b/test/golden/mysql2/ruby31_gemfile/other_sql.rb index a2bd1aa..c55138e 100644 --- a/test/golden/mysql2/ruby31_gemfile/other_sql.rb +++ b/test/golden/mysql2/ruby31_gemfile/other_sql.rb @@ -47,6 +47,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = !client.in_transaction + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.query("SET TRANSACTION ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}") if isolation @@ -144,7 +146,13 @@ def with_client(name) instrument(name) do @db.with do |client| client.extend(ConnectionState) unless client.is_a?(ConnectionState) - yield client + begin + yield client + rescue Mysql2::Error + # a connection closed by the server (restart, KILL, wait_timeout) is replaced by the pool + @db.discard_current_connection if client.closed? && @db.respond_to?(:discard_current_connection) + raise + end end end end diff --git a/test/golden/mysql2/ruby31_gemfile/query_sql.rb b/test/golden/mysql2/ruby31_gemfile/query_sql.rb index e767e3e..da336be 100644 --- a/test/golden/mysql2/ruby31_gemfile/query_sql.rb +++ b/test/golden/mysql2/ruby31_gemfile/query_sql.rb @@ -246,6 +246,58 @@ module Mysql2Codegen TRUNCATE TABLE regressions SQL + CopyCopyTypesArgs = Struct.new( + :id, + :code, + :doc, + :flags, + :amount, + keyword_init: true + ) + + CopyCopyTypesIgnoreArgs = Struct.new(:id, :code, keyword_init: true) + + CopyCopyTypesReplaceArgs = Struct.new(:id, :code, keyword_init: true) + + CREATE_COPY_TYPES_BATCH_SQL = <<~'SQL' + INSERT INTO copy_types (id, code, flags, amount) VALUES (?, ?, ?, ?) + SQL + + CreateCopyTypesBatchArgs = Struct.new( + :id, + :code, + :flags, + :amount, + keyword_init: true + ) + + COUNT_COPY_TYPES_BY_CODES_SQL = <<~'SQL' + SELECT count(*) AS matches FROM copy_types WHERE code = ? OR code = ? + SQL + + CountCopyTypesByCodesRow = Struct.new(:matches, keyword_init: true) + + # rubocop:disable Naming/VariableNumber + CountCopyTypesByCodesArgs = Struct.new(:code, :code_2, keyword_init: true) + # rubocop:enable Naming/VariableNumber + + LIST_COPY_TYPES_SQL = <<~'SQL' + SELECT id, code, doc, flags, amount FROM copy_types ORDER BY id + SQL + + ListCopyTypesRow = Struct.new( + :id, + :code, + :doc, + :flags, + :amount, + keyword_init: true + ) + + TRUNCATE_COPY_TYPES_SQL = <<~'SQL' + TRUNCATE TABLE copy_types + SQL + MANY_PARAMS_SQL = <<~'SQL' SELECT 1 -- ? ? WHERE a = ? @@ -303,6 +355,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = !client.in_transaction + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.query("SET TRANSACTION ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}") if isolation @@ -712,48 +766,48 @@ def create_author_result(name:, bio:) def create_authors_batch(rows) with_client('create_authors_batch') do |client| - stmt = client.prepare(CREATE_AUTHORS_BATCH_SQL) - begin - rows.each do |args| - query_params = [args.name, args.bio] + rows.each do |args| + query_params = [args.name, args.bio] + stmt = client.prepare(CREATE_AUTHORS_BATCH_SQL) + begin stmt.execute(*query_params) + ensure + stmt.close end - nil - ensure - stmt.close end + nil end end def get_author_batch(rows) with_client('get_author_batch') do |client| - stmt = client.prepare(GET_AUTHOR_BATCH_SQL) - begin - rows.map do |args| - query_params = [args.id] + rows.map do |args| + query_params = [args.id] + stmt = client.prepare(GET_AUTHOR_BATCH_SQL) + begin result = stmt.execute(*query_params, as: :array) row = result.first row && GetAuthorBatchRow.new(id: row[0], name: row[1], bio: row[2]) + ensure + stmt.close end - ensure - stmt.close end end end def list_authors_by_name_batch(rows) with_client('list_authors_by_name_batch') do |client| - stmt = client.prepare(LIST_AUTHORS_BY_NAME_BATCH_SQL) - begin - rows.map do |args| - query_params = [args.name] + rows.map do |args| + query_params = [args.name] + stmt = client.prepare(LIST_AUTHORS_BY_NAME_BATCH_SQL) + begin result = stmt.execute(*query_params, as: :array) result.map do |row| ListAuthorsByNameBatchRow.new(id: row[0], name: row[1], bio: row[2]) end + ensure + stmt.close end - ensure - stmt.close end end end @@ -848,6 +902,114 @@ def truncate_regressions end end + def copy_copy_types(rows) + with_client('copy_copy_types') do |client| + return if rows.empty? + + Tempfile.create(['copy_copy_types', '.csv']) do |file| + rows.each do |row| + file.write(csv_line([row.id, row.code, row.doc, row.flags, row.amount])) + end + file.flush + load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' INTO TABLE `copy_types` CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (`id`, `code`, @sqlc_v2, `flags`, `amount`) SET `doc` = CONVERT(@sqlc_v2 USING utf8mb4)") + end + nil + end + end + + def copy_copy_types_ignore(rows) + with_client('copy_copy_types_ignore') do |client| + return if rows.empty? + + Tempfile.create(['copy_copy_types_ignore', '.csv']) do |file| + rows.each do |row| + file.write(csv_line([row.id, row.code])) + end + file.flush + load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' IGNORE INTO TABLE `copy_types` CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (`id`, `code`)", strict: false) + end + nil + end + end + + def copy_copy_types_replace(rows) + with_client('copy_copy_types_replace') do |client| + return if rows.empty? + + Tempfile.create(['copy_copy_types_replace', '.csv']) do |file| + rows.each do |row| + file.write(csv_line([row.id, row.code])) + end + file.flush + load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' REPLACE INTO TABLE `copy_types` CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (`id`, `code`)") + end + nil + end + end + + def create_copy_types_batch(rows) + with_client('create_copy_types_batch') do |client| + rows.each do |args| + query_params = [args.id, args.code, args.flags, args.amount] + stmt = client.prepare(CREATE_COPY_TYPES_BATCH_SQL) + begin + stmt.execute(*query_params) + ensure + stmt.close + end + end + nil + end + end + + def count_copy_types_by_codes(rows) + with_client('count_copy_types_by_codes') do |client| + rows.map do |args| + query_params = [args.code, args.code_2] + stmt = client.prepare(COUNT_COPY_TYPES_BY_CODES_SQL) + begin + result = stmt.execute(*query_params, as: :array) + row = result.first + row && CountCopyTypesByCodesRow.new(matches: row[0]) + ensure + stmt.close + end + end + end + end + + def list_copy_types + with_client('list_copy_types') do |client| + stmt = client.prepare(LIST_COPY_TYPES_SQL) + begin + result = stmt.execute(as: :array) + result.map do |row| + ListCopyTypesRow.new( + id: row[0], + code: row[1], + doc: row[2], + flags: row[3], + amount: row[4] + ) + end + ensure + stmt.close + end + end + end + + def truncate_copy_types + with_client('truncate_copy_types') do |client| + stmt = client.prepare(TRUNCATE_COPY_TYPES_SQL) + begin + stmt.execute + nil + ensure + stmt.close + end + end + end + def many_params(c0:, c1:, c2:, c3:, c4:) with_client('many_params') do |client| query_params = [ @@ -926,7 +1088,13 @@ def with_client(name) instrument(name) do @db.with do |client| client.extend(ConnectionState) unless client.is_a?(ConnectionState) - yield client + begin + yield client + rescue Mysql2::Error + # a connection closed by the server (restart, KILL, wait_timeout) is replaced by the pool + @db.discard_current_connection if client.closed? && @db.respond_to?(:discard_current_connection) + raise + end end end end @@ -945,15 +1113,15 @@ def csv_line(values) "#{fields.join(',')}\n" end - # Runs LOAD DATA in a transaction (a savepoint inside #transaction): all rows or none, and an error instead - # of the warnings LOAD DATA LOCAL gives for duplicate keys and invalid values. - def load_data(client, sql) + # Runs LOAD DATA in a transaction (a savepoint inside #transaction): all rows or none, and (strict) an error + # instead of the warnings LOAD DATA LOCAL gives for duplicate keys and invalid values. + def load_data(client, sql, strict: true) outermost = !client.in_transaction client.query(outermost ? 'BEGIN' : 'SAVEPOINT sqlc_copy') done = false begin client.query(sql) - unless client.warning_count.zero? + if strict && !client.warning_count.zero? warnings = client.query('SHOW WARNINGS', as: :array).reject { |level, *| level == 'Note' } raise Mysql2::Error, "LOAD DATA: #{warnings.map { |w| w[2] }.join('; ')}" unless warnings.empty? end diff --git a/test/golden/mysql2/ruby31_gemfile/query_sql.rbs b/test/golden/mysql2/ruby31_gemfile/query_sql.rbs index 1409ee8..e5c05c5 100644 --- a/test/golden/mysql2/ruby31_gemfile/query_sql.rbs +++ b/test/golden/mysql2/ruby31_gemfile/query_sql.rbs @@ -333,6 +333,94 @@ module Mysql2Codegen TRUNCATE_REGRESSIONS_SQL: String + class CopyCopyTypesArgs < ::Struct[untyped] + attr_accessor id: Integer + attr_accessor code: String + attr_accessor doc: String? + attr_accessor flags: String? + attr_accessor amount: Integer? + def initialize: ( + id: Integer, + code: String, + doc: String?, + flags: String?, + amount: Integer? + ) -> void + def self.new: ( + id: Integer, + code: String, + doc: String?, + flags: String?, + amount: Integer? + ) -> CopyCopyTypesArgs + end + + class CopyCopyTypesIgnoreArgs < ::Struct[untyped] + attr_accessor id: Integer + attr_accessor code: String + def initialize: ( id: Integer, code: String) -> void + def self.new: ( id: Integer, code: String) -> CopyCopyTypesIgnoreArgs + end + + class CopyCopyTypesReplaceArgs < ::Struct[untyped] + attr_accessor id: Integer + attr_accessor code: String + def initialize: ( id: Integer, code: String) -> void + def self.new: ( id: Integer, code: String) -> CopyCopyTypesReplaceArgs + end + + CREATE_COPY_TYPES_BATCH_SQL: String + + class CreateCopyTypesBatchArgs < ::Struct[untyped] + attr_accessor id: Integer + attr_accessor code: String + attr_accessor flags: String? + attr_accessor amount: Integer? + def initialize: ( id: Integer, code: String, flags: String?, amount: Integer?) -> void + def self.new: ( id: Integer, code: String, flags: String?, amount: Integer?) -> CreateCopyTypesBatchArgs + end + + COUNT_COPY_TYPES_BY_CODES_SQL: String + + class CountCopyTypesByCodesRow < ::Struct[untyped] + attr_accessor matches: Integer + def initialize: ( matches: Integer) -> void + def self.new: ( matches: Integer) -> CountCopyTypesByCodesRow + end + + class CountCopyTypesByCodesArgs < ::Struct[untyped] + attr_accessor code: String + attr_accessor code_2: String + def initialize: ( code: String, code_2: String) -> void + def self.new: ( code: String, code_2: String) -> CountCopyTypesByCodesArgs + end + + LIST_COPY_TYPES_SQL: String + + class ListCopyTypesRow < ::Struct[untyped] + attr_accessor id: Integer + attr_accessor code: String + attr_accessor doc: String? + attr_accessor flags: String? + attr_accessor amount: Integer? + def initialize: ( + id: Integer, + code: String, + doc: String?, + flags: String?, + amount: Integer? + ) -> void + def self.new: ( + id: Integer, + code: String, + doc: String?, + flags: String?, + amount: Integer? + ) -> ListCopyTypesRow + end + + TRUNCATE_COPY_TYPES_SQL: String + MANY_PARAMS_SQL: String class ManyParamsRow < ::Struct[untyped] @@ -464,6 +552,20 @@ module Mysql2Codegen def truncate_regressions: -> void + def copy_copy_types: (Array[CopyCopyTypesArgs] rows) -> void + + def copy_copy_types_ignore: (Array[CopyCopyTypesIgnoreArgs] rows) -> void + + def copy_copy_types_replace: (Array[CopyCopyTypesReplaceArgs] rows) -> void + + def create_copy_types_batch: (Array[CreateCopyTypesBatchArgs] rows) -> void + + def count_copy_types_by_codes: (Array[CountCopyTypesByCodesArgs] rows) -> Array[CountCopyTypesByCodesRow?] + + def list_copy_types: -> Array[ListCopyTypesRow] + + def truncate_copy_types: -> void + def many_params: (c0: Integer, c1: String, c2: Float, c3: String, c4: Integer) -> ManyParamsRow? def many_params!: (c0: Integer, c1: String, c2: Float, c3: String, c4: Integer) -> ManyParamsRow @@ -480,7 +582,7 @@ module Mysql2Codegen def csv_line: (Array[untyped] values) -> String - def load_data: (untyped client, String sql) -> void + def load_data: (untyped client, String sql, ?strict: bool) -> void def slice_placeholders: (Array[untyped] values) -> String end diff --git a/test/golden/mysql2/ruby32_notypes/models.rb b/test/golden/mysql2/ruby32_notypes/models.rb index 95ad226..07b421a 100644 --- a/test/golden/mysql2/ruby32_notypes/models.rb +++ b/test/golden/mysql2/ruby32_notypes/models.rb @@ -69,4 +69,13 @@ class ExtendedBio < Data.define(:author_name, :name, :bio_type, :author_type) class Regression < Data.define(:id, :order, :note, :payload) end + + class CopyType < Data.define( + :id, + :code, + :doc, + :flags, + :amount + ) + end end diff --git a/test/golden/mysql2/ruby32_notypes/my_queries_sql.rb b/test/golden/mysql2/ruby32_notypes/my_queries_sql.rb index 2f677b4..7e3b3a8 100644 --- a/test/golden/mysql2/ruby32_notypes/my_queries_sql.rb +++ b/test/golden/mysql2/ruby32_notypes/my_queries_sql.rb @@ -35,6 +35,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = !client.in_transaction + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.query("SET TRANSACTION ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}") if isolation @@ -95,7 +97,13 @@ def with_client(name) instrument(name) do @db.with do |client| client.extend(ConnectionState) unless client.is_a?(ConnectionState) - yield client + begin + yield client + rescue Mysql2::Error + # a connection closed by the server (restart, KILL, wait_timeout) is replaced by the pool + @db.discard_current_connection if client.closed? && @db.respond_to?(:discard_current_connection) + raise + end end end end diff --git a/test/golden/mysql2/ruby32_notypes/other_sql.rb b/test/golden/mysql2/ruby32_notypes/other_sql.rb index 3d5d7c8..c067da1 100644 --- a/test/golden/mysql2/ruby32_notypes/other_sql.rb +++ b/test/golden/mysql2/ruby32_notypes/other_sql.rb @@ -48,6 +48,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = !client.in_transaction + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.query("SET TRANSACTION ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}") if isolation @@ -145,7 +147,13 @@ def with_client(name) instrument(name) do @db.with do |client| client.extend(ConnectionState) unless client.is_a?(ConnectionState) - yield client + begin + yield client + rescue Mysql2::Error + # a connection closed by the server (restart, KILL, wait_timeout) is replaced by the pool + @db.discard_current_connection if client.closed? && @db.respond_to?(:discard_current_connection) + raise + end end end end diff --git a/test/golden/mysql2/ruby32_notypes/query_sql.rb b/test/golden/mysql2/ruby32_notypes/query_sql.rb index 6ad9117..d49bcfa 100644 --- a/test/golden/mysql2/ruby32_notypes/query_sql.rb +++ b/test/golden/mysql2/ruby32_notypes/query_sql.rb @@ -249,6 +249,57 @@ class ListRegressionPayloadsRow < Data.define(:id, :payload) TRUNCATE TABLE regressions SQL + class CopyCopyTypesArgs < Data.define( + :id, + :code, + :doc, + :flags, + :amount + ) + end + + class CopyCopyTypesIgnoreArgs < Data.define(:id, :code) + end + + class CopyCopyTypesReplaceArgs < Data.define(:id, :code) + end + + CREATE_COPY_TYPES_BATCH_SQL = <<~'SQL' + INSERT INTO copy_types (id, code, flags, amount) VALUES (?, ?, ?, ?) + SQL + + class CreateCopyTypesBatchArgs < Data.define(:id, :code, :flags, :amount) + end + + COUNT_COPY_TYPES_BY_CODES_SQL = <<~'SQL' + SELECT count(*) AS matches FROM copy_types WHERE code = ? OR code = ? + SQL + + class CountCopyTypesByCodesRow < Data.define(:matches) + end + + # rubocop:disable Naming/VariableNumber + class CountCopyTypesByCodesArgs < Data.define(:code, :code_2) + end + # rubocop:enable Naming/VariableNumber + + LIST_COPY_TYPES_SQL = <<~'SQL' + SELECT id, code, doc, flags, amount FROM copy_types ORDER BY id + SQL + + class ListCopyTypesRow < Data.define( + :id, + :code, + :doc, + :flags, + :amount + ) + end + + TRUNCATE_COPY_TYPES_SQL = <<~'SQL' + TRUNCATE TABLE copy_types + SQL + MANY_PARAMS_SQL = <<~'SQL' SELECT 1 -- ? ? WHERE a = ? @@ -302,6 +353,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = !client.in_transaction + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.query("SET TRANSACTION ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}") if isolation @@ -699,48 +752,48 @@ def create_author_result(name:, bio:) def create_authors_batch(rows) with_client('create_authors_batch') do |client| - stmt = client.prepare(CREATE_AUTHORS_BATCH_SQL) - begin - rows.each do |args| - query_params = [args.name, args.bio] + rows.each do |args| + query_params = [args.name, args.bio] + stmt = client.prepare(CREATE_AUTHORS_BATCH_SQL) + begin stmt.execute(*query_params) + ensure + stmt.close end - nil - ensure - stmt.close end + nil end end def get_author_batch(rows) with_client('get_author_batch') do |client| - stmt = client.prepare(GET_AUTHOR_BATCH_SQL) - begin - rows.map do |args| - query_params = [args.id] + rows.map do |args| + query_params = [args.id] + stmt = client.prepare(GET_AUTHOR_BATCH_SQL) + begin result = stmt.execute(*query_params, as: :array) row = result.first row && GetAuthorBatchRow.new(row[0], row[1], row[2]) + ensure + stmt.close end - ensure - stmt.close end end end def list_authors_by_name_batch(rows) with_client('list_authors_by_name_batch') do |client| - stmt = client.prepare(LIST_AUTHORS_BY_NAME_BATCH_SQL) - begin - rows.map do |args| - query_params = [args.name] + rows.map do |args| + query_params = [args.name] + stmt = client.prepare(LIST_AUTHORS_BY_NAME_BATCH_SQL) + begin result = stmt.execute(*query_params, as: :array) result.map do |row| ListAuthorsByNameBatchRow.new(row[0], row[1], row[2]) end + ensure + stmt.close end - ensure - stmt.close end end end @@ -835,6 +888,114 @@ def truncate_regressions end end + def copy_copy_types(rows) + with_client('copy_copy_types') do |client| + return if rows.empty? + + Tempfile.create(['copy_copy_types', '.csv']) do |file| + rows.each do |row| + file.write(csv_line([row.id, row.code, row.doc, row.flags, row.amount])) + end + file.flush + load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' INTO TABLE `copy_types` CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (`id`, `code`, @sqlc_v2, `flags`, `amount`) SET `doc` = CONVERT(@sqlc_v2 USING utf8mb4)") + end + nil + end + end + + def copy_copy_types_ignore(rows) + with_client('copy_copy_types_ignore') do |client| + return if rows.empty? + + Tempfile.create(['copy_copy_types_ignore', '.csv']) do |file| + rows.each do |row| + file.write(csv_line([row.id, row.code])) + end + file.flush + load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' IGNORE INTO TABLE `copy_types` CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (`id`, `code`)", strict: false) + end + nil + end + end + + def copy_copy_types_replace(rows) + with_client('copy_copy_types_replace') do |client| + return if rows.empty? + + Tempfile.create(['copy_copy_types_replace', '.csv']) do |file| + rows.each do |row| + file.write(csv_line([row.id, row.code])) + end + file.flush + load_data(client, "LOAD DATA LOCAL INFILE '#{client.escape(file.path)}' REPLACE INTO TABLE `copy_types` CHARACTER SET binary FIELDS TERMINATED BY ',' OPTIONALLY ENCLOSED BY '\"' ESCAPED BY '\\\\' LINES TERMINATED BY '\\n' (`id`, `code`)") + end + nil + end + end + + def create_copy_types_batch(rows) + with_client('create_copy_types_batch') do |client| + rows.each do |args| + query_params = [args.id, args.code, args.flags, args.amount] + stmt = client.prepare(CREATE_COPY_TYPES_BATCH_SQL) + begin + stmt.execute(*query_params) + ensure + stmt.close + end + end + nil + end + end + + def count_copy_types_by_codes(rows) + with_client('count_copy_types_by_codes') do |client| + rows.map do |args| + query_params = [args.code, args.code_2] + stmt = client.prepare(COUNT_COPY_TYPES_BY_CODES_SQL) + begin + result = stmt.execute(*query_params, as: :array) + row = result.first + row && CountCopyTypesByCodesRow.new(row[0]) + ensure + stmt.close + end + end + end + end + + def list_copy_types + with_client('list_copy_types') do |client| + stmt = client.prepare(LIST_COPY_TYPES_SQL) + begin + result = stmt.execute(as: :array) + result.map do |row| + ListCopyTypesRow.new( + row[0], + row[1], + row[2], + row[3], + row[4] + ) + end + ensure + stmt.close + end + end + end + + def truncate_copy_types + with_client('truncate_copy_types') do |client| + stmt = client.prepare(TRUNCATE_COPY_TYPES_SQL) + begin + stmt.execute + nil + ensure + stmt.close + end + end + end + def many_params(c0:, c1:, c2:, c3:, c4:) with_client('many_params') do |client| query_params = [ @@ -913,7 +1074,13 @@ def with_client(name) instrument(name) do @db.with do |client| client.extend(ConnectionState) unless client.is_a?(ConnectionState) - yield client + begin + yield client + rescue Mysql2::Error + # a connection closed by the server (restart, KILL, wait_timeout) is replaced by the pool + @db.discard_current_connection if client.closed? && @db.respond_to?(:discard_current_connection) + raise + end end end end @@ -932,15 +1099,15 @@ def csv_line(values) "#{fields.join(',')}\n" end - # Runs LOAD DATA in a transaction (a savepoint inside #transaction): all rows or none, and an error instead - # of the warnings LOAD DATA LOCAL gives for duplicate keys and invalid values. - def load_data(client, sql) + # Runs LOAD DATA in a transaction (a savepoint inside #transaction): all rows or none, and (strict) an error + # instead of the warnings LOAD DATA LOCAL gives for duplicate keys and invalid values. + def load_data(client, sql, strict: true) outermost = !client.in_transaction client.query(outermost ? 'BEGIN' : 'SAVEPOINT sqlc_copy') done = false begin client.query(sql) - unless client.warning_count.zero? + if strict && !client.warning_count.zero? warnings = client.query('SHOW WARNINGS', as: :array).reject { |level, *| level == 'Note' } raise Mysql2::Error, "LOAD DATA: #{warnings.map { |w| w[2] }.join('; ')}" unless warnings.empty? end diff --git a/test/golden/pg/default/models.rb b/test/golden/pg/default/models.rb index 02758eb..b9a3073 100644 --- a/test/golden/pg/default/models.rb +++ b/test/golden/pg/default/models.rb @@ -76,6 +76,18 @@ class Regression < Data.define( ) end + class BatchType < Data.define( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps + ) + end + class ExtendedBio < Data.define(:author_name, :name, :bio_type) end end diff --git a/test/golden/pg/default/models.rbs b/test/golden/pg/default/models.rbs index 69c5313..edc0520 100644 --- a/test/golden/pg/default/models.rbs +++ b/test/golden/pg/default/models.rbs @@ -130,6 +130,27 @@ module PgCodegen ) -> void end + class BatchType + attr_reader id: Integer + attr_reader ok: bool? + attr_reader ratio: Float? + attr_reader blobs: Array[String]? + attr_reader docs: Array[untyped]? + attr_reader counts: Array[Integer]? + attr_reader amounts: Array[BigDecimal]? + attr_reader stamps: Array[Time]? + def initialize: ( + Integer id, + bool? ok, + Float? ratio, + Array[String]? blobs, + Array[untyped]? docs, + Array[Integer]? counts, + Array[BigDecimal]? amounts, + Array[Time]? stamps + ) -> void + end + class ExtendedBio attr_reader author_name: String attr_reader name: String diff --git a/test/golden/pg/default/my_queries_sql.rb b/test/golden/pg/default/my_queries_sql.rb index c162668..9c3f794 100644 --- a/test/golden/pg/default/my_queries_sql.rb +++ b/test/golden/pg/default/my_queries_sql.rb @@ -35,6 +35,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = client.transaction_status == PG::PQTRANS_IDLE + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.exec(isolation ? "BEGIN ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}" : 'BEGIN') @@ -51,11 +53,12 @@ def transaction(isolation: nil) raise ensure unless failed + # a statement failed and the block rescued the error: PostgreSQL can only roll back + rescued = client.transaction_status == PG::PQTRANS_INERROR commit = outermost ? 'COMMIT' : "RELEASE SAVEPOINT #{savepoint}" - result = client.exec(Thread.current.status == 'aborting' ? rollback : commit) - # COMMIT of a transaction aborted by an error the block rescued is a ROLLBACK - rolled_back = outermost && result.cmd_status == 'ROLLBACK' && Thread.current.status != 'aborting' - raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rolled_back + client.exec(rescued || Thread.current.status == 'aborting' ? rollback : commit) + client.exec("RELEASE SAVEPOINT #{savepoint}") if rescued && savepoint + raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rescued && Thread.current.status != 'aborting' end end end @@ -90,6 +93,7 @@ def instrument(name) def with_client(name) instrument(name) do @db.with do |client| + client.reset if client.status == PG::CONNECTION_BAD # closed by the server (restart, failover, killed) configure(client) unless client.is_a?(ConnectionState) yield client end @@ -101,7 +105,10 @@ def configure(client) type_map = PG::BasicTypeMapForResults.new(client) type_map.default_type_map = string_arrays(client) # uuid, enums, money, ... come back as Strings client.type_map_for_results = type_map - client.type_map_for_queries = PG::BasicTypeMapForQueries.new(client) + queries = PG::BasicTypeMapForQueries.new(client) + # true / false as text ('t' / 'f'), also valid in COPY (the default binary encoder is not) + queries[TrueClass] = queries[FalseClass] = PG::TextEncoder::Boolean.new(oid: 16) + client.type_map_for_queries = queries client.extend(ConnectionState) end diff --git a/test/golden/pg/default/other_sql.rb b/test/golden/pg/default/other_sql.rb index 8bd3b15..21eb5b6 100644 --- a/test/golden/pg/default/other_sql.rb +++ b/test/golden/pg/default/other_sql.rb @@ -48,6 +48,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = client.transaction_status == PG::PQTRANS_IDLE + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.exec(isolation ? "BEGIN ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}" : 'BEGIN') @@ -64,11 +66,12 @@ def transaction(isolation: nil) raise ensure unless failed + # a statement failed and the block rescued the error: PostgreSQL can only roll back + rescued = client.transaction_status == PG::PQTRANS_INERROR commit = outermost ? 'COMMIT' : "RELEASE SAVEPOINT #{savepoint}" - result = client.exec(Thread.current.status == 'aborting' ? rollback : commit) - # COMMIT of a transaction aborted by an error the block rescued is a ROLLBACK - rolled_back = outermost && result.cmd_status == 'ROLLBACK' && Thread.current.status != 'aborting' - raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rolled_back + client.exec(rescued || Thread.current.status == 'aborting' ? rollback : commit) + client.exec("RELEASE SAVEPOINT #{savepoint}") if rescued && savepoint + raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rescued && Thread.current.status != 'aborting' end end end @@ -80,7 +83,7 @@ def no_cols(c0:, c1:, c2:, c3:, c4:) c0, c1, c2, - c3, + encode_float(c3), encode_bytea(c4) ] unless client.prepared_statements.include?('other_sql_no_cols') @@ -138,6 +141,7 @@ def instrument(name) def with_client(name) instrument(name) do @db.with do |client| + client.reset if client.status == PG::CONNECTION_BAD # closed by the server (restart, failover, killed) configure(client) unless client.is_a?(ConnectionState) yield client end @@ -149,7 +153,10 @@ def configure(client) type_map = PG::BasicTypeMapForResults.new(client) type_map.default_type_map = string_arrays(client) # uuid, enums, money, ... come back as Strings client.type_map_for_results = type_map - client.type_map_for_queries = PG::BasicTypeMapForQueries.new(client) + queries = PG::BasicTypeMapForQueries.new(client) + # true / false as text ('t' / 'f'), also valid in COPY (the default binary encoder is not) + queries[TrueClass] = queries[FalseClass] = PG::TextEncoder::Boolean.new(oid: 16) + client.type_map_for_queries = queries client.extend(ConnectionState) end @@ -169,5 +176,10 @@ def string_arrays(client) def encode_bytea(value) value && PG::TextEncoder::Bytea.new.encode(value) end + + # Float#to_s is the shortest text that reads back as the same Float (the pg gem's encoder drops digits). + def encode_float(value) + value.is_a?(Float) ? value.to_s : value + end end end diff --git a/test/golden/pg/default/other_sql.rbs b/test/golden/pg/default/other_sql.rbs index 4275907..b4d4038 100644 --- a/test/golden/pg/default/other_sql.rbs +++ b/test/golden/pg/default/other_sql.rbs @@ -40,5 +40,7 @@ module PgCodegen def string_arrays: (untyped client) -> untyped def encode_bytea: (String? value) -> String? + + def encode_float: (untyped value) -> untyped end end diff --git a/test/golden/pg/default/query_sql.rb b/test/golden/pg/default/query_sql.rb index 050e2d4..462945e 100644 --- a/test/golden/pg/default/query_sql.rb +++ b/test/golden/pg/default/query_sql.rb @@ -343,6 +343,61 @@ class ListRegressionsRow < Data.define( TRUNCATE TABLE regressions SQL + CREATE_BATCH_TYPES_BATCH_SQL = <<~'SQL' + INSERT INTO batch_types (id, ok, ratio, blobs, docs, counts, amounts, stamps) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + SQL + + class CreateBatchTypesBatchArgs < Data.define( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps + ) + end + + class CopyBatchTypesArgs < Data.define( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps + ) + end + + LIST_BATCH_TYPES_SQL = <<~'SQL' + SELECT id, ok, ratio, blobs, docs, counts, amounts, stamps FROM batch_types ORDER BY id + SQL + + class ListBatchTypesRow < Data.define( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps + ) + end + + COUNT_BATCH_TYPES_ABOVE_SQL = <<~'SQL' + SELECT count(*) FROM batch_types WHERE id > $1 + SQL + + class CountBatchTypesAboveRow < Data.define(:count) + end + + TRUNCATE_BATCH_TYPES_SQL = <<~'SQL' + TRUNCATE TABLE batch_types + SQL + MANY_PARAMS_SQL = <<~'SQL' SELECT 1 -- ? ? WHERE a = ? @@ -396,6 +451,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = client.transaction_status == PG::PQTRANS_IDLE + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.exec(isolation ? "BEGIN ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}" : 'BEGIN') @@ -412,11 +469,12 @@ def transaction(isolation: nil) raise ensure unless failed + # a statement failed and the block rescued the error: PostgreSQL can only roll back + rescued = client.transaction_status == PG::PQTRANS_INERROR commit = outermost ? 'COMMIT' : "RELEASE SAVEPOINT #{savepoint}" - result = client.exec(Thread.current.status == 'aborting' ? rollback : commit) - # COMMIT of a transaction aborted by an error the block rescued is a ROLLBACK - rolled_back = outermost && result.cmd_status == 'ROLLBACK' && Thread.current.status != 'aborting' - raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rolled_back + client.exec(rescued || Thread.current.status == 'aborting' ? rollback : commit) + client.exec("RELEASE SAVEPOINT #{savepoint}") if rescued && savepoint + raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rescued && Thread.current.status != 'aborting' end end end @@ -579,7 +637,7 @@ def update_author_bios(bio:) def get_authors_by_ids(param1:) with_client('get_authors_by_ids') do |client| - query_params = [param1] + query_params = [encode_array(param1)] unless client.prepared_statements.include?('query_sql_get_authors_by_ids') client.prepare('query_sql_get_authors_by_ids', GET_AUTHORS_BY_IDS_SQL) client.prepared_statements.add('query_sql_get_authors_by_ids') @@ -593,7 +651,7 @@ def get_authors_by_ids(param1:) def get_authors_by_ids_and_names(param1:, param2:) with_client('get_authors_by_ids_and_names') do |client| - query_params = [param1, param2] + query_params = [encode_array(param1), encode_array(param2)] unless client.prepared_statements.include?('query_sql_get_authors_by_ids_and_names') client.prepare('query_sql_get_authors_by_ids_and_names', GET_AUTHORS_BY_IDS_AND_NAMES_SQL) client.prepared_statements.add('query_sql_get_authors_by_ids_and_names') @@ -731,8 +789,8 @@ def insert_extra_types(c_uuid:, c_jsonb:, c_text_array:, c_integer_array:, c_tim query_params = [ c_uuid, encode_json(c_jsonb), - c_text_array, - c_integer_array, + encode_array(c_text_array), + encode_array(c_integer_array), c_timestamptz, c_inet, c_money, @@ -786,7 +844,7 @@ def truncate_extra_types def insert_array_types(c_uuid_array:, c_bio_type_array:, c_integer_matrix:) with_client('insert_array_types') do |client| - query_params = [c_uuid_array, c_bio_type_array, c_integer_matrix] + query_params = [encode_array(c_uuid_array), encode_array(c_bio_type_array), encode_array(c_integer_matrix)] unless client.prepared_statements.include?('query_sql_insert_array_types') client.prepare('query_sql_insert_array_types', INSERT_ARRAY_TYPES_SQL) client.prepared_statements.add('query_sql_insert_array_types') @@ -974,8 +1032,8 @@ def create_regression(id:, order:, payload:, doc:, bio_types:, uids:) order, encode_bytea(payload), encode_json(doc), - bio_types, - uids + encode_array(bio_types), + encode_array(uids) ] unless client.prepared_statements.include?('query_sql_create_regression') client.prepare('query_sql_create_regression', CREATE_REGRESSION_SQL) @@ -1070,13 +1128,99 @@ def truncate_regressions end end + def create_batch_types_batch(rows) + with_client('create_batch_types_batch') do |client| + unless client.prepared_statements.include?('query_sql_create_batch_types_batch') + client.prepare('query_sql_create_batch_types_batch', CREATE_BATCH_TYPES_BATCH_SQL) + client.prepared_statements.add('query_sql_create_batch_types_batch') + end + rows.each do |args| + query_params = [ + args.id, + args.ok, + encode_float(args.ratio), + encode_array(args.blobs, :bytea), + encode_array(args.docs, :json), + encode_array(args.counts), + encode_array(args.amounts), + encode_array(args.stamps) + ] + client.exec_prepared('query_sql_create_batch_types_batch', query_params) + end + nil + end + end + + def copy_batch_types(rows) + with_client('copy_batch_types') do |client| + encoder = PG::TextEncoder::CopyRow.new(type_map: client.type_map_for_queries) + client.copy_data('COPY "batch_types" ("id", "ok", "ratio", "blobs", "docs", "counts", "amounts", "stamps") FROM STDIN', encoder) do + rows.each do |row| + client.put_copy_data([row.id, row.ok, encode_float(row.ratio), encode_array(row.blobs, :bytea), encode_array(row.docs, :json), encode_array(row.counts), encode_array(row.amounts), encode_array(row.stamps)]) + end + end + nil + end + end + + def list_batch_types + with_client('list_batch_types') do |client| + unless client.prepared_statements.include?('query_sql_list_batch_types') + client.prepare('query_sql_list_batch_types', LIST_BATCH_TYPES_SQL) + client.prepared_statements.add('query_sql_list_batch_types') + end + result = client.exec_prepared('query_sql_list_batch_types') + result.map do |row| + ListBatchTypesRow.new( + row['id'], + row['ok'], + row['ratio'], + row['blobs'], + row['docs'], + row['counts'], + row['amounts'], + row['stamps'] + ) + end + end + end + + def count_batch_types_above(min_id:) + with_client('count_batch_types_above') do |client| + query_params = [min_id] + unless client.prepared_statements.include?('query_sql_count_batch_types_above') + client.prepare('query_sql_count_batch_types_above', COUNT_BATCH_TYPES_ABOVE_SQL) + client.prepared_statements.add('query_sql_count_batch_types_above') + end + result = client.exec_prepared('query_sql_count_batch_types_above', query_params) + row = result.first + row && CountBatchTypesAboveRow.new(row['count']) + end + end + + # Like #count_batch_types_above, but raises NotFound when there is no row. + def count_batch_types_above!(...) + count_batch_types_above(...) || raise(NotFound, 'CountBatchTypesAbove returned no row') + end + + def truncate_batch_types + with_client('truncate_batch_types') do |client| + unless client.prepared_statements.include?('query_sql_truncate_batch_types') + client.prepare('query_sql_truncate_batch_types', TRUNCATE_BATCH_TYPES_SQL) + client.prepared_statements.add('query_sql_truncate_batch_types') + end + client.exec_prepared('query_sql_truncate_batch_types') + nil + end + end + def many_params(c0:, c1:, c2:, c3:, c4:) with_client('many_params') do |client| query_params = [ c0, c1, c2, - c3, + encode_float(c3), encode_bytea(c4) ] unless client.prepared_statements.include?('query_sql_many_params') @@ -1144,6 +1288,7 @@ def instrument(name) def with_client(name) instrument(name) do @db.with do |client| + client.reset if client.status == PG::CONNECTION_BAD # closed by the server (restart, failover, killed) configure(client) unless client.is_a?(ConnectionState) yield client end @@ -1155,7 +1300,10 @@ def configure(client) type_map = PG::BasicTypeMapForResults.new(client) type_map.default_type_map = string_arrays(client) # uuid, enums, money, ... come back as Strings client.type_map_for_results = type_map - client.type_map_for_queries = PG::BasicTypeMapForQueries.new(client) + queries = PG::BasicTypeMapForQueries.new(client) + # true / false as text ('t' / 'f'), also valid in COPY (the default binary encoder is not) + queries[TrueClass] = queries[FalseClass] = PG::TextEncoder::Boolean.new(oid: 16) + client.type_map_for_queries = queries client.extend(ConnectionState) end @@ -1180,5 +1328,31 @@ def encode_bytea(value) def encode_json(value) value.is_a?(Array) ? PG::TextEncoder::JSON.new.encode(value) : value end + + # Float#to_s is the shortest text that reads back as the same Float (the pg gem's encoder drops digits). + def encode_float(value) + value.is_a?(Float) ? value.to_s : value + end + + # Array parameters are sent as array literals, which the server converts to the parameter's type (the pg gem + # would type them from their first element: wrong for [], [nil, ...], bytea[], json[]). + def encode_array(value, kind = :value) + return value unless value.is_a?(Array) + + PG::TextEncoder::Array.new(elements_type: PG::TextEncoder::String.new).encode(array_elements(value, kind)) + end + + def array_elements(values, kind) + values.map do |value| + case value + when nil then nil + when Array then kind == :json ? PG::TextEncoder::JSON.new.encode(value) : array_elements(value, kind) + when Hash then PG::TextEncoder::JSON.new.encode(value) + when String then kind == :bytea ? PG::TextEncoder::Bytea.new.encode(value) : value + when Time then PG::TextEncoder::TimestampWithTimeZone.new.encode(value) + else defined?(IPAddr) && value.is_a?(IPAddr) ? "#{value}/#{value.prefix}" : value.to_s + end + end + end end end diff --git a/test/golden/pg/default/query_sql.rbs b/test/golden/pg/default/query_sql.rbs index c39e328..dd1cc04 100644 --- a/test/golden/pg/default/query_sql.rbs +++ b/test/golden/pg/default/query_sql.rbs @@ -356,6 +356,82 @@ module PgCodegen TRUNCATE_REGRESSIONS_SQL: String + CREATE_BATCH_TYPES_BATCH_SQL: String + + class CreateBatchTypesBatchArgs + attr_reader id: Integer + attr_reader ok: bool? + attr_reader ratio: Float? + attr_reader blobs: Array[String]? + attr_reader docs: Array[untyped]? + attr_reader counts: Array[Integer]? + attr_reader amounts: Array[BigDecimal]? + attr_reader stamps: Array[Time]? + def initialize: ( + Integer id, + bool? ok, + Float? ratio, + Array[String]? blobs, + Array[untyped]? docs, + Array[Integer]? counts, + Array[BigDecimal]? amounts, + Array[Time]? stamps + ) -> void + end + + class CopyBatchTypesArgs + attr_reader id: Integer + attr_reader ok: bool? + attr_reader ratio: Float? + attr_reader blobs: Array[String]? + attr_reader docs: Array[untyped]? + attr_reader counts: Array[Integer]? + attr_reader amounts: Array[BigDecimal]? + attr_reader stamps: Array[Time]? + def initialize: ( + Integer id, + bool? ok, + Float? ratio, + Array[String]? blobs, + Array[untyped]? docs, + Array[Integer]? counts, + Array[BigDecimal]? amounts, + Array[Time]? stamps + ) -> void + end + + LIST_BATCH_TYPES_SQL: String + + class ListBatchTypesRow + attr_reader id: Integer + attr_reader ok: bool? + attr_reader ratio: Float? + attr_reader blobs: Array[String]? + attr_reader docs: Array[untyped]? + attr_reader counts: Array[Integer]? + attr_reader amounts: Array[BigDecimal]? + attr_reader stamps: Array[Time]? + def initialize: ( + Integer id, + bool? ok, + Float? ratio, + Array[String]? blobs, + Array[untyped]? docs, + Array[Integer]? counts, + Array[BigDecimal]? amounts, + Array[Time]? stamps + ) -> void + end + + COUNT_BATCH_TYPES_ABOVE_SQL: String + + class CountBatchTypesAboveRow + attr_reader count: Integer + def initialize: ( Integer count) -> void + end + + TRUNCATE_BATCH_TYPES_SQL: String + MANY_PARAMS_SQL: String class ManyParamsRow @@ -505,6 +581,18 @@ module PgCodegen def truncate_regressions: -> void + def create_batch_types_batch: (Array[CreateBatchTypesBatchArgs] rows) -> void + + def copy_batch_types: (Array[CopyBatchTypesArgs] rows) -> void + + def list_batch_types: -> Array[ListBatchTypesRow] + + def count_batch_types_above: (min_id: Integer) -> CountBatchTypesAboveRow? + + def count_batch_types_above!: (min_id: Integer) -> CountBatchTypesAboveRow + + def truncate_batch_types: -> void + def many_params: (c0: bool, c1: Integer, c2: String, c3: Float, c4: String) -> ManyParamsRow? def many_params!: (c0: bool, c1: Integer, c2: String, c3: Float, c4: String) -> ManyParamsRow @@ -526,5 +614,11 @@ module PgCodegen def encode_bytea: (String? value) -> String? def encode_json: (untyped value) -> untyped + + def encode_float: (untyped value) -> untyped + + def encode_array: (Array[untyped]? value, ?Symbol kind) -> untyped + + def array_elements: (Array[untyped] values, Symbol kind) -> Array[untyped] end end diff --git a/test/golden/pg/ruby31_gemfile/models.rb b/test/golden/pg/ruby31_gemfile/models.rb index a385132..3912a47 100644 --- a/test/golden/pg/ruby31_gemfile/models.rb +++ b/test/golden/pg/ruby31_gemfile/models.rb @@ -79,5 +79,17 @@ module ExtendedBioType keyword_init: true ) + BatchType = Struct.new( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps, + keyword_init: true + ) + ExtendedBio = Struct.new(:author_name, :name, :bio_type, keyword_init: true) end diff --git a/test/golden/pg/ruby31_gemfile/models.rbs b/test/golden/pg/ruby31_gemfile/models.rbs index b4f038d..e9b3bc8 100644 --- a/test/golden/pg/ruby31_gemfile/models.rbs +++ b/test/golden/pg/ruby31_gemfile/models.rbs @@ -180,6 +180,37 @@ module PgCodegen ) -> Regression end + class BatchType < ::Struct[untyped] + attr_accessor id: Integer + attr_accessor ok: bool? + attr_accessor ratio: Float? + attr_accessor blobs: Array[String]? + attr_accessor docs: Array[untyped]? + attr_accessor counts: Array[Integer]? + attr_accessor amounts: Array[BigDecimal]? + attr_accessor stamps: Array[Time]? + def initialize: ( + id: Integer, + ok: bool?, + ratio: Float?, + blobs: Array[String]?, + docs: Array[untyped]?, + counts: Array[Integer]?, + amounts: Array[BigDecimal]?, + stamps: Array[Time]? + ) -> void + def self.new: ( + id: Integer, + ok: bool?, + ratio: Float?, + blobs: Array[String]?, + docs: Array[untyped]?, + counts: Array[Integer]?, + amounts: Array[BigDecimal]?, + stamps: Array[Time]? + ) -> BatchType + end + class ExtendedBio < ::Struct[untyped] attr_accessor author_name: String attr_accessor name: String diff --git a/test/golden/pg/ruby31_gemfile/my_queries_sql.rb b/test/golden/pg/ruby31_gemfile/my_queries_sql.rb index 044886b..289d380 100644 --- a/test/golden/pg/ruby31_gemfile/my_queries_sql.rb +++ b/test/golden/pg/ruby31_gemfile/my_queries_sql.rb @@ -36,6 +36,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = client.transaction_status == PG::PQTRANS_IDLE + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.exec(isolation ? "BEGIN ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}" : 'BEGIN') @@ -52,11 +54,12 @@ def transaction(isolation: nil) raise ensure unless failed + # a statement failed and the block rescued the error: PostgreSQL can only roll back + rescued = client.transaction_status == PG::PQTRANS_INERROR commit = outermost ? 'COMMIT' : "RELEASE SAVEPOINT #{savepoint}" - result = client.exec(Thread.current.status == 'aborting' ? rollback : commit) - # COMMIT of a transaction aborted by an error the block rescued is a ROLLBACK - rolled_back = outermost && result.cmd_status == 'ROLLBACK' && Thread.current.status != 'aborting' - raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rolled_back + client.exec(rescued || Thread.current.status == 'aborting' ? rollback : commit) + client.exec("RELEASE SAVEPOINT #{savepoint}") if rescued && savepoint + raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rescued && Thread.current.status != 'aborting' end end end @@ -91,6 +94,7 @@ def instrument(name) def with_client(name) instrument(name) do @db.with do |client| + client.reset if client.status == PG::CONNECTION_BAD # closed by the server (restart, failover, killed) configure(client) unless client.is_a?(ConnectionState) yield client end @@ -102,7 +106,10 @@ def configure(client) type_map = PG::BasicTypeMapForResults.new(client) type_map.default_type_map = string_arrays(client) # uuid, enums, money, ... come back as Strings client.type_map_for_results = type_map - client.type_map_for_queries = PG::BasicTypeMapForQueries.new(client) + queries = PG::BasicTypeMapForQueries.new(client) + # true / false as text ('t' / 'f'), also valid in COPY (the default binary encoder is not) + queries[TrueClass] = queries[FalseClass] = PG::TextEncoder::Boolean.new(oid: 16) + client.type_map_for_queries = queries client.extend(ConnectionState) end diff --git a/test/golden/pg/ruby31_gemfile/other_sql.rb b/test/golden/pg/ruby31_gemfile/other_sql.rb index 571ea0e..4e2cd5d 100644 --- a/test/golden/pg/ruby31_gemfile/other_sql.rb +++ b/test/golden/pg/ruby31_gemfile/other_sql.rb @@ -48,6 +48,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = client.transaction_status == PG::PQTRANS_IDLE + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.exec(isolation ? "BEGIN ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}" : 'BEGIN') @@ -64,11 +66,12 @@ def transaction(isolation: nil) raise ensure unless failed + # a statement failed and the block rescued the error: PostgreSQL can only roll back + rescued = client.transaction_status == PG::PQTRANS_INERROR commit = outermost ? 'COMMIT' : "RELEASE SAVEPOINT #{savepoint}" - result = client.exec(Thread.current.status == 'aborting' ? rollback : commit) - # COMMIT of a transaction aborted by an error the block rescued is a ROLLBACK - rolled_back = outermost && result.cmd_status == 'ROLLBACK' && Thread.current.status != 'aborting' - raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rolled_back + client.exec(rescued || Thread.current.status == 'aborting' ? rollback : commit) + client.exec("RELEASE SAVEPOINT #{savepoint}") if rescued && savepoint + raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rescued && Thread.current.status != 'aborting' end end end @@ -80,7 +83,7 @@ def no_cols(c0:, c1:, c2:, c3:, c4:) c0, c1, c2, - c3, + encode_float(c3), encode_bytea(c4) ] unless client.prepared_statements.include?('other_sql_no_cols') @@ -138,6 +141,7 @@ def instrument(name) def with_client(name) instrument(name) do @db.with do |client| + client.reset if client.status == PG::CONNECTION_BAD # closed by the server (restart, failover, killed) configure(client) unless client.is_a?(ConnectionState) yield client end @@ -149,7 +153,10 @@ def configure(client) type_map = PG::BasicTypeMapForResults.new(client) type_map.default_type_map = string_arrays(client) # uuid, enums, money, ... come back as Strings client.type_map_for_results = type_map - client.type_map_for_queries = PG::BasicTypeMapForQueries.new(client) + queries = PG::BasicTypeMapForQueries.new(client) + # true / false as text ('t' / 'f'), also valid in COPY (the default binary encoder is not) + queries[TrueClass] = queries[FalseClass] = PG::TextEncoder::Boolean.new(oid: 16) + client.type_map_for_queries = queries client.extend(ConnectionState) end @@ -169,5 +176,10 @@ def string_arrays(client) def encode_bytea(value) value && PG::TextEncoder::Bytea.new.encode(value) end + + # Float#to_s is the shortest text that reads back as the same Float (the pg gem's encoder drops digits). + def encode_float(value) + value.is_a?(Float) ? value.to_s : value + end end end diff --git a/test/golden/pg/ruby31_gemfile/other_sql.rbs b/test/golden/pg/ruby31_gemfile/other_sql.rbs index c4212d8..0324878 100644 --- a/test/golden/pg/ruby31_gemfile/other_sql.rbs +++ b/test/golden/pg/ruby31_gemfile/other_sql.rbs @@ -41,5 +41,7 @@ module PgCodegen def string_arrays: (untyped client) -> untyped def encode_bytea: (String? value) -> String? + + def encode_float: (untyped value) -> untyped end end diff --git a/test/golden/pg/ruby31_gemfile/query_sql.rb b/test/golden/pg/ruby31_gemfile/query_sql.rb index 8d8dc80..8cd07dd 100644 --- a/test/golden/pg/ruby31_gemfile/query_sql.rb +++ b/test/golden/pg/ruby31_gemfile/query_sql.rb @@ -337,6 +337,60 @@ module PgCodegen TRUNCATE TABLE regressions SQL + CREATE_BATCH_TYPES_BATCH_SQL = <<~'SQL' + INSERT INTO batch_types (id, ok, ratio, blobs, docs, counts, amounts, stamps) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + SQL + + CreateBatchTypesBatchArgs = Struct.new( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps, + keyword_init: true + ) + + CopyBatchTypesArgs = Struct.new( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps, + keyword_init: true + ) + + LIST_BATCH_TYPES_SQL = <<~'SQL' + SELECT id, ok, ratio, blobs, docs, counts, amounts, stamps FROM batch_types ORDER BY id + SQL + + ListBatchTypesRow = Struct.new( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps, + keyword_init: true + ) + + COUNT_BATCH_TYPES_ABOVE_SQL = <<~'SQL' + SELECT count(*) FROM batch_types WHERE id > $1 + SQL + + CountBatchTypesAboveRow = Struct.new(:count_, keyword_init: true) + + TRUNCATE_BATCH_TYPES_SQL = <<~'SQL' + TRUNCATE TABLE batch_types + SQL + MANY_PARAMS_SQL = <<~'SQL' SELECT 1 -- ? ? WHERE a = ? @@ -394,6 +448,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = client.transaction_status == PG::PQTRANS_IDLE + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.exec(isolation ? "BEGIN ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}" : 'BEGIN') @@ -410,11 +466,12 @@ def transaction(isolation: nil) raise ensure unless failed + # a statement failed and the block rescued the error: PostgreSQL can only roll back + rescued = client.transaction_status == PG::PQTRANS_INERROR commit = outermost ? 'COMMIT' : "RELEASE SAVEPOINT #{savepoint}" - result = client.exec(Thread.current.status == 'aborting' ? rollback : commit) - # COMMIT of a transaction aborted by an error the block rescued is a ROLLBACK - rolled_back = outermost && result.cmd_status == 'ROLLBACK' && Thread.current.status != 'aborting' - raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rolled_back + client.exec(rescued || Thread.current.status == 'aborting' ? rollback : commit) + client.exec("RELEASE SAVEPOINT #{savepoint}") if rescued && savepoint + raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rescued && Thread.current.status != 'aborting' end end end @@ -576,7 +633,7 @@ def update_author_bios(bio:) def get_authors_by_ids(param1:) with_client('get_authors_by_ids') do |client| - query_params = [param1] + query_params = [encode_array(param1)] unless client.prepared_statements.include?('query_sql_get_authors_by_ids') client.prepare('query_sql_get_authors_by_ids', GET_AUTHORS_BY_IDS_SQL) client.prepared_statements.add('query_sql_get_authors_by_ids') @@ -590,7 +647,7 @@ def get_authors_by_ids(param1:) def get_authors_by_ids_and_names(param1:, param2:) with_client('get_authors_by_ids_and_names') do |client| - query_params = [param1, param2] + query_params = [encode_array(param1), encode_array(param2)] unless client.prepared_statements.include?('query_sql_get_authors_by_ids_and_names') client.prepare('query_sql_get_authors_by_ids_and_names', GET_AUTHORS_BY_IDS_AND_NAMES_SQL) client.prepared_statements.add('query_sql_get_authors_by_ids_and_names') @@ -740,8 +797,8 @@ def insert_extra_types(c_uuid:, c_jsonb:, c_text_array:, c_integer_array:, c_tim query_params = [ c_uuid, encode_json(c_jsonb), - c_text_array, - c_integer_array, + encode_array(c_text_array), + encode_array(c_integer_array), c_timestamptz, c_inet, c_money, @@ -795,7 +852,7 @@ def truncate_extra_types def insert_array_types(c_uuid_array:, c_bio_type_array:, c_integer_matrix:) with_client('insert_array_types') do |client| - query_params = [c_uuid_array, c_bio_type_array, c_integer_matrix] + query_params = [encode_array(c_uuid_array), encode_array(c_bio_type_array), encode_array(c_integer_matrix)] unless client.prepared_statements.include?('query_sql_insert_array_types') client.prepare('query_sql_insert_array_types', INSERT_ARRAY_TYPES_SQL) client.prepared_statements.add('query_sql_insert_array_types') @@ -987,8 +1044,8 @@ def create_regression(id:, order:, payload:, doc:, bio_types:, uids:) order, encode_bytea(payload), encode_json(doc), - bio_types, - uids + encode_array(bio_types), + encode_array(uids) ] unless client.prepared_statements.include?('query_sql_create_regression') client.prepare('query_sql_create_regression', CREATE_REGRESSION_SQL) @@ -1083,13 +1140,99 @@ def truncate_regressions end end + def create_batch_types_batch(rows) + with_client('create_batch_types_batch') do |client| + unless client.prepared_statements.include?('query_sql_create_batch_types_batch') + client.prepare('query_sql_create_batch_types_batch', CREATE_BATCH_TYPES_BATCH_SQL) + client.prepared_statements.add('query_sql_create_batch_types_batch') + end + rows.each do |args| + query_params = [ + args.id, + args.ok, + encode_float(args.ratio), + encode_array(args.blobs, :bytea), + encode_array(args.docs, :json), + encode_array(args.counts), + encode_array(args.amounts), + encode_array(args.stamps) + ] + client.exec_prepared('query_sql_create_batch_types_batch', query_params) + end + nil + end + end + + def copy_batch_types(rows) + with_client('copy_batch_types') do |client| + encoder = PG::TextEncoder::CopyRow.new(type_map: client.type_map_for_queries) + client.copy_data('COPY "batch_types" ("id", "ok", "ratio", "blobs", "docs", "counts", "amounts", "stamps") FROM STDIN', encoder) do + rows.each do |row| + client.put_copy_data([row.id, row.ok, encode_float(row.ratio), encode_array(row.blobs, :bytea), encode_array(row.docs, :json), encode_array(row.counts), encode_array(row.amounts), encode_array(row.stamps)]) + end + end + nil + end + end + + def list_batch_types + with_client('list_batch_types') do |client| + unless client.prepared_statements.include?('query_sql_list_batch_types') + client.prepare('query_sql_list_batch_types', LIST_BATCH_TYPES_SQL) + client.prepared_statements.add('query_sql_list_batch_types') + end + result = client.exec_prepared('query_sql_list_batch_types') + result.map do |row| + ListBatchTypesRow.new( + id: row['id'], + ok: row['ok'], + ratio: row['ratio'], + blobs: row['blobs'], + docs: row['docs'], + counts: row['counts'], + amounts: row['amounts'], + stamps: row['stamps'] + ) + end + end + end + + def count_batch_types_above(min_id:) + with_client('count_batch_types_above') do |client| + query_params = [min_id] + unless client.prepared_statements.include?('query_sql_count_batch_types_above') + client.prepare('query_sql_count_batch_types_above', COUNT_BATCH_TYPES_ABOVE_SQL) + client.prepared_statements.add('query_sql_count_batch_types_above') + end + result = client.exec_prepared('query_sql_count_batch_types_above', query_params) + row = result.first + row && CountBatchTypesAboveRow.new(count_: row['count']) + end + end + + # Like #count_batch_types_above, but raises NotFound when there is no row. + def count_batch_types_above!(...) + count_batch_types_above(...) || raise(NotFound, 'CountBatchTypesAbove returned no row') + end + + def truncate_batch_types + with_client('truncate_batch_types') do |client| + unless client.prepared_statements.include?('query_sql_truncate_batch_types') + client.prepare('query_sql_truncate_batch_types', TRUNCATE_BATCH_TYPES_SQL) + client.prepared_statements.add('query_sql_truncate_batch_types') + end + client.exec_prepared('query_sql_truncate_batch_types') + nil + end + end + def many_params(c0:, c1:, c2:, c3:, c4:) with_client('many_params') do |client| query_params = [ c0, c1, c2, - c3, + encode_float(c3), encode_bytea(c4) ] unless client.prepared_statements.include?('query_sql_many_params') @@ -1157,6 +1300,7 @@ def instrument(name) def with_client(name) instrument(name) do @db.with do |client| + client.reset if client.status == PG::CONNECTION_BAD # closed by the server (restart, failover, killed) configure(client) unless client.is_a?(ConnectionState) yield client end @@ -1168,7 +1312,10 @@ def configure(client) type_map = PG::BasicTypeMapForResults.new(client) type_map.default_type_map = string_arrays(client) # uuid, enums, money, ... come back as Strings client.type_map_for_results = type_map - client.type_map_for_queries = PG::BasicTypeMapForQueries.new(client) + queries = PG::BasicTypeMapForQueries.new(client) + # true / false as text ('t' / 'f'), also valid in COPY (the default binary encoder is not) + queries[TrueClass] = queries[FalseClass] = PG::TextEncoder::Boolean.new(oid: 16) + client.type_map_for_queries = queries client.extend(ConnectionState) end @@ -1193,5 +1340,31 @@ def encode_bytea(value) def encode_json(value) value.is_a?(Array) ? PG::TextEncoder::JSON.new.encode(value) : value end + + # Float#to_s is the shortest text that reads back as the same Float (the pg gem's encoder drops digits). + def encode_float(value) + value.is_a?(Float) ? value.to_s : value + end + + # Array parameters are sent as array literals, which the server converts to the parameter's type (the pg gem + # would type them from their first element: wrong for [], [nil, ...], bytea[], json[]). + def encode_array(value, kind = :value) + return value unless value.is_a?(Array) + + PG::TextEncoder::Array.new(elements_type: PG::TextEncoder::String.new).encode(array_elements(value, kind)) + end + + def array_elements(values, kind) + values.map do |value| + case value + when nil then nil + when Array then kind == :json ? PG::TextEncoder::JSON.new.encode(value) : array_elements(value, kind) + when Hash then PG::TextEncoder::JSON.new.encode(value) + when String then kind == :bytea ? PG::TextEncoder::Bytea.new.encode(value) : value + when Time then PG::TextEncoder::TimestampWithTimeZone.new.encode(value) + else defined?(IPAddr) && value.is_a?(IPAddr) ? "#{value}/#{value.prefix}" : value.to_s + end + end + end end end diff --git a/test/golden/pg/ruby31_gemfile/query_sql.rbs b/test/golden/pg/ruby31_gemfile/query_sql.rbs index a747ec7..c1aa9c6 100644 --- a/test/golden/pg/ruby31_gemfile/query_sql.rbs +++ b/test/golden/pg/ruby31_gemfile/query_sql.rbs @@ -436,6 +436,113 @@ module PgCodegen TRUNCATE_REGRESSIONS_SQL: String + CREATE_BATCH_TYPES_BATCH_SQL: String + + class CreateBatchTypesBatchArgs < ::Struct[untyped] + attr_accessor id: Integer + attr_accessor ok: bool? + attr_accessor ratio: Float? + attr_accessor blobs: Array[String]? + attr_accessor docs: Array[untyped]? + attr_accessor counts: Array[Integer]? + attr_accessor amounts: Array[BigDecimal]? + attr_accessor stamps: Array[Time]? + def initialize: ( + id: Integer, + ok: bool?, + ratio: Float?, + blobs: Array[String]?, + docs: Array[untyped]?, + counts: Array[Integer]?, + amounts: Array[BigDecimal]?, + stamps: Array[Time]? + ) -> void + def self.new: ( + id: Integer, + ok: bool?, + ratio: Float?, + blobs: Array[String]?, + docs: Array[untyped]?, + counts: Array[Integer]?, + amounts: Array[BigDecimal]?, + stamps: Array[Time]? + ) -> CreateBatchTypesBatchArgs + end + + class CopyBatchTypesArgs < ::Struct[untyped] + attr_accessor id: Integer + attr_accessor ok: bool? + attr_accessor ratio: Float? + attr_accessor blobs: Array[String]? + attr_accessor docs: Array[untyped]? + attr_accessor counts: Array[Integer]? + attr_accessor amounts: Array[BigDecimal]? + attr_accessor stamps: Array[Time]? + def initialize: ( + id: Integer, + ok: bool?, + ratio: Float?, + blobs: Array[String]?, + docs: Array[untyped]?, + counts: Array[Integer]?, + amounts: Array[BigDecimal]?, + stamps: Array[Time]? + ) -> void + def self.new: ( + id: Integer, + ok: bool?, + ratio: Float?, + blobs: Array[String]?, + docs: Array[untyped]?, + counts: Array[Integer]?, + amounts: Array[BigDecimal]?, + stamps: Array[Time]? + ) -> CopyBatchTypesArgs + end + + LIST_BATCH_TYPES_SQL: String + + class ListBatchTypesRow < ::Struct[untyped] + attr_accessor id: Integer + attr_accessor ok: bool? + attr_accessor ratio: Float? + attr_accessor blobs: Array[String]? + attr_accessor docs: Array[untyped]? + attr_accessor counts: Array[Integer]? + attr_accessor amounts: Array[BigDecimal]? + attr_accessor stamps: Array[Time]? + def initialize: ( + id: Integer, + ok: bool?, + ratio: Float?, + blobs: Array[String]?, + docs: Array[untyped]?, + counts: Array[Integer]?, + amounts: Array[BigDecimal]?, + stamps: Array[Time]? + ) -> void + def self.new: ( + id: Integer, + ok: bool?, + ratio: Float?, + blobs: Array[String]?, + docs: Array[untyped]?, + counts: Array[Integer]?, + amounts: Array[BigDecimal]?, + stamps: Array[Time]? + ) -> ListBatchTypesRow + end + + COUNT_BATCH_TYPES_ABOVE_SQL: String + + class CountBatchTypesAboveRow < ::Struct[untyped] + attr_accessor count_: Integer + def initialize: ( count_: Integer) -> void + def self.new: ( count_: Integer) -> CountBatchTypesAboveRow + end + + TRUNCATE_BATCH_TYPES_SQL: String + MANY_PARAMS_SQL: String class ManyParamsRow < ::Struct[untyped] @@ -595,6 +702,18 @@ module PgCodegen def truncate_regressions: -> void + def create_batch_types_batch: (Array[CreateBatchTypesBatchArgs] rows) -> void + + def copy_batch_types: (Array[CopyBatchTypesArgs] rows) -> void + + def list_batch_types: -> Array[ListBatchTypesRow] + + def count_batch_types_above: (min_id: Integer) -> CountBatchTypesAboveRow? + + def count_batch_types_above!: (min_id: Integer) -> CountBatchTypesAboveRow + + def truncate_batch_types: -> void + def many_params: (c0: bool, c1: Integer, c2: String, c3: Float, c4: String) -> ManyParamsRow? def many_params!: (c0: bool, c1: Integer, c2: String, c3: Float, c4: String) -> ManyParamsRow @@ -616,5 +735,11 @@ module PgCodegen def encode_bytea: (String? value) -> String? def encode_json: (untyped value) -> untyped + + def encode_float: (untyped value) -> untyped + + def encode_array: (Array[untyped]? value, ?Symbol kind) -> untyped + + def array_elements: (Array[untyped] values, Symbol kind) -> Array[untyped] end end diff --git a/test/golden/pg/ruby32_notypes/models.rb b/test/golden/pg/ruby32_notypes/models.rb index 02758eb..b9a3073 100644 --- a/test/golden/pg/ruby32_notypes/models.rb +++ b/test/golden/pg/ruby32_notypes/models.rb @@ -76,6 +76,18 @@ class Regression < Data.define( ) end + class BatchType < Data.define( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps + ) + end + class ExtendedBio < Data.define(:author_name, :name, :bio_type) end end diff --git a/test/golden/pg/ruby32_notypes/my_queries_sql.rb b/test/golden/pg/ruby32_notypes/my_queries_sql.rb index c162668..9c3f794 100644 --- a/test/golden/pg/ruby32_notypes/my_queries_sql.rb +++ b/test/golden/pg/ruby32_notypes/my_queries_sql.rb @@ -35,6 +35,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = client.transaction_status == PG::PQTRANS_IDLE + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.exec(isolation ? "BEGIN ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}" : 'BEGIN') @@ -51,11 +53,12 @@ def transaction(isolation: nil) raise ensure unless failed + # a statement failed and the block rescued the error: PostgreSQL can only roll back + rescued = client.transaction_status == PG::PQTRANS_INERROR commit = outermost ? 'COMMIT' : "RELEASE SAVEPOINT #{savepoint}" - result = client.exec(Thread.current.status == 'aborting' ? rollback : commit) - # COMMIT of a transaction aborted by an error the block rescued is a ROLLBACK - rolled_back = outermost && result.cmd_status == 'ROLLBACK' && Thread.current.status != 'aborting' - raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rolled_back + client.exec(rescued || Thread.current.status == 'aborting' ? rollback : commit) + client.exec("RELEASE SAVEPOINT #{savepoint}") if rescued && savepoint + raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rescued && Thread.current.status != 'aborting' end end end @@ -90,6 +93,7 @@ def instrument(name) def with_client(name) instrument(name) do @db.with do |client| + client.reset if client.status == PG::CONNECTION_BAD # closed by the server (restart, failover, killed) configure(client) unless client.is_a?(ConnectionState) yield client end @@ -101,7 +105,10 @@ def configure(client) type_map = PG::BasicTypeMapForResults.new(client) type_map.default_type_map = string_arrays(client) # uuid, enums, money, ... come back as Strings client.type_map_for_results = type_map - client.type_map_for_queries = PG::BasicTypeMapForQueries.new(client) + queries = PG::BasicTypeMapForQueries.new(client) + # true / false as text ('t' / 'f'), also valid in COPY (the default binary encoder is not) + queries[TrueClass] = queries[FalseClass] = PG::TextEncoder::Boolean.new(oid: 16) + client.type_map_for_queries = queries client.extend(ConnectionState) end diff --git a/test/golden/pg/ruby32_notypes/other_sql.rb b/test/golden/pg/ruby32_notypes/other_sql.rb index 8bd3b15..21eb5b6 100644 --- a/test/golden/pg/ruby32_notypes/other_sql.rb +++ b/test/golden/pg/ruby32_notypes/other_sql.rb @@ -48,6 +48,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = client.transaction_status == PG::PQTRANS_IDLE + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.exec(isolation ? "BEGIN ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}" : 'BEGIN') @@ -64,11 +66,12 @@ def transaction(isolation: nil) raise ensure unless failed + # a statement failed and the block rescued the error: PostgreSQL can only roll back + rescued = client.transaction_status == PG::PQTRANS_INERROR commit = outermost ? 'COMMIT' : "RELEASE SAVEPOINT #{savepoint}" - result = client.exec(Thread.current.status == 'aborting' ? rollback : commit) - # COMMIT of a transaction aborted by an error the block rescued is a ROLLBACK - rolled_back = outermost && result.cmd_status == 'ROLLBACK' && Thread.current.status != 'aborting' - raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rolled_back + client.exec(rescued || Thread.current.status == 'aborting' ? rollback : commit) + client.exec("RELEASE SAVEPOINT #{savepoint}") if rescued && savepoint + raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rescued && Thread.current.status != 'aborting' end end end @@ -80,7 +83,7 @@ def no_cols(c0:, c1:, c2:, c3:, c4:) c0, c1, c2, - c3, + encode_float(c3), encode_bytea(c4) ] unless client.prepared_statements.include?('other_sql_no_cols') @@ -138,6 +141,7 @@ def instrument(name) def with_client(name) instrument(name) do @db.with do |client| + client.reset if client.status == PG::CONNECTION_BAD # closed by the server (restart, failover, killed) configure(client) unless client.is_a?(ConnectionState) yield client end @@ -149,7 +153,10 @@ def configure(client) type_map = PG::BasicTypeMapForResults.new(client) type_map.default_type_map = string_arrays(client) # uuid, enums, money, ... come back as Strings client.type_map_for_results = type_map - client.type_map_for_queries = PG::BasicTypeMapForQueries.new(client) + queries = PG::BasicTypeMapForQueries.new(client) + # true / false as text ('t' / 'f'), also valid in COPY (the default binary encoder is not) + queries[TrueClass] = queries[FalseClass] = PG::TextEncoder::Boolean.new(oid: 16) + client.type_map_for_queries = queries client.extend(ConnectionState) end @@ -169,5 +176,10 @@ def string_arrays(client) def encode_bytea(value) value && PG::TextEncoder::Bytea.new.encode(value) end + + # Float#to_s is the shortest text that reads back as the same Float (the pg gem's encoder drops digits). + def encode_float(value) + value.is_a?(Float) ? value.to_s : value + end end end diff --git a/test/golden/pg/ruby32_notypes/query_sql.rb b/test/golden/pg/ruby32_notypes/query_sql.rb index 050e2d4..462945e 100644 --- a/test/golden/pg/ruby32_notypes/query_sql.rb +++ b/test/golden/pg/ruby32_notypes/query_sql.rb @@ -343,6 +343,61 @@ class ListRegressionsRow < Data.define( TRUNCATE TABLE regressions SQL + CREATE_BATCH_TYPES_BATCH_SQL = <<~'SQL' + INSERT INTO batch_types (id, ok, ratio, blobs, docs, counts, amounts, stamps) VALUES ($1, $2, $3, $4, $5, $6, $7, $8) + SQL + + class CreateBatchTypesBatchArgs < Data.define( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps + ) + end + + class CopyBatchTypesArgs < Data.define( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps + ) + end + + LIST_BATCH_TYPES_SQL = <<~'SQL' + SELECT id, ok, ratio, blobs, docs, counts, amounts, stamps FROM batch_types ORDER BY id + SQL + + class ListBatchTypesRow < Data.define( + :id, + :ok, + :ratio, + :blobs, + :docs, + :counts, + :amounts, + :stamps + ) + end + + COUNT_BATCH_TYPES_ABOVE_SQL = <<~'SQL' + SELECT count(*) FROM batch_types WHERE id > $1 + SQL + + class CountBatchTypesAboveRow < Data.define(:count) + end + + TRUNCATE_BATCH_TYPES_SQL = <<~'SQL' + TRUNCATE TABLE batch_types + SQL + MANY_PARAMS_SQL = <<~'SQL' SELECT 1 -- ? ? WHERE a = ? @@ -396,6 +451,8 @@ def initialize(db, logger: nil) def transaction(isolation: nil) with_client('transaction') do |client| outermost = client.transaction_status == PG::PQTRANS_IDLE + raise ArgumentError, 'isolation: cannot be set on a nested transaction' if isolation && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" if outermost client.exec(isolation ? "BEGIN ISOLATION LEVEL #{ISOLATION_LEVELS.fetch(isolation)}" : 'BEGIN') @@ -412,11 +469,12 @@ def transaction(isolation: nil) raise ensure unless failed + # a statement failed and the block rescued the error: PostgreSQL can only roll back + rescued = client.transaction_status == PG::PQTRANS_INERROR commit = outermost ? 'COMMIT' : "RELEASE SAVEPOINT #{savepoint}" - result = client.exec(Thread.current.status == 'aborting' ? rollback : commit) - # COMMIT of a transaction aborted by an error the block rescued is a ROLLBACK - rolled_back = outermost && result.cmd_status == 'ROLLBACK' && Thread.current.status != 'aborting' - raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rolled_back + client.exec(rescued || Thread.current.status == 'aborting' ? rollback : commit) + client.exec("RELEASE SAVEPOINT #{savepoint}") if rescued && savepoint + raise PG::InFailedSqlTransaction, 'transaction rolled back: a statement in it failed' if rescued && Thread.current.status != 'aborting' end end end @@ -579,7 +637,7 @@ def update_author_bios(bio:) def get_authors_by_ids(param1:) with_client('get_authors_by_ids') do |client| - query_params = [param1] + query_params = [encode_array(param1)] unless client.prepared_statements.include?('query_sql_get_authors_by_ids') client.prepare('query_sql_get_authors_by_ids', GET_AUTHORS_BY_IDS_SQL) client.prepared_statements.add('query_sql_get_authors_by_ids') @@ -593,7 +651,7 @@ def get_authors_by_ids(param1:) def get_authors_by_ids_and_names(param1:, param2:) with_client('get_authors_by_ids_and_names') do |client| - query_params = [param1, param2] + query_params = [encode_array(param1), encode_array(param2)] unless client.prepared_statements.include?('query_sql_get_authors_by_ids_and_names') client.prepare('query_sql_get_authors_by_ids_and_names', GET_AUTHORS_BY_IDS_AND_NAMES_SQL) client.prepared_statements.add('query_sql_get_authors_by_ids_and_names') @@ -731,8 +789,8 @@ def insert_extra_types(c_uuid:, c_jsonb:, c_text_array:, c_integer_array:, c_tim query_params = [ c_uuid, encode_json(c_jsonb), - c_text_array, - c_integer_array, + encode_array(c_text_array), + encode_array(c_integer_array), c_timestamptz, c_inet, c_money, @@ -786,7 +844,7 @@ def truncate_extra_types def insert_array_types(c_uuid_array:, c_bio_type_array:, c_integer_matrix:) with_client('insert_array_types') do |client| - query_params = [c_uuid_array, c_bio_type_array, c_integer_matrix] + query_params = [encode_array(c_uuid_array), encode_array(c_bio_type_array), encode_array(c_integer_matrix)] unless client.prepared_statements.include?('query_sql_insert_array_types') client.prepare('query_sql_insert_array_types', INSERT_ARRAY_TYPES_SQL) client.prepared_statements.add('query_sql_insert_array_types') @@ -974,8 +1032,8 @@ def create_regression(id:, order:, payload:, doc:, bio_types:, uids:) order, encode_bytea(payload), encode_json(doc), - bio_types, - uids + encode_array(bio_types), + encode_array(uids) ] unless client.prepared_statements.include?('query_sql_create_regression') client.prepare('query_sql_create_regression', CREATE_REGRESSION_SQL) @@ -1070,13 +1128,99 @@ def truncate_regressions end end + def create_batch_types_batch(rows) + with_client('create_batch_types_batch') do |client| + unless client.prepared_statements.include?('query_sql_create_batch_types_batch') + client.prepare('query_sql_create_batch_types_batch', CREATE_BATCH_TYPES_BATCH_SQL) + client.prepared_statements.add('query_sql_create_batch_types_batch') + end + rows.each do |args| + query_params = [ + args.id, + args.ok, + encode_float(args.ratio), + encode_array(args.blobs, :bytea), + encode_array(args.docs, :json), + encode_array(args.counts), + encode_array(args.amounts), + encode_array(args.stamps) + ] + client.exec_prepared('query_sql_create_batch_types_batch', query_params) + end + nil + end + end + + def copy_batch_types(rows) + with_client('copy_batch_types') do |client| + encoder = PG::TextEncoder::CopyRow.new(type_map: client.type_map_for_queries) + client.copy_data('COPY "batch_types" ("id", "ok", "ratio", "blobs", "docs", "counts", "amounts", "stamps") FROM STDIN', encoder) do + rows.each do |row| + client.put_copy_data([row.id, row.ok, encode_float(row.ratio), encode_array(row.blobs, :bytea), encode_array(row.docs, :json), encode_array(row.counts), encode_array(row.amounts), encode_array(row.stamps)]) + end + end + nil + end + end + + def list_batch_types + with_client('list_batch_types') do |client| + unless client.prepared_statements.include?('query_sql_list_batch_types') + client.prepare('query_sql_list_batch_types', LIST_BATCH_TYPES_SQL) + client.prepared_statements.add('query_sql_list_batch_types') + end + result = client.exec_prepared('query_sql_list_batch_types') + result.map do |row| + ListBatchTypesRow.new( + row['id'], + row['ok'], + row['ratio'], + row['blobs'], + row['docs'], + row['counts'], + row['amounts'], + row['stamps'] + ) + end + end + end + + def count_batch_types_above(min_id:) + with_client('count_batch_types_above') do |client| + query_params = [min_id] + unless client.prepared_statements.include?('query_sql_count_batch_types_above') + client.prepare('query_sql_count_batch_types_above', COUNT_BATCH_TYPES_ABOVE_SQL) + client.prepared_statements.add('query_sql_count_batch_types_above') + end + result = client.exec_prepared('query_sql_count_batch_types_above', query_params) + row = result.first + row && CountBatchTypesAboveRow.new(row['count']) + end + end + + # Like #count_batch_types_above, but raises NotFound when there is no row. + def count_batch_types_above!(...) + count_batch_types_above(...) || raise(NotFound, 'CountBatchTypesAbove returned no row') + end + + def truncate_batch_types + with_client('truncate_batch_types') do |client| + unless client.prepared_statements.include?('query_sql_truncate_batch_types') + client.prepare('query_sql_truncate_batch_types', TRUNCATE_BATCH_TYPES_SQL) + client.prepared_statements.add('query_sql_truncate_batch_types') + end + client.exec_prepared('query_sql_truncate_batch_types') + nil + end + end + def many_params(c0:, c1:, c2:, c3:, c4:) with_client('many_params') do |client| query_params = [ c0, c1, c2, - c3, + encode_float(c3), encode_bytea(c4) ] unless client.prepared_statements.include?('query_sql_many_params') @@ -1144,6 +1288,7 @@ def instrument(name) def with_client(name) instrument(name) do @db.with do |client| + client.reset if client.status == PG::CONNECTION_BAD # closed by the server (restart, failover, killed) configure(client) unless client.is_a?(ConnectionState) yield client end @@ -1155,7 +1300,10 @@ def configure(client) type_map = PG::BasicTypeMapForResults.new(client) type_map.default_type_map = string_arrays(client) # uuid, enums, money, ... come back as Strings client.type_map_for_results = type_map - client.type_map_for_queries = PG::BasicTypeMapForQueries.new(client) + queries = PG::BasicTypeMapForQueries.new(client) + # true / false as text ('t' / 'f'), also valid in COPY (the default binary encoder is not) + queries[TrueClass] = queries[FalseClass] = PG::TextEncoder::Boolean.new(oid: 16) + client.type_map_for_queries = queries client.extend(ConnectionState) end @@ -1180,5 +1328,31 @@ def encode_bytea(value) def encode_json(value) value.is_a?(Array) ? PG::TextEncoder::JSON.new.encode(value) : value end + + # Float#to_s is the shortest text that reads back as the same Float (the pg gem's encoder drops digits). + def encode_float(value) + value.is_a?(Float) ? value.to_s : value + end + + # Array parameters are sent as array literals, which the server converts to the parameter's type (the pg gem + # would type them from their first element: wrong for [], [nil, ...], bytea[], json[]). + def encode_array(value, kind = :value) + return value unless value.is_a?(Array) + + PG::TextEncoder::Array.new(elements_type: PG::TextEncoder::String.new).encode(array_elements(value, kind)) + end + + def array_elements(values, kind) + values.map do |value| + case value + when nil then nil + when Array then kind == :json ? PG::TextEncoder::JSON.new.encode(value) : array_elements(value, kind) + when Hash then PG::TextEncoder::JSON.new.encode(value) + when String then kind == :bytea ? PG::TextEncoder::Bytea.new.encode(value) : value + when Time then PG::TextEncoder::TimestampWithTimeZone.new.encode(value) + else defined?(IPAddr) && value.is_a?(IPAddr) ? "#{value}/#{value.prefix}" : value.to_s + end + end + end end end diff --git a/test/golden/sqlite3/default/models.rb b/test/golden/sqlite3/default/models.rb index a880702..498e262 100644 --- a/test/golden/sqlite3/default/models.rb +++ b/test/golden/sqlite3/default/models.rb @@ -22,4 +22,7 @@ class DeclaredType < Data.define( class Regression < Data.define(:id, :order, :price) end + + class Tag < Data.define(:name, :uses) + end end diff --git a/test/golden/sqlite3/default/models.rbs b/test/golden/sqlite3/default/models.rbs index 7b76c71..60532b3 100644 --- a/test/golden/sqlite3/default/models.rbs +++ b/test/golden/sqlite3/default/models.rbs @@ -40,4 +40,10 @@ module Sqlite3Codegen attr_reader price: Float? def initialize: ( Integer id, String order, Float? price) -> void end + + class Tag + attr_reader name: String + attr_reader uses: Integer + def initialize: ( String name, Integer uses) -> void + end end diff --git a/test/golden/sqlite3/default/my_queries_sql.rb b/test/golden/sqlite3/default/my_queries_sql.rb index 2e11fbe..5298177 100644 --- a/test/golden/sqlite3/default/my_queries_sql.rb +++ b/test/golden/sqlite3/default/my_queries_sql.rb @@ -35,6 +35,8 @@ def transaction(mode: nil) instrument('transaction') do @lock.synchronize do outermost = !@db.transaction_active? + raise ArgumentError, 'mode: cannot be set on a nested transaction' if mode && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" @db.execute(outermost ? "BEGIN#{" #{mode.to_s.upcase}" if mode}" : "SAVEPOINT #{savepoint}") rollback = outermost ? 'ROLLBACK' : "ROLLBACK TO SAVEPOINT #{savepoint}" diff --git a/test/golden/sqlite3/default/other_sql.rb b/test/golden/sqlite3/default/other_sql.rb index 8089748..5d31abc 100644 --- a/test/golden/sqlite3/default/other_sql.rb +++ b/test/golden/sqlite3/default/other_sql.rb @@ -48,6 +48,8 @@ def transaction(mode: nil) instrument('transaction') do @lock.synchronize do outermost = !@db.transaction_active? + raise ArgumentError, 'mode: cannot be set on a nested transaction' if mode && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" @db.execute(outermost ? "BEGIN#{" #{mode.to_s.upcase}" if mode}" : "SAVEPOINT #{savepoint}") rollback = outermost ? 'ROLLBACK' : "ROLLBACK TO SAVEPOINT #{savepoint}" diff --git a/test/golden/sqlite3/default/query_sql.rb b/test/golden/sqlite3/default/query_sql.rb index d44d1f0..5fe3381 100644 --- a/test/golden/sqlite3/default/query_sql.rb +++ b/test/golden/sqlite3/default/query_sql.rb @@ -218,6 +218,26 @@ class GetMaxRegressionPriceRow < Data.define(:max_price) DELETE FROM regressions SQL + class CopyTagsOrReplaceArgs < Data.define(:name, :uses) + end + + class CopyTagsOrIgnoreArgs < Data.define(:name, :uses) + end + + class CopyTagsAddingUsesArgs < Data.define(:name, :uses) + end + + LIST_TAGS_SQL = <<~'SQL' + SELECT name, uses FROM tags ORDER BY name + SQL + + class ListTagsRow < Data.define(:name, :uses) + end + + TRUNCATE_TAGS_SQL = <<~'SQL' + DELETE FROM tags + SQL + MANY_PARAMS_SQL = <<~'SQL' SELECT 1 -- ? ? WHERE a = ? @@ -270,6 +290,8 @@ def transaction(mode: nil) instrument('transaction') do @lock.synchronize do outermost = !@db.transaction_active? + raise ArgumentError, 'mode: cannot be set on a nested transaction' if mode && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" @db.execute(outermost ? "BEGIN#{" #{mode.to_s.upcase}" if mode}" : "SAVEPOINT #{savepoint}") rollback = outermost ? 'ROLLBACK' : "ROLLBACK TO SAVEPOINT #{savepoint}" @@ -731,6 +753,68 @@ def truncate_regressions end end + def copy_tags_or_replace(rows) + with_client('copy_tags_or_replace') do |client| + atomically(client) do + rows.each_slice(499) do |batch| + sql = %(INSERT OR REPLACE INTO "tags" ("name", "uses") VALUES #{Array.new(batch.size, '(?, ?)').join(', ')}) + client.execute(sql, batch.flat_map { |row| [row.name, row.uses] }) + end + end + nil + end + end + + def copy_tags_or_ignore(rows) + with_client('copy_tags_or_ignore') do |client| + atomically(client) do + rows.each_slice(499) do |batch| + sql = %(INSERT OR IGNORE INTO "tags" ("name", "uses") VALUES #{Array.new(batch.size, '(?, ?)').join(', ')}) + client.execute(sql, batch.flat_map { |row| [row.name, row.uses] }) + end + end + nil + end + end + + def copy_tags_adding_uses(rows) + with_client('copy_tags_adding_uses') do |client| + atomically(client) do + rows.each_slice(499) do |batch| + sql = %(INSERT INTO "tags" ("name", "uses") VALUES #{Array.new(batch.size, '(?, ?)').join(', ')} ON CONFLICT (name) DO UPDATE SET uses = tags.uses + excluded.uses) + client.execute(sql, batch.flat_map { |row| [row.name, row.uses] }) + end + end + nil + end + end + + def list_tags + with_client('list_tags') do |client| + stmt = client.prepare(LIST_TAGS_SQL) + begin + result = stmt.execute + result.map do |row| + ListTagsRow.new(row[0], row[1]) + end + ensure + stmt.close + end + end + end + + def truncate_tags + with_client('truncate_tags') do |client| + stmt = client.prepare(TRUNCATE_TAGS_SQL) + begin + stmt.execute + nil + ensure + stmt.close + end + end + end + def many_params(c0:, c1:, c2:, c3:, c4:) with_client('many_params') do |client| query_params = [ diff --git a/test/golden/sqlite3/default/query_sql.rbs b/test/golden/sqlite3/default/query_sql.rbs index 0d6b211..8d2029f 100644 --- a/test/golden/sqlite3/default/query_sql.rbs +++ b/test/golden/sqlite3/default/query_sql.rbs @@ -209,6 +209,34 @@ module Sqlite3Codegen TRUNCATE_REGRESSIONS_SQL: String + class CopyTagsOrReplaceArgs + attr_reader name: String + attr_reader uses: Integer + def initialize: ( String name, Integer uses) -> void + end + + class CopyTagsOrIgnoreArgs + attr_reader name: String + attr_reader uses: Integer + def initialize: ( String name, Integer uses) -> void + end + + class CopyTagsAddingUsesArgs + attr_reader name: String + attr_reader uses: Integer + def initialize: ( String name, Integer uses) -> void + end + + LIST_TAGS_SQL: String + + class ListTagsRow + attr_reader name: String + attr_reader uses: Integer + def initialize: ( String name, Integer uses) -> void + end + + TRUNCATE_TAGS_SQL: String + MANY_PARAMS_SQL: String class ManyParamsRow @@ -322,6 +350,16 @@ module Sqlite3Codegen def truncate_regressions: -> void + def copy_tags_or_replace: (Array[CopyTagsOrReplaceArgs] rows) -> void + + def copy_tags_or_ignore: (Array[CopyTagsOrIgnoreArgs] rows) -> void + + def copy_tags_adding_uses: (Array[CopyTagsAddingUsesArgs] rows) -> void + + def list_tags: -> Array[ListTagsRow] + + def truncate_tags: -> void + def many_params: (c0: Integer, c1: String, c2: Float, c3: String, c4: Integer) -> ManyParamsRow? def many_params!: (c0: Integer, c1: String, c2: Float, c3: String, c4: Integer) -> ManyParamsRow diff --git a/test/golden/sqlite3/ruby31_gemfile/models.rb b/test/golden/sqlite3/ruby31_gemfile/models.rb index 5c3baaf..622dde5 100644 --- a/test/golden/sqlite3/ruby31_gemfile/models.rb +++ b/test/golden/sqlite3/ruby31_gemfile/models.rb @@ -25,4 +25,6 @@ module Sqlite3Codegen ) Regression = Struct.new(:id, :order, :price, keyword_init: true) + + Tag = Struct.new(:name, :uses, keyword_init: true) end diff --git a/test/golden/sqlite3/ruby31_gemfile/models.rbs b/test/golden/sqlite3/ruby31_gemfile/models.rbs index 18006c9..27900f5 100644 --- a/test/golden/sqlite3/ruby31_gemfile/models.rbs +++ b/test/golden/sqlite3/ruby31_gemfile/models.rbs @@ -52,4 +52,11 @@ module Sqlite3Codegen def initialize: ( id: Integer, order: String, price: Float?) -> void def self.new: ( id: Integer, order: String, price: Float?) -> Regression end + + class Tag < ::Struct[untyped] + attr_accessor name: String + attr_accessor uses: Integer + def initialize: ( name: String, uses: Integer) -> void + def self.new: ( name: String, uses: Integer) -> Tag + end end diff --git a/test/golden/sqlite3/ruby31_gemfile/my_queries_sql.rb b/test/golden/sqlite3/ruby31_gemfile/my_queries_sql.rb index 2e11fbe..5298177 100644 --- a/test/golden/sqlite3/ruby31_gemfile/my_queries_sql.rb +++ b/test/golden/sqlite3/ruby31_gemfile/my_queries_sql.rb @@ -35,6 +35,8 @@ def transaction(mode: nil) instrument('transaction') do @lock.synchronize do outermost = !@db.transaction_active? + raise ArgumentError, 'mode: cannot be set on a nested transaction' if mode && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" @db.execute(outermost ? "BEGIN#{" #{mode.to_s.upcase}" if mode}" : "SAVEPOINT #{savepoint}") rollback = outermost ? 'ROLLBACK' : "ROLLBACK TO SAVEPOINT #{savepoint}" diff --git a/test/golden/sqlite3/ruby31_gemfile/other_sql.rb b/test/golden/sqlite3/ruby31_gemfile/other_sql.rb index a8cb909..ce4c088 100644 --- a/test/golden/sqlite3/ruby31_gemfile/other_sql.rb +++ b/test/golden/sqlite3/ruby31_gemfile/other_sql.rb @@ -47,6 +47,8 @@ def transaction(mode: nil) instrument('transaction') do @lock.synchronize do outermost = !@db.transaction_active? + raise ArgumentError, 'mode: cannot be set on a nested transaction' if mode && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" @db.execute(outermost ? "BEGIN#{" #{mode.to_s.upcase}" if mode}" : "SAVEPOINT #{savepoint}") rollback = outermost ? 'ROLLBACK' : "ROLLBACK TO SAVEPOINT #{savepoint}" diff --git a/test/golden/sqlite3/ruby31_gemfile/query_sql.rb b/test/golden/sqlite3/ruby31_gemfile/query_sql.rb index a0ac386..088a2e4 100644 --- a/test/golden/sqlite3/ruby31_gemfile/query_sql.rb +++ b/test/golden/sqlite3/ruby31_gemfile/query_sql.rb @@ -202,6 +202,22 @@ module Sqlite3Codegen DELETE FROM regressions SQL + CopyTagsOrReplaceArgs = Struct.new(:name, :uses, keyword_init: true) + + CopyTagsOrIgnoreArgs = Struct.new(:name, :uses, keyword_init: true) + + CopyTagsAddingUsesArgs = Struct.new(:name, :uses, keyword_init: true) + + LIST_TAGS_SQL = <<~'SQL' + SELECT name, uses FROM tags ORDER BY name + SQL + + ListTagsRow = Struct.new(:name, :uses, keyword_init: true) + + TRUNCATE_TAGS_SQL = <<~'SQL' + DELETE FROM tags + SQL + MANY_PARAMS_SQL = <<~'SQL' SELECT 1 -- ? ? WHERE a = ? @@ -258,6 +274,8 @@ def transaction(mode: nil) instrument('transaction') do @lock.synchronize do outermost = !@db.transaction_active? + raise ArgumentError, 'mode: cannot be set on a nested transaction' if mode && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" @db.execute(outermost ? "BEGIN#{" #{mode.to_s.upcase}" if mode}" : "SAVEPOINT #{savepoint}") rollback = outermost ? 'ROLLBACK' : "ROLLBACK TO SAVEPOINT #{savepoint}" @@ -726,6 +744,68 @@ def truncate_regressions end end + def copy_tags_or_replace(rows) + with_client('copy_tags_or_replace') do |client| + atomically(client) do + rows.each_slice(499) do |batch| + sql = %(INSERT OR REPLACE INTO "tags" ("name", "uses") VALUES #{Array.new(batch.size, '(?, ?)').join(', ')}) + client.execute(sql, batch.flat_map { |row| [row.name, row.uses] }) + end + end + nil + end + end + + def copy_tags_or_ignore(rows) + with_client('copy_tags_or_ignore') do |client| + atomically(client) do + rows.each_slice(499) do |batch| + sql = %(INSERT OR IGNORE INTO "tags" ("name", "uses") VALUES #{Array.new(batch.size, '(?, ?)').join(', ')}) + client.execute(sql, batch.flat_map { |row| [row.name, row.uses] }) + end + end + nil + end + end + + def copy_tags_adding_uses(rows) + with_client('copy_tags_adding_uses') do |client| + atomically(client) do + rows.each_slice(499) do |batch| + sql = %(INSERT INTO "tags" ("name", "uses") VALUES #{Array.new(batch.size, '(?, ?)').join(', ')} ON CONFLICT (name) DO UPDATE SET uses = tags.uses + excluded.uses) + client.execute(sql, batch.flat_map { |row| [row.name, row.uses] }) + end + end + nil + end + end + + def list_tags + with_client('list_tags') do |client| + stmt = client.prepare(LIST_TAGS_SQL) + begin + result = stmt.execute + result.map do |row| + ListTagsRow.new(name: row[0], uses: row[1]) + end + ensure + stmt.close + end + end + end + + def truncate_tags + with_client('truncate_tags') do |client| + stmt = client.prepare(TRUNCATE_TAGS_SQL) + begin + stmt.execute + nil + ensure + stmt.close + end + end + end + def many_params(c0:, c1:, c2:, c3:, c4:) with_client('many_params') do |client| query_params = [ diff --git a/test/golden/sqlite3/ruby31_gemfile/query_sql.rbs b/test/golden/sqlite3/ruby31_gemfile/query_sql.rbs index 4f449d8..e939847 100644 --- a/test/golden/sqlite3/ruby31_gemfile/query_sql.rbs +++ b/test/golden/sqlite3/ruby31_gemfile/query_sql.rbs @@ -240,6 +240,38 @@ module Sqlite3Codegen TRUNCATE_REGRESSIONS_SQL: String + class CopyTagsOrReplaceArgs < ::Struct[untyped] + attr_accessor name: String + attr_accessor uses: Integer + def initialize: ( name: String, uses: Integer) -> void + def self.new: ( name: String, uses: Integer) -> CopyTagsOrReplaceArgs + end + + class CopyTagsOrIgnoreArgs < ::Struct[untyped] + attr_accessor name: String + attr_accessor uses: Integer + def initialize: ( name: String, uses: Integer) -> void + def self.new: ( name: String, uses: Integer) -> CopyTagsOrIgnoreArgs + end + + class CopyTagsAddingUsesArgs < ::Struct[untyped] + attr_accessor name: String + attr_accessor uses: Integer + def initialize: ( name: String, uses: Integer) -> void + def self.new: ( name: String, uses: Integer) -> CopyTagsAddingUsesArgs + end + + LIST_TAGS_SQL: String + + class ListTagsRow < ::Struct[untyped] + attr_accessor name: String + attr_accessor uses: Integer + def initialize: ( name: String, uses: Integer) -> void + def self.new: ( name: String, uses: Integer) -> ListTagsRow + end + + TRUNCATE_TAGS_SQL: String + MANY_PARAMS_SQL: String class ManyParamsRow < ::Struct[untyped] @@ -363,6 +395,16 @@ module Sqlite3Codegen def truncate_regressions: -> void + def copy_tags_or_replace: (Array[CopyTagsOrReplaceArgs] rows) -> void + + def copy_tags_or_ignore: (Array[CopyTagsOrIgnoreArgs] rows) -> void + + def copy_tags_adding_uses: (Array[CopyTagsAddingUsesArgs] rows) -> void + + def list_tags: -> Array[ListTagsRow] + + def truncate_tags: -> void + def many_params: (c0: Integer, c1: String, c2: Float, c3: String, c4: Integer) -> ManyParamsRow? def many_params!: (c0: Integer, c1: String, c2: Float, c3: String, c4: Integer) -> ManyParamsRow diff --git a/test/golden/sqlite3/ruby32_notypes/models.rb b/test/golden/sqlite3/ruby32_notypes/models.rb index a880702..498e262 100644 --- a/test/golden/sqlite3/ruby32_notypes/models.rb +++ b/test/golden/sqlite3/ruby32_notypes/models.rb @@ -22,4 +22,7 @@ class DeclaredType < Data.define( class Regression < Data.define(:id, :order, :price) end + + class Tag < Data.define(:name, :uses) + end end diff --git a/test/golden/sqlite3/ruby32_notypes/my_queries_sql.rb b/test/golden/sqlite3/ruby32_notypes/my_queries_sql.rb index 2e11fbe..5298177 100644 --- a/test/golden/sqlite3/ruby32_notypes/my_queries_sql.rb +++ b/test/golden/sqlite3/ruby32_notypes/my_queries_sql.rb @@ -35,6 +35,8 @@ def transaction(mode: nil) instrument('transaction') do @lock.synchronize do outermost = !@db.transaction_active? + raise ArgumentError, 'mode: cannot be set on a nested transaction' if mode && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" @db.execute(outermost ? "BEGIN#{" #{mode.to_s.upcase}" if mode}" : "SAVEPOINT #{savepoint}") rollback = outermost ? 'ROLLBACK' : "ROLLBACK TO SAVEPOINT #{savepoint}" diff --git a/test/golden/sqlite3/ruby32_notypes/other_sql.rb b/test/golden/sqlite3/ruby32_notypes/other_sql.rb index 8089748..5d31abc 100644 --- a/test/golden/sqlite3/ruby32_notypes/other_sql.rb +++ b/test/golden/sqlite3/ruby32_notypes/other_sql.rb @@ -48,6 +48,8 @@ def transaction(mode: nil) instrument('transaction') do @lock.synchronize do outermost = !@db.transaction_active? + raise ArgumentError, 'mode: cannot be set on a nested transaction' if mode && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" @db.execute(outermost ? "BEGIN#{" #{mode.to_s.upcase}" if mode}" : "SAVEPOINT #{savepoint}") rollback = outermost ? 'ROLLBACK' : "ROLLBACK TO SAVEPOINT #{savepoint}" diff --git a/test/golden/sqlite3/ruby32_notypes/query_sql.rb b/test/golden/sqlite3/ruby32_notypes/query_sql.rb index d44d1f0..5fe3381 100644 --- a/test/golden/sqlite3/ruby32_notypes/query_sql.rb +++ b/test/golden/sqlite3/ruby32_notypes/query_sql.rb @@ -218,6 +218,26 @@ class GetMaxRegressionPriceRow < Data.define(:max_price) DELETE FROM regressions SQL + class CopyTagsOrReplaceArgs < Data.define(:name, :uses) + end + + class CopyTagsOrIgnoreArgs < Data.define(:name, :uses) + end + + class CopyTagsAddingUsesArgs < Data.define(:name, :uses) + end + + LIST_TAGS_SQL = <<~'SQL' + SELECT name, uses FROM tags ORDER BY name + SQL + + class ListTagsRow < Data.define(:name, :uses) + end + + TRUNCATE_TAGS_SQL = <<~'SQL' + DELETE FROM tags + SQL + MANY_PARAMS_SQL = <<~'SQL' SELECT 1 -- ? ? WHERE a = ? @@ -270,6 +290,8 @@ def transaction(mode: nil) instrument('transaction') do @lock.synchronize do outermost = !@db.transaction_active? + raise ArgumentError, 'mode: cannot be set on a nested transaction' if mode && !outermost + savepoint = outermost ? nil : "sqlc_sp_#{rand(0xffff_ffff).to_s(16)}" @db.execute(outermost ? "BEGIN#{" #{mode.to_s.upcase}" if mode}" : "SAVEPOINT #{savepoint}") rollback = outermost ? 'ROLLBACK' : "ROLLBACK TO SAVEPOINT #{savepoint}" @@ -731,6 +753,68 @@ def truncate_regressions end end + def copy_tags_or_replace(rows) + with_client('copy_tags_or_replace') do |client| + atomically(client) do + rows.each_slice(499) do |batch| + sql = %(INSERT OR REPLACE INTO "tags" ("name", "uses") VALUES #{Array.new(batch.size, '(?, ?)').join(', ')}) + client.execute(sql, batch.flat_map { |row| [row.name, row.uses] }) + end + end + nil + end + end + + def copy_tags_or_ignore(rows) + with_client('copy_tags_or_ignore') do |client| + atomically(client) do + rows.each_slice(499) do |batch| + sql = %(INSERT OR IGNORE INTO "tags" ("name", "uses") VALUES #{Array.new(batch.size, '(?, ?)').join(', ')}) + client.execute(sql, batch.flat_map { |row| [row.name, row.uses] }) + end + end + nil + end + end + + def copy_tags_adding_uses(rows) + with_client('copy_tags_adding_uses') do |client| + atomically(client) do + rows.each_slice(499) do |batch| + sql = %(INSERT INTO "tags" ("name", "uses") VALUES #{Array.new(batch.size, '(?, ?)').join(', ')} ON CONFLICT (name) DO UPDATE SET uses = tags.uses + excluded.uses) + client.execute(sql, batch.flat_map { |row| [row.name, row.uses] }) + end + end + nil + end + end + + def list_tags + with_client('list_tags') do |client| + stmt = client.prepare(LIST_TAGS_SQL) + begin + result = stmt.execute + result.map do |row| + ListTagsRow.new(row[0], row[1]) + end + ensure + stmt.close + end + end + end + + def truncate_tags + with_client('truncate_tags') do |client| + stmt = client.prepare(TRUNCATE_TAGS_SQL) + begin + stmt.execute + nil + ensure + stmt.close + end + end + end + def many_params(c0:, c1:, c2:, c3:, c4:) with_client('many_params') do |client| query_params = [ diff --git a/tests/end2end_mysql2.rb b/tests/end2end_mysql2.rb index 62fd135..fe81bea 100755 --- a/tests/end2end_mysql2.rb +++ b/tests/end2end_mysql2.rb @@ -3,6 +3,7 @@ require 'minitest/autorun' require 'logger' require 'stringio' +require 'json' require 'async' require 'async/barrier' require_relative 'consts' @@ -28,6 +29,7 @@ def setup @query_sql.truncate_authors # cascades to books @query_sql.truncate_extended_bios @query_sql.truncate_regressions + @query_sql.truncate_copy_types end def create_author(name, bio = nil) @@ -428,4 +430,45 @@ def test_transaction_left_with_break_or_throw_commits_and_frees_the_connection pooled.instance_variable_get(:@db).with { |client| refute client.in_transaction } assert_equal ['break 0', 'break 1', 'throw'], @query_sql.list_authors.map(&:name).sort end + + # ---- final sweep fixes: :copyfrom into JSON, IGNORE / REPLACE, reused statements, dead connections ------------ + + def copy_type_rows = @query_sql.list_copy_types.map { |r| [r.id, r.code, r.doc && JSON.parse(r.doc), r.flags, r.amount] } + + def test_copyfrom_loads_json_next_to_binary_values + @query_sql.copy_copy_types([CopyCopyTypesArgs.new(id: 1, code: 'é', doc: '{"a": [1, "é"]}', flags: "\x83".b, amount: 4_000_000_000), + CopyCopyTypesArgs.new(id: 2, code: 'b', doc: nil, flags: nil, amount: nil)]) + assert_equal [[1, 'é', { 'a' => [1, 'é'] }, "\x83".b, 4_000_000_000], [2, 'b', nil, nil, nil]], copy_type_rows + end + + def test_copyfrom_keeps_insert_ignore_and_replace + @query_sql.copy_copy_types_ignore([CopyCopyTypesIgnoreArgs.new(id: 1, code: 'a'), CopyCopyTypesIgnoreArgs.new(id: 1, code: 'b')]) + assert_equal [[1, 'a']], copy_type_rows.map { |r| r.first(2) } + @query_sql.copy_copy_types_replace([CopyCopyTypesReplaceArgs.new(id: 1, code: 'c'), CopyCopyTypesReplaceArgs.new(id: 2, code: 'd')]) + assert_equal [[1, 'c'], [2, 'd']], copy_type_rows.map { |r| r.first(2) } + end + + def test_batch_with_binary_parameters_after_an_unsigned_int + rows = (1..5).map { |i| CreateCopyTypesBatchArgs.new(id: i, code: "c#{i}", flags: [0x83 + i].pack('C'), amount: 586_988_122 + i) } + @query_sql.create_copy_types_batch(rows) + assert_equal(rows.map { |r| [r.id, r.flags, r.amount] }, copy_type_rows.map { |r| [r[0], r[3], r[4]] }) + counts = @query_sql.count_copy_types_by_codes([CountCopyTypesByCodesArgs.new(code: 'c1', code_2: 'c2')]) + assert_equal [2], counts.map(&:matches) + end + + def test_nested_transaction_rejects_an_isolation_level + @query_sql.transaction do |tx| + assert_raises(ArgumentError) { tx.transaction(isolation: :serializable) { nil } } + end + end + + def test_a_connection_closed_by_the_server_is_replaced + pooled = QuerySql.connect(mysql2_params, size: 1) + id = pooled.instance_variable_get(:@db).with(&:thread_id) + admin = Mysql2::Client.new(**mysql2_params.transform_keys(&:to_sym)) + admin.query("KILL #{id}") + admin.close + assert_raises(Mysql2::Error) { pooled.list_authors } + assert_equal [], pooled.list_authors + end end diff --git a/tests/end2end_pg.rb b/tests/end2end_pg.rb index 0bec6df..04e99a6 100755 --- a/tests/end2end_pg.rb +++ b/tests/end2end_pg.rb @@ -34,6 +34,7 @@ def setup @query_sql.truncate_array_types @query_sql.truncate_special_types @query_sql.truncate_regressions + @query_sql.truncate_batch_types end def create_author(name, bio = nil) @@ -554,4 +555,82 @@ def test_transaction_left_with_break_or_throw_commits_and_frees_the_connection pooled.instance_variable_get(:@db).with { |client| assert_equal PG::PQTRANS_IDLE, client.transaction_status } assert_equal ['break 0', 'break 1', 'throw'], @query_sql.list_authors.map(&:name).sort end + + # ---- final sweep fixes: :batch* / :copyfrom parameter encoding, nested rescued errors, dead connections --------- + + def batch_type_rows + stamp = Time.at(1_700_000_000, 123_456, :usec) + [{ id: 1, ok: true, ratio: 115_610.38729014261, blobs: ["\x00\xFF".b, nil], docs: [{ 'a' => 1 }, [1, 2], nil], + counts: [], amounts: [nil, BigDecimal('1.25')], stamps: [nil, stamp] }, + { id: 2, ok: false, ratio: 1.0e-305, blobs: [], docs: [], counts: [nil], amounts: [], stamps: [] }, + { id: 3, ok: nil, ratio: nil, blobs: nil, docs: nil, counts: nil, amounts: nil, stamps: nil }] + end + + def assert_batch_types_round_trip(rows) + got = @query_sql.list_batch_types.map(&:to_h) + assert_equal rows.size, got.size + rows.zip(got).each do |want, row| + want.each do |key, value| + if key == :stamps && value + assert_equal value.map { |t| t&.to_r }, row[key].map { |t| t&.to_r }, "#{want[:id]} #{key}" + elsif value.nil? + assert_nil row[key], "#{want[:id]} #{key}" + else + assert_equal value, row[key], "#{want[:id]} #{key}" + end + end + end + end + + def test_copyfrom_encodes_booleans_floats_and_arrays + @query_sql.copy_batch_types(batch_type_rows.map { |r| CopyBatchTypesArgs.new(**r) }) + assert_batch_types_round_trip(batch_type_rows) + end + + def test_batchexec_encodes_booleans_floats_and_arrays + @query_sql.create_batch_types_batch(batch_type_rows.map { |r| CreateBatchTypesBatchArgs.new(**r) }) + assert_batch_types_round_trip(batch_type_rows) + end + + def test_array_parameters_need_no_server_side_parameter_types + # what preparedStatements: false sends: exec_params without prepared parameter types + params = [9, nil, nil, @query_sql.send(:encode_array, ["\x00".b], :bytea), @query_sql.send(:encode_array, [{ 'a' => 1 }], :json), + @query_sql.send(:encode_array, []), @query_sql.send(:encode_array, [nil]), @query_sql.send(:encode_array, [nil, Time.now])] + @query_sql.instance_variable_get(:@db).with { |client| client.exec_params(CREATE_BATCH_TYPES_BATCH_SQL, params) } + row = @query_sql.list_batch_types.first + assert_equal [["\x00".b], [{ 'a' => 1 }], [], [nil]], [row.blobs, row.docs, row.counts, row.amounts] + end + + def test_all_caps_parameter_names_are_snake_cased + assert_equal 0, @query_sql.count_batch_types_above!(min_id: 0).count + end + + def test_nested_transaction_rescuing_a_failed_statement_rolls_back_only_itself + @query_sql.transaction do |tx| + tx.create_author(name: 'outer', bio: nil) + assert_raises(PG::InFailedSqlTransaction) do + tx.transaction do |inner| + inner.create_author(name: 'inner', bio: nil) + assert_raises(PG::Error) { inner.create_book(name: 'b', author_id: -1) } # foreign key violation + end + end + tx.create_author(name: 'outer 2', bio: nil) + end + assert_equal ['outer', 'outer 2'], @query_sql.list_authors.map(&:name).sort + end + + def test_nested_transaction_rejects_an_isolation_level + @query_sql.transaction do |tx| + assert_raises(ArgumentError) { tx.transaction(isolation: :serializable) { nil } } + end + end + + def test_a_connection_closed_by_the_server_is_reconnected + pooled = QuerySql.connect(pg_params, size: 1) + assert_equal [], pooled.list_authors + pid = pooled.instance_variable_get(:@db).with(&:backend_pid) + PG.connect(**pg_params.transform_keys(&:to_sym)) { |admin| admin.exec_params('SELECT pg_terminate_backend($1)', [pid]) } + assert_raises(PG::Error) { pooled.list_authors } + assert_equal [], pooled.list_authors + end end diff --git a/tests/end2end_sqlite3.rb b/tests/end2end_sqlite3.rb index 7c33379..2961ee5 100644 --- a/tests/end2end_sqlite3.rb +++ b/tests/end2end_sqlite3.rb @@ -338,4 +338,21 @@ def test_transaction_left_with_break_or_throw_commits refute @db.transaction_active? assert_equal ['break 0', 'break 1', 'throw'], @query_sql.list_authors.map(&:name).sort end + + # ---- final sweep fixes: :copyfrom conflict clauses, nested modes ------------------------------------------------ + + def test_copyfrom_keeps_conflict_clauses + @query_sql.copy_tags_or_ignore([CopyTagsOrIgnoreArgs.new('a', 1), CopyTagsOrIgnoreArgs.new('a', 2)]) + assert_equal [['a', 1]], @query_sql.list_tags.map { |t| [t.name, t.uses] } + @query_sql.copy_tags_or_replace([CopyTagsOrReplaceArgs.new('a', 5), CopyTagsOrReplaceArgs.new('b', 1)]) + assert_equal [['a', 5], ['b', 1]], @query_sql.list_tags.map { |t| [t.name, t.uses] } + @query_sql.copy_tags_adding_uses([CopyTagsAddingUsesArgs.new('a', 1), CopyTagsAddingUsesArgs.new('c', 1)]) + assert_equal [['a', 6], ['b', 1], ['c', 1]], @query_sql.list_tags.map { |t| [t.name, t.uses] } + end + + def test_nested_transaction_rejects_a_mode + @query_sql.transaction do |tx| + assert_raises(ArgumentError) { tx.transaction(mode: :immediate) { nil } } + end + end end