From 7ef39adf7b455853fa89cd72ff0d3a87b96fb41b Mon Sep 17 00:00:00 2001 From: Stoica Alex Date: Mon, 3 Jul 2023 10:04:46 +0100 Subject: [PATCH 1/5] Add new `fetch_with_database` method This will allow us to simplify our code, and not have to continously call `connect` function; but allow the re-use of connections. Currently, there is no DB pool configuration specified, this will be a follow-up. --- ext/client.go | 97 +++++++++++++++-- ext/result.go | 12 +++ ext/ruby_snowflake.go | 2 + lib/ruby_snowflake_client.rb | 12 +++ spec/snowflake/client_spec.rb | 190 ++++++++++++++++++++++++++++++++-- 5 files changed, 296 insertions(+), 17 deletions(-) diff --git a/ext/client.go b/ext/client.go index 7fb1b6d..fa873c2 100644 --- a/ext/client.go +++ b/ext/client.go @@ -21,16 +21,25 @@ import ( ) type SnowflakeClient struct { - db *sql.DB + db *sql.DB + cfg *sf.Config } -func (x SnowflakeClient) Fetch(statement C.VALUE) C.VALUE { +func (x SnowflakeClient) Fetch(statement string) C.VALUE { t1 := time.Now() if LOG_LEVEL > 0 { - fmt.Println("statement", RbGoString(statement)) + fmt.Println("statement", statement) } - rows, err := x.db.QueryContext(sf.WithHigherPrecision(context.Background()), RbGoString(statement)) + if LOG_LEVEL > 0 { + fmt.Println("getting conn") + } + dbCtx := context.Background() + conn, _ := x.db.Conn(dbCtx) + if LOG_LEVEL > 0 { + fmt.Println("got conn") + } + rows, err := conn.QueryContext(sf.WithHigherPrecision(context.Background()), statement) if err != nil { result := C.rb_class_new_instance(0, &empty, rbSnowflakeResultClass) errStr := fmt.Sprintf("Query error: '%s'", err.Error()) @@ -42,6 +51,56 @@ func (x SnowflakeClient) Fetch(statement C.VALUE) C.VALUE { if LOG_LEVEL > 0 { fmt.Printf("Query duration: %s\n", time.Now().Sub(t1)) } + + result := C.rb_class_new_instance(0, &empty, rbSnowflakeResultClass) + if LOG_LEVEL > 0 { + fmt.Println("create new instance") + } + cols, _ := rows.Columns() + for idx, col := range cols { + col := col + cols[idx] = strings.ToLower(col) + } + rs := SnowflakeResult{rows, cols, conn} + resultMap[result] = &rs + if LOG_LEVEL > 0 { + fmt.Println("after for & map") + } + C.rb_ivar_set(result, RESULT_DURATION, RbNumFromDouble(C.double(duration))) + if LOG_LEVEL > 0 { + fmt.Println("end of func") + } + return result +} + +func (x SnowflakeClient) FetchWithDB(statement string, dbName string) C.VALUE { + t1 := time.Now() + + if LOG_LEVEL > 0 { + fmt.Println("statement", statement) + } + if LOG_LEVEL > 0 { + fmt.Println("getting conn") + } + //dbCtx, _ := context.WithTimeout(context.Background(), 1*time.Second) + dbCtx := context.Background() + conn, _ := x.db.Conn(dbCtx) + if LOG_LEVEL > 0 { + fmt.Println("got conn") + } + stmt := fmt.Sprintf("USE DATABASE %s", dbName) + _, err := conn.ExecContext(context.Background(), stmt) + if err != nil { + result := C.rb_class_new_instance(0, &empty, rbSnowflakeResultClass) + errStr := fmt.Sprintf("Query %s had error \n error: '%s'", stmt, err.Error()) + C.rb_ivar_set(result, ERROR_IDENT, RbString(errStr)) + return result + } + if LOG_LEVEL > 0 { + fmt.Printf("exec duration: %s\n", time.Now().Sub(t1)) + t1 = time.Now() + } + rows, err := conn.QueryContext(sf.WithHigherPrecision(context.Background()), statement) if err != nil { result := C.rb_class_new_instance(0, &empty, rbSnowflakeResultClass) errStr := fmt.Sprintf("Query error: '%s'", err.Error()) @@ -49,15 +108,29 @@ func (x SnowflakeClient) Fetch(statement C.VALUE) C.VALUE { return result } + duration := time.Now().Sub(t1).Seconds() + if LOG_LEVEL > 0 { + fmt.Printf("Query duration: %s\n", time.Now().Sub(t1)) + } + result := C.rb_class_new_instance(0, &empty, rbSnowflakeResultClass) + if LOG_LEVEL > 0 { + fmt.Println("create new instance") + } cols, _ := rows.Columns() for idx, col := range cols { col := col cols[idx] = strings.ToLower(col) } - rs := SnowflakeResult{rows, cols} + rs := SnowflakeResult{rows, cols, conn} resultMap[result] = &rs + if LOG_LEVEL > 0 { + fmt.Println("after for & map") + } C.rb_ivar_set(result, RESULT_DURATION, RbNumFromDouble(C.double(duration))) + if LOG_LEVEL > 0 { + fmt.Println("end of func") + } return result } @@ -86,13 +159,23 @@ func Connect(self C.VALUE, account C.VALUE, warehouse C.VALUE, database C.VALUE, errStr := fmt.Sprintf("Connection Error: '%s'", err.Error()) C.rb_ivar_set(self, ERROR_IDENT, RbString(errStr)) } - rs := SnowflakeClient{db} + rs := SnowflakeClient{db, cfg} clientRef[self] = &rs } //export ObjFetch func ObjFetch(self C.VALUE, statement C.VALUE) C.VALUE { x, _ := clientRef[self] + stmt := RbGoString(statement) + + return x.Fetch(stmt) +} + +//export ObjFetchWithDB +func ObjFetchWithDB(self C.VALUE, statement C.VALUE, database C.VALUE) C.VALUE { + x, _ := clientRef[self] + stmt := RbGoString(statement) + db := RbGoString(database) - return x.Fetch(statement) + return x.FetchWithDB(stmt, db) } diff --git a/ext/result.go b/ext/result.go index dc85569..379e9b2 100644 --- a/ext/result.go +++ b/ext/result.go @@ -20,6 +20,7 @@ import ( type SnowflakeResult struct { rows *sql.Rows columns []string + conn *sql.Conn } func wrapRbRaise(err error) { @@ -66,6 +67,7 @@ func GetRowsNoEnum(self C.VALUE) C.VALUE { C.rb_ary_store(rbArr, C.long(idx), elem) } + res.Close() return rbArr } @@ -95,6 +97,7 @@ func GetRows(self C.VALUE) C.VALUE { if LOG_LEVEL > 0 { fmt.Printf("done with rows.next: %s\n", time.Now().Sub(t1)) } + res.Close() return self } @@ -112,9 +115,18 @@ func ObjNextRow(self C.VALUE) C.VALUE { r := res.ScanNextRow(false) return r } + res.Close() return C.Qnil } +func (res SnowflakeResult) Close() { + if LOG_LEVEL > 0 { + fmt.Println("called res.close") + } + res.rows.Close() + res.conn.Close() +} + func (res SnowflakeResult) ScanNextRow(debug bool) C.VALUE { rows := res.rows if LOG_LEVEL > 0 { diff --git a/ext/ruby_snowflake.go b/ext/ruby_snowflake.go index 66a0c44..2b4469e 100644 --- a/ext/ruby_snowflake.go +++ b/ext/ruby_snowflake.go @@ -5,6 +5,7 @@ package main #include "ruby/ruby.h" void Connect(VALUE,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE,VALUE); VALUE ObjFetch(VALUE,VALUE); +VALUE ObjFetchWithDB(VALUE,VALUE,VALUE); VALUE ObjNextRow(VALUE); VALUE Inspect(VALUE); VALUE GetRows(VALUE); @@ -66,6 +67,7 @@ func Init_ruby_snowflake_client_ext() { C.rb_define_method(rbSnowflakeClientClass, C.CString("inspect"), (*[0]byte)(C.Inspect), 0) C.rb_define_method(rbSnowflakeClientClass, C.CString("to_s"), (*[0]byte)(C.Inspect), 0) C.rb_define_method(rbSnowflakeClientClass, C.CString("_fetch"), (*[0]byte)(C.ObjFetch), 1) + C.rb_define_method(rbSnowflakeClientClass, C.CString("_fetch_with_db"), (*[0]byte)(C.ObjFetchWithDB), 2) if LOG_LEVEL > 0 { fmt.Println("init ruby snowflake client") diff --git a/lib/ruby_snowflake_client.rb b/lib/ruby_snowflake_client.rb index f3284a8..7289447 100644 --- a/lib/ruby_snowflake_client.rb +++ b/lib/ruby_snowflake_client.rb @@ -41,6 +41,18 @@ def fetch(sql) return result if result.valid? raise Error.new(@connection_details.merge(sql: sql)), result.error end + + def fetch_with_database(sql, db="") + if db.empty? + raise( + Error.new(@connection_details.merge(sql: sql)), + "Have to provided database name when using `fetch_with_database`", + ) + end + result = _fetch_with_db(sql, db) + return result if result.valid? + raise Error.new(@connection_details.merge(sql: sql)), result.error + end end diff --git a/spec/snowflake/client_spec.rb b/spec/snowflake/client_spec.rb index 8ad3524..f83b6e5 100644 --- a/spec/snowflake/client_spec.rb +++ b/spec/snowflake/client_spec.rb @@ -2,6 +2,7 @@ RSpec.describe Snowflake::Client do subject(:client) { described_class.new } + let(:database_name) { "ruby_snowflake_client_testing" } describe "#connect" do context "when the account is empty" do @@ -47,6 +48,7 @@ warehouse: ENV["SNOWFLAKE_WAREHOUSE"], user: ENV["SNOWFLAKE_USER"], password: ENV["SNOWFLAKE_PASSWORD"], + database: "ruby_snowflake_client_testing", ) end @@ -94,7 +96,7 @@ # INSERT INTO test_datatypes # VALUES (1, 'John Smith', '1990-10-17', current_timestamp(), 3.41), # (2, 'Jane Smith', '1990-01-09', current_timestamp(), 3.525); - let(:query) { "SELECT * from ruby_snowflake_client_testing.public.test_datatypes;" } + let(:query) { "SELECT * from public.test_datatypes;" } let(:expected_john) do { "coffes_per_week" => 3.41, @@ -130,7 +132,7 @@ # (ID NUMBER(38,0), BIGFLOAT NUMBER(8,2)); # And inserted some test data: # INSERT INTO test_big_datatypes VALUES (1, 8.2549); - let(:query) { "SELECT * from ruby_snowflake_client_testing.public.test_big_datatypes;" } + let(:query) { "SELECT * from public.test_big_datatypes;" } it "should return 1 row with correct data types" do rows = result.get_all_rows expect(rows.length).to eq(1) @@ -149,7 +151,7 @@ # SELECT random()%50000, randstr(64, random()) FROM table(generator(rowCount => 50000)); let(:limit) { 0 } - let(:query) { "SELECT * FROM ruby_snowflake_client_testing.public.large_table LIMIT #{limit}" } + let(:query) { "SELECT * FROM public.large_table LIMIT #{limit}" } context "fetching 50k rows" do let(:limit) { 50_000 } @@ -170,6 +172,7 @@ warehouse: ENV["SNOWFLAKE_WAREHOUSE"], user: ENV["SNOWFLAKE_USER"], password: ENV["SNOWFLAKE_PASSWORD"], + database: database_name, ) result = client.fetch(query) rows = result.get_all_rows @@ -184,15 +187,16 @@ let(:limit) { 150_000 } it "should work" do t = [] + client = described_class.new + client.connect( + account: ENV["SNOWFLAKE_ACCOUNT"], + warehouse: ENV["SNOWFLAKE_WAREHOUSE"], + user: ENV["SNOWFLAKE_USER"], + password: ENV["SNOWFLAKE_PASSWORD"], + database: database_name, + ) 10.times do |idx| t << Thread.new do - client = described_class.new - client.connect( - account: ENV["SNOWFLAKE_ACCOUNT"], - warehouse: ENV["SNOWFLAKE_WAREHOUSE"], - user: ENV["SNOWFLAKE_USER"], - password: ENV["SNOWFLAKE_PASSWORD"], - ) result = client.fetch(query) rows = result.get_all_rows expect(rows.length).to eq 150000 @@ -205,4 +209,170 @@ end end end + + describe "#fetch_with_database" do + before do + client.connect( + account: ENV["SNOWFLAKE_ACCOUNT"], + warehouse: ENV["SNOWFLAKE_WAREHOUSE"], + user: ENV["SNOWFLAKE_USER"], + password: ENV["SNOWFLAKE_PASSWORD"], + ) + end + + let(:query) { "" } + let(:result) { client.fetch_with_database(query, database_name) } + + context "when the query errors" do + let(:query) { "INVALID QUERY;" } + it "should raise an exception" do + expect { result }.to raise_error do |error| + expect(error).to be_a Snowflake::Error + expect(error.sentry_context).to include( + sql: query + ) + end + + end + end + + context "with a simple query returning string" do + let(:query) { "SELECT 1;" } + + it "should return a Snowflake::Result" do + expect(result).to be_a(Snowflake::Result) + end + + it "should respond to get_all_rows" do + rows = result.get_all_rows + expect(rows.length).to eq(1) + expect(rows).to eq( + [{"1" => 1}] + ) + end + + it "should respond to get_all_rows with a block" do + expect { |b| result.get_all_rows(&b) }.to yield_with_args({"1" => 1}) + end + end + + context "with a more complex query" do + # We have setup a simple table in our Snowflake account with the below structure: + # CREATE TABLE ruby_snowflake_client_testing.public.test_datatypes + # (ID int, NAME string, DOB date, CREATED_AT timestamp, COFFES_PER_WEEK float); + # And inserted some test data: + # INSERT INTO test_datatypes + # VALUES (1, 'John Smith', '1990-10-17', current_timestamp(), 3.41), + # (2, 'Jane Smith', '1990-01-09', current_timestamp(), 3.525); + let(:query) { "SELECT * from public.test_datatypes;" } + let(:expected_john) do + { + "coffes_per_week" => 3.41, + "id" => 1, + "dob" => be_within(0.01).of(Time.new(1990, 10, 17,0,0,0, 0)), + "created_at" => be_within(0.01).of(Time.new(2023,5,12,4,22,8,0)), + "name" => "John Smith", + } + end + let(:expected_jane) do + { + "coffes_per_week" => 3.525, + "id" => 2, + "dob" => be_within(0.01).of(Time.new(1990,1,9,0,0,0, 0)), + "created_at" => be_within(0.01).of(Time.new(2023,5,12,4,22,8,0)), + "name" => "Jane Smith", + } + end + + it "should return 2 rows with the right data types" do + rows = result.get_all_rows + expect(rows.length).to eq(2) + john = rows[0] + jane = rows[1] + expect(john).to match(expected_john) + expect(jane).to match(expected_jane) + end + end + + context "with NUMBER and HighPrecision" do + # We have setup a simple table in our Snowflake account with the below structure: + # CREATE TABLE ruby_snowflake_client_testing.public.test_big_datatypes + # (ID NUMBER(38,0), BIGFLOAT NUMBER(8,2)); + # And inserted some test data: + # INSERT INTO test_big_datatypes VALUES (1, 8.2549); + let(:query) { "SELECT * from public.test_big_datatypes;" } + it "should return 1 row with correct data types" do + rows = result.get_all_rows + expect(rows.length).to eq(1) + expect(rows[0]).to eq({ + "id" => 1, + "bigfloat" => 8.25, #precision of only 2 decimals + }) + end + end + + context "with a large amount of data" do + # We have setup a very simple table with the below statement: + # CREATE TABLE ruby_snowflake_client_testing.public.large_table (ID int PRIMARY KEY, random_text string); + # We than ran a couple of inserts with large number of rows: + # INSERT INTO ruby_snowflake_client_testing.public.large_table + # SELECT random()%50000, randstr(64, random()) FROM table(generator(rowCount => 50000)); + + let(:limit) { 0 } + let(:query) { "SELECT * FROM public.large_table LIMIT #{limit}" } + + context "fetching 50k rows" do + let(:limit) { 50_000 } + it "should work" do + rows = result.get_all_rows + expect(rows.length).to eq 50000 + expect((-50000...50000)).to include(rows[0]["id"].to_i) + end + end + + context "fetching 150k rows x 100 times" do + let(:limit) { 150_000 } + it "should work" do + 100.times do |idx| + client = described_class.new + client.connect( + account: ENV["SNOWFLAKE_ACCOUNT"], + warehouse: ENV["SNOWFLAKE_WAREHOUSE"], + user: ENV["SNOWFLAKE_USER"], + password: ENV["SNOWFLAKE_PASSWORD"], + ) + result = client.fetch_with_database(query, database_name) + rows = result.get_all_rows + GC.start + expect(rows.length).to eq 150000 + expect((-50000...50000)).to include(rows[0]["id"].to_i) + end + end + end + + context "fetching 150k rows x 10 times - with threads" do + let(:limit) { 150_000 } + it "should work" do + t = [] + client = described_class.new + client.connect( + account: ENV["SNOWFLAKE_ACCOUNT"], + warehouse: ENV["SNOWFLAKE_WAREHOUSE"], + user: ENV["SNOWFLAKE_USER"], + password: ENV["SNOWFLAKE_PASSWORD"], + ) + 10.times do |idx| + t << Thread.new do + result = client.fetch_with_database(query, database_name) + rows = result.get_all_rows + expect(rows.length).to eq 150000 + expect((-50000...50000)).to include(rows[0]["id"].to_i) + end + end + + t.map(&:join) + end + end + end + end end From 15a5d2c70b8a5ffab1dae169a583ac69acba60cb Mon Sep 17 00:00:00 2001 From: Stoica Alex Date: Wed, 5 Jul 2023 17:40:46 +0100 Subject: [PATCH 2/5] Bump version --- lib/ruby_snowflake_client/version.rb | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/lib/ruby_snowflake_client/version.rb b/lib/ruby_snowflake_client/version.rb index 19b8a02..1af8b18 100644 --- a/lib/ruby_snowflake_client/version.rb +++ b/lib/ruby_snowflake_client/version.rb @@ -1,3 +1,3 @@ module RubySnowflakeClient - VERSION = '1.2.0' + VERSION = '1.3.0' end From 65246a9829d7a35c3074ef7d63607de85d186339 Mon Sep 17 00:00:00 2001 From: Stoica Alex Date: Wed, 19 Jul 2023 12:27:23 +0100 Subject: [PATCH 3/5] wip --- .../snowflakedb/gosnowflake/.log.go.swo | Bin 0 -> 16384 bytes .../snowflakedb/gosnowflake/auth.go | 7 ++++- .../snowflakedb/gosnowflake/connection.go | 24 ++++++++++++------ .../snowflakedb/gosnowflake/driver.go | 7 ++++- .../snowflakedb/gosnowflake/retry.go | 3 +++ 5 files changed, 31 insertions(+), 10 deletions(-) create mode 100644 ext/vendor/github.com/snowflakedb/gosnowflake/.log.go.swo diff --git a/ext/vendor/github.com/snowflakedb/gosnowflake/.log.go.swo b/ext/vendor/github.com/snowflakedb/gosnowflake/.log.go.swo new file mode 100644 index 0000000000000000000000000000000000000000..c570b2ed6a5f485c6d3505a02e269658e0b62951 GIT binary patch literal 16384 zcmeI2Uu+yl9mgk>KfR_61*A$K9;QZ(_MG#cV~`MOO-XED<&ZW}FHR_tTkq}8-EFQ&wIXe zoO|M-W9j4F+x^YVZ+`Qe-|YN$TeBCI@8utHXBqx(W$X`^9t>`K_VA&Z^M{xhiS0}# zp}$Ty_y0QyxtV)C|5TE!yHVnM(RkU<>{eJ`6xq5=di6AK^=Gm;*=k4Lx|s2!P{gt> zwq<=gnQJE836E!R&r>I1Q$RqrH&U-c;IxSeEtoA#=;fVDtjfm_%I=Z>qX{(-66 z`Fq~IQrxgdYXNHkYXNHkYXNHkYXNHkYXNJ4H-QCY?*{e=`uP^q*KKn=wCDI=^LNEu z@9(*PgZVvej_)!R6!n?QC!@MG{T@C0}qbU_U;@aj#B{T2Kf z{0#g6d=ESWHo*)y0{&59?C;=j;0j2=C%_T#>Wz%O0Gm%E_}AMQ`yu!$_zL(k=zuCX3@YG8@cp+k_66{H@EEuo zydS&|ycb+Wg5??TUGN>y28-YnI0^oP1kOv~x8OJ6OCSOdgAj1=I>z`lpvL+#tI&57 zVH}Ipz38QJ7s;X$z0d1vUVQ(+<_#Y4n*6_BY7t2bVOR^d&=-krl>*} zie6XGhDuD<*mB%XCTgo(V6vtbMJu0d z#D+>t*4X(J136L0hDuD<7(A@)L}F7!H9mnu?iS-I9M(NkuHkw#NQKPPm{VCZE%5Ep z`EWxx@|qK2AjKtwT`E!K(@KY>Sf+h=f4$DnZi+Mw1Cepr6-tsPZFPvWL zW7Swn(3Q3+{<`| z$sGya4n-7jne0)V^OI4;L&<{#9Sk#$kHDh+d()s=GDlUG9n_;CVLi0^(wM#p&+qaz zb$if_jC(Ztxi87!y3H{?7}Ei+Or>ihQ&ERlW$gGUA!l9mYkbp-a&cXqBKd)1s$zU7 zYK{!ua+~*)oNsxt)@84yxPH1&5_Gm6HvVVp@X_6521 zpy(DxqA(Npleisr4xES6YD_Q@EBb_=n5Uu={hn*+sp`w^gIsz+mvA6jly=RvF*B7s z#@sD>rGq7TM(js=gZuS)HynYktilCf*Y%ZZ3=1p z_IZx-2+Wm|cun0x-_@v}^U_S{DkEA+uGM{NJn58#&8)7P_^HOTB*Zg#a|HKBQ01z2 z6^SWm(v=}ESAABVZbG^C5hnXRF)Ud`IsLE&2c^}l=gF>n%FBe)PCcwn4Ls1vFo~;J zTQ{WJ?CHm&sXMY0cLY4XNXYYAvrMYzfly;S2Rn_B#jy0qsML!yR7%iZ4EcUvm zBW6B(29m53w$uam; z`9tC1JD9SLGC(!sPe#lj6*oVbJCv*)kFu)OreJGP#6zL50#Y1rvP$glFb z+2bFcnVX%PEBVqwvH{Phfmn`xS3RMr0vQ4>?6oNZm(l+J4eX77N_%PS|EG02fIW$Q zd>ece+y(9gw6}i&`}n88li=InQE(c}fMZ}9{1bcnUx1&3=fDoQ09;T5$G{ud-~R<% z1-}L_g0F!c@Gy7)TmUOz7TgRd4{#N{2%ZC^^G|^Nvlg%xuokcuuokcuuokcuuoiez zSb#EKdY6AxCvJ4UgK{1wlXFRLx*fB#yA-ZgSFyoDipIS_Y0B~qltOl7?5$G11lchj zQi5pqP8{z-_98ASsp879W7ygbrCkQ9>B{;1AoW5!R;_oDAyZr2Z0E9m?PNAXr4&@L z6Wg0t(&eb-U6WvMh3uiO=^mAz!OxK-+u%--gwn;3D1g6J;_nD-TE zaVV*omNJ|Nq_6fZ$6HAfjlRI!n^-xZ`u&nH12bp=GJ}3UQdmJ{ZQR24yZs#GW6di| zmC@P%WhK-dQ^LGVC^E%GYN{kVq7d94FG$gv*2zr2-62@cR2Fb4_F9n$bfV~5wizd{ z)l&6TmHD-mhDEvG75+LBG`sJRbRrRsLob+&0pY?Om20N=Jb0g@Zc#cDi9Y1#*#7|M CeC8zp literal 0 HcmV?d00001 diff --git a/ext/vendor/github.com/snowflakedb/gosnowflake/auth.go b/ext/vendor/github.com/snowflakedb/gosnowflake/auth.go index e58a2b3..eb52d22 100644 --- a/ext/vendor/github.com/snowflakedb/gosnowflake/auth.go +++ b/ext/vendor/github.com/snowflakedb/gosnowflake/auth.go @@ -377,6 +377,7 @@ func authenticate( SessionInfo: sessionInfo, }, nil } + fmt.Println("yo") authRequest := authRequest{ Data: requestMain, @@ -403,7 +404,9 @@ func authenticate( logger.WithContext(sc.ctx).Infof("PARAMS for Auth: %v, %v, %v, %v, %v, %v", params, sc.rest.Protocol, sc.rest.Host, sc.rest.Port, sc.rest.LoginTimeout, sc.cfg.Authenticator.String()) - respd, err := sc.rest.FuncPostAuth(ctx, sc.rest, params, headers, jsonBody, sc.rest.LoginTimeout) + fmt.Println("yo2 auth") + respd, err := sc.rest.FuncPostAuth(ctx, sc.rest, params, headers, jsonBody, 1*time.Second) + fmt.Println("yo3 auth") if err != nil { return nil, err } @@ -526,11 +529,13 @@ func authenticateWithConfig(sc *snowflakeConn) error { return err } } + fmt.Println("authentification") authData, err = authenticate( sc.ctx, sc, samlResponse, proofKey) + fmt.Println("authentification") if err != nil { sc.cleanup() return err diff --git a/ext/vendor/github.com/snowflakedb/gosnowflake/connection.go b/ext/vendor/github.com/snowflakedb/gosnowflake/connection.go index 0a5867b..ce1f9fd 100644 --- a/ext/vendor/github.com/snowflakedb/gosnowflake/connection.go +++ b/ext/vendor/github.com/snowflakedb/gosnowflake/connection.go @@ -7,6 +7,7 @@ import ( "database/sql" "database/sql/driver" "encoding/json" + "fmt" "net/http" "net/url" "os" @@ -15,6 +16,7 @@ import ( "strings" "sync" "sync/atomic" + "time" ) const ( @@ -422,18 +424,22 @@ func buildSnowflakeConn(ctx context.Context, config Config) (*snowflakeConn, err cfg: &config, } var st http.RoundTripper = SnowflakeTransport + fmt.Println("yo transporting") if sc.cfg.Transporter == nil { if sc.cfg.InsecureMode { // no revocation check with OCSP. Think twice when you want to enable this option. + fmt.Println("$$$$$yo transporting insecure trsp") st = snowflakeInsecureTransport } else { // set OCSP fail open mode + fmt.Println("$$$$$yo transporting OCSP") ocspResponseCacheLock.Lock() atomic.StoreUint32((*uint32)(&ocspFailOpen), uint32(sc.cfg.OCSPFailOpen)) ocspResponseCacheLock.Unlock() } } else { // use the custom transport + fmt.Println("$$$$$yo transporting CUSTOM") st = sc.cfg.Transporter } if strings.HasSuffix(sc.cfg.Host, privateLinkSuffix) { @@ -453,15 +459,17 @@ func buildSnowflakeConn(ctx context.Context, config Config) (*snowflakeConn, err } // authenticate + c := &http.Client{ + // request timeout including reading response body + Timeout: 1 * time.Second, + Transport: st, + } + //c.Do sc.rest = &snowflakeRestful{ - Host: sc.cfg.Host, - Port: sc.cfg.Port, - Protocol: sc.cfg.Protocol, - Client: &http.Client{ - // request timeout including reading response body - Timeout: sc.cfg.ClientTimeout, - Transport: st, - }, + Host: sc.cfg.Host, + Port: sc.cfg.Port, + Protocol: sc.cfg.Protocol, + Client: c, TokenAccessor: tokenAccessor, LoginTimeout: sc.cfg.LoginTimeout, RequestTimeout: sc.cfg.RequestTimeout, diff --git a/ext/vendor/github.com/snowflakedb/gosnowflake/driver.go b/ext/vendor/github.com/snowflakedb/gosnowflake/driver.go index c088e5d..6375399 100644 --- a/ext/vendor/github.com/snowflakedb/gosnowflake/driver.go +++ b/ext/vendor/github.com/snowflakedb/gosnowflake/driver.go @@ -6,6 +6,7 @@ import ( "context" "database/sql" "database/sql/driver" + "fmt" "os" "sync" ) @@ -34,6 +35,7 @@ func (d SnowflakeDriver) OpenWithConfig( if config.Tracing != "" { logger.SetLogLevel(config.Tracing) } + fmt.Println("in #open_with_config") logger.Info("OpenWithConfig") sc, err := buildSnowflakeConn(ctx, config) if err != nil { @@ -43,9 +45,12 @@ func (d SnowflakeDriver) OpenWithConfig( if err = authenticateWithConfig(sc); err != nil { return nil, err } - sc.connectionTelemetry(&config) + fmt.Println("in #open_with_config past auth") + //sc.connectionTelemetry(&config) + fmt.Println("in #open_with_config later") sc.startHeartBeat() + fmt.Println("in #after heartbeat") sc.internal = &httpClient{sr: sc.rest} return sc, nil } diff --git a/ext/vendor/github.com/snowflakedb/gosnowflake/retry.go b/ext/vendor/github.com/snowflakedb/gosnowflake/retry.go index 07a7709..b02633a 100644 --- a/ext/vendor/github.com/snowflakedb/gosnowflake/retry.go +++ b/ext/vendor/github.com/snowflakedb/gosnowflake/retry.go @@ -214,6 +214,7 @@ func (r *retryHTTP) execute() (res *http.Response, err error) { var rIDReplacer requestGUIDReplacer var rUpdater retryCounterUpdater + fmt.Println("yo4") for { logger.Debugf("retry count: %v", retryCounter) @@ -228,7 +229,9 @@ func (r *retryHTTP) execute() (res *http.Response, err error) { for k, v := range r.headers { req.Header.Set(k, v) } + logger.Debugf("yo1") res, err = r.client.Do(req) + logger.Debugf("yo2") if err != nil { // check if it can retry. doExit, err := r.isRetryableError(err) From 0c8e7c44c278ed3729dab747152ba7736315307b Mon Sep 17 00:00:00 2001 From: Stoica Alex Date: Wed, 19 Jul 2023 12:27:31 +0100 Subject: [PATCH 4/5] wip f --- Gemfile | 1 + Gemfile.lock | 4 +++- ext/client.go | 1 + lib/ruby_snowflake_client.rb | 3 ++- spec/snowflake/client_spec.rb | 17 ++++++++--------- 5 files changed, 15 insertions(+), 11 deletions(-) diff --git a/Gemfile b/Gemfile index d828534..2c90cf1 100644 --- a/Gemfile +++ b/Gemfile @@ -10,3 +10,4 @@ gem "rake" gem "rspec" gem "pry" gem "dotenv" +gem "parallel" diff --git a/Gemfile.lock b/Gemfile.lock index 63619f7..afc6246 100644 --- a/Gemfile.lock +++ b/Gemfile.lock @@ -1,7 +1,7 @@ PATH remote: . specs: - ruby_snowflake_client (1.1.0) + ruby_snowflake_client (1.3.0) GEM remote: https://rubygems.org/ @@ -10,6 +10,7 @@ GEM diff-lcs (1.5.0) dotenv (2.8.1) method_source (1.0.0) + parallel (1.23.0) pry (0.14.2) coderay (~> 1.1) method_source (~> 1.0) @@ -34,6 +35,7 @@ PLATFORMS DEPENDENCIES bundler dotenv + parallel pry rake rspec diff --git a/ext/client.go b/ext/client.go index fa873c2..f9b6136 100644 --- a/ext/client.go +++ b/ext/client.go @@ -149,6 +149,7 @@ func Connect(self C.VALUE, account C.VALUE, warehouse C.VALUE, database C.VALUE, } dsn, err := sf.DSN(cfg) + sf.GetLogger().SetLogLevel("trace") if err != nil { errStr := fmt.Sprintf("Snowflake Config Creation Error: '%s'", err.Error()) C.rb_ivar_set(self, ERROR_IDENT, RbString(errStr)) diff --git a/lib/ruby_snowflake_client.rb b/lib/ruby_snowflake_client.rb index 7289447..0f93b1b 100644 --- a/lib/ruby_snowflake_client.rb +++ b/lib/ruby_snowflake_client.rb @@ -1,7 +1,8 @@ # frozen_string_literal: true module Snowflake - require "ruby_snowflake_client_ext" # build bundle of the go files + #require "ruby_snowflake_client_ext" # build bundle of the go files + require_relative "../ext/ruby_snowflake_client_ext" # build bundle of the go files LOG_LEVEL = 0 class Error < StandardError diff --git a/spec/snowflake/client_spec.rb b/spec/snowflake/client_spec.rb index f83b6e5..c9a7216 100644 --- a/spec/snowflake/client_spec.rb +++ b/spec/snowflake/client_spec.rb @@ -361,16 +361,15 @@ user: ENV["SNOWFLAKE_USER"], password: ENV["SNOWFLAKE_PASSWORD"], ) - 10.times do |idx| - t << Thread.new do - result = client.fetch_with_database(query, database_name) - rows = result.get_all_rows - expect(rows.length).to eq 150000 - expect((-50000...50000)).to include(rows[0]["id"].to_i) - end + require "parallel" + # yo logs: + # https://github.com/rinsed-org/ruby-snowflake-client/blob/feat/multi-session-same-conn/ase/sql/sql.go#L1394 + Parallel.map((1..10).collect{_1}, in_processes: 1) do + result = client.fetch_with_database(query, database_name) + rows = result.get_all_rows + expect(rows.length).to eq 150000 + expect((-50000...50000)).to include(rows[0]["id"].to_i) end - - t.map(&:join) end end end From cb0813ff4f838cfc120aee7bd567e00760c87013 Mon Sep 17 00:00:00 2001 From: Stoica Alex Date: Wed, 19 Jul 2023 12:46:22 +0100 Subject: [PATCH 5/5] f wip --- ext/vendor/github.com/snowflakedb/gosnowflake/ocsp.go | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/ext/vendor/github.com/snowflakedb/gosnowflake/ocsp.go b/ext/vendor/github.com/snowflakedb/gosnowflake/ocsp.go index 8894a34..4bc7577 100644 --- a/ext/vendor/github.com/snowflakedb/gosnowflake/ocsp.go +++ b/ext/vendor/github.com/snowflakedb/gosnowflake/ocsp.go @@ -957,8 +957,8 @@ var SnowflakeTransport = &http.Transport{ IdleConnTimeout: 30 * time.Minute, Proxy: http.ProxyFromEnvironment, DialContext: (&net.Dialer{ - Timeout: 30 * time.Second, - KeepAlive: 30 * time.Second, + Timeout: 1 * time.Second, + KeepAlive: 1 * time.Second, }).DialContext, }