From 46649014402dd1d5d92c90fe3382f0c3852537ad Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Wed, 22 Apr 2026 00:38:53 +0300 Subject: [PATCH 1/7] add bfloat16 --- lib/ch.ex | 2 ++ lib/ch/row_binary.ex | 25 +++++++++++++++++++----- lib/ch/types.ex | 1 + test/ch/bfloat16_test.exs | 39 +++++++++++++++++++++++++++++++++++++ test/ch/ecto_type_test.exs | 18 +++++++++++++++++ test/ch/row_binary_test.exs | 4 ++++ test/ch/types_test.exs | 1 + test/test_helper.exs | 4 ++-- 8 files changed, 87 insertions(+), 7 deletions(-) create mode 100644 test/ch/bfloat16_test.exs diff --git a/lib/ch.ex b/lib/ch.ex index 60c26bd1..0d688487 100644 --- a/lib/ch.ex +++ b/lib/ch.ex @@ -168,6 +168,8 @@ defmodule Ch do def cast(value, unquote(:"f#{size}")), do: Ecto.Type.cast(:float, value) end + def cast(value, :bf16), do: Ecto.Type.cast(:float, value) + def cast(value, {:decimal = type, _p, _s}), do: Ecto.Type.cast(type, value) for size <- [32, 64, 128, 256] do diff --git a/lib/ch/row_binary.ex b/lib/ch/row_binary.ex index 98fc7db0..aa404f83 100644 --- a/lib/ch/row_binary.ex +++ b/lib/ch/row_binary.ex @@ -103,7 +103,8 @@ defmodule Ch.RowBinary do :ipv4, :ipv6, :point, - :nothing + :nothing, + :bf16 ], do: t @@ -252,12 +253,18 @@ defmodule Ch.RowBinary do type = :"f#{size}" def encode(unquote(type), f) when is_number(f) do - <> + <> end def encode(unquote(type), nil), do: <<0::unquote(size)>> end + def encode(:bf16, f) when is_number(f) do + <> + end + + def encode(:bf16, nil), do: <<0::16>> + def encode({:decimal, precision, scale}, decimal) do type = case decimal_size(precision) do @@ -722,7 +729,8 @@ defmodule Ch.RowBinary do :ipv4, :ipv6, :point, - :nothing + :nothing, + :bf16 ], do: t @@ -1004,7 +1012,8 @@ defmodule Ch.RowBinary do uuid: 0x1D, ipv4: 0x28, ipv6: 0x29, - boolean: 0x2D + boolean: 0x2D, + bf16: 0x31 ] # TODO compile inline? @@ -1195,7 +1204,6 @@ defmodule Ch.RowBinary do other_dynamic_types = [ datetime: 0x11, set: 0x21, - bfloat16: 0x31, time: 0x32 ] @@ -1302,6 +1310,10 @@ defmodule Ch.RowBinary do %{pattern: quote(do: <>), value: quote(do: f)}, %{pattern: quote(do: <<_nan_or_inf::64>>), value: quote(do: nil)} ], + bf16: [ + %{pattern: quote(do: <>), value: quote(do: f)}, + %{pattern: quote(do: <<_nan_or_inf::16>>), value: quote(do: nil)} + ], uuid: %{ pattern: quote(do: <>), value: quote(do: <>) @@ -1396,6 +1408,9 @@ defmodule Ch.RowBinary do :f64 -> decode_f64_decode_rows(bin, types_rest, row, rows, types) + :bf16 -> + decode_bf16_decode_rows(bin, types_rest, row, rows, types) + :string -> decode_string_decode_rows(bin, types_rest, row, rows, types) diff --git a/lib/ch/types.ex b/lib/ch/types.ex index 9ab18fa4..d1edd8be 100644 --- a/lib/ch/types.ex +++ b/lib/ch/types.ex @@ -16,6 +16,7 @@ defmodule Ch.Types do for size <- [32, 64] do {"Float#{size}", :"f#{size}", []} end, + {"BFloat16", :bf16, []}, {"Array", :array, [:type]}, {"Tuple", :tuple, [:maybe_named_column]}, {"Variant", :variant, [:type]}, diff --git a/test/ch/bfloat16_test.exs b/test/ch/bfloat16_test.exs new file mode 100644 index 00000000..68cd36aa --- /dev/null +++ b/test/ch/bfloat16_test.exs @@ -0,0 +1,39 @@ +defmodule Ch.BFloat16Test do + use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] + + @moduletag :bf16 + + setup ctx do + {:ok, + query_options: ctx[:query_options] || [], + conn: start_supervised!({Ch, database: Ch.Test.database()})} + end + + test "plain", %{conn: conn, query_options: query_options} do + assert Ch.query!(conn, "select 1.75::BFloat16", _no_params = %{}, query_options).rows == [ + [1.75] + ] + end + + test "send and read back via params", %{conn: conn, query_options: query_options} do + assert Ch.query!(conn, "select {value:BFloat16} as value", %{"value" => 1.75}, query_options).rows == + [[1.75]] + end + + test "send and read back via rowbinary", %{conn: conn, query_options: query_options} do + rows = [ + [1.75], + [-1.75], + [0] + ] + + query_options = Keyword.merge(query_options, types: ["BFloat16"]) + + assert Ch.query!( + conn, + "select bf16 from input('bf16 BFloat16') format RowBinary", + rows, + query_options + ).rows == rows + end +end diff --git a/test/ch/ecto_type_test.exs b/test/ch/ecto_type_test.exs index 0cbc1019..be2776e3 100644 --- a/test/ch/ecto_type_test.exs +++ b/test/ch/ecto_type_test.exs @@ -257,6 +257,24 @@ defmodule Ch.EctoTypeTest do end end + test "BFloat16" do + assert {:parameterized, {Ch, :bf16}} = + type = Ecto.ParameterizedType.init(Ch, type: unquote("BFloat16")) + + assert Ecto.Type.type(type) == type + assert Ecto.Type.format(type) == "#Ch" + + assert {:ok, 1.0} = Ecto.Type.cast(type, 1.0) + assert {:ok, 1.0} = Ecto.Type.cast(type, 1) + assert {:ok, 1.0} = Ecto.Type.cast(type, "1.0") + assert {:ok, nil} = Ecto.Type.cast(type, nil) + + assert :error = Ecto.Type.cast(type, "asdf") + + assert {:ok, 1.0} = Ecto.Type.dump(type, 1.0) + assert {:ok, 1.0} = Ecto.Type.load(type, 1.0) + end + test "Date" do assert {:parameterized, {Ch, :date}} = type = Ecto.ParameterizedType.init(Ch, type: "Date") diff --git a/test/ch/row_binary_test.exs b/test/ch/row_binary_test.exs index 3d5e453a..2db6d7e3 100644 --- a/test/ch/row_binary_test.exs +++ b/test/ch/row_binary_test.exs @@ -45,6 +45,7 @@ defmodule Ch.RowBinaryTest do {:i256, 0x7FFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFFF}, {:f32, 1.2345678806304932}, {:f64, 1.234567898762738492830000503040030202020433}, + {:bf16, 1.75}, {:date, ~D[2022-01-01]}, {:date, ~D[2042-01-01]}, {:date, ~D[1970-01-01]}, @@ -76,6 +77,7 @@ defmodule Ch.RowBinaryTest do 0, 1.234567898762738492830000503040030202020433 ]}, + {{:array, :bf16}, [-1.75, 0, 1.75]}, {{:array, :date}, [~D[2022-01-01], ~D[2042-01-01], ~D[1970-01-01]]}, {{:array, :datetime}, [~N[1970-01-01 12:23:34], ~N[2022-01-01 22:12:59], ~N[2042-01-01 04:23:01]]}, @@ -145,6 +147,7 @@ defmodule Ch.RowBinaryTest do assert encode(:i64, nil) == <<0, 0, 0, 0, 0, 0, 0, 0>> assert encode(:f32, nil) == <<0, 0, 0, 0>> assert encode(:f64, nil) == <<0, 0, 0, 0, 0, 0, 0, 0>> + assert encode(:bf16, nil) == <<0, 0>> assert encode(:boolean, nil) == 0 assert encode({:array, :string}, nil) == 0 assert encode(:date, nil) == <<0, 0>> @@ -209,6 +212,7 @@ defmodule Ch.RowBinaryTest do {"Int256", :i256}, {"Float32", :f32}, {"Float64", :f64}, + {"BFloat16", :bf16}, {"Decimal(9, 4)", {:decimal, _size = 32, _scale = 4}}, {"Decimal(23, 11)", {:decimal, _size = 128, _scale = 11}}, {"Bool", :boolean}, diff --git a/test/ch/types_test.exs b/test/ch/types_test.exs index 5ec1f5e5..a08d4db4 100644 --- a/test/ch/types_test.exs +++ b/test/ch/types_test.exs @@ -25,6 +25,7 @@ defmodule Ch.TypesTest do assert decode("Float32") == :f32 assert decode("Float64") == :f64 + assert decode("BFloat16") == :bf16 assert decode("Date") == :date assert decode("DateTime") == :datetime diff --git a/test/test_helper.exs b/test/test_helper.exs index 97caedd8..c1b235d6 100644 --- a/test/test_helper.exs +++ b/test/test_helper.exs @@ -39,8 +39,8 @@ extra_exclude = if ch_version >= "25" do [] else - # Time, Variant, JSON, and Dynamic types are not supported in older ClickHouse versions we have in the CI - [:time, :variant, :json, :dynamic] + # Time, Variant, JSON, BFloat16, and Dynamic types are not supported in older ClickHouse versions we have in the CI + [:time, :variant, :json, :bf16, :dynamic] end ExUnit.start(exclude: [:slow | extra_exclude]) From 19097af36e14d5821b0ed2a8249539b6f5fbabcb Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Wed, 22 Apr 2026 00:40:00 +0300 Subject: [PATCH 2/7] changelog --- CHANGELOG.md | 1 + 1 file changed, 1 insertion(+) diff --git a/CHANGELOG.md b/CHANGELOG.md index cee3cc24..80f8adc6 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -9,6 +9,7 @@ - use `DateTime.to_unix/2` + `DateTime.to_naive/1` for naive datetime decoding in RowBinary https://github.com/plausible/ch/pull/313 - allow non-UTC timezones for DateTime64 RowBinary encoding https://github.com/plausible/ch/pull/315 - use gregorian days in RowBinary dates https://github.com/plausible/ch/pull/318 +- support `BFloat16` https://github.com/plausible/ch/pull/321 ## 0.7.1 (2026-01-15) From 3cb9fcbd1a15186be3bd9be2a81c52d7e9dd19d8 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 7 May 2026 20:05:29 +0300 Subject: [PATCH 3/7] Fix BFloat16 row binary support --- lib/ch/row_binary.ex | 25 ++++++++++++++++++++++--- test/ch/bfloat16_test.exs | 19 +++++++++++++------ 2 files changed, 35 insertions(+), 9 deletions(-) diff --git a/lib/ch/row_binary.ex b/lib/ch/row_binary.ex index aa404f83..d32cc727 100644 --- a/lib/ch/row_binary.ex +++ b/lib/ch/row_binary.ex @@ -260,7 +260,7 @@ defmodule Ch.RowBinary do end def encode(:bf16, f) when is_number(f) do - <> + <> end def encode(:bf16, nil), do: <<0::16>> @@ -477,6 +477,26 @@ defmodule Ch.RowBinary do end end + defp float_to_bf16(f) do + <> = <> + + upper = bits >>> 16 + lower = bits &&& 0xFFFF + + if lower > 0x8000 or (lower == 0x8000 and (upper &&& 1) == 1) do + upper + 1 + else + upper + end + end + + defp bfloat16_to_float(bits) when (bits &&& 0x7F80) == 0x7F80, do: nil + + defp bfloat16_to_float(bits) do + <> = <> + f + end + defp encode_varint_cont(i) when i < 128, do: <> defp encode_varint_cont(i) do @@ -1311,8 +1331,7 @@ defmodule Ch.RowBinary do %{pattern: quote(do: <<_nan_or_inf::64>>), value: quote(do: nil)} ], bf16: [ - %{pattern: quote(do: <>), value: quote(do: f)}, - %{pattern: quote(do: <<_nan_or_inf::16>>), value: quote(do: nil)} + %{pattern: quote(do: <>), value: quote(do: bfloat16_to_float(bits))} ], uuid: %{ pattern: quote(do: <>), diff --git a/test/ch/bfloat16_test.exs b/test/ch/bfloat16_test.exs index 68cd36aa..47d24fb4 100644 --- a/test/ch/bfloat16_test.exs +++ b/test/ch/bfloat16_test.exs @@ -21,6 +21,11 @@ defmodule Ch.BFloat16Test do end test "send and read back via rowbinary", %{conn: conn, query_options: query_options} do + table = "bf16_#{System.unique_integer([:positive])}" + + Ch.query!(conn, "create table #{table} (bf16 BFloat16) engine Memory") + on_exit(fn -> Ch.Test.query("drop table if exists #{table}") end) + rows = [ [1.75], [-1.75], @@ -29,11 +34,13 @@ defmodule Ch.BFloat16Test do query_options = Keyword.merge(query_options, types: ["BFloat16"]) - assert Ch.query!( - conn, - "select bf16 from input('bf16 BFloat16') format RowBinary", - rows, - query_options - ).rows == rows + assert %{num_rows: 3} = + Ch.query!(conn, "insert into #{table} (bf16) format RowBinary", rows, query_options) + + assert Ch.query!(conn, "select bf16 from #{table} order by bf16").rows == [ + [-1.75], + [0.0], + [1.75] + ] end end From 9649b1701afb57462517d403fdbaaed3a5eef663 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 7 May 2026 20:18:04 +0300 Subject: [PATCH 4/7] Address BFloat16 review feedback --- CHANGELOG.md | 5 +++- lib/ch.ex | 2 +- test/ch/bfloat16_test.exs | 47 ++++++++++++++++++++++++++++++-------- test/ch/ecto_type_test.exs | 2 +- 4 files changed, 43 insertions(+), 13 deletions(-) diff --git a/CHANGELOG.md b/CHANGELOG.md index 143d4f77..bc85470b 100644 --- a/CHANGELOG.md +++ b/CHANGELOG.md @@ -1,5 +1,9 @@ # Changelog +## Unreleased + +- support `BFloat16` https://github.com/plausible/ch/pull/321 + ## 0.8.2 (2026-05-07) - use scientific decimals rendering in params https://github.com/plausible/ch/pull/333 @@ -15,7 +19,6 @@ - use `DateTime.to_unix/2` + `DateTime.to_naive/1` for naive datetime decoding in RowBinary https://github.com/plausible/ch/pull/313 - allow non-UTC timezones for DateTime64 RowBinary encoding https://github.com/plausible/ch/pull/315 - use gregorian days in RowBinary dates https://github.com/plausible/ch/pull/318 -- support `BFloat16` https://github.com/plausible/ch/pull/321 - fix `Ch.type/1` callback https://github.com/plausible/ch/pull/331 ## 0.7.1 (2026-01-15) diff --git a/lib/ch.ex b/lib/ch.ex index 86924500..0f326131 100644 --- a/lib/ch.ex +++ b/lib/ch.ex @@ -149,7 +149,7 @@ defmodule Ch do def type(unquote(:"f#{size}")), do: :float end - def type(:bf16), do: {:parameterized, {Ch, :bf16}} + def type(:bf16), do: :float def type({:decimal, _p, _s}), do: :decimal diff --git a/test/ch/bfloat16_test.exs b/test/ch/bfloat16_test.exs index 81419971..f9f5bfac 100644 --- a/test/ch/bfloat16_test.exs +++ b/test/ch/bfloat16_test.exs @@ -12,15 +12,28 @@ defmodule Ch.BFloat16Test do conn: start_supervised!({Ch, database: Ch.Test.database()})} end - test "plain", %{conn: conn, query_options: query_options} do - assert Ch.query!(conn, "select 1.75::BFloat16", _no_params = %{}, query_options).rows == [ - [1.75] - ] + property "plain", %{conn: conn, query_options: query_options} do + check all value <- bfloat16() do + assert Ch.query!( + conn, + "select #{Float.to_string(value)}::BFloat16", + _no_params = %{}, + query_options + ).rows == + [[value]] + end end - test "send and read back via params", %{conn: conn, query_options: query_options} do - assert Ch.query!(conn, "select {value:BFloat16} as value", %{"value" => 1.75}, query_options).rows == - [[1.75]] + property "send and read back via params", %{conn: conn, query_options: query_options} do + check all value <- bfloat16() do + assert Ch.query!( + conn, + "select {value:BFloat16} as value", + %{"value" => value}, + query_options + ).rows == + [[value]] + end end property "send and read back via rowbinary", %{conn: conn, query_options: query_options} do @@ -31,7 +44,7 @@ defmodule Ch.BFloat16Test do query_options = Keyword.merge(query_options, types: ["UInt8", "BFloat16"]) - check all values <- list_of(bfloat16(), length: 20), max_runs: 10 do + check all values <- list_of(bfloat16(), length: 20) do Ch.query!(conn, "truncate table #{table}") rows = @@ -53,11 +66,25 @@ defmodule Ch.BFloat16Test do end defp bfloat16 do - integer(0..0xFFFF) - |> filter(fn bits -> (bits &&& 0x7F80) != 0x7F80 end) + integer(-1_000_000..1_000_000) + |> map(&(&1 / 16)) + |> map(&float_to_bfloat16/1) |> map(&bfloat16_to_float/1) end + defp float_to_bfloat16(float) do + <> = <> + + upper = bits >>> 16 + lower = bits &&& 0xFFFF + + if lower > 0x8000 or (lower == 0x8000 and (upper &&& 1) == 1) do + upper + 1 + else + upper + end + end + defp bfloat16_to_float(bits) do <> = <> float diff --git a/test/ch/ecto_type_test.exs b/test/ch/ecto_type_test.exs index 8ec1dcad..db01071d 100644 --- a/test/ch/ecto_type_test.exs +++ b/test/ch/ecto_type_test.exs @@ -261,7 +261,7 @@ defmodule Ch.EctoTypeTest do assert {:parameterized, {Ch, :bf16}} = type = Ecto.ParameterizedType.init(Ch, type: unquote("BFloat16")) - assert Ecto.Type.type(type) == type + assert Ecto.Type.type(type) == :float assert Ecto.Type.format(type) == "#Ch" assert {:ok, 1.0} = Ecto.Type.cast(type, 1.0) From 946901686772bd0fc901c9a8dff9eca8a3de2c61 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 7 May 2026 20:24:42 +0300 Subject: [PATCH 5/7] Expand BFloat16 tests --- test/ch/bfloat16_test.exs | 98 ++++++++++++++++++++++++++++++++++++--- 1 file changed, 91 insertions(+), 7 deletions(-) diff --git a/test/ch/bfloat16_test.exs b/test/ch/bfloat16_test.exs index f9f5bfac..9127a0e6 100644 --- a/test/ch/bfloat16_test.exs +++ b/test/ch/bfloat16_test.exs @@ -2,18 +2,35 @@ defmodule Ch.BFloat16Test do use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] use ExUnitProperties + import Ch.RowBinary import Bitwise @moduletag :bf16 + @bf16_edges [ + 0x0000, + 0x8000, + 0x0001, + 0x8001, + 0x007F, + 0x807F, + 0x0080, + 0x8080, + 0x3F80, + 0xBF80, + 0x3FE0, + 0x7F7F, + 0xFF7F + ] + setup ctx do {:ok, query_options: ctx[:query_options] || [], conn: start_supervised!({Ch, database: Ch.Test.database()})} end - property "plain", %{conn: conn, query_options: query_options} do - check all value <- bfloat16() do + property "plain finite values", %{conn: conn, query_options: query_options} do + check all value <- bounded_bfloat16() do assert Ch.query!( conn, "select #{Float.to_string(value)}::BFloat16", @@ -24,8 +41,11 @@ defmodule Ch.BFloat16Test do end end - property "send and read back via params", %{conn: conn, query_options: query_options} do - check all value <- bfloat16() do + property "finite params round-trip through ClickHouse casts", %{ + conn: conn, + query_options: query_options + } do + check all value <- bounded_bfloat16() do assert Ch.query!( conn, "select {value:BFloat16} as value", @@ -36,7 +56,49 @@ defmodule Ch.BFloat16Test do end end - property "send and read back via rowbinary", %{conn: conn, query_options: query_options} do + test "special values decode as nil", %{conn: conn, query_options: query_options} do + assert Ch.query!( + conn, + "select 'nan'::BFloat16, 'inf'::BFloat16, '-inf'::BFloat16", + [], + query_options + ).rows == + [[nil, nil, nil]] + end + + property "RowBinary encodes finite values as their BFloat16 bits" do + check all bits <- finite_bfloat16_bits() do + value = bfloat16_to_float(bits) + + assert encode(:bf16, value) == <> + end + end + + property "RowBinary decodes finite BFloat16 bit patterns" do + check all bits <- finite_bfloat16_bits() do + assert decode_rows(<>, [:bf16]) == [[bfloat16_to_float(bits)]] + end + end + + property "RowBinary decodes non-finite BFloat16 bit patterns as nil" do + check all bits <- non_finite_bfloat16_bits() do + assert decode_rows(<>, [:bf16]) == [[nil]] + end + end + + test "RowBinary covers BFloat16 edge bit patterns" do + for bits <- @bf16_edges do + value = bfloat16_to_float(bits) + + assert encode(:bf16, value) == <> + assert decode_rows(<>, [:bf16]) == [[value]] + end + end + + property "finite RowBinary values round-trip through ClickHouse", %{ + conn: conn, + query_options: query_options + } do table = "bf16_#{System.unique_integer([:positive])}" Ch.query!(conn, "create table #{table} (idx UInt8, bf16 BFloat16) engine Memory") @@ -44,9 +106,16 @@ defmodule Ch.BFloat16Test do query_options = Keyword.merge(query_options, types: ["UInt8", "BFloat16"]) - check all values <- list_of(bfloat16(), length: 20) do + check all bits <- list_of(finite_bfloat16_bits(), length: 20) do + Ch.query!( + conn, + "create table if not exists #{table} (idx UInt8, bf16 BFloat16) engine Memory" + ) + Ch.query!(conn, "truncate table #{table}") + values = Enum.map(bits, &bfloat16_to_float/1) + rows = values |> Enum.with_index() @@ -65,13 +134,28 @@ defmodule Ch.BFloat16Test do end end - defp bfloat16 do + defp bounded_bfloat16 do integer(-1_000_000..1_000_000) |> map(&(&1 / 16)) |> map(&float_to_bfloat16/1) |> map(&bfloat16_to_float/1) end + defp finite_bfloat16_bits do + gen all sign <- integer(0..1), + exponent <- integer(0..0xFE), + fraction <- integer(0..0x7F) do + sign <<< 15 ||| exponent <<< 7 ||| fraction + end + end + + defp non_finite_bfloat16_bits do + gen all sign <- integer(0..1), + fraction <- integer(0..0x7F) do + sign <<< 15 ||| 0x7F80 ||| fraction + end + end + defp float_to_bfloat16(float) do <> = <> From 724d3951c65a86d19f485ad6d50097610c99de32 Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 7 May 2026 20:28:07 +0300 Subject: [PATCH 6/7] Run BFloat16 RowBinary tests through ClickHouse --- test/ch/bfloat16_test.exs | 83 ++++++++++++++++----------------------- 1 file changed, 34 insertions(+), 49 deletions(-) diff --git a/test/ch/bfloat16_test.exs b/test/ch/bfloat16_test.exs index 9127a0e6..7bc50b09 100644 --- a/test/ch/bfloat16_test.exs +++ b/test/ch/bfloat16_test.exs @@ -2,7 +2,6 @@ defmodule Ch.BFloat16Test do use ExUnit.Case, parameterize: [%{query_options: []}, %{query_options: [multipart: true]}] use ExUnitProperties - import Ch.RowBinary import Bitwise @moduletag :bf16 @@ -66,33 +65,18 @@ defmodule Ch.BFloat16Test do [[nil, nil, nil]] end - property "RowBinary encodes finite values as their BFloat16 bits" do - check all bits <- finite_bfloat16_bits() do - value = bfloat16_to_float(bits) - - assert encode(:bf16, value) == <> - end - end - - property "RowBinary decodes finite BFloat16 bit patterns" do - check all bits <- finite_bfloat16_bits() do - assert decode_rows(<>, [:bf16]) == [[bfloat16_to_float(bits)]] - end - end + test "RowBinary edge values round-trip through ClickHouse", %{ + conn: conn, + query_options: query_options + } do + table = "bf16_#{System.unique_integer([:positive])}" - property "RowBinary decodes non-finite BFloat16 bit patterns as nil" do - check all bits <- non_finite_bfloat16_bits() do - assert decode_rows(<>, [:bf16]) == [[nil]] - end - end + Ch.query!(conn, "create table #{table} (idx UInt8, bf16 BFloat16) engine Memory") + on_exit(fn -> Ch.Test.query("drop table if exists #{table}") end) - test "RowBinary covers BFloat16 edge bit patterns" do - for bits <- @bf16_edges do - value = bfloat16_to_float(bits) + values = Enum.map(@bf16_edges, &bfloat16_to_float/1) - assert encode(:bf16, value) == <> - assert decode_rows(<>, [:bf16]) == [[value]] - end + assert_rowbinary_round_trip(conn, table, query_options, values) end property "finite RowBinary values round-trip through ClickHouse", %{ @@ -112,25 +96,9 @@ defmodule Ch.BFloat16Test do "create table if not exists #{table} (idx UInt8, bf16 BFloat16) engine Memory" ) - Ch.query!(conn, "truncate table #{table}") - values = Enum.map(bits, &bfloat16_to_float/1) - rows = - values - |> Enum.with_index() - |> Enum.map(fn {value, idx} -> [idx, value] end) - - assert %{num_rows: 20} = - Ch.query!( - conn, - "insert into #{table} (idx, bf16) format RowBinary", - rows, - query_options - ) - - assert Ch.query!(conn, "select bf16 from #{table} order by idx").rows == - Enum.map(values, &[&1]) + assert_rowbinary_round_trip(conn, table, query_options, values) end end @@ -149,13 +117,6 @@ defmodule Ch.BFloat16Test do end end - defp non_finite_bfloat16_bits do - gen all sign <- integer(0..1), - fraction <- integer(0..0x7F) do - sign <<< 15 ||| 0x7F80 ||| fraction - end - end - defp float_to_bfloat16(float) do <> = <> @@ -173,4 +134,28 @@ defmodule Ch.BFloat16Test do <> = <> float end + + defp assert_rowbinary_round_trip(conn, table, query_options, values) do + Ch.query!(conn, "truncate table #{table}") + + query_options = Keyword.merge(query_options, types: ["UInt8", "BFloat16"]) + + rows = + values + |> Enum.with_index() + |> Enum.map(fn {value, idx} -> [idx, value] end) + + assert %{num_rows: count} = + Ch.query!( + conn, + "insert into #{table} (idx, bf16) format RowBinary", + rows, + query_options + ) + + assert count == length(values) + + assert Ch.query!(conn, "select bf16 from #{table} order by idx").rows == + Enum.map(values, &[&1]) + end end From 47ef9d0205e34f4d2b2e013b47a0f5eb2a9331fa Mon Sep 17 00:00:00 2001 From: ruslandoga Date: Thu, 7 May 2026 20:33:43 +0300 Subject: [PATCH 7/7] Use identifier params in BFloat16 table queries --- test/ch/bfloat16_test.exs | 47 ++++++++++++++++++++++++++------------- 1 file changed, 31 insertions(+), 16 deletions(-) diff --git a/test/ch/bfloat16_test.exs b/test/ch/bfloat16_test.exs index 7bc50b09..7d41bd98 100644 --- a/test/ch/bfloat16_test.exs +++ b/test/ch/bfloat16_test.exs @@ -69,36 +69,41 @@ defmodule Ch.BFloat16Test do conn: conn, query_options: query_options } do - table = "bf16_#{System.unique_integer([:positive])}" + table = "bf16_edges" + insert = "insert into bf16_edges (idx, bf16) format RowBinary" - Ch.query!(conn, "create table #{table} (idx UInt8, bf16 BFloat16) engine Memory") - on_exit(fn -> Ch.Test.query("drop table if exists #{table}") end) + create_table(conn, table) + + on_exit(fn -> + Ch.Test.query("drop table if exists {table:Identifier}", %{"table" => table}) + end) values = Enum.map(@bf16_edges, &bfloat16_to_float/1) - assert_rowbinary_round_trip(conn, table, query_options, values) + assert_rowbinary_round_trip(conn, table, insert, query_options, values) end property "finite RowBinary values round-trip through ClickHouse", %{ conn: conn, query_options: query_options } do - table = "bf16_#{System.unique_integer([:positive])}" + table = "bf16_finite" + insert = "insert into bf16_finite (idx, bf16) format RowBinary" + + create_table(conn, table) - Ch.query!(conn, "create table #{table} (idx UInt8, bf16 BFloat16) engine Memory") - on_exit(fn -> Ch.Test.query("drop table if exists #{table}") end) + on_exit(fn -> + Ch.Test.query("drop table if exists {table:Identifier}", %{"table" => table}) + end) query_options = Keyword.merge(query_options, types: ["UInt8", "BFloat16"]) check all bits <- list_of(finite_bfloat16_bits(), length: 20) do - Ch.query!( - conn, - "create table if not exists #{table} (idx UInt8, bf16 BFloat16) engine Memory" - ) + create_table(conn, table, if_not_exists: true) values = Enum.map(bits, &bfloat16_to_float/1) - assert_rowbinary_round_trip(conn, table, query_options, values) + assert_rowbinary_round_trip(conn, table, insert, query_options, values) end end @@ -135,8 +140,18 @@ defmodule Ch.BFloat16Test do float end - defp assert_rowbinary_round_trip(conn, table, query_options, values) do - Ch.query!(conn, "truncate table #{table}") + defp create_table(conn, table, opts \\ []) do + exists = if opts[:if_not_exists], do: " if not exists", else: "" + + Ch.query!( + conn, + "create table#{exists} {table:Identifier} (idx UInt8, bf16 BFloat16) engine Memory", + %{"table" => table} + ) + end + + defp assert_rowbinary_round_trip(conn, table, insert, query_options, values) do + Ch.query!(conn, "truncate table {table:Identifier}", %{"table" => table}) query_options = Keyword.merge(query_options, types: ["UInt8", "BFloat16"]) @@ -148,14 +163,14 @@ defmodule Ch.BFloat16Test do assert %{num_rows: count} = Ch.query!( conn, - "insert into #{table} (idx, bf16) format RowBinary", + insert, rows, query_options ) assert count == length(values) - assert Ch.query!(conn, "select bf16 from #{table} order by idx").rows == + assert Ch.query!(conn, "select bf16 from {table:Identifier} order by idx", %{"table" => table}).rows == Enum.map(values, &[&1]) end end