Skip to content

Commit 673aa45

Browse files
committed
Fix seed boolean null handling
Signed-off-by: nkwork9999 <143652584+nkwork9999@users.noreply.github.com>
1 parent 5d2bb26 commit 673aa45

2 files changed

Lines changed: 29 additions & 3 deletions

File tree

sqlmesh/core/model/definition.py

Lines changed: 4 additions & 3 deletions
Original file line numberDiff line numberDiff line change
@@ -1706,6 +1706,7 @@ def render(
17061706

17071707
def render_seed(self) -> t.Iterator[QueryOrDF]:
17081708
import numpy as np
1709+
import pandas as pd
17091710

17101711
self._ensure_hydrated()
17111712

@@ -1746,8 +1747,6 @@ def render_seed(self) -> t.Iterator[QueryOrDF]:
17461747

17471748
# convert all date/time types to native pandas timestamp
17481749
for column in [*date_columns, *datetime_columns]:
1749-
import pandas as pd
1750-
17511750
df[column] = pd.to_datetime(df[column], infer_datetime_format=True, errors="ignore") # type: ignore
17521751

17531752
# extract datetime.date from pandas timestamp for DATE columns
@@ -1763,7 +1762,9 @@ def render_seed(self) -> t.Iterator[QueryOrDF]:
17631762
)
17641763

17651764
for column in bool_columns:
1766-
df[column] = df[column].apply(lambda i: str_to_bool(str(i)))
1765+
df[column] = df[column].apply(
1766+
lambda i: None if pd.isna(i) else str_to_bool(str(i))
1767+
)
17671768

17681769
df.loc[:, string_columns] = df[string_columns].mask(
17691770
cond=lambda x: x.notna(), # type: ignore

tests/dbt/test_transformation.py

Lines changed: 25 additions & 0 deletions
Original file line numberDiff line numberDiff line change
@@ -947,6 +947,31 @@ def test_seed_single_whitespace_is_na(tmp_path):
947947
assert df["col_b"].to_list() == [1, None]
948948

949949

950+
def test_seed_boolean_nulls_are_preserved(tmp_path):
951+
seed_csv = tmp_path / "seed.csv"
952+
with open(seed_csv, "w", encoding="utf-8") as fd:
953+
fd.write("id,test_ind\n")
954+
fd.write("1,null\n")
955+
fd.write("2,false\n")
956+
fd.write("3,true\n")
957+
fd.write("4,null\n")
958+
959+
seed = SeedConfig(
960+
name="test_model",
961+
package="foo",
962+
path=Path(seed_csv),
963+
column_types={"test_ind": "boolean"},
964+
)
965+
966+
context = DbtContext()
967+
context.project_name = "foo"
968+
context.target = DuckDbConfig(name="target", schema="test")
969+
sqlmesh_seed = seed.to_sqlmesh(context)
970+
971+
df = next(sqlmesh_seed.render_seed())
972+
assert df["test_ind"].to_list() == [None, False, True, None]
973+
974+
950975
def test_seed_partial_column_inference(tmp_path):
951976
seed_csv = tmp_path / "seed.csv"
952977
with open(seed_csv, "w", encoding="utf-8") as fd:

0 commit comments

Comments
 (0)