Skip to content

Commit ca97964

Browse files
committed
fix(upsert): clarify join column error messages
Name the Iceberg table "the table" and the Arrow argument "the input" consistently, point nested paths at the struct they live in instead of saying "top-level", and finish each message with what to do about it.
1 parent 79cad45 commit ca97964

3 files changed

Lines changed: 47 additions & 44 deletions

File tree

‎mkdocs/docs/api.md‎

Lines changed: 1 addition & 1 deletion
Original file line numberDiff line numberDiff line change
@@ -522,7 +522,7 @@ PyIceberg supports upsert operations, meaning that it is able to merge an Arrow
522522
<!-- markdownlint-disable MD046 -- Allowing indented multi-line formatting in admonition-->
523523

524524
!!! note "Join columns"
525-
Use `join_cols` to pick the top-level columns to match on; when omitted, the table's identifier fields are used. Unsupported join columns are rejected with a descriptive error before anything is written.
525+
Use `join_cols` to pick the columns to match on; when omitted, the table's identifier fields are used. Unsupported join columns are rejected with a descriptive error before anything is written.
526526

527527
- **Supported**: boolean, integer, long, decimal, date, time, timestamp, string, and binary columns.
528528
- **Not supported**: `float` and `double` columns, because floating-point equality is unreliable; nested columns (structs, lists, and maps), including identifier fields nested inside a struct; and UUID columns.

‎pyiceberg/table/upsert_util.py‎

Lines changed: 20 additions & 15 deletions
Original file line numberDiff line numberDiff line change
@@ -37,54 +37,59 @@ def validate_join_cols(df: pyarrow_table, join_cols: list[str], table_schema: pa
3737
raise ValueError(f"join_cols must be a list of column names, got {type(join_cols).__name__}.")
3838
duplicates = sorted(col for col, count in Counter(join_cols).items() if count > 1)
3939
if duplicates:
40-
raise ValueError(f"Duplicate join columns: {', '.join(duplicates)}.")
40+
raise ValueError(f"join_cols contains duplicates: {', '.join(duplicates)}.")
4141

4242
df_column_names = set(df.schema.names)
4343

4444
for col in join_cols:
4545
if col not in table_schema.names:
46+
parent = col.split(".", 1)[0]
47+
if "." in col and parent in table_schema.names and pa.types.is_nested(table_schema.field(parent).type):
48+
raise ValueError(f"Join column '{col}' is a field inside struct '{parent}' and cannot be used as a join key.")
4649
raise ValueError(
47-
f"Join column '{col}' does not exist in the table schema. Only top-level columns can be used as join keys. "
48-
f"Available columns: {', '.join(table_schema.names)}."
50+
f"Join column '{col}' does not exist in the table. Available columns: {', '.join(table_schema.names)}."
4951
)
5052
table_field = table_schema.field(col)
5153
# Table-level rejections: These types are fundamentally unreliable or
5254
# unsupported as join keys regardless of the input data format.
5355
if pa.types.is_floating(table_field.type):
5456
raise ValueError(
55-
f"Floating point column '{col}' cannot be used as a join key in upsert. "
56-
"Floating point equality is unreliable; choose a different join column."
57+
f"Join column '{col}' is floating point and cannot be used as a join key "
58+
"because floating point equality is unreliable."
5759
)
5860
if pa.types.is_nested(table_field.type):
5961
raise ValueError(
60-
f"Nested column '{col}' of type '{table_field.type}' cannot be used as a join key in upsert. "
61-
"Only primitive types are supported."
62+
f"Join column '{col}' has nested type '{table_field.type}'; only primitive columns can be join keys."
6263
)
6364
if isinstance(table_field.type, pa.BaseExtensionType):
6465
raise NotImplementedError(
65-
f"Column '{col}' of type '{table_field.type}' is not currently supported as a join key in upsert."
66+
f"Join column '{col}' has type '{table_field.type}', which is not yet supported as a join key."
6667
)
6768

6869
# Schema compatibility permits missing optional fields, but upsert needs every join key.
6970
if col not in df_column_names:
70-
raise ValueError(f"Join column '{col}' does not exist in the source schema.")
71-
# Some source representations are unsupported even when the table type is valid.
71+
raise ValueError(f"Join column '{col}' is missing from the input.")
72+
# Some input representations are unsupported even when the table type is valid.
7273
arr = df.column(col)
7374
if pa.types.is_dictionary(arr.type):
74-
raise NotImplementedError(f"Dictionary-encoded column '{col}' is not currently supported as a join key in upsert.")
75+
raise NotImplementedError(
76+
f"Input column '{col}' is dictionary-encoded, which is not yet supported for join keys. Decode it first."
77+
)
7578
if pa.types.is_null(arr.type):
76-
raise ValueError(f"Null-type column '{col}' cannot be used as a join key in upsert.")
79+
raise ValueError(f"Input column '{col}' has the null type and cannot be used as a join key.")
7780
if pa.types.is_string_view(arr.type) or pa.types.is_binary_view(arr.type):
81+
plain = "string" if pa.types.is_string_view(arr.type) else "binary"
7882
raise NotImplementedError(
79-
f"View-typed column '{col}' of type '{arr.type}' is not currently supported as a join key in upsert."
83+
f"Input column '{col}' has type '{arr.type}', which is not yet supported for join keys. "
84+
f"Cast it to '{plain}' first."
8085
)
8186
if isinstance(arr.type, pa.BaseExtensionType):
8287
raise NotImplementedError(
83-
f"Extension type '{arr.type}' for column '{col}' is not currently supported as a join key in upsert."
88+
f"Input column '{col}' has extension type '{arr.type}', which is not yet supported for join keys."
8489
)
8590
# Null keys cannot be expressed as Iceberg literals in the match filter.
8691
if arr.null_count > 0:
87-
raise ValueError(f"Join column '{col}' contains null values, which cannot be used as join keys in upsert.")
92+
raise ValueError(f"Input column '{col}' contains null values, which cannot be join keys.")
8893

8994

9095
def create_match_filter(df: pyarrow_table, join_cols: list[str]) -> BooleanExpression:

‎tests/table/test_upsert.py‎

Lines changed: 26 additions & 28 deletions
Original file line numberDiff line numberDiff line change
@@ -345,7 +345,7 @@ def test_key_cols_misaligned(catalog: Catalog) -> None:
345345

346346
df_src = ctx.sql("select 1 as item_id, date '2021-05-01' as order_date, 'B' as order_type").to_arrow_table()
347347

348-
with pytest.raises(ValueError, match="Join column 'order_id' does not exist in the source schema"):
348+
with pytest.raises(ValueError, match="Join column 'order_id' is missing from the input"):
349349
table.upsert(df=df_src, join_cols=["order_id"])
350350

351351

@@ -690,8 +690,8 @@ def test_upsert_with_struct_field_as_join_key(catalog: Catalog) -> None:
690690
with pytest.raises(
691691
ValueError,
692692
match=(
693-
"Nested column 'nested_type' of type 'struct<sub1: large_string not null, sub2: large_string not null>' "
694-
"cannot be used as a join key in upsert"
693+
"Join column 'nested_type' has nested type 'struct<sub1: large_string not null, sub2: large_string not null>'; "
694+
"only primitive columns can be join keys"
695695
),
696696
):
697697
_ = tbl.upsert(update_data, join_cols=["nested_type"])
@@ -960,69 +960,69 @@ def test_upsert_snapshot_properties(catalog: Catalog) -> None:
960960
@pytest.mark.parametrize(
961961
"table_type, source_key, expected_error, match",
962962
[
963-
pytest.param(pa.float32(), pa.array([1.0], pa.float32()), ValueError, "Floating point column 'k'", id="float32"),
964-
pytest.param(pa.float64(), pa.array([1.0], pa.float64()), ValueError, "Floating point column 'k'", id="float64"),
963+
pytest.param(pa.float32(), pa.array([1.0], pa.float32()), ValueError, "Join column 'k' is floating point", id="float32"),
964+
pytest.param(pa.float64(), pa.array([1.0], pa.float64()), ValueError, "Join column 'k' is floating point", id="float64"),
965965
pytest.param(
966-
_STRUCT, pa.array([{"a": 1}], _STRUCT), ValueError, "Nested column 'k' of type 'struct<a: int32>'", id="struct"
966+
_STRUCT, pa.array([{"a": 1}], _STRUCT), ValueError, "Join column 'k' has nested type 'struct<a: int32>'", id="struct"
967967
),
968968
pytest.param(
969969
pa.list_(pa.int32()),
970970
pa.array([[1]], pa.list_(pa.int32())),
971971
ValueError,
972-
"Nested column 'k' of type 'large_list",
972+
"Join column 'k' has nested type 'large_list",
973973
id="list",
974974
),
975975
pytest.param(
976-
_MAP, pa.array([[("a", 1)]], _MAP), ValueError, "Nested column 'k' of type 'map<large_string, int32>'", id="map"
976+
_MAP, pa.array([[("a", 1)]], _MAP), ValueError, "Join column 'k' has nested type 'map<large_string, int32>'", id="map"
977977
),
978978
pytest.param(
979979
pa.uuid(),
980980
pa.array([_UUID_BYTES], pa.uuid()),
981981
NotImplementedError,
982-
"Column 'k' of type 'extension<arrow.uuid>'",
982+
"Join column 'k' has type 'extension<arrow.uuid>'",
983983
id="uuid-table",
984984
),
985985
pytest.param(
986986
pa.uuid(),
987987
pa.array([_UUID_BYTES], pa.binary(16)),
988988
NotImplementedError,
989-
"Column 'k' of type 'extension<arrow.uuid>'",
989+
"Join column 'k' has type 'extension<arrow.uuid>'",
990990
id="uuid-table-fixed-source",
991991
),
992992
pytest.param(
993993
pa.string(),
994994
pa.array([_UUID_BYTES], pa.uuid()),
995995
NotImplementedError,
996-
"Extension type 'extension<arrow.uuid>' for column 'k'",
996+
"Input column 'k' has extension type 'extension<arrow.uuid>'",
997997
id="uuid-extension-source",
998998
),
999999
pytest.param(
10001000
pa.string(),
10011001
pa.array(["a"]).dictionary_encode(),
10021002
NotImplementedError,
1003-
"Dictionary-encoded column 'k'",
1003+
"Input column 'k' is dictionary-encoded",
10041004
id="dictionary-string",
10051005
),
10061006
pytest.param(
10071007
pa.int64(),
10081008
pa.array([1]).dictionary_encode(),
10091009
NotImplementedError,
1010-
"Dictionary-encoded column 'k'",
1010+
"Input column 'k' is dictionary-encoded",
10111011
id="dictionary-int",
10121012
),
1013-
pytest.param(pa.int32(), pa.array([None], pa.null()), ValueError, "Null-type column 'k'", id="null-type"),
1013+
pytest.param(pa.int32(), pa.array([None], pa.null()), ValueError, "Input column 'k' has the null type", id="null-type"),
10141014
pytest.param(
10151015
pa.string(),
10161016
pa.array(["a"], pa.string_view()),
10171017
NotImplementedError,
1018-
"View-typed column 'k' of type 'string_view'",
1018+
"Input column 'k' has type 'string_view'",
10191019
id="string-view",
10201020
),
10211021
pytest.param(
10221022
pa.binary(),
10231023
pa.array([b"a"], pa.binary_view()),
10241024
NotImplementedError,
1025-
"View-typed column 'k' of type 'binary_view'",
1025+
"Input column 'k' has type 'binary_view'",
10261026
id="binary-view",
10271027
),
10281028
pytest.param(
@@ -1033,10 +1033,10 @@ def test_upsert_snapshot_properties(catalog: Catalog) -> None:
10331033
id="run-end-encoded",
10341034
),
10351035
pytest.param(
1036-
pa.int32(), pa.array([1, None], pa.int32()), ValueError, "Join column 'k' contains null values", id="null-values"
1036+
pa.int32(), pa.array([1, None], pa.int32()), ValueError, "Input column 'k' contains null values", id="null-values"
10371037
),
10381038
pytest.param(
1039-
pa.int32(), pa.array([None], pa.int32()), ValueError, "Join column 'k' contains null values", id="all-null-values"
1039+
pa.int32(), pa.array([None], pa.int32()), ValueError, "Input column 'k' contains null values", id="all-null-values"
10401040
),
10411041
pytest.param(pa.int32(), pa.array(["1"]), ValueError, "Mismatch in fields", id="wrong-type-source"),
10421042
],
@@ -1059,16 +1059,14 @@ def test_upsert_rejects_unsupported_join_key(
10591059
@pytest.mark.parametrize(
10601060
"join_cols, identifier_field_ids, drop_source_columns, match",
10611061
[
1062-
pytest.param(["missing"], [], [], "Join column 'missing' does not exist in the table schema", id="not-in-table"),
1063-
pytest.param(["K"], [], [], "Join column 'K' does not exist in the table schema", id="case-mismatch"),
1064-
pytest.param(["k"], [], ["k"], "Join column 'k' does not exist in the source schema", id="required-not-in-source"),
1065-
pytest.param(["opt"], [], ["opt"], "Join column 'opt' does not exist in the source schema", id="optional-not-in-source"),
1066-
pytest.param(
1067-
["k", "opt"], [], ["opt"], "Join column 'opt' does not exist in the source schema", id="composite-second-missing"
1068-
),
1069-
pytest.param(["k", "k"], [], [], "Duplicate join columns: k", id="duplicate-join-cols"),
1070-
pytest.param(["s.x"], [], [], "Only top-level columns can be used as join keys", id="nested-path"),
1071-
pytest.param(None, [4], [], "Only top-level columns can be used as join keys", id="nested-identifier-field"),
1062+
pytest.param(["missing"], [], [], "Join column 'missing' does not exist in the table", id="not-in-table"),
1063+
pytest.param(["K"], [], [], "Join column 'K' does not exist in the table", id="case-mismatch"),
1064+
pytest.param(["k"], [], ["k"], "Join column 'k' is missing from the input", id="required-not-in-source"),
1065+
pytest.param(["opt"], [], ["opt"], "Join column 'opt' is missing from the input", id="optional-not-in-source"),
1066+
pytest.param(["k", "opt"], [], ["opt"], "Join column 'opt' is missing from the input", id="composite-second-missing"),
1067+
pytest.param(["k", "k"], [], [], "join_cols contains duplicates: k", id="duplicate-join-cols"),
1068+
pytest.param(["s.x"], [], [], "Join column 's.x' is a field inside struct 's'", id="nested-path"),
1069+
pytest.param(None, [4], [], "Join column 's.x' is a field inside struct 's'", id="nested-identifier-field"),
10721070
pytest.param([], [], [], "Join columns could not be found", id="empty-join-cols"),
10731071
pytest.param(None, [], [], "Join columns could not be found", id="no-identifier-fields"),
10741072
pytest.param("k", [], [], "join_cols must be a list of column names, got str", id="string-not-list"),

0 commit comments

Comments
 (0)