From 974ec7c017ea9fd5dcb6bffba5226431debe4bfa Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 29 May 2026 10:44:57 +0000
Subject: [PATCH 1/2] Improve missing-value filtering for dataframe conversion

---
 dataverse_api/utils/data.py | 22 ++++++++++++++++++++--
 tests/test_entity.py        | 18 ++++++++++++++++++
 2 files changed, 38 insertions(+), 2 deletions(-)

diff --git a/dataverse_api/utils/data.py b/dataverse_api/utils/data.py
index d2544d7..0fb41e0 100644
--- a/dataverse_api/utils/data.py
+++ b/dataverse_api/utils/data.py
@@ -1,4 +1,5 @@
 import json
+import math
 from collections.abc import Collection, Mapping
 from typing import Any
 
@@ -14,7 +15,24 @@ def isoformat(self) -> str: ...
 
 def is_not_none(value: Any) -> bool:
     """Checks if a value is not None or NaN."""
-    return value == value and value is not None
+    if value is None:
+        return False
+
+    try:
+        if math.isnan(value):
+            return False
+    except TypeError:
+        pass
+
+    try:
+        is_self_equal = value == value
+    except Exception:
+        return True
+
+    try:
+        return bool(is_self_equal)
+    except Exception:
+        return False
 
 
 def dict_of_lists_to_list_of_dicts(data: dict[str, list[Any]]) -> list[dict[str, Any]]:
@@ -37,7 +55,7 @@ def dict_of_lists_to_list_of_dicts(data: dict[str, list[Any]]) -> list[dict[str,
     keys = list(data.keys())
     values = zip(*data.values())
     zipped = [dict(zip(keys, v)) for v in values]
-    return [{k: v for k, v in row.items() if v == v and v is not None} for row in zipped]
+    return [{k: v for k, v in row.items() if is_not_none(v)} for row in zipped]
 
 
 def convert_dataframe_to_dict(data: IntoDataFrame) -> list[dict[str, Any]]:
diff --git a/tests/test_entity.py b/tests/test_entity.py
index 66de9e3..1fcdeec 100644
--- a/tests/test_entity.py
+++ b/tests/test_entity.py
@@ -785,3 +785,21 @@ def test_entity_upsert_dataframe_with_none(
 
     for row in resp:
         assert row.status_code == 204
+
+
+def test_convert_dataframe_to_dict_filters_nan_like_values():
+    df = pd.DataFrame(
+        {
+            "id": ["a", "b", "c"],
+            "float_data": [1.0, float("nan"), 3.0],
+            "nullable_float_data": pd.Series([1.0, pd.NA, 3.0], dtype="Float64"),
+        }
+    )
+
+    data = convert_dataframe_to_dict(df)
+
+    assert data == [
+        {"id": "a", "float_data": 1.0, "nullable_float_data": 1.0},
+        {"id": "b"},
+        {"id": "c", "float_data": 3.0, "nullable_float_data": 3.0},
+    ]

From 9b2fb28b057ea64b21382fbcd42491e609eb5969 Mon Sep 17 00:00:00 2001
From: "copilot-swe-agent[bot]" <198982749+Copilot@users.noreply.github.com>
Date: Fri, 29 May 2026 10:47:35 +0000
Subject: [PATCH 2/2] Refine null-filter comparison handling

---
 dataverse_api/utils/data.py | 5 +++--
 1 file changed, 3 insertions(+), 2 deletions(-)

diff --git a/dataverse_api/utils/data.py b/dataverse_api/utils/data.py
index 0fb41e0..d05acbe 100644
--- a/dataverse_api/utils/data.py
+++ b/dataverse_api/utils/data.py
@@ -26,12 +26,13 @@ def is_not_none(value: Any) -> bool:
 
     try:
         is_self_equal = value == value
-    except Exception:
+    except (TypeError, ValueError):
+        # Keep values that do not support scalar self-comparison.
         return True
 
     try:
         return bool(is_self_equal)
-    except Exception:
+    except (TypeError, ValueError):
         return False