Skip to content
Merged
Show file tree
Hide file tree
Changes from all commits
Commits
File filter

Filter by extension

Filter by extension

Conversations
Failed to load comments.
Loading
Jump to
Jump to file
Failed to load files.
Loading
Diff view
Diff view
15 changes: 11 additions & 4 deletions src/freshdata/parsers/base.py
Original file line number Diff line number Diff line change
Expand Up @@ -96,17 +96,24 @@ class Parser(ABC):
def parse(self, source: Any) -> ParseResult:
"""Parse *source* (Path, text, bytes, or file-like) into a :class:`ParseResult`."""

def read_text(self, source: Any, *, encoding: str = "utf-8") -> str:
"""Read *source* into text, accepting a Path, str content, bytes, or file-like."""
def read_text(self, source: Any, *, encoding: str = "utf-8-sig") -> str:
"""Read *source* into text, accepting a Path, str content, bytes, or file-like.

A leading UTF-8 byte-order mark (common in files exported by Windows tools) is
dropped: ``utf-8-sig`` strips it when decoding bytes, and already-decoded text
has it removed explicitly.
"""
if isinstance(source, (bytes, bytearray)):
return bytes(source).decode(encoding)
if hasattr(source, "read"):
data = source.read()
return data.decode(encoding) if isinstance(data, (bytes, bytearray)) else data
if isinstance(data, (bytes, bytearray)):
return bytes(data).decode(encoding)
return data.lstrip("\ufeff") if isinstance(data, str) else data
if isinstance(source, Path):
return source.read_text(encoding=encoding)
if isinstance(source, str):
return source
return source.lstrip("\ufeff")
raise TypeError(f"cannot read a {type(source).__name__} source")

def open_binary(self, source: Any) -> io.BufferedIOBase | io.BytesIO:
Expand Down
56 changes: 39 additions & 17 deletions src/freshdata/parsers/fhir.py
Original file line number Diff line number Diff line change
Expand Up @@ -21,12 +21,24 @@
from .base import Parser, ParseResult


def _as_list(value: Any) -> list[Any]:
"""Coerce a FHIR repeating element: a dict becomes ``[dict]``, a non-list ``[]``."""
if isinstance(value, dict):
return [value]
return value if isinstance(value, list) else []


def _as_dict(value: Any) -> dict[str, Any]:
"""Coerce a FHIR complex element: a list yields its first item if that is a dict."""
items = _as_list(value)
return items[0] if items and isinstance(items[0], dict) else {}


def _coding(concept: Any) -> tuple[Any, Any, Any]:
"""Return ``(system, code, display)`` from a FHIR CodeableConcept's first coding."""
if not isinstance(concept, dict):
return None, None, None
codings = concept.get("coding") or []
first = codings[0] if codings and isinstance(codings[0], dict) else {}
first = _as_dict(concept.get("coding"))
return first.get("system"), first.get("code"), first.get("display") or concept.get("text")


Expand All @@ -43,8 +55,7 @@ def _ref_id(reference: Any) -> Any:


def _flatten_patient(r: dict[str, Any]) -> dict[str, Any]:
addresses = r.get("address") or [{}]
addr = addresses[0] if isinstance(addresses[0], dict) else {}
addr = _as_dict(r.get("address"))
deceased_dt = r.get("deceasedDateTime")
deceased = r.get("deceasedBoolean")
if deceased is None and deceased_dt is not None:
Expand All @@ -63,8 +74,8 @@ def _flatten_patient(r: dict[str, Any]) -> dict[str, Any]:

def _flatten_observation(r: dict[str, Any]) -> dict[str, Any]:
system, code, display = _coding(r.get("code"))
vq = r.get("valueQuantity") or {}
interpretation = (r.get("interpretation") or [None])[0]
vq = _as_dict(r.get("valueQuantity"))
interpretation = _as_dict(r.get("interpretation"))
return {
"observation_id": r.get("id"),
"patient_id": _ref_id(r.get("subject")),
Expand All @@ -73,7 +84,7 @@ def _flatten_observation(r: dict[str, Any]) -> dict[str, Any]:
"code_value": code,
"display": display,
"effective_date": (r.get("effectiveDateTime")
or (r.get("effectivePeriod") or {}).get("start")),
or _as_dict(r.get("effectivePeriod")).get("start")),
"value_quantity": vq.get("value"),
"value_unit": vq.get("unit") or vq.get("code"),
"value_string": r.get("valueString"),
Expand All @@ -82,10 +93,10 @@ def _flatten_observation(r: dict[str, Any]) -> dict[str, Any]:


def _flatten_encounter(r: dict[str, Any]) -> dict[str, Any]:
cls = r.get("class") or {}
period = r.get("period") or {}
rsystem, rcode, _ = _coding((r.get("reasonCode") or [None])[0])
hosp = r.get("hospitalization") or {}
cls = _as_dict(r.get("class"))
period = _as_dict(r.get("period"))
rsystem, rcode, _ = _coding(_as_dict(r.get("reasonCode")))
hosp = _as_dict(r.get("hospitalization"))
return {
"encounter_id": r.get("id"),
"patient_id": _ref_id(r.get("subject")),
Expand All @@ -107,11 +118,11 @@ def _flatten_condition(r: dict[str, Any]) -> dict[str, Any]:
"patient_id": _ref_id(r.get("subject")),
"clinical_status": _coding_code(r.get("clinicalStatus")),
"verification_status": _coding_code(r.get("verificationStatus")),
"category": _coding_code((r.get("category") or [None])[0]),
"category": _coding_code(_as_dict(r.get("category"))),
"code_system": system,
"code_value": code,
"display": display,
"onset_date": r.get("onsetDateTime") or (r.get("onsetPeriod") or {}).get("start"),
"onset_date": r.get("onsetDateTime") or _as_dict(r.get("onsetPeriod")).get("start"),
"recorded_date": r.get("recordedDate"),
}

Expand Down Expand Up @@ -167,13 +178,24 @@ def parse(self, source: Any) -> ParseResult:

for res in self._iter_resources(data, warnings):
rtype = res.get("resourceType")
counts[str(rtype)] += 1
if not isinstance(rtype, str):
warnings.append(
f"resource with non-string resourceType ({type(rtype).__name__}) skipped")
continue
counts[rtype] += 1
handler = _FLATTENERS.get(rtype)
if handler is None:
unsupported[str(rtype)] += 1
unsupported[rtype] += 1
continue
frame_name, flatten = handler
rows[frame_name].append(flatten(res))
try:
row = flatten(res)
except (AttributeError, KeyError, TypeError, IndexError, ValueError) as exc:
warnings.append(
f"{rtype} resource {res.get('id')!r} skipped: malformed "
f"({type(exc).__name__})")
continue
rows[frame_name].append(row)

if unsupported:
listed = ", ".join(f"{k}({v})" for k, v in sorted(unsupported.items()))
Expand Down Expand Up @@ -203,7 +225,7 @@ def _load_json(self, source: Any) -> Any:
def _iter_resources(self, data: Any, warnings: list[str]):
"""Yield resource dicts from a Bundle, a single resource, or a list."""
if isinstance(data, dict) and data.get("resourceType") == "Bundle":
for entry in data.get("entry") or []:
for entry in _as_list(data.get("entry")):
resource = entry.get("resource") if isinstance(entry, dict) else None
if isinstance(resource, dict):
yield resource
Expand Down
4 changes: 4 additions & 0 deletions src/freshdata/parsers/gpx.py
Original file line number Diff line number Diff line change
Expand Up @@ -53,6 +53,10 @@ def _point(elem: ET.Element, kind: str, **extra: Any) -> None:
try:
row["lat"] = float(elem.get("lat")) # type: ignore[arg-type]
row["lon"] = float(elem.get("lon")) # type: ignore[arg-type]
# float() accepts "nan"/"inf"; NaN fails every comparison, so this range
# check also rejects non-finite coordinates.
if not (-90.0 <= row["lat"] <= 90.0 and -180.0 <= row["lon"] <= 180.0):
raise ValueError("coordinates not finite or out of range")
except (TypeError, ValueError):
warnings.append(f"{_local(elem.tag)} with missing/invalid lat/lon skipped")
return
Expand Down
Loading
Loading