synthetic dataset generation by housetype - m2

[DesmondCampbell]

Hi Vlad,

Household type m2 fails to complete the first iteration
The output is

INFO:  2025-07-27 16:12:03.514682 sHhTypeCurr = m2 minb = 0
INFO:  2025-07-27 16:12:03.514751 sHhTypeCurr = m2 minb = 0 micb = 0
Sampling rows: 100%|██████████| 10000/10000 [00:34<00:00, 292.01it/s]
INFO:  2025-07-27 16:12:37.770126 pq.write_to_dataset a set of files called - part-m2-0-0-0.parquet

The error is

---------------------------------------------------------------------------
ArrowInvalid                              Traceback (most recent call last)
Cell In[9], line 31
     28      bSchemaGenerated = True
     30 print("INFO: ", datetime.datetime.now(), "pq.write_to_dataset a set of files called -", sFileOut)
---> 31 pq.write_to_dataset(pa.Table.from_pandas(df=new_data, schema=_schema, preserve_index=False),
     32                                      root_path=ROOT_DIR, # location to store data is defined elsewhere
     33                                      partition_cols=["category_household_location",
     34                                                      "ordinal_household_total_cars",
     35                                                      "category_household_house_ownership",
     36                                                      "indicator_household_has_central_heating"],
     37                                      basename_template=f"part-{sHhTypeCurr}-{minb}-{micb}-{{i}}.parquet",
     38                                      existing_data_behavior="overwrite_or_ignore")
     39 # record completion of this iteration in manifest file
     40 with open( sPathManifest, "a") as f:

File ~/miniconda3/envs/synthwave/lib/python3.12/site-packages/pyarrow/table.pxi:4751, in pyarrow.lib.Table.from_pandas()

File ~/miniconda3/envs/synthwave/lib/python3.12/site-packages/pyarrow/pandas_compat.py:639, in dataframe_to_arrays(df, schema, preserve_index, nthreads, columns, safe)
    634     return (isinstance(arr, np.ndarray) and
    635             arr.flags.contiguous and
    636             issubclass(arr.dtype.type, np.integer))
    638 if nthreads == 1:
--> 639     arrays = [convert_column(c, f)
    640               for c, f in zip(columns_to_convert, convert_fields)]
    641 else:
    642     arrays = []

File ~/miniconda3/envs/synthwave/lib/python3.12/site-packages/pyarrow/pandas_compat.py:626, in dataframe_to_arrays.<locals>.convert_column(col, field)
    621 except (pa.ArrowInvalid,
    622         pa.ArrowNotImplementedError,
    623         pa.ArrowTypeError) as e:
    624     e.args += ("Conversion failed for column {!s} with type {!s}"
    625                .format(col.name, col.dtype),)
--> 626     raise e
    627 if not field_nullable and result.null_count > 0:
    628     raise ValueError("Field {} was non-nullable but pandas column "
    629                      "had {} null values".format(str(field),
    630                                                  result.null_count))

File ~/miniconda3/envs/synthwave/lib/python3.12/site-packages/pyarrow/pandas_compat.py:620, in dataframe_to_arrays.<locals>.convert_column(col, field)
    617     type_ = field.type
    619 try:
--> 620     result = pa.array(col, type=type_, from_pandas=True, safe=safe)
    621 except (pa.ArrowInvalid,
    622         pa.ArrowNotImplementedError,
    623         pa.ArrowTypeError) as e:
    624     e.args += ("Conversion failed for column {!s} with type {!s}"
    625                .format(col.name, col.dtype),)

File ~/miniconda3/envs/synthwave/lib/python3.12/site-packages/pyarrow/array.pxi:362, in pyarrow.lib.array()

File ~/miniconda3/envs/synthwave/lib/python3.12/site-packages/pyarrow/array.pxi:87, in pyarrow.lib._ndarray_to_array()

File ~/miniconda3/envs/synthwave/lib/python3.12/site-packages/pyarrow/error.pxi:92, in pyarrow.lib.check_status()

ArrowInvalid: ('Integer value -70 not in range: 0 to 65535', 'Conversion failed for column income_person_self_employment_a1 with type int64')