SDV 2.0: Update metadata auto-detection to the new V2 format

sdv/_utils.py

@@ -678,14 +678,19 @@ def _metadata_range_exceeds_real(data, metadata):
     return False
 
 
-def _validate_data_single_table(data):
+def _validate_data_single_table(data, table_name=None):
     """Validate that the data is a dictionary with a single table."""
     if len(data) != 1:
         raise InvalidDataTypeError(
             'The `data` parameter must be a dictionary containing exactly one table name '
             'mapped to a pandas DataFrame.'
         )
 
+    if table_name is not None and table_name not in data:
+        raise InvalidDataTypeError(
+            f"The specified table name '{table_name}' is not present in the data."
+        )
+
 
 def _get_single_table_data(data):
     """Return the single table DataFrame from the data dictionary."""

sdv/data_processing/data_processor.py

@@ -218,7 +218,7 @@ def create_anonymized_transformer(sdtype, column_metadata, cardinality_rule, loc
         """
         kwargs = {'locales': locales, 'cardinality_rule': cardinality_rule}
         for key, value in column_metadata.items():
-            if key not in ['pii', 'sdtype']:
+            if key not in ['pii', 'sdtype', 'range_is_nullable']:
                 kwargs[key] = value
 
         try:

sdv/metadata/_single_table.py

@@ -12,7 +12,7 @@
 import pandas as pd
 from rdt.transformers._validators import AddressValidator, GPSValidator
 from rdt.transformers.pii.anonymization import SDTYPE_ANONYMIZERS, is_faker_function
-from rdt.transformers.utils import learn_rounding_digits
+from rdt.transformers.utils import MAX_DECIMALS, learn_rounding_digits
 
 from sdv._utils import (
     _cast_to_datetime64,
@@ -51,6 +51,7 @@
     'is stored as an int but the Regex allows it to start with "0". Please remove the Regex '
     'or update it to correspond to valid ints.'
 )
+MAX_RANGE_VALUES = 500
 
 
 class _SingleTableMetadata:
@@ -573,6 +574,34 @@ def _detect_id_column(self, column_name):
 
         return None
 
+    def _detect_ordinal_sdtype(self, data):
+        """Detect whether a numerical column should have the ordinal sdtype.
+
+        A numerical column is considered ordinal if:
+        - It contains only whole numbers
+        - It has low cardinality, defined as having at most 10% unique values relative
+          to the total number of rows, capped at 10 unique values.
+
+        Args:
+            data (pandas.Series):
+                The data to be analyzed.
+        """
+        if len(data) <= self._MIN_ROWS_FOR_PREDICTION:
+            return None
+
+        clean_data = data.dropna()
+        if clean_data.empty:
+            return None
+
+        whole_values = (clean_data == clean_data.round()).all()
+        unique_values = clean_data.nunique()
+        ordinal_threshold = min(round(len(data) / 10), 10)
+        low_cardinality = unique_values <= ordinal_threshold
+        if whole_values and low_cardinality:
+            return 'ordinal'
+
+        return None
+
     def _determine_sdtype_for_numbers(self, data, valid_potential_primary_key):
         """Determine the sdtype for a numerical column.
 
@@ -582,22 +611,16 @@ def _determine_sdtype_for_numbers(self, data, valid_potential_primary_key):
             valid_potential_primary_key(bool):
                 If the column is unique and doesn't have NaNs.
         """
-        sdtype = 'numerical'
+        sdtype = self._detect_ordinal_sdtype(data) or 'numerical'
         pk_candidate = False
+
         if len(data) > self._MIN_ROWS_FOR_PREDICTION:
-            is_not_null = ~data.isna()
-            clean_data = (data == data.round()).loc[is_not_null]
+            clean_data = data.dropna()
             if clean_data.empty:
                 return sdtype, pk_candidate
 
-            whole_values = clean_data.all()
-            positive_values = (data >= 0).loc[is_not_null].all()
-
-            unique_values = data.nunique()
-            unique_lt_categorical_threshold = unique_values <= min(round(len(data) / 10), 10)
-
-            if whole_values and positive_values and unique_lt_categorical_threshold:
-                sdtype = 'categorical'
+            whole_values = (clean_data == clean_data.round()).all()
+            positive_values = (clean_data >= 0).all()
 
             pk_candidate = valid_potential_primary_key and whole_values and positive_values
 
@@ -733,15 +756,7 @@ def _select_primary_key(
                 A list of primary key candidates that are pii.
             table_name (str):
                 The name of the table to be analyzed. Defaults to ``None``.
-            verbose (bool):
-                A boolean that determines if information should be printed regarding detection.
-                If True, it prints out information about what is detected.
-                If False, it does not print out any information about what is detected.
-                Defaults to False.
         """
-        if verbose:
-            table_str = f" for table '{table_name}'" if table_name else ''
-            sys.stdout.write(f'\nDetecting primary key{table_str}:\n')
         chosen_pk = None
         sdtype_updated = False
         pii_removed = False
@@ -767,7 +782,86 @@ def _select_primary_key(
                 del self.columns[self.primary_key]['pii']
                 pii_removed = True
 
-        if verbose:
+        return chosen_pk, sdtype_updated, pii_removed
+
+    def _detect_range_values(self, data):
+        """Detect the range values for a column.
+
+        This method detects the unique values in a column if there are fewer than
+        `MAX_RANGE_VALUES` unique values.
+
+        Args:
+            data (pandas.Series):
+                The data to be analyzed.
+        """
+        range_values = data.dropna().unique()
+        if len(range_values) < MAX_RANGE_VALUES:
+            return range_values.tolist()
+
+        return None
+
+    def _detect_ranges(self, data):
+        """Detect the range information for all columns.
+
+        Args:
+            data (pandas.DataFrame):
+                The data to be analyzed.
+        """
+        for column_name, column_metadata in self.columns.items():
+            if column_name == self.primary_key:
+                continue
+
+            column_data = data[column_name]
+            sdtype = column_metadata['sdtype']
+            if sdtype == 'unknown':
+                continue
+
+            column_metadata['range_is_nullable'] = bool(column_data.isna().any())
+            clean_data = column_data.dropna()
+            if clean_data.empty:
+                continue
+
+            if sdtype == 'numerical':
+                ranges = clean_data.agg(['min', 'max']).to_dict()
+                column_metadata['range_min'] = ranges['min']
+                column_metadata['range_max'] = ranges['max']
+                digits = learn_rounding_digits(column_data)
+                column_metadata['decimal_places'] = digits if digits is not None else MAX_DECIMALS
+
+            elif sdtype == 'datetime':
+                datetime_format = column_metadata.get('datetime_format')
+                clean_data = pd.to_datetime(clean_data, format=datetime_format, errors='coerce')
+
+                range_min = clean_data.min()
+                range_max = clean_data.max()
+                if datetime_format:
+                    range_min = range_min.strftime(datetime_format)
+                    range_max = range_max.strftime(datetime_format)
+                else:
+                    range_min = str(range_min)
+                    range_max = str(range_max)
+
+                column_metadata['range_min'] = range_min
+                column_metadata['range_max'] = range_max
+
+            elif sdtype in {'categorical', 'ordinal'}:
+                range_values = self._detect_range_values(column_data)
+                if range_values is not None:
+                    column_metadata['range_values'] = range_values
+
+    def _print_detection(
+        self, table_name, data, infer_sdtypes, infer_keys, chosen_pk, sdtype_updated, pii_removed
+    ):
+        if infer_sdtypes:
+            table_str = f"table '{table_name}'" if table_name else 'table'
+            sys.stdout.write(f'\nDetecting {table_str}:\n')
+            for field in data:
+                column_metadata = _format_column_metadata(self.columns[field])
+                sys.stdout.write(f"- Column '{field}': {column_metadata}\n")
+
+        if infer_keys == 'primary_only':
+            table_str = f" for table '{table_name}'" if table_name else ''
+            sys.stdout.write(f'\nDetecting primary key{table_str}:\n')
             _print_primary_key_detection(chosen_pk, sdtype_updated, pii_removed)
 
     def _detect_columns(
@@ -796,10 +890,6 @@ def _detect_columns(
                 If False, it does not print out any information about what is detected.
                 Defaults to False.
         """
-        if verbose and infer_sdtypes:
-            table_str = f"table '{table_name}'" if table_name else 'table'
-            sys.stdout.write(f'\nDetecting {table_str}:\n')
-
         old_columns = data.columns
         data.columns = data.columns.astype(str)
         pk_candidates = []
@@ -823,24 +913,29 @@ def _detect_columns(
                 if sdtype == 'datetime' and dtype == 'O':
                     datetime_format = _get_datetime_format(column_data.iloc[:100])
                     column_dict['datetime_format'] = datetime_format
+
             else:
                 sdtype = 'unknown'
                 column_dict['pii'] = True
 
             column_dict['sdtype'] = sdtype
-
-            if verbose and infer_sdtypes:
-                column_metadata = _format_column_metadata(column_dict)
-                sys.stdout.write(f"- Column '{field}': {column_metadata}\n")
-
             self.columns[field] = deepcopy(column_dict)
+
+        chosen_pk = None
+        sdtype_updated = False
+        pii_removed = False
         if infer_keys == 'primary_only':
-            self._select_primary_key(
+            chosen_pk, sdtype_updated, pii_removed = self._select_primary_key(
                 infer_sdtypes=infer_sdtypes,
                 pk_candidates=pk_candidates,
                 pii_pk_candidates=pii_pk_candidates,
                 table_name=table_name,
-                verbose=verbose,
+            )
+
+        self._detect_ranges(data)
+        if verbose:
+            self._print_detection(
+                table_name, data, infer_sdtypes, infer_keys, chosen_pk, sdtype_updated, pii_removed
             )
 
         self._updated = True
@@ -1472,7 +1567,7 @@ def _validate_column_data(self, column, sdtype_warnings):
             if decimal_places is not None:
                 column_values = column.dropna()
                 data_digits = learn_rounding_digits(column_values)
-                if data_digits > decimal_places:
+                if data_digits is not None and data_digits > decimal_places:
                     errors += [
                         f"Values found for numerical column '{column.name}' exceed the allowed "
                         f'decimal places ({decimal_places}).'

sdv/metadata/metadata.py

@@ -728,10 +728,16 @@ def _detect_foreign_keys_by_column_name(self, data, verbose=False):
                     try:
                         sdtype_updated = False
                         if pk_sdtype == 'id' and original_fk_sdtype != 'id':
+                            update_kwargs = {'sdtype': 'id'}
+                            if 'range_is_nullable' in original_fk_meta:
+                                update_kwargs['range_is_nullable'] = original_fk_meta[
+                                    'range_is_nullable'
+                                ]
+
                             self.update_column(
                                 table_name=child_candidate,
                                 column_name=primary_key,
-                                sdtype='id',
+                                **update_kwargs,
                             )
                             sdtype_updated = True
                         self.add_relationship(
@@ -779,7 +785,7 @@ def _detect_relationships(
         if foreign_key_inference_algorithm == 'column_name_match':
             self._detect_foreign_keys_by_column_name(data, verbose)
 
-    def detect_table_from_dataframe(
+    def _detect_table_from_dataframe(
         self,
         table_name,
         data,
@@ -844,7 +850,7 @@ def _detect_from_dataframes(
 
         metadata = Metadata()
         for table_name, dataframe in data.items():
-            metadata.detect_table_from_dataframe(
+            metadata._detect_table_from_dataframe(
                 table_name,
                 dataframe,
                 infer_sdtypes,
@@ -931,7 +937,7 @@ def detect_from_csvs(self, folder_name, read_csv_parameters=None):
         for csv_file in csv_files:
             table_name = csv_file.stem
             data[table_name] = _load_data_from_csv(csv_file, read_csv_parameters)
-            self.detect_table_from_dataframe(table_name, data[table_name])
+            self._detect_table_from_dataframe(table_name, data[table_name])
 
         self._detect_relationships(data)
 
@@ -948,57 +954,9 @@ def _detect_from_dataframe(
 
         _validate_boolean_parameter(infer_sdtypes, 'infer_sdtypes')
         metadata = Metadata()
-        metadata.detect_table_from_dataframe(table_name, data, infer_sdtypes, infer_keys, verbose)
+        metadata._detect_table_from_dataframe(table_name, data, infer_sdtypes, infer_keys, verbose)
         return metadata
 
-    @classmethod
-    def detect_from_dataframe(
-        cls,
-        data,
-        table_name=DEFAULT_SINGLE_TABLE_NAME,
-        infer_sdtypes=True,
-        infer_keys='primary_only',
-        verbose=False,
-    ):
-        """Detect the metadata for a DataFrame.
-
-        This method automatically detects the ``sdtypes`` for the given ``pandas.DataFrame``.
-        All data column names are converted to strings.
-
-        Args:
-            data (pandas.DataFrame):
-                The data to detect metadata from.
-            table_name (str):
-                The name of the table to detect. If None, a default name will be used.
-                Defaults to None.
-            infer_sdtypes (bool):
-                A boolean describing whether to infer the sdtypes of each column.
-                If True it infers the sdtypes based on the data.
-                If False it does not infer the sdtypes and all columns are marked as unknown.
-                Defaults to True.
-            infer_keys (str):
-                A string describing whether to infer the primary keys. Options are:
-                    - 'primary_only': Infer only the primary keys of each table
-                    - None: Do not infer any keys
-                Defaults to 'primary_only'.
-            verbose (bool):
-                A boolean that determines if information should be printed regarding detection.
-                If True, it prints out information about what is detected.
-                If False, it does not print out any information about what is detected.
-                Defaults to False.
-
-        Returns:
-            Metadata:
-                A new metadata object with the sdtypes detected from the data.
-        """
-        return cls._detect_from_dataframe(
-            data=data,
-            table_name=table_name,
-            infer_sdtypes=infer_sdtypes,
-            infer_keys=infer_keys,
-            verbose=verbose,
-        )
-
     def set_primary_key(self, column_name, table_name=None):
         """Set the primary key of a table.