Distinguish int/float in NumericInspector

sdgx/data_models/inspectors/numeric.py

@@ -11,7 +11,52 @@
 class NumericInspector(Inspector):
     def __init__(self, *args, **kwargs):
         super().__init__(*args, **kwargs)
-        self.numeric_columns: set[str] = set()
+        self.int_columns: set[str] = set()
+        self.float_columns: set[str] = set()
+        self._int_rate = 0.9
+
+    def is_int_column(self, col_series: pd.Series):
+        """
+        Determine whether a column of pd.DataFrame is of type int
+        In the original pd.DataFrame automatically updated dtype, some int types will be marked as float.
+        In fact, we can make an accurate result by getting the decimal part of the value.
+
+        Args:
+            col_series (pd.Series): One single column of the raw data.
+        """
+
+        def is_decimal_part_zero(num: float):
+            """
+            Is the decimal part == 0.0 ?
+
+            Args:
+                col_series (float): The number.
+            """
+            try:
+                decimal_part = num - int(num)
+            except ValueError:
+                return None
+            if decimal_part == 0.0:
+                return True
+            else:
+                return False
+
+        int_cnt = 0
+        col_length = self.df_length
+        for each_val in col_series:
+            decimal_zer0 = is_decimal_part_zero(each_val)
+            if decimal_zer0 is True:
+                int_cnt += 1
+                continue
+            if decimal_zer0 is None:
+                col_length -= 1
+                continue
+
+        int_rate = int_cnt / col_length
+        if int_rate > self._int_rate:
+            return True
+        else:
+            return False
 
     def fit(self, raw_data: pd.DataFrame, *args, **kwargs):
         """Fit the inspector.
@@ -22,15 +67,31 @@ def fit(self, raw_data: pd.DataFrame, *args, **kwargs):
             raw_data (pd.DataFrame): Raw data
         """
 
-        self.numeric_columns = self.numeric_columns.union(
-            set(raw_data.select_dtypes(include=["float64", "int64"]).columns)
+        self.df_length = len(raw_data)
+
+        float_candidate = self.float_columns.union(
+            set(raw_data.select_dtypes(include=["float64"]).columns)
         )
+
+        for candidate in float_candidate:
+            if self.is_int_column(raw_data[candidate]):
+                self.int_columns.add(candidate)
+            else:
+                self.float_columns.add(candidate)
+
+        self.int_columns = self.int_columns.union(
+            set(raw_data.select_dtypes(include=["int64"]).columns)
+        )
+
         self.ready = True
 
     def inspect(self, *args, **kwargs) -> dict[str, Any]:
         """Inspect raw data and generate metadata."""
 
-        return {"numeric_columns": list(self.numeric_columns)}
+        return {
+            "int_columns": list(self.int_columns),
+            "float_columns": list(self.float_columns),
+        }
 
 
 @hookimpl

sdgx/data_models/metadata.py

@@ -55,9 +55,10 @@ class Metadata(BaseModel):
     """
 
     # other columns lists are used to store column information
-    # here are 5 basic data types
+    # here are 6 basic data types
     id_columns: Set[str] = set()
-    numeric_columns: Set[str] = set()
+    int_columns: Set[str] = set()
+    float_columns: Set[str] = set()
     bool_columns: Set[str] = set()
     discrete_columns: Set[str] = set()
     datetime_columns: Set[str] = set()

tests/data_models/inspector/test_numeric.py

@@ -17,10 +17,11 @@ def raw_data(demo_single_table_path):
 def test_inspector(inspector: NumericInspector, raw_data):
     inspector.fit(raw_data)
     assert inspector.ready
-    assert inspector.numeric_columns
-    assert sorted(inspector.inspect()["numeric_columns"]) == sorted(
+    assert inspector.int_columns
+    assert sorted(inspector.inspect()["int_columns"]) == sorted(
         ["educational-num", "fnlwgt", "hours-per-week", "age", "capital-gain", "capital-loss"]
     )
+    assert not inspector.float_columns
     assert inspector.inspect_level == 10
 
 

tests/data_models/test_metadata.py

@@ -26,7 +26,8 @@ def test_metadata(metadata: Metadata):
     assert metadata.id_columns == metadata.get("id_columns")
     assert metadata.datetime_columns == metadata.get("datetime_columns")
     assert metadata.bool_columns == metadata.get("bool_columns")
-    assert metadata.numeric_columns == metadata.get("numeric_columns")
+    assert metadata.int_columns == metadata.get("int_columns")
+    assert metadata.float_columns == metadata.get("float_columns")
 
     metadata.set("a", "something")
     assert metadata.get("a") == set(["something"])
@@ -66,14 +67,13 @@ def test_demo_multi_table_data_metadata_parent(demo_multi_data_parent_matadata):
     assert demo_multi_data_parent_matadata.get_column_data_type("Store") == "id"
     assert demo_multi_data_parent_matadata.get_column_data_type("StoreType") == "discrete"
     assert demo_multi_data_parent_matadata.get_column_data_type("Assortment") == "discrete"
-    assert demo_multi_data_parent_matadata.get_column_data_type("CompetitionDistance") == "numeric"
+    assert demo_multi_data_parent_matadata.get_column_data_type("CompetitionDistance") == "int"
     assert (
-        demo_multi_data_parent_matadata.get_column_data_type("CompetitionOpenSinceMonth")
-        == "numeric"
+        demo_multi_data_parent_matadata.get_column_data_type("CompetitionOpenSinceMonth") == "int"
     )
-    assert demo_multi_data_parent_matadata.get_column_data_type("Promo2") == "numeric"
-    assert demo_multi_data_parent_matadata.get_column_data_type("Promo2SinceWeek") == "numeric"
-    assert demo_multi_data_parent_matadata.get_column_data_type("Promo2SinceYear") == "numeric"
+    assert demo_multi_data_parent_matadata.get_column_data_type("Promo2") == "int"
+    assert demo_multi_data_parent_matadata.get_column_data_type("Promo2SinceWeek") == "int"
+    assert demo_multi_data_parent_matadata.get_column_data_type("Promo2SinceYear") == "int"
     assert demo_multi_data_parent_matadata.get_column_data_type("PromoInterval") == "discrete"
     # check pii
     for each_col in demo_multi_data_parent_matadata.column_list:
@@ -87,15 +87,15 @@ def test_demo_multi_table_data_metadata_child(demo_multi_data_child_matadata):
     # self check
     demo_multi_data_child_matadata.check()
     # check each col's data type
-    assert demo_multi_data_child_matadata.get_column_data_type("Store") == "numeric"
+    assert demo_multi_data_child_matadata.get_column_data_type("Store") == "int"
     assert demo_multi_data_child_matadata.get_column_data_type("Date") == "datetime"
-    assert demo_multi_data_child_matadata.get_column_data_type("Customers") == "numeric"
-    assert demo_multi_data_child_matadata.get_column_data_type("StateHoliday") == "numeric"
-    assert demo_multi_data_child_matadata.get_column_data_type("Sales") == "numeric"
-    assert demo_multi_data_child_matadata.get_column_data_type("Promo") == "numeric"
-    assert demo_multi_data_child_matadata.get_column_data_type("DayOfWeek") == "numeric"
-    assert demo_multi_data_child_matadata.get_column_data_type("Open") == "numeric"
-    assert demo_multi_data_child_matadata.get_column_data_type("SchoolHoliday") == "numeric"
+    assert demo_multi_data_child_matadata.get_column_data_type("Customers") == "int"
+    assert demo_multi_data_child_matadata.get_column_data_type("StateHoliday") == "int"
+    assert demo_multi_data_child_matadata.get_column_data_type("Sales") == "int"
+    assert demo_multi_data_child_matadata.get_column_data_type("Promo") == "int"
+    assert demo_multi_data_child_matadata.get_column_data_type("DayOfWeek") == "int"
+    assert demo_multi_data_child_matadata.get_column_data_type("Open") == "int"
+    assert demo_multi_data_child_matadata.get_column_data_type("SchoolHoliday") == "int"
     # check pii
     for each_col in demo_multi_data_child_matadata.column_list:
         assert demo_multi_data_child_matadata.get_column_pii(each_col) is False