[0.2.0] Metadata Implementation

sdgx/data_models/inspectors/bool.py

@@ -0,0 +1,39 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+from pandas._libs.tslibs.parsing import DateParseError
+
+from sdgx.data_models.inspectors.base import Inspector
+from sdgx.data_models.inspectors.extension import hookimpl
+
+
+class BoolInspector(Inspector):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.bool_columns: set[str] = set()
+
+    def fit(self, raw_data: pd.DataFrame):
+        """Fit the inspector.
+
+        Gets the list of discrete columns from the raw data.
+
+        Args:
+            raw_data (pd.DataFrame): Raw data
+        """
+        self.bool_columns = self.bool_columns.union(
+            set(raw_data.infer_objects().select_dtypes(include=["bool"]).columns)
+        )
+
+        self.ready = True
+
+    def inspect(self) -> dict[str, Any]:
+        """Inspect raw data and generate metadata."""
+
+        return {"bool_columns": list(self.bool_columns)}
+
+
+@hookimpl
+def register(manager):
+    manager.register("BoolInspector", BoolInspector)

sdgx/data_models/inspectors/datetime.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+from pandas._libs.tslibs.parsing import DateParseError
+
+from sdgx.data_models.inspectors.base import Inspector
+from sdgx.data_models.inspectors.extension import hookimpl
+
+
+class DatetimeInspector(Inspector):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.datetime_columns: set[str] = set()
+
+    @classmethod
+    def can_convert_to_datetime(cls, input_col: pd.Series):
+        """Whether a df column can be converted to datetime.
+
+        Args:
+            input_col(pd.Series): A column of a dataframe.
+        """
+        try:
+            pd.to_datetime(input_col)
+            return True
+        except DateParseError:
+            return False
+
+    def fit(self, raw_data: pd.DataFrame):
+        """Fit the inspector.
+
+        Gets the list of discrete columns from the raw data.
+
+        Args:
+            raw_data (pd.DataFrame): Raw data
+        """
+        self.datetime_columns = self.datetime_columns.union(
+            set(raw_data.infer_objects().select_dtypes(include=["datetime64"]).columns)
+        )
+
+        # for some other case
+        # Some columns containing dates after infer are still marked as object
+        candidate_columns = set(raw_data.select_dtypes(include=["object"]).columns)
+        for col_name in candidate_columns:
+            each_col = raw_data[col_name]
+            if DatetimeInspector.can_convert_to_datetime(each_col):
+                self.datetime_columns.add(col_name)
+
+        self.ready = True
+
+    def inspect(self) -> dict[str, Any]:
+        """Inspect raw data and generate metadata."""
+
+        return {"datetime_columns": list(self.datetime_columns)}
+
+
+@hookimpl
+def register(manager):
+    manager.register("DatetimeInspector", DatetimeInspector)

sdgx/data_models/inspectors/id.py

@@ -0,0 +1,44 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+
+from sdgx.data_models.inspectors.base import Inspector
+from sdgx.data_models.inspectors.extension import hookimpl
+
+
+class IDInspector(Inspector):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.ID_columns: set[str] = set()
+
+    def fit(self, raw_data: pd.DataFrame):
+        """Fit the inspector.
+
+        Gets the list of discrete columns from the raw data.
+
+        Args:
+            raw_data (pd.DataFrame): Raw data
+        """
+
+        df_length = len(raw_data)
+        candidate_columns = set(raw_data.select_dtypes(include=["object", "int64"]).columns)
+
+        for each_col_name in candidate_columns:
+            target_col = raw_data[each_col_name]
+            col_set_length = len(set(target_col))
+            if col_set_length == df_length:
+                self.ID_columns.add(each_col_name)
+
+        self.ready = True
+
+    def inspect(self) -> dict[str, Any]:
+        """Inspect raw data and generate metadata."""
+
+        return {"id_columns": list(self.ID_columns)}
+
+
+@hookimpl
+def register(manager):
+    manager.register("IDInspector", IDInspector)

sdgx/data_models/inspectors/numeric.py

@@ -0,0 +1,38 @@
+from __future__ import annotations
+
+from typing import Any
+
+import pandas as pd
+
+from sdgx.data_models.inspectors.base import Inspector
+from sdgx.data_models.inspectors.extension import hookimpl
+
+
+class NumericInspector(Inspector):
+    def __init__(self, *args, **kwargs):
+        super().__init__(*args, **kwargs)
+        self.numeric_columns: set[str] = set()
+
+    def fit(self, raw_data: pd.DataFrame):
+        """Fit the inspector.
+
+        Gets the list of discrete columns from the raw data.
+
+        Args:
+            raw_data (pd.DataFrame): Raw data
+        """
+
+        self.numeric_columns = self.numeric_columns.union(
+            set(raw_data.select_dtypes(include=["float64", "int64"]).columns)
+        )
+        self.ready = True
+
+    def inspect(self) -> dict[str, Any]:
+        """Inspect raw data and generate metadata."""
+
+        return {"numeric_columns": list(self.numeric_columns)}
+
+
+@hookimpl
+def register(manager):
+    manager.register("NumericInspector", NumericInspector)

sdgx/data_models/metadata.py

@@ -13,22 +13,46 @@
 from sdgx.exceptions import MetadataInitError
 from sdgx.utils import logger
 
-# TODO: Design metadata for relationships...
-# class DType(Enum):
-#     datetime = "datetime"
-#     timestamp = "timestamp"
-#     numeric = "numeric"
-#     category = "category"
 
+class Metadata(BaseModel):
+    """Metadata
 
-# class Relationship:
-#     pass
+    This metadata is mainly used to describe the data types of all columns in a single data table.
 
+    For each column, there should be an instance of the Data Type object.
 
-class Metadata(BaseModel):
+    Args:
+        primary_key(str): The primary key, a field used to uniquely identify each row in the table.
+        The primary key of each row must be unique and not empty.
+
+        composite_primary_key(bool): Whether to enable the composite primary key feature.
+
+        primary_key_list(bool): List of composite primary keys.
+
+        column_list(list[str]): list of the comlumn name in the table, other columns lists are used to store column information.
+    """
+
+    # for primary key
+    # compatible with single primary key or composite primary key
+    primary_keys: List[str] = []
+
+    # variables related to columns
+    # column_list is used to store all columns' name
+    column_list: List[str] = []
+    # other columns lists are used to store column information
+    # here are 5 basic data types
+    id_columns: List[str] = []
+    numeric_columns: List[str] = []
+    bool_columns: List[str] = []
     discrete_columns: List[str] = []
+    datetime_columns: List[str] = []
+
+    # _column_dict = {}
     _extend: Dict[str, Any] = {}
 
+    # version info
+    metadata_version: str = "1.0"
+
     def get(self, key: str, default=None) -> Any:
         return getattr(self, key, getattr(self._extend, key, default))
 
@@ -52,10 +76,27 @@ def from_dataloader(
         cls,
         dataloader: DataLoader,
         max_chunk: int = 10,
+        primary_key: str = None,
         include_inspectors: list[str] | None = None,
         exclude_inspectors: list[str] | None = None,
         inspector_init_kwargs: dict[str, Any] | None = None,
     ) -> "Metadata":
+        """Initialize a metadata from DataLoader and Inspectors
+
+        Args:
+            dataloader(DataLoader): the input DataLoader.
+
+            max_chunk(int): max chunk count.
+
+            primary_key(list(str) | str): the primary key of this table.
+            Use the first column in table by default.
+
+            include_inspectors(list[str]): data type inspectors that should included in this metadata (table).
+
+            exclude_inspectors(list[str]): data type inspectors that should NOT included in this metadata (table).
+
+            inspector_init_kwargs(dict): inspector args.
+        """
         logger.info("Inspecting metadata...")
         inspectors = InspectorManager().init_inspcetors(
             include_inspectors, exclude_inspectors, **(inspector_init_kwargs or {})
@@ -66,7 +107,11 @@ def from_dataloader(
             if all(i.ready for i in inspectors) or i > max_chunk:
                 break
 
-        metadata = Metadata()
+        # If primary_key is not specified, use the first column.
+        if primary_key is None:
+            primary_key = dataloader.columns()[0]
+
+        metadata = Metadata(primary_key=primary_key, column_list=dataloader.columns())
         for inspector in inspectors:
             metadata.update(inspector.inspect())
 
@@ -86,7 +131,7 @@ def from_dataframe(
         for inspector in inspectors:
             inspector.fit(df)
 
-        metadata = Metadata()
+        metadata = Metadata(primary_key=df.columns[0], column_list=list(df.columns))
         for inspector in inspectors:
             metadata.update(inspector.inspect())
 
@@ -101,3 +146,42 @@ def load(cls, path: str | Path) -> "Metadata":
         path = Path(path).expanduser().resolve()
         attributes = json.load(path.open("r"))
         return Metadata().update(attributes)
+
+    def check(self):
+        """Checks column info.
+
+        When passing as input to the next module, perform necessary checks, including:
+            -Is the primary key correctly defined.
+            -Is there any missing definition of the column.
+            -Are there any  unknown columns that have been incorrectly updated.
+        """
+        # Not implemented yet
+
+        pass
+
+    def update_primary_key(self, primary_key: str | list[str], composite_primary_key: bool = False):
+        """Update the primary key of the table
+
+        When update the primary key, the original primary key will be erased.
+
+        Args:
+            primary_key(str | list[str]): the primary key or key list.
+
+            composite_primary_key(bool): whether this table use composite primary key.
+        """
+
+        if composite_primary_key is False and not isinstance(primary_key, str):
+            raise ValueError("Primary key should be a string")
+
+        if composite_primary_key is True and len(primary_key) == 0:
+            raise ValueError("Composite primary key list shoud NOT be empty.")
+
+        if composite_primary_key is True:
+            self._composite_primary_key = True
+            self.primary_key = None
+            self.primary_key_list = primary_key
+        else:
+            self._composite_primary_key = False
+            self.primary_key = primary_key
+
+        logger.info(f"Primary Key updated: {primary_key}.")

sdgx/data_models/multi_table_combiner.py

@@ -0,0 +1,54 @@
+from typing import Any, Dict, List
+
+from pydantic import BaseModel
+
+from sdgx.data_models.metadata import Metadata
+from sdgx.data_models.relationship import Relationship
+
+
+class MultiTableCombiner(BaseModel):
+    """MultiTableCombiner: combine different tables using relationship
+
+    Args:
+        metadata_dict (Dict[str, Any]):
+
+        relationships (List[Any]):
+    """
+
+    metadata_version: str = "1.0"
+
+    metadata_dict: Dict[str, Any] = {}
+    relationships: List[Any] = []
+
+    def check(self):
+        """Do necessary checks:
+
+        - Whether number of tables corresponds to relationships.
+        - Whether table names corresponds to the relationship between tables;
+        """
+
+        # count check
+        relationship_cnt = len(self.relationships)
+        metadata_cnt = len(self.metadata_dict.keys())
+        if metadata_cnt != relationship_cnt + 1:
+            raise ValueError("Number of tables should corresponds to relationships.")
+
+        # table name check
+        table_names_from_relationships = set()
+
+        # each relationship's table must have metadata
+        table_names = list(self.metadata_dict.keys())
+        for each_r in self.relationships:
+            if each_r.parent_table not in table_names:
+                raise ValueError(f"Metadata of parent table {each_r.parent_table} is missing.")
+            if each_r.child_table not in table_names:
+                raise ValueError(f"Metadata of child table {each_r.child_table} is missing.")
+            table_names_from_relationships.add(each_r.parent_table)
+            table_names_from_relationships.add(each_r.child_table)
+
+        # each table in metadata must in a relationship
+        for each_t in table_names:
+            if each_t not in table_names_from_relationships:
+                raise ValueError(f"Table {each_t} has not relationship.")
+
+        return True