Add Regex Inspector and Email Inspector example.

LICENSE

@@ -1,4 +1,4 @@
-                                 Apache License
+                                 Apache License, Version 2.0
                            Version 2.0, January 2004
                         http://www.apache.org/licenses/
 
@@ -188,7 +188,7 @@
 
    Copyright 2023 hitsz-ids
 
-   Licensed under the Apache License, Version 2.0 (the "License");
+   Licensed under the Apache License, Version 2.0, Version 2.0 (the "License");
    you may not use this file except in compliance with the License.
    You may obtain a copy of the License at
 

docs/source/design/motivation.rst

@@ -58,11 +58,11 @@ What can we do?
 To address these challenges, we intend to design a new system that is efficient, scalable,
 and capable of simulating databases at the scale of tens of millions.
 This new system will be designed with a flexible architecture
-that can easily incorporate additional algorithms and support different types of data.
-Furthermore, it will be licensed under the `Apache 2.0 license <https://github.com/hitsz-ids/synthetic-data-generator/blob/main/LICENSE>`_,
-which allows for greater freedom in terms of modifications and derivative works.
+that can easily incorporate additional algorithms, support different types of data, and provide efficient scalability.
+Furthermore, it will be now licensed under the Apache License, Version 2.0
+which allows for greater freedom in terms of modifications and derivative works and is more permissive for open source contributions.
 
-By developing this new system, we aim to advance the research and development of synthetic data,
+In response to these challenges, we aim to design a new system, we aim to advance the research and development of synthetic data,
 providing a more robust and flexible tool for data scientists and researchers.
 This will not only enhance the quality and representativeness of synthetic data but also promote its use
 in a wider range of applications,

docs/source/developer_guides/extension/index.rst

@@ -10,7 +10,7 @@ which is based on the `entry-points of Python project <https://packaging.python.
 
 A plugin project is made up of three parts:
 
-- A class, inherits from the ``register_type`` of :ref:`Manager <manager>`, which contains your own logic.
+- A class that inherits from the ``register_type`` of :ref:`Manager <manager>`, containing your own logic.
 - A register function, which's name is defined(decorated) by ``@hookspec``.
   and you need to implement it and use ``@hookimp`` to declare it as a registed hook.
 - A ``entry-points`` in ``pyproject.toml``, which pointing to the hookimp function. The subdomain of the entry-point
@@ -26,6 +26,7 @@ Plugin-supported modules
 - :ref:`API Reference for extended Data Connector <api_reference/data-connectors-extension>`:
   :ref:`Data Connector <Data Connector>` is used to connect to data sources.
 - :ref:`API Reference for extended Cacher for DataLoader <api_reference/cachers-extension>`:
+  :ref:`Cacher <Cacher>` is used for improving performance, reducing network overhead, and supporting large datasets.:
   :ref:`Cacher <Cacher>` is used for improving performance,
   reducing network overhead and support large datasets.
 - :ref:`API Reference for extended Data Processor <api_reference/data-processors-extension>`:
@@ -34,7 +35,7 @@ Plugin-supported modules
 - :ref:`API Reference for extended Inspector for Metadata <api_reference/data-models-inspectors-extension>`:
   :ref:`Inspector <Inspector>` is used to extract metadata such as patterns, types, etc. from raw data.
 - :ref:`API Reference for extended Model <api_reference/models-extension>`:
-  :ref:`Model <SynthesizerModel>`, the model fitted by processed data and used to generate synthetic data.
+  :ref:`Model <SynthesizerModel>`: The model fitted by processed data and used to generate synthetic data., the model fitted by processed data and used to generate synthetic data.
 - :ref:`API Reference for extended Data Exporter <api_reference/data-exporters-extension>`:
   :ref:`Data Exporter <Data Exporter>` is used to export data to somewhere.
   Use it in CLI or library way to save your processed data or synthetic data.

sdgx/data_models/inspectors/personal.py

@@ -0,0 +1,28 @@
+from sdgx.data_models.inspectors.extension import hookimpl
+from sdgx.data_models.inspectors.regex import RegexInspector
+
+
+class EmailInspector(RegexInspector):
+    pattern = r"^[a-zA-Z0-9_.+-]+@[a-zA-Z0-9-]+\.[a-zA-Z0-9-.]+$"
+    data_type_name = "email"
+
+
+class ChinaMainlandIDInspector(RegexInspector):
+    pattern = r"(^[1-9]\\d{5}(18|19|20)\\d{2}((0[1-9])|(10|11|12))(([0-2][1-9])|10|20|30|31)\\d{3}[0-9Xx]$)|"
+
+    data_type_name = "china_mainland_id"
+
+
+class ChinaMainlandMobilePhoneInspector(RegexInspector):
+    pattern = r"^1[3-9]\d{9}$"
+
+    data_type_name = "china_mainland_mobile_phone"
+
+
+@hookimpl
+def register(manager):
+    manager.register("EmailInspector", EmailInspector)
+
+    manager.register("ChinaMainlandIDInspector", ChinaMainlandIDInspector)
+
+    manager.register("ChinaMainlandMobilePhoneInspector", ChinaMainlandMobilePhoneInspector)

sdgx/data_models/inspectors/regex.py

@@ -0,0 +1,113 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+import pandas as pd
+
+from sdgx.data_models.inspectors.base import Inspector
+from sdgx.exceptions import InspectorInitError
+
+# By default, we will not directly register the RegexInspector to the Inspector Manager
+# Instead, use it as a baseclass or user-defined regex, then put it into the Inspector Manager or use it alone
+
+
+class RegexInspector(Inspector):
+    """RegexInspector
+    RegexInspector is a sdgx inspector that uses regular expression rules to detect column data types. It can be initialized with a custom expression, or it can be inherited and applied to specific data types,such as email, US address, HKID etc.
+    """
+
+    pattern: str = None
+    """
+    pattern is the regular expression string of current inspector.
+    """
+
+    data_type_name: str = None
+    """
+    data_type_name is the name of the data type, such as email, US address, HKID etc.
+    """
+
+    _match_percentage: float = 0.8
+    """
+    match_percentage shoud > 0.5 and < 1.
+
+    Due to the existence of empty data, wrong data, etc., the match_percentage is the proportion of the current regular expression compound. When the number of compound regular expressions is higher than this ratio, the column can be considered fit the current data type.
+    """
+
+    @property
+    def match_percentage(self):
+        return self._match_percentage
+
+    @match_percentage.setter
+    def match_percentage(self, value):
+        if value > 0.5 and value <= 1:
+            self._match_percentage = value
+        else:
+            raise InspectorInitError("The match_percentage should be set in (0.5, 1].")
+
+    def __init__(
+        self,
+        pattern: str = None,
+        data_type_name: str = None,
+        match_percentage: float = None,
+        *args,
+        **kwargs,
+    ):
+        super().__init__(*args, **kwargs)
+        self.regex_columns: set[str] = set()
+
+        # this pattern should be a re pattern
+        if pattern:
+            self.pattern = pattern
+        # check pattern
+        if self.pattern is None:
+            raise InspectorInitError("Regular expression NOT found.")
+        self.p = re.compile(self.pattern)
+
+        # set data_type_name
+        if data_type_name:
+            if data_type_name.endswith("_columns"):
+                self.data_type_name = data_type_name[:-8]
+            else:
+                self.data_type_name = data_type_name
+        elif not self.data_type_name:
+            self.data_type_name = f"regex_{self.pattern}_columns"
+        # then chech the data type name
+        if self.data_type_name is None:
+            raise InspectorInitError("Inspector's data type undefined.")
+
+        # set percentage
+        if match_percentage:
+            self.match_percentage = match_percentage
+
+    def fit(self, raw_data: pd.DataFrame, *args, **kwargs):
+        """Fit the inspector.
+
+        Finds the list of regex columns from the raw data.
+
+        Args:
+            raw_data (pd.DataFrame): Raw data
+        """
+        for each_col in raw_data.columns:
+            each_match_rate = self._fit_column(raw_data[each_col])
+            if each_match_rate > self.match_percentage:
+                self.regex_columns.add(each_col)
+
+        self.ready = True
+
+    def _fit_column(self, column_data: pd.Series):
+        """
+        Regular expression matching for a single column, returning the matching ratio.
+        """
+        length = len(column_data)
+        match_cnt = 0
+        for i in column_data:
+            m = re.match(self.p, str(i))
+            if m:
+                match_cnt += 1
+        return match_cnt / length
+
+    def inspect(self, *args, **kwargs) -> dict[str, Any]:
+        """Inspect raw data and generate metadata."""
+
+        return {self.data_type_name + "_columns": list(self.regex_columns)}

sdgx/exceptions.py

@@ -137,3 +137,7 @@ class MetadataCombinerInvalidError(MetadataCombinerError):
 
 class MetadataCombinerInitError(MetadataCombinerError):
     ERROR_CODE = 9006
+
+
+class InspectorInitError(DataModelError):
+    ERROR_CODE = 9007