0.1.0: Intro DataConnector and DataLoader (#64)

This PR contains the following main elements:
- `Data Connector` and `DataLoader` implemented
- `Cacher` for `DataLoader` and `CacherManager` for extensions
- Some chore: docs build test, example...

.github/workflows/docs.yml

@@ -0,0 +1,31 @@
+name: Testing docs build
+
+on:
+  push:
+    branches: ["master", "main"]
+  pull_request:
+    branches: ["master", "main"]
+
+jobs:
+  build:
+    runs-on: ubuntu-latest
+    strategy:
+      fail-fast: false
+      matrix:
+        python-version: ["3.8", "3.11"]
+
+    steps:
+      - uses: actions/checkout@v3
+      - name: Set up Python ${{ matrix.python-version }}
+        uses: actions/setup-python@v5
+        with:
+          python-version: ${{ matrix.python-version }}
+          cache: "pip"
+      - name: Install dependencies
+        run: |
+          python -m pip install --upgrade pip
+          python -m pip install -e .[docs]
+      - name: Test build docs
+        run: |
+          cd docs
+          make clean && make html

.github/workflows/extension.yml

@@ -27,6 +27,7 @@ jobs:
           python -m pip install -e .[test]
       - name: Install all packages in example/extension
         run: |
+          python -m pip install -e example/extension/dummycache[test]
           python -m pip install -e example/extension/dummydataconnector[test]
           python -m pip install -e example/extension/dummydataprocessor[test]
           python -m pip install -e example/extension/dummymodel[test]

.pre-commit-config.yaml

@@ -52,5 +52,5 @@ repos:
     rev: "v1.7.1"
     hooks:
       - id: mypy
-        files: duetector
+        files: sdgx
         stages: [manual]

docs/source/api_reference/cachers/base.rst

@@ -0,0 +1,21 @@
+Cacher
+=======================
+
+.. autoclass:: sdgx.cachers.base.Cacher
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :show-inheritance:
+   :private-members:
+
+
+
+NoCache
+=======================
+
+.. autoclass:: sdgx.cachers.base.NoCache
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :show-inheritance:
+   :private-members:

docs/source/api_reference/cachers/disk_cache.rst

@@ -0,0 +1,9 @@
+DiskCache
+=======================
+
+.. autoclass:: sdgx.cachers.disk_cache.DiskCache
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :show-inheritance:
+   :private-members:

docs/source/api_reference/cachers/extension.rst

@@ -0,0 +1,11 @@
+.. _api_reference/cacher-extension:
+
+Extension hookspec
+============================
+
+.. automodule:: sdgx.cachers.extension
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :show-inheritance:
+   :private-members:

docs/source/api_reference/cachers/index.rst

@@ -0,0 +1,21 @@
+Cacher
+========================================================
+
+Built-in Cacher
+-----------------------------
+
+.. toctree::
+    :maxdepth: 2
+
+    base <base>
+    MemoryCache <memory_cache>
+    DiskCache <disk_cache>
+
+Custom Cacher Relevant
+-----------------------------
+
+.. toctree::
+    :maxdepth: 2
+
+    Extension hookspec <extension>
+    CacherManager <manager>

docs/source/api_reference/cachers/manager.rst

@@ -0,0 +1,9 @@
+CacherManager
+=================================
+
+.. autoclass:: sdgx.cachers.manager.CacherManager
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :show-inheritance:
+   :private-members:

docs/source/api_reference/cachers/memory_cache.rst

@@ -0,0 +1,9 @@
+MemoryCache
+==========================
+
+.. autoclass:: sdgx.cachers.memory_cache.MemoryCache
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :show-inheritance:
+   :private-members:

docs/source/api_reference/data_connectors/base.rst

@@ -0,0 +1,9 @@
+DataConnector
+=======================
+
+.. autoclass:: sdgx.data_connectors.base.DataConnector
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :show-inheritance:
+   :private-members:

docs/source/api_reference/data_connectors/csv_connector.rst

@@ -0,0 +1,10 @@
+CsvConnector
+=====================================
+
+
+.. autoclass:: sdgx.data_connectors.csv_connector.CsvConnector
+    :members:
+    :undoc-members:
+    :inherited-members:
+    :show-inheritance:
+    :private-members:

docs/source/api_reference/data_connectors/extension.rst

@@ -0,0 +1,11 @@
+.. _api_reference/data-connector-extension:
+
+Extension hookspec
+============================
+
+.. automodule:: sdgx.data_connectors.extension
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :show-inheritance:
+   :private-members:

docs/source/api_reference/data_connectors/index.rst

@@ -0,0 +1,20 @@
+DataConnector
+========================================================
+
+Built-in DataConnector
+-----------------------------
+
+.. toctree::
+    :maxdepth: 2
+
+    DataConnector <base>
+    CsvConnector <csv_connector>
+
+Custom DataConnector Relevant
+-----------------------------
+
+.. toctree::
+    :maxdepth: 2
+
+    Extension hookspec <extension>
+    DataConnectorManager <manager>

docs/source/api_reference/data_connectors/manager.rst

@@ -0,0 +1,9 @@
+DataConnectorManager
+=================================
+
+.. autoclass:: sdgx.data_connectors.manager.DataConnectorManager
+   :members:
+   :undoc-members:
+   :inherited-members:
+   :show-inheritance:
+   :private-members:

docs/source/api_reference/data_loader.rst

@@ -0,0 +1,8 @@
+Data Loader
+============================
+
+.. automodule:: sdgx.data_loader
+   :members:
+   :undoc-members:
+   :private-members:
+   :show-inheritance:

docs/source/api_reference/exceptions.rst

@@ -0,0 +1,7 @@
+Exceptions
+============================
+
+.. automodule:: sdgx.exceptions
+    :members:
+    :undoc-members:
+    :inherited-members:

docs/source/api_reference/index.rst

@@ -5,3 +5,10 @@ API Reference
 
 .. toctree::
     :maxdepth: 2
+
+    Synthesizer <synthesizer>
+    Data Loader <data_loader>
+    Cacher for DataLoader <cachers/index>
+    Data Connector <data_connectors/index>
+    Manager <manager>
+    Exceptions <exceptions>

docs/source/api_reference/manager.rst

@@ -0,0 +1,11 @@
+.. _manager:
+
+Manager
+=======================
+
+
+.. autoclass:: sdgx.manager.Manager
+   :members:
+   :undoc-members:
+   :private-members:
+   :show-inheritance:

docs/source/api_reference/synthesizer.rst

@@ -0,0 +1,8 @@
+Synthesizer
+==================================
+
+.. autoclass:: sdgx.synthesizer.Synthesizer
+   :members:
+   :undoc-members:
+   :private-members:
+   :show-inheritance:

docs/source/conf.py

@@ -35,6 +35,7 @@
     "sphinx.ext.autodoc",
     "sphinx.ext.napoleon",
     "sphinx.ext.viewcode",
+    "sphinx.ext.todo",
     "sphinx.ext.autosectionlabel",
     "sphinx_click",
     "sphinxcontrib.autodoc_pydantic",
@@ -66,3 +67,5 @@
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
 html_static_path = ["_static"]
+
+todo_include_todos = True

docs/source/developer_guides/extension/index.rst

@@ -0,0 +1,29 @@
+Extented Synthetic Data Generator
+=====================================
+
+.. NOTE::
+
+    Understand the purpose of each component from the :ref:`architecture`.
+
+SDG uses `pluggy <https://github.com/pytest-dev/pluggy>`_ to develop plug-in systems,
+which is based on the `entry-points of Python project <https://packaging.python.org/en/latest/specifications/entry-points/#entry-points>`_.
+
+A plugin project is made up of three parts:
+
+- A class, inherits from the ``register_type`` of :ref:`Manager <manager>`, which contains your own logic.
+- A register function, which's name is defined(decorated) by ``@hookspec``.
+  and you need to implement it and use ``@hookimp`` to declare it as a registed hook.
+- A ``entry-points`` in ``pyproject.toml``, which pointing to the hookimp function. The subdomain of the entry-point
+  is the ``PROJECT_NAME`` you can find in :ref:`manager`.
+
+
+View latest extension example on `GitHub <https://github.com/hitsz-ids/synthetic-data-generator/tree/main/example/extension>`_.
+
+
+Plugin-supported modules
+------------------------
+
+- :ref:`Cacher for DataLoader <api_reference/cacher-extension>`
+- :ref:`Data Connector <_api_reference/data-connector-extension>`
+- :ref:`Data Processor <_api_reference/data-processor-extension>`
+- :ref:`Model <_api_reference/model-extension>`

docs/source/developer_guides/index.rst

@@ -8,6 +8,6 @@ Before you start, maybe you are interested in :ref:`architecture` and :ref:`Our
 .. toctree::
     :maxdepth: 2
 
-    Extented Synthetic Data Generator <extension>
+    Extented Synthetic Data Generator <extension/index>
     Contributing to our project directly <contributing>
     Data Models Specification <data_models>

example/extension/dummycache/dummycache/__init__.py

@@ -0,0 +1 @@
+__version__ = "0.1.0"

example/extension/dummycache/dummycache/dummycache.py

@@ -0,0 +1,15 @@
+from __future__ import annotations
+
+from sdgx.cachers.base import Cacher
+
+
+class MyOwnCache(Cacher):
+    ...
+
+
+from sdgx.cachers.extension import hookimpl
+
+
+@hookimpl
+def register(manager):
+    manager.register("DummyCache", MyOwnCache)

example/extension/dummycache/pyproject.toml

@@ -0,0 +1,27 @@
+[build-system]
+requires = ["hatchling"]
+build-backend = "hatchling.build"
+
+[project]
+name = "dummycache"
+dependencies = ["sdgx"]
+dynamic = ["version"]
+requires-python = ">=3.8"
+classifiers = [
+    "Programming Language :: Python :: 3",
+    'Programming Language :: Python :: 3.8',
+    'Programming Language :: Python :: 3.9',
+    'Programming Language :: Python :: 3.10',
+    'Programming Language :: Python :: 3.11',
+]
+[project.optional-dependencies]
+test = ["pytest"]
+
+[tool.check-manifest]
+ignore = [".*"]
+
+[tool.hatch.version]
+path = "dummycache/__init__.py"
+
+[project.entry-points."sdgx.cacher"]
+dummycache = "dummycache.dummycache"

example/extension/dummycache/tests/test_registed_cacher.py

@@ -0,0 +1,16 @@
+import pytest
+
+from sdgx.cachers.manager import CacherManager
+
+
+@pytest.fixture
+def manager():
+    yield CacherManager()
+
+
+def test_registed_cacher(manager: CacherManager):
+    assert manager._normalize_name("DummyCache") in manager.registed_cachers
+
+
+if __name__ == "__main__":
+    pytest.main(["-vv", "-s", __file__])

example/extension/dummydataconnector/tests/test_registed_data_connector.py

@@ -10,7 +10,6 @@ def manager():
 
 def test_registed_data_connector(manager: DataConnectorManager):
     assert manager._normalize_name("DummyDataConnector") in manager.registed_data_connectors
-    manager.init_data_connector("DummyDataConnector")
 
 
 if __name__ == "__main__":

example/extension/dummydataprocessor/tests/test_registed_data_processor.py

@@ -10,7 +10,6 @@ def manager():
 
 def test_registed_data_processor(manager: DataProcessorManager):
     assert manager._normalize_name("DummyDataProcessor") in manager.registed_data_processors
-    manager.init_data_processor("DummyDataProcessor")
 
 
 if __name__ == "__main__":

example/extension/dummymodel/tests/test_registed_model.py

@@ -10,7 +10,6 @@ def manager():
 
 def test_registed_model(manager: ModelManager):
     assert manager._normalize_name("DummyModel") in manager.registed_models
-    manager.init_model("DummyModel")
 
 
 if __name__ == "__main__":