FIX-#3823: Fix TypeError when creating Series from SparseArray (#5377)

Signed-off-by: Andrey Pavlenko <andrey.a.pavlenko@gmail.com>
modin-project · Dec 8, 2022 · 4f62964 · 4f62964
1 parent 36f2436
commit 4f62964
Show file tree

Hide file tree

Showing 2 changed files with 22 additions and 0 deletions.
diff --git a/...tal/core/execution/native/implementations/hdk_on_native/partitioning/partition_manager.py b/...tal/core/execution/native/implementations/hdk_on_native/partitioning/partition_manager.py
@@ -13,6 +13,7 @@
 
 """Module provides a partition manager class for ``HdkOnNativeDataframe`` frame."""
 
+from modin.error_message import ErrorMessage
 from modin.pandas.utils import is_scalar
 import numpy as np
 
@@ -184,13 +185,26 @@ def _get_unsupported_cols(cls, obj):
                 pyarrow.lib.ArrowTypeError,
                 pyarrow.lib.ArrowInvalid,
                 ValueError,
+                TypeError,
             ) as err:
+                # The TypeError could be raised when converting a sparse data to
+                # arrow table - https://github.com/apache/arrow/pull/4497. If this
+                # is the case - fall back to pandas, otherwise - rethrow the error.
+                if type(err) == TypeError:
+                    if any([isinstance(t, pandas.SparseDtype) for t in obj.dtypes]):
+                        ErrorMessage.single_warning(
+                            "Sparse data is not currently supported!"
+                        )
+                    else:
+                        raise err
+
                 # The ValueError is raised by pyarrow in case of duplicate columns.
                 # We catch and handle this error here. If there are no duplicates
                 # (is_unique is True), then the error is caused by something different
                 # and we just rethrow it.
                 if (type(err) == ValueError) and obj.columns.is_unique:
                     raise err
+
                 regex = r"Conversion failed for column ([^\W]*)"
                 unsupported_cols = []
                 for msg in err.args:

diff --git a/...n/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py b/...n/experimental/core/execution/native/implementations/hdk_on_native/test/test_dataframe.py
@@ -2428,5 +2428,13 @@ def test_dict(self):
         assert type(mdt._new(at, at.column(2)._name)) == pandas.CategoricalDtype
 
 
+class TestSparseArray:
+    def test_sparse_series(self):
+        data = pandas.arrays.SparseArray(np.array([3, 1, 2, 3, 4, np.nan]))
+        mds = pd.Series(data)
+        pds = pandas.Series(data)
+        df_equals(mds, pds)
+
+
 if __name__ == "__main__":
     pytest.main(["-v", __file__])