From 297423209cdc5b157f8e1ae2468ee547b51ae043 Mon Sep 17 00:00:00 2001 From: Kernc Date: Wed, 12 Jul 2017 15:14:09 +0200 Subject: [PATCH] fixup! BUG: Made SparseDataFrame.fillna() fill all NaNs --- doc/source/whatsnew/v0.21.0.txt | 1 + pandas/core/sparse/array.py | 5 ++--- pandas/tests/sparse/test_frame.py | 14 ++++++-------- 3 files changed, 9 insertions(+), 11 deletions(-) diff --git a/doc/source/whatsnew/v0.21.0.txt b/doc/source/whatsnew/v0.21.0.txt index 015fdf1f45f47..e628578de9894 100644 --- a/doc/source/whatsnew/v0.21.0.txt +++ b/doc/source/whatsnew/v0.21.0.txt @@ -178,6 +178,7 @@ Groupby/Resample/Rolling Sparse ^^^^^^ +- Bug in :func:`SparseDataFrame.fillna` not filling all NaNs when frame was instantiated from SciPy sparse matrix (:issue:`16112`) Reshaping diff --git a/pandas/core/sparse/array.py b/pandas/core/sparse/array.py index 9025f248e26b6..42fc5189eebd8 100644 --- a/pandas/core/sparse/array.py +++ b/pandas/core/sparse/array.py @@ -595,9 +595,8 @@ def fillna(self, value, downcast=None): if issubclass(self.dtype.type, np.floating): value = float(value) - new_values = self.sp_values.copy() - new_values[isnull(new_values)] = value - fill_value = value if isnull(self.fill_value) else self.fill_value + new_values = np.where(isnull(self.sp_values), value, self.sp_values) + fill_value = value if self._null_fill_value else self.fill_value return self._simple_new(new_values, self.sp_index, fill_value=fill_value) diff --git a/pandas/tests/sparse/test_frame.py b/pandas/tests/sparse/test_frame.py index 5d75616464fa1..619e646fbfff3 100644 --- a/pandas/tests/sparse/test_frame.py +++ b/pandas/tests/sparse/test_frame.py @@ -1252,7 +1252,6 @@ def test_from_scipy_correct_ordering(spmatrix): tm.skip_if_no_package('scipy') arr = np.arange(1, 5).reshape(2, 2) - try: spm = spmatrix(arr) assert spm.dtype == arr.dtype @@ -1268,9 +1267,9 @@ def test_from_scipy_correct_ordering(spmatrix): tm.assert_frame_equal(sdf.to_dense(), expected.to_dense()) -def test_from_scipy_object_fillna(spmatrix): +def test_from_scipy_fillna(spmatrix): # GH 16112 - tm.skip_if_no_package('scipy', max_version='0.19.0') + tm.skip_if_no_package('scipy') arr = np.eye(3) arr[1:, 0] = np.nan @@ -1287,12 +1286,11 @@ def test_from_scipy_object_fillna(spmatrix): sdf = pd.SparseDataFrame(spm).fillna(-1.0) # Returning frame should fill all nan values with -1.0 - expected = pd.SparseDataFrame({0: {0: 1.0, 1: np.nan, 2: np.nan}, - 1: {0: np.nan, 1: 1.0, 2: np.nan}, - 2: {0: np.nan, 1: np.nan, 2: 1.0}} - ).fillna(-1.0) + expected = pd.SparseDataFrame([[1, -1, -1], + [-1, 1, -1], + [-1, -1, 1.]]) - tm.assert_frame_equal(sdf.to_dense(), expected.to_dense()) + tm.assert_numpy_array_equal(sdf.values, expected.values) class TestSparseDataFrameArithmetic(object):