From f1a1a03ae0009f10b073c891507075c756186df9 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sat, 23 Nov 2019 14:58:18 -0800 Subject: [PATCH 1/8] CLN: astype --- pandas/core/generic.py | 2 +- pandas/core/internals/blocks.py | 14 ++++++-------- pandas/core/internals/managers.py | 8 ++++---- 3 files changed, 11 insertions(+), 13 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 2e2ae4e1dfa0a..8e3573d2500ec 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -5664,7 +5664,7 @@ def _to_dict_of_blocks(self, copy=True): for k, v, in self._data.to_dict(copy=copy).items() } - def astype(self, dtype, copy=True, errors="raise"): + def astype(self, dtype, copy: bool = True, errors: str = "raise"): """ Cast a pandas object to a specified dtype ``dtype``. diff --git a/pandas/core/internals/blocks.py b/pandas/core/internals/blocks.py index 2d6ffb7277742..243786d3ba759 100644 --- a/pandas/core/internals/blocks.py +++ b/pandas/core/internals/blocks.py @@ -523,16 +523,14 @@ def f(mask, val, idx): return self.split_and_operate(None, f, False) - def astype(self, dtype, copy=False, errors="raise", **kwargs): - return self._astype(dtype, copy=copy, errors=errors, **kwargs) - - def _astype(self, dtype, copy=False, errors="raise", **kwargs): - """Coerce to the new type + def astype(self, dtype, copy: bool = False, errors: str = "raise"): + """ + Coerce to the new dtype. Parameters ---------- dtype : str, dtype convertible - copy : boolean, default False + copy : bool, default False copy if indicated errors : str, {'raise', 'ignore'}, default 'ignore' - ``raise`` : allow exceptions to be raised @@ -2142,7 +2140,7 @@ def _maybe_coerce_values(self, values): assert isinstance(values, np.ndarray), type(values) return values - def _astype(self, dtype, **kwargs): + def astype(self, dtype, copy: bool = False, errors: str = "raise"): """ these automatically copy, so copy=True has no effect raise on an except if raise == True @@ -2158,7 +2156,7 @@ def _astype(self, dtype, **kwargs): return self.make_block(values) # delegate - return super()._astype(dtype=dtype, **kwargs) + return super().astype(dtype=dtype, copy=copy, errors=errors) def _can_hold_element(self, element: Any) -> bool: tipo = maybe_infer_dtype_type(element) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 5e60440f1577e..d8c21086c0367 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -543,8 +543,8 @@ def get_axe(block, qs, axes): [make_block(values, ndim=1, placement=np.arange(len(values)))], axes[0] ) - def isna(self, func, **kwargs): - return self.apply("apply", func=func, **kwargs) + def isna(self, func): + return self.apply("apply", func=func) def where(self, **kwargs): return self.apply("where", **kwargs) @@ -570,8 +570,8 @@ def fillna(self, **kwargs): def downcast(self, **kwargs): return self.apply("downcast", **kwargs) - def astype(self, dtype, **kwargs): - return self.apply("astype", dtype=dtype, **kwargs) + def astype(self, dtype, copy: bool = False, errors: str = "raise"): + return self.apply("astype", dtype=dtype, copy=copy, errors=errors) def convert(self, **kwargs): return self.apply("convert", **kwargs) From c164a515b0b2d3235aff0fa2b0264e9284131f96 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sat, 23 Nov 2019 15:44:02 -0800 Subject: [PATCH 2/8] CLN: remove redundant convert method --- pandas/core/internals/managers.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index d8c21086c0367..b843c01c808e4 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -1532,10 +1532,6 @@ def get_slice(self, slobj, axis=0): def index(self): return self.axes[0] - def convert(self, **kwargs): - """ convert the whole block as one """ - return self.apply("convert", **kwargs) - @property def dtype(self): return self._block.dtype From 6aef8ed75bc30d57f4652a843ec3b3746e3a0502 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sun, 24 Nov 2019 10:48:59 -0800 Subject: [PATCH 3/8] CLN: BlockManager.apply --- pandas/core/internals/managers.py | 16 ++++++---------- 1 file changed, 6 insertions(+), 10 deletions(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index b843c01c808e4..91133be397a17 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -348,12 +348,11 @@ def apply( f, axes=None, filter=None, - do_integrity_check=False, consolidate=True, **kwargs, ): """ - iterate over the blocks, collect and create a new block manager + Iterate over the blocks, collect and create a new BlockManager. Parameters ---------- @@ -361,15 +360,12 @@ def apply( axes : optional (if not supplied, use self.axes) filter : list, if supplied, only call the block if the filter is in the block - do_integrity_check : boolean, default False. Do the block manager - integrity check - consolidate: boolean, default True. Join together blocks having same - dtype + consolidate: bool, default True. + Join together blocks having same dtype. Returns ------- - Block Manager (new object) - + BlockManager """ result_blocks = [] @@ -434,7 +430,7 @@ def apply( if len(result_blocks) == 0: return self.make_empty(axes or self.axes) bm = self.__class__( - result_blocks, axes or self.axes, do_integrity_check=do_integrity_check + result_blocks, axes or self.axes, do_integrity_check=False ) bm._consolidate_inplace() return bm @@ -778,7 +774,7 @@ def copy(self, deep=True): new_axes = [copy(ax) for ax in self.axes] else: new_axes = list(self.axes) - return self.apply("copy", axes=new_axes, deep=deep, do_integrity_check=False) + return self.apply("copy", axes=new_axes, deep=deep) def as_array(self, transpose=False, items=None): """Convert the blockmanager data into an numpy array. From 05ace2e6d4aff345ebef191aa99774c2792a617d Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sun, 24 Nov 2019 10:50:11 -0800 Subject: [PATCH 4/8] CLN: appyl signature --- pandas/core/internals/managers.py | 11 ++++------- 1 file changed, 4 insertions(+), 7 deletions(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 91133be397a17..848d34b6e575c 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -345,10 +345,9 @@ def _verify_integrity(self): def apply( self, - f, + f: str, axes=None, filter=None, - consolidate=True, **kwargs, ): """ @@ -356,12 +355,11 @@ def apply( Parameters ---------- - f : the callable or function name to operate on at the block level + f : str + Name of the Block method to apply. axes : optional (if not supplied, use self.axes) filter : list, if supplied, only call the block if the filter is in the block - consolidate: bool, default True. - Join together blocks having same dtype. Returns ------- @@ -379,8 +377,7 @@ def apply( else: kwargs["filter"] = filter_locs - if consolidate: - self._consolidate_inplace() + self._consolidate_inplace() if f == "where": align_copy = True From 9a6401b561931ec4c4d90244dc2a7cacf9174cc5 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sun, 24 Nov 2019 10:52:34 -0800 Subject: [PATCH 5/8] CLN: BlockManager.copy --- pandas/core/internals/managers.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 848d34b6e575c..1d340e5971297 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -346,7 +346,6 @@ def _verify_integrity(self): def apply( self, f: str, - axes=None, filter=None, **kwargs, ): @@ -357,7 +356,6 @@ def apply( ---------- f : str Name of the Block method to apply. - axes : optional (if not supplied, use self.axes) filter : list, if supplied, only call the block if the filter is in the block @@ -425,9 +423,9 @@ def apply( result_blocks = _extend_blocks(applied, result_blocks) if len(result_blocks) == 0: - return self.make_empty(axes or self.axes) + return self.make_empty(self.axes) bm = self.__class__( - result_blocks, axes or self.axes, do_integrity_check=False + result_blocks, self.axes, do_integrity_check=False ) bm._consolidate_inplace() return bm @@ -765,13 +763,15 @@ def copy(self, deep=True): # this preserves the notion of view copying of axes if deep: if deep == "all": - copy = lambda ax: ax.copy(deep=True) + copy_func = lambda ax: ax.copy(deep=True) else: - copy = lambda ax: ax.view() - new_axes = [copy(ax) for ax in self.axes] + copy_func = lambda ax: ax.view() + new_axes = [copy_func(ax) for ax in self.axes] else: new_axes = list(self.axes) - return self.apply("copy", axes=new_axes, deep=deep) + res = self.apply("copy", deep=deep) + res.axes = new_axes + return res def as_array(self, transpose=False, items=None): """Convert the blockmanager data into an numpy array. From 22e28080ccaf821e66fbcb6485cb8b590a90b343 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sun, 24 Nov 2019 13:01:49 -0800 Subject: [PATCH 6/8] blackify --- pandas/core/internals/managers.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index 1d340e5971297..d89de362bbc6a 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -343,12 +343,7 @@ def _verify_integrity(self): "tot_items: {1}".format(len(self.items), tot_items) ) - def apply( - self, - f: str, - filter=None, - **kwargs, - ): + def apply(self, f: str, filter=None, **kwargs): """ Iterate over the blocks, collect and create a new BlockManager. @@ -424,9 +419,7 @@ def apply( if len(result_blocks) == 0: return self.make_empty(self.axes) - bm = self.__class__( - result_blocks, self.axes, do_integrity_check=False - ) + bm = self.__class__(result_blocks, self.axes, do_integrity_check=False) bm._consolidate_inplace() return bm From 6f5591ccb9fdbaa0eb0e5f36089ce2a0b5f2d54d Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Sun, 24 Nov 2019 13:31:13 -0800 Subject: [PATCH 7/8] mypy fixups --- pandas/core/generic.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/pandas/core/generic.py b/pandas/core/generic.py index 8e3573d2500ec..22bc856069b2c 100644 --- a/pandas/core/generic.py +++ b/pandas/core/generic.py @@ -5664,7 +5664,7 @@ def _to_dict_of_blocks(self, copy=True): for k, v, in self._data.to_dict(copy=copy).items() } - def astype(self, dtype, copy: bool = True, errors: str = "raise"): + def astype(self, dtype, copy: bool_t = True, errors: str = "raise"): """ Cast a pandas object to a specified dtype ``dtype``. @@ -5790,10 +5790,10 @@ def astype(self, dtype, copy: bool = True, errors: str = "raise"): elif is_extension_array_dtype(dtype) and self.ndim > 1: # GH 18099/22869: columnwise conversion to extension dtype # GH 24704: use iloc to handle duplicate column names - results = ( + results = [ self.iloc[:, i].astype(dtype, copy=copy) for i in range(len(self.columns)) - ) + ] else: # else, only a single dtype is given From cb95d640f9c1f28df7e2e5195f25bee1fc6893d6 Mon Sep 17 00:00:00 2001 From: jbrockmendel Date: Mon, 25 Nov 2019 15:57:42 -0800 Subject: [PATCH 8/8] make copy_func non-lambda --- pandas/core/internals/managers.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/pandas/core/internals/managers.py b/pandas/core/internals/managers.py index efff67815b523..bb7c02c6690fb 100644 --- a/pandas/core/internals/managers.py +++ b/pandas/core/internals/managers.py @@ -757,10 +757,13 @@ def copy(self, deep=True): """ # this preserves the notion of view copying of axes if deep: - if deep == "all": - copy_func = lambda ax: ax.copy(deep=True) - else: - copy_func = lambda ax: ax.view() + + def copy_func(ax): + if deep == "all": + return ax.copy(deep=True) + else: + return ax.view() + new_axes = [copy_func(ax) for ax in self.axes] else: new_axes = list(self.axes)