Skip to content
New issue

Have a question about this project? Sign up for a free GitHub account to open an issue and contact its maintainers and the community.

By clicking “Sign up for GitHub”, you agree to our terms of service and privacy statement. We’ll occasionally send you account related emails.

Already on GitHub? Sign in to your account

CLN: BlockManager.apply #29825

Merged
merged 13 commits into from
Dec 1, 2019
Merged
6 changes: 3 additions & 3 deletions pandas/core/generic.py
Original file line number Diff line number Diff line change
Expand Up @@ -5609,7 +5609,7 @@ def _to_dict_of_blocks(self, copy=True):
for k, v, in self._data.to_dict(copy=copy).items()
}

def astype(self, dtype, copy=True, errors="raise"):
def astype(self, dtype, copy: bool_t = True, errors: str = "raise"):
Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

did you mean to use bool_t here?

Copy link
Member Author

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

Yes. We can't use bool here because NDFrame defines bool, see note on L142

Copy link
Contributor

Choose a reason for hiding this comment

The reason will be displayed to describe this comment to others. Learn more.

ok, interesting

"""
Cast a pandas object to a specified dtype ``dtype``.

Expand Down Expand Up @@ -5735,10 +5735,10 @@ def astype(self, dtype, copy=True, errors="raise"):
elif is_extension_array_dtype(dtype) and self.ndim > 1:
# GH 18099/22869: columnwise conversion to extension dtype
# GH 24704: use iloc to handle duplicate column names
results = (
results = [
self.iloc[:, i].astype(dtype, copy=copy)
for i in range(len(self.columns))
)
]

else:
# else, only a single dtype is given
Expand Down
14 changes: 6 additions & 8 deletions pandas/core/internals/blocks.py
Original file line number Diff line number Diff line change
Expand Up @@ -523,16 +523,14 @@ def f(mask, val, idx):

return self.split_and_operate(None, f, False)

def astype(self, dtype, copy=False, errors="raise", **kwargs):
return self._astype(dtype, copy=copy, errors=errors, **kwargs)

def _astype(self, dtype, copy=False, errors="raise", **kwargs):
"""Coerce to the new type
def astype(self, dtype, copy: bool = False, errors: str = "raise"):
"""
Coerce to the new dtype.

Parameters
----------
dtype : str, dtype convertible
copy : boolean, default False
copy : bool, default False
copy if indicated
errors : str, {'raise', 'ignore'}, default 'ignore'
- ``raise`` : allow exceptions to be raised
Expand Down Expand Up @@ -2142,7 +2140,7 @@ def _maybe_coerce_values(self, values):
assert isinstance(values, np.ndarray), type(values)
return values

def _astype(self, dtype, **kwargs):
def astype(self, dtype, copy: bool = False, errors: str = "raise"):
"""
these automatically copy, so copy=True has no effect
raise on an except if raise == True
Expand All @@ -2158,7 +2156,7 @@ def _astype(self, dtype, **kwargs):
return self.make_block(values)

# delegate
return super()._astype(dtype=dtype, **kwargs)
return super().astype(dtype=dtype, copy=copy, errors=errors)

def _can_hold_element(self, element: Any) -> bool:
tipo = maybe_infer_dtype_type(element)
Expand Down
62 changes: 23 additions & 39 deletions pandas/core/internals/managers.py
Original file line number Diff line number Diff line change
Expand Up @@ -340,33 +340,20 @@ def _verify_integrity(self):
"tot_items: {1}".format(len(self.items), tot_items)
)

def apply(
self,
f,
axes=None,
filter=None,
do_integrity_check=False,
consolidate=True,
**kwargs,
):
def apply(self, f: str, filter=None, **kwargs):
"""
iterate over the blocks, collect and create a new block manager
Iterate over the blocks, collect and create a new BlockManager.

Parameters
----------
f : the callable or function name to operate on at the block level
axes : optional (if not supplied, use self.axes)
f : str
Name of the Block method to apply.
filter : list, if supplied, only call the block if the filter is in
the block
do_integrity_check : boolean, default False. Do the block manager
integrity check
consolidate: boolean, default True. Join together blocks having same
dtype

Returns
-------
Block Manager (new object)

BlockManager
"""

result_blocks = []
Expand All @@ -380,8 +367,7 @@ def apply(
else:
kwargs["filter"] = filter_locs

if consolidate:
self._consolidate_inplace()
self._consolidate_inplace()

if f == "where":
align_copy = True
Expand Down Expand Up @@ -429,11 +415,8 @@ def apply(
result_blocks = _extend_blocks(applied, result_blocks)

if len(result_blocks) == 0:
return self.make_empty(axes or self.axes)
bm = type(self)(
result_blocks, axes or self.axes, do_integrity_check=do_integrity_check
)
bm._consolidate_inplace()
return self.make_empty(self.axes)
bm = type(self)(result_blocks, self.axes, do_integrity_check=False)
return bm

def quantile(
Expand Down Expand Up @@ -540,8 +523,8 @@ def get_axe(block, qs, axes):
[make_block(values, ndim=1, placement=np.arange(len(values)))], axes[0]
)

def isna(self, func, **kwargs):
return self.apply("apply", func=func, **kwargs)
def isna(self, func):
return self.apply("apply", func=func)

def where(self, **kwargs):
return self.apply("where", **kwargs)
Expand All @@ -567,8 +550,8 @@ def fillna(self, **kwargs):
def downcast(self, **kwargs):
return self.apply("downcast", **kwargs)

def astype(self, dtype, **kwargs):
return self.apply("astype", dtype=dtype, **kwargs)
def astype(self, dtype, copy: bool = False, errors: str = "raise"):
return self.apply("astype", dtype=dtype, copy=copy, errors=errors)

def convert(self, **kwargs):
return self.apply("convert", **kwargs)
Expand Down Expand Up @@ -768,14 +751,19 @@ def copy(self, deep=True):
"""
# this preserves the notion of view copying of axes
if deep:
if deep == "all":
copy = lambda ax: ax.copy(deep=True)
else:
copy = lambda ax: ax.view()
new_axes = [copy(ax) for ax in self.axes]

def copy_func(ax):
if deep == "all":
return ax.copy(deep=True)
else:
return ax.view()

new_axes = [copy_func(ax) for ax in self.axes]
else:
new_axes = list(self.axes)
return self.apply("copy", axes=new_axes, deep=deep, do_integrity_check=False)
res = self.apply("copy", deep=deep)
res.axes = new_axes
return res

def as_array(self, transpose=False, items=None):
"""Convert the blockmanager data into an numpy array.
Expand Down Expand Up @@ -1527,10 +1515,6 @@ def get_slice(self, slobj, axis=0):
def index(self):
return self.axes[0]

def convert(self, **kwargs):
""" convert the whole block as one """
return self.apply("convert", **kwargs)

@property
def dtype(self):
return self._block.dtype
Expand Down