diff --git a/pandas/core/arrays/base.py b/pandas/core/arrays/base.py index 7bf13fb2fecc0..f7c4ee35adfe4 100644 --- a/pandas/core/arrays/base.py +++ b/pandas/core/arrays/base.py @@ -739,14 +739,22 @@ def _create_method(cls, op, coerce_to_dtype=True): ---------- op : function An operator that takes arguments op(a, b) - coerce_to_dtype : bool + coerce_to_dtype : bool, default True boolean indicating whether to attempt to convert - the result to the underlying ExtensionArray dtype - (default True) + the result to the underlying ExtensionArray dtype. + If it's not possible to create a new ExtensionArray with the + values, an ndarray is returned instead. Returns ------- - A method that can be bound to a method of a class + Callable[[Any, Any], Union[ndarray, ExtensionArray]] + A method that can be bound to a class. When used, the method + receives the two arguments, one of which is the instance of + this class, and should return an ExtensionArray or an ndarray. + + Returning an ndarray may be necessary when the result of the + `op` cannot be stored in the ExtensionArray. The dtype of the + ndarray uses NumPy's normal inference rules. Example ------- @@ -757,7 +765,6 @@ def _create_method(cls, op, coerce_to_dtype=True): in the class definition of MyExtensionArray to create the operator for addition, that will be based on the operator implementation of the underlying elements of the ExtensionArray - """ def _binop(self, other): @@ -777,8 +784,13 @@ def convert_values(param): if coerce_to_dtype: try: res = self._from_sequence(res) - except TypeError: - pass + except Exception: + # https://github.com/pandas-dev/pandas/issues/22850 + # We catch all regular exceptions here, and fall back + # to an ndarray. + res = np.asarray(res) + else: + res = np.asarray(res) return res diff --git a/pandas/core/series.py b/pandas/core/series.py index 82198c2b3edd5..2e22e4e6e1bfc 100644 --- a/pandas/core/series.py +++ b/pandas/core/series.py @@ -2323,10 +2323,14 @@ def combine(self, other, func, fill_value=None): pass elif is_extension_array_dtype(self.values): # The function can return something of any type, so check - # if the type is compatible with the calling EA + # if the type is compatible with the calling EA. try: new_values = self._values._from_sequence(new_values) - except TypeError: + except Exception: + # https://github.com/pandas-dev/pandas/issues/22850 + # pandas has no control over what 3rd-party ExtensionArrays + # do in _values_from_sequence. We still want ops to work + # though, so we catch any regular Exception. pass return self._constructor(new_values, index=new_index, name=new_name) diff --git a/pandas/tests/extension/decimal/__init__.py b/pandas/tests/extension/decimal/__init__.py index e69de29bb2d1d..c37aad0af8407 100644 --- a/pandas/tests/extension/decimal/__init__.py +++ b/pandas/tests/extension/decimal/__init__.py @@ -0,0 +1,4 @@ +from .array import DecimalArray, DecimalDtype, to_decimal, make_data + + +__all__ = ['DecimalArray', 'DecimalDtype', 'to_decimal', 'make_data'] diff --git a/pandas/tests/extension/decimal/array.py b/pandas/tests/extension/decimal/array.py index 387942234e6fd..79e1a692f744a 100644 --- a/pandas/tests/extension/decimal/array.py +++ b/pandas/tests/extension/decimal/array.py @@ -1,5 +1,6 @@ import decimal import numbers +import random import sys import numpy as np @@ -138,5 +139,13 @@ def _concat_same_type(cls, to_concat): return cls(np.concatenate([x._data for x in to_concat])) +def to_decimal(values, context=None): + return DecimalArray([decimal.Decimal(x) for x in values], context=context) + + +def make_data(): + return [decimal.Decimal(random.random()) for _ in range(100)] + + DecimalArray._add_arithmetic_ops() DecimalArray._add_comparison_ops() diff --git a/pandas/tests/extension/decimal/test_decimal.py b/pandas/tests/extension/decimal/test_decimal.py index 93b8ea786ef5b..dd625d6e1eb3c 100644 --- a/pandas/tests/extension/decimal/test_decimal.py +++ b/pandas/tests/extension/decimal/test_decimal.py @@ -1,6 +1,6 @@ +import operator import decimal -import random import numpy as np import pandas as pd import pandas.util.testing as tm @@ -8,11 +8,7 @@ from pandas.tests.extension import base -from .array import DecimalDtype, DecimalArray - - -def make_data(): - return [decimal.Decimal(random.random()) for _ in range(100)] +from .array import DecimalDtype, DecimalArray, make_data @pytest.fixture @@ -275,3 +271,47 @@ def test_compare_array(self, data, all_compare_operators): other = pd.Series(data) * [decimal.Decimal(pow(2.0, i)) for i in alter] self._compare_other(s, data, op_name, other) + + +class DecimalArrayWithoutFromSequence(DecimalArray): + """Helper class for testing error handling in _from_sequence.""" + def _from_sequence(cls, scalars, dtype=None, copy=False): + raise KeyError("For the test") + + +class DecimalArrayWithoutCoercion(DecimalArrayWithoutFromSequence): + @classmethod + def _create_arithmetic_method(cls, op): + return cls._create_method(op, coerce_to_dtype=False) + + +DecimalArrayWithoutCoercion._add_arithmetic_ops() + + +def test_combine_from_sequence_raises(): + # https://github.com/pandas-dev/pandas/issues/22850 + ser = pd.Series(DecimalArrayWithoutFromSequence([ + decimal.Decimal("1.0"), + decimal.Decimal("2.0") + ])) + result = ser.combine(ser, operator.add) + + # note: object dtype + expected = pd.Series([decimal.Decimal("2.0"), + decimal.Decimal("4.0")], dtype="object") + tm.assert_series_equal(result, expected) + + +@pytest.mark.parametrize("class_", [DecimalArrayWithoutFromSequence, + DecimalArrayWithoutCoercion]) +def test_scalar_ops_from_sequence_raises(class_): + # op(EA, EA) should return an EA, or an ndarray if it's not possible + # to return an EA with the return values. + arr = class_([ + decimal.Decimal("1.0"), + decimal.Decimal("2.0") + ]) + result = arr + arr + expected = np.array([decimal.Decimal("2.0"), decimal.Decimal("4.0")], + dtype="object") + tm.assert_numpy_array_equal(result, expected) diff --git a/pandas/tests/extension/json/__init__.py b/pandas/tests/extension/json/__init__.py index e69de29bb2d1d..f2679d087c841 100644 --- a/pandas/tests/extension/json/__init__.py +++ b/pandas/tests/extension/json/__init__.py @@ -0,0 +1,3 @@ +from .array import JSONArray, JSONDtype, make_data + +__all__ = ['JSONArray', 'JSONDtype', 'make_data'] diff --git a/pandas/tests/extension/json/array.py b/pandas/tests/extension/json/array.py index 6ce0d63eb63ec..87876d84bef99 100644 --- a/pandas/tests/extension/json/array.py +++ b/pandas/tests/extension/json/array.py @@ -13,6 +13,8 @@ import collections import itertools import numbers +import random +import string import sys import numpy as np @@ -179,3 +181,10 @@ def _values_for_argsort(self): # cast them to an (N, P) array, instead of an (N,) array of tuples. frozen = [()] + [tuple(x.items()) for x in self] return np.array(frozen, dtype=object)[1:] + + +def make_data(): + # TODO: Use a regular dict. See _NDFrameIndexer._setitem_with_indexer + return [collections.UserDict([ + (random.choice(string.ascii_letters), random.randint(0, 100)) + for _ in range(random.randint(0, 10))]) for _ in range(100)] diff --git a/pandas/tests/extension/json/test_json.py b/pandas/tests/extension/json/test_json.py index 93f10b7fbfc23..bcbc3e9109182 100644 --- a/pandas/tests/extension/json/test_json.py +++ b/pandas/tests/extension/json/test_json.py @@ -1,7 +1,5 @@ import operator import collections -import random -import string import pytest @@ -10,18 +8,11 @@ from pandas.compat import PY2, PY36 from pandas.tests.extension import base -from .array import JSONArray, JSONDtype +from .array import JSONArray, JSONDtype, make_data pytestmark = pytest.mark.skipif(PY2, reason="Py2 doesn't have a UserDict") -def make_data(): - # TODO: Use a regular dict. See _NDFrameIndexer._setitem_with_indexer - return [collections.UserDict([ - (random.choice(string.ascii_letters), random.randint(0, 100)) - for _ in range(random.randint(0, 10))]) for _ in range(100)] - - @pytest.fixture def dtype(): return JSONDtype()