BUG: Bug in loc did not change dtype when complete column was assigned

doc/source/whatsnew/v1.3.0.rst

@@ -339,6 +339,7 @@ Indexing
 - Bug in :meth:`DataFrame.__setitem__` raising ``ValueError`` with empty :class:`DataFrame` and specified columns for string indexer and non empty :class:`DataFrame` to set (:issue:`38831`)
 - Bug in :meth:`DataFrame.loc.__setitem__` raising ValueError when expanding unique column for :class:`DataFrame` with duplicate columns (:issue:`38521`)
 - Bug in :meth:`DataFrame.iloc.__setitem__` and :meth:`DataFrame.loc.__setitem__` with mixed dtypes when setting with a dictionary value (:issue:`38335`)
+- Bug in :meth:`DataFrame.loc` not preserving dtype of new values, when complete columns was assigned (:issue:`20635`, :issue:`20511`, :issue:`27583`)
 - Bug in :meth:`DataFrame.__setitem__` not raising ``ValueError`` when right hand side is a :class:`DataFrame` with wrong number of columns (:issue:`38604`)
 - Bug in :meth:`Series.__setitem__` raising ``ValueError`` when setting a :class:`Series` with a scalar indexer (:issue:`38303`)
 - Bug in :meth:`DataFrame.loc` dropping levels of :class:`MultiIndex` when :class:`DataFrame` used as input has only one row (:issue:`10521`)

pandas/core/indexing.py

@@ -13,9 +13,11 @@
 from pandas.errors import AbstractMethodError, InvalidIndexError
 from pandas.util._decorators import doc
 
+from pandas.core.dtypes.cast import infer_dtype_from_scalar
 from pandas.core.dtypes.common import (
     is_array_like,
     is_bool_dtype,
+    is_dtype_equal,
     is_hashable,
     is_integer,
     is_iterator,
@@ -1559,6 +1561,26 @@ def _setitem_with_indexer(self, indexer, value, name="iloc"):
             val = list(value.values()) if isinstance(value, dict) else value
             blk = self.obj._mgr.blocks[0]
             take_split_path = not blk._can_hold_element(val)
+            if not take_split_path:
+                if (
+                    isinstance(indexer, tuple)
+                    and is_integer(indexer[0])
+                    and is_integer(indexer[1])
+                    and not is_scalar(value)
+                ):
+                    # GH#37749 this is for listlikes to be treated as scalars, can
+                    # not take split path here
+                    pass
+                elif is_scalar(value):
+                    dtype, _ = infer_dtype_from_scalar(value)
+                    take_split_path = not is_dtype_equal(dtype, blk.dtype)
+                elif isinstance(value, ABCSeries):
+                    take_split_path = not (is_dtype_equal(value.dtype, blk.dtype))
+                elif isinstance(value, ABCDataFrame):
+                    dtypes = list(value.dtypes.unique())
+                    take_split_path = not (
+                        len(dtypes) == 1 and is_dtype_equal(dtypes[0], blk.dtype)
+                    )
 
         # if we have any multi-indexes that have non-trivial slices
         # (not null slices) then we must take the split path, xref

pandas/tests/frame/indexing/test_indexing.py

@@ -1492,7 +1492,7 @@ def test_at_time_between_time_datetimeindex(self):
         result.loc[akey] = 0
         result = result.loc[akey]
         expected = df.loc[akey].copy()
-        expected.loc[:] = 0
+        expected.loc[:] = 0.0
         tm.assert_frame_equal(result, expected)
 
         result = df.copy()
@@ -1504,7 +1504,7 @@ def test_at_time_between_time_datetimeindex(self):
         result.loc[bkey] = 0
         result = result.loc[bkey]
         expected = df.loc[bkey].copy()
-        expected.loc[:] = 0
+        expected.loc[:] = 0.0
         tm.assert_frame_equal(result, expected)
 
         result = df.copy()

pandas/tests/frame/indexing/test_setitem.py

@@ -395,6 +395,18 @@ def test_setitem_listlike_indexer_duplicate_columns_not_equal_length(self):
         with pytest.raises(ValueError, match=msg):
             df[["a", "b"]] = rhs
 
+    def test_setitem_scalar_dtype_change(self):
+        # GH#27583
+        df = DataFrame({"a": [0.0], "b": [0.0]})
+        df[["a", "b"]] = 0
+        expected = DataFrame({"a": [0], "b": [0]})
+        tm.assert_frame_equal(df, expected)
+
+        df = DataFrame({"a": [0.0], "b": [0.0]})
+        df["b"] = 0
+        expected = DataFrame({"a": [0.0], "b": [0]})
+        tm.assert_frame_equal(df, expected)
+
 
 class TestDataFrameSetItemWithExpansion:
     def test_setitem_listlike_views(self):

pandas/tests/indexing/multiindex/test_partial.py

@@ -127,7 +127,7 @@ def test_partial_set(self, multiindex_year_month_day_dataframe_random_data):
         exp["A"].loc[2000, 4].values[:] = 1
         tm.assert_frame_equal(df, exp)
 
-        df.loc[2000] = 5
+        df.loc[2000] = 5.0
         exp.loc[2000].values[:] = 5
         tm.assert_frame_equal(df, exp)
 

pandas/tests/indexing/test_iloc.py

@@ -14,10 +14,12 @@
     Index,
     NaT,
     Series,
+    Timestamp,
     array as pd_array,
     concat,
     date_range,
     isna,
+    to_datetime,
 )
 import pandas._testing as tm
 from pandas.api.types import is_scalar
@@ -987,6 +989,32 @@ def test_iloc_setitem_dictionary_value(self):
         expected = DataFrame({"x": [1, 9], "y": [2.0, 99.0]})
         tm.assert_frame_equal(df, expected)
 
+    def test_iloc_setitem_conversion_to_datetime(self):
+        # GH#20511
+        df = DataFrame(
+            [["2015-01-01", "2016-01-01"], ["2016-01-01", "2015-01-01"]],
+            columns=["date0", "date1"],
+        )
+        df.iloc[:, [0]] = df.iloc[:, [0]].apply(
+            lambda x: to_datetime(x, errors="coerce")
+        )
+        expected = DataFrame(
+            {
+                "date0": [Timestamp("2015-01-01"), Timestamp("2016-01-01")],
+                "date1": ["2016-01-01", "2015-01-01"],
+            }
+        )
+        tm.assert_frame_equal(df, expected)
+
+    def test_iloc_conversion_to_float_32_for_columns_list(self):
+        # GH#33198
+        arr = np.random.randn(10 ** 2).reshape(5, 20).astype(np.float64)
+        df = DataFrame(arr)
+        df.iloc[:, 11:] = df.iloc[:, 11:].astype(np.float32)
+        result = df.dtypes.value_counts()
+        expected = Series([11, 9], index=[np.dtype("float64"), np.dtype("float32")])
+        tm.assert_series_equal(result, expected)
+
 
 class TestILocErrors:
     # NB: this test should work for _any_ Series we can pass as

pandas/tests/indexing/test_loc.py

@@ -1149,6 +1149,23 @@ def test_loc_setitem_listlike_with_timedelta64index(self, indexer, expected):
 
         tm.assert_frame_equal(expected, df)
 
+    def test_loc_setitem_null_slice_single_column_series_value_different_dtype(self):
+        # GH#20635
+        df = DataFrame({"A": ["a", "b"], "B": ["1", "2"], "C": ["3", "4"]})
+        df.loc[:, "C"] = df["C"].astype("int64")
+        expected = DataFrame({"A": ["a", "b"], "B": ["1", "2"], "C": [3, 4]})
+        tm.assert_frame_equal(df, expected)
+
+    @pytest.mark.parametrize("dtype", ["int64", "Int64"])
+    def test_loc_setitem_null_slice_different_dtypes(self, dtype):
+        # GH#20635
+        df = DataFrame({"A": ["a", "b"], "B": ["1", "2"], "C": ["3", "4"], "D": [1, 2]})
+        rhs = df[["B", "C"]].astype("int64").astype(dtype)
+        df.loc[:, ["B", "C"]] = rhs
+        expected = DataFrame({"A": ["a", "b"], "B": [1, 2], "C": [3, 4], "D": [1, 2]})
+        expected[["B", "C"]] = expected[["B", "C"]].astype(dtype)
+        tm.assert_frame_equal(df, expected)
+
 
 class TestLocWithMultiIndex:
     @pytest.mark.parametrize(
@@ -2117,6 +2134,14 @@ def test_loc_setitem_dt64tz_values(self):
         result = s2["a"]
         assert result == expected
 
+    @pytest.mark.parametrize("dtype", ["int64", "Int64"])
+    def test_loc_setitem_series_null_slice_different_dtypes(self, dtype):
+        # GH#20635
+        ser = Series(["3", "4"], name="A")
+        ser.loc[:] = ser.astype("int64").astype(dtype)
+        expected = Series([3, 4], name="A", dtype=dtype)
+        tm.assert_series_equal(ser, expected)
+
     @pytest.mark.parametrize("array_fn", [np.array, pd.array, list, tuple])
     @pytest.mark.parametrize("size", [0, 4, 5, 6])
     def test_loc_iloc_setitem_with_listlike(self, size, array_fn):