Skip to content

Commit

Permalink
Begin merge, combine.
Browse files Browse the repository at this point in the history
  • Loading branch information
dcherian committed Aug 22, 2019
1 parent 39f68fc commit 820463b
Show file tree
Hide file tree
Showing 5 changed files with 54 additions and 9 deletions.
13 changes: 11 additions & 2 deletions xarray/core/combine.py
Original file line number Diff line number Diff line change
Expand Up @@ -243,11 +243,12 @@ def _combine_1d(
dim=concat_dim,
data_vars=data_vars,
coords=coords,
compat=compat,
fill_value=fill_value,
join=join,
)
except ValueError as err:
if "encountered unexpected variable" in str(err):
if "Encountered unexpected variable" in str(err):
raise ValueError(
"These objects cannot be combined using only "
"xarray.combine_nested, instead either use "
Expand Down Expand Up @@ -598,6 +599,7 @@ def combine_by_coords(
concat_dims=concat_dims,
data_vars=data_vars,
coords=coords,
compat=compat,
fill_value=fill_value,
join=join,
)
Expand Down Expand Up @@ -832,6 +834,7 @@ def _old_auto_combine(
dim=dim,
data_vars=data_vars,
coords=coords,
compat=compat,
fill_value=fill_value,
join=join,
)
Expand All @@ -850,6 +853,7 @@ def _auto_concat(
coords="different",
fill_value=dtypes.NA,
join="outer",
compat="no_conflicts",
):
if len(datasets) == 1 and dim is None:
# There is nothing more to combine, so kick out early.
Expand All @@ -876,5 +880,10 @@ def _auto_concat(
)
dim, = concat_dims
return concat(
datasets, dim=dim, data_vars=data_vars, coords=coords, fill_value=fill_value
datasets,
dim=dim,
data_vars=data_vars,
coords=coords,
fill_value=fill_value,
compat=compat,
)
11 changes: 8 additions & 3 deletions xarray/core/concat.py
Original file line number Diff line number Diff line change
Expand Up @@ -70,7 +70,7 @@ def concat(
* 'identical' means that variable attributes and global attributes
must also be equal.
* 'override' means that checks are skipped and values from the first dataset
are used.
are used. This cannot be used with coords='different' or data_vars='different'.
positions : None or list of integer arrays, optional
List of integer arrays which specifies the integer positions to which
to assign each dataset along the concatenated dimension. If not
Expand Down Expand Up @@ -145,7 +145,7 @@ def concat(
"`data_vars` and `coords` arguments"
)

if compat not in ["equals", "identical", "override"]:
if compat not in ["equals", "identical", "override", "no_conflicts"]:
raise ValueError(
"compat=%r invalid: must be 'equals', 'identical or 'override'" % compat
)
Expand Down Expand Up @@ -207,6 +207,11 @@ def _calc_concat_over(datasets, dim, data_vars, coords):
def process_subset_opt(opt, subset):
if isinstance(opt, str):
if opt == "different":
if compat == "override":
raise ValueError(
"Cannot specify both %s='different' and compat='override'."
% subset
)
# all nonindexes that are not the same in each dataset
for k in getattr(datasets[0], subset):
if k not in concat_over:
Expand Down Expand Up @@ -395,7 +400,7 @@ def _dataarray_concat(

if data_vars != "all":
raise ValueError(
"data_vars is not a valid argument when " "concatenating DataArray objects"
"data_vars is not a valid argument when concatenating DataArray objects"
)

datasets = []
Expand Down
9 changes: 5 additions & 4 deletions xarray/core/merge.py
Original file line number Diff line number Diff line change
Expand Up @@ -44,6 +44,7 @@
"broadcast_equals": 2,
"minimal": 3,
"no_conflicts": 4,
"override": 5,
}
)

Expand Down Expand Up @@ -81,7 +82,7 @@ def unique_variable(name, variables, compat="broadcast_equals"):
variables : list of xarray.Variable
List of Variable objects, all of which go by the same name in different
inputs.
compat : {'identical', 'equals', 'broadcast_equals', 'no_conflicts'}, optional
compat : {'identical', 'equals', 'broadcast_equals', 'no_conflicts', 'override'}, optional
Type of equality check to use.
Returns
Expand All @@ -93,7 +94,7 @@ def unique_variable(name, variables, compat="broadcast_equals"):
MergeError: if any of the variables are not equal.
""" # noqa
out = variables[0]
if len(variables) > 1:
if len(variables) > 1 and compat != "override":
combine_method = None

if compat == "minimal":
Expand Down Expand Up @@ -152,7 +153,7 @@ def merge_variables(
priority_vars : mapping with Variable or None values, optional
If provided, variables are always taken from this dict in preference to
the input variable dictionaries, without checking for conflicts.
compat : {'identical', 'equals', 'broadcast_equals', 'minimal', 'no_conflicts'}, optional
compat : {'identical', 'equals', 'broadcast_equals', 'minimal', 'no_conflicts', 'override'}, optional
Type of equality check to use when checking for conflicts.
Returns
Expand Down Expand Up @@ -449,7 +450,7 @@ def merge_core(
----------
objs : list of mappings
All values must be convertable to labeled arrays.
compat : {'identical', 'equals', 'broadcast_equals', 'no_conflicts'}, optional
compat : {'identical', 'equals', 'broadcast_equals', 'no_conflicts', 'override'}, optional
Compatibility checks to use when merging variables.
join : {'outer', 'inner', 'left', 'right'}, optional
How to combine objects with different indexes.
Expand Down
28 changes: 28 additions & 0 deletions xarray/tests/test_concat.py
Original file line number Diff line number Diff line change
Expand Up @@ -17,6 +17,30 @@
from .test_dataset import create_test_data


def test_concat_compat():
ds1 = Dataset(
{
"has_x_y": (("y", "x"), [[1, 2]]),
"has_x": ("x", [1, 2]),
"no_x_y": ("z", [1, 2]),
},
coords={"x": [0, 1], "y": [0], "z": [-1, -2]},
)
ds2 = Dataset(
{
"has_x_y": (("y", "x"), [[3, 4]]),
"has_x": ("x", [1, 2]),
"no_x_y": (("q", "z"), [[1, 2]]),
},
coords={"x": [0, 1], "y": [1], "z": [-1, -2], "q": [0]},
)

result = concat([ds1, ds2], dim="y", compat="equals")

for var in ["has_x", "no_x_y", "const1"]:
assert "y" not in result[var]


class TestConcatDataset:
@pytest.fixture
def data(self):
Expand Down Expand Up @@ -404,6 +428,10 @@ def test_compat_override(self, data_vars, coords):
for var in equal_to_first_ds:
assert_equal(actual[var], self.dsets[0][var])

def test_compat_override_different_error(self):
with raises_regex(ValueError, "Cannot specify both .*='different'"):
concat(self.dsets, data_vars="different", compat="override")


class TestConcatDataArray:
def test_concat(self):
Expand Down
2 changes: 2 additions & 0 deletions xarray/tests/test_merge.py
Original file line number Diff line number Diff line change
Expand Up @@ -196,6 +196,8 @@ def test_merge_compat(self):
with raises_regex(ValueError, "compat=.* invalid"):
ds1.merge(ds2, compat="foobar")

assert ds1.identical(ds1.merge(ds2, compat="override"))

def test_merge_auto_align(self):
ds1 = xr.Dataset({"a": ("x", [1, 2]), "x": [0, 1]})
ds2 = xr.Dataset({"b": ("x", [3, 4]), "x": [1, 2]})
Expand Down

0 comments on commit 820463b

Please sign in to comment.