diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java index d6b59b6866c307a..75a03fade2971ed 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/EquivalenceClass.java @@ -21,12 +21,9 @@ import java.util.ArrayList; import java.util.HashMap; -import java.util.HashSet; import java.util.LinkedHashMap; -import java.util.LinkedHashSet; import java.util.List; import java.util.Map; -import java.util.Set; /** * EquivalenceClass, this is used for equality propagation when predicate compensation @@ -40,14 +37,19 @@ public class EquivalenceClass { * a: [a, b], * b: [a, b] * } + * or column a = a, + * this would be + * { + * a: [a, a] + * } */ - private Map> equivalenceSlotMap = new LinkedHashMap<>(); - private List> equivalenceSlotList; + private Map> equivalenceSlotMap = new LinkedHashMap<>(); + private List> equivalenceSlotList; public EquivalenceClass() { } - public EquivalenceClass(Map> equivalenceSlotMap) { + public EquivalenceClass(Map> equivalenceSlotMap) { this.equivalenceSlotMap = equivalenceSlotMap; } @@ -56,13 +58,13 @@ public EquivalenceClass(Map> equivalenceSlotMa */ public void addEquivalenceClass(SlotReference leftSlot, SlotReference rightSlot) { - Set leftSlotSet = equivalenceSlotMap.get(leftSlot); - Set rightSlotSet = equivalenceSlotMap.get(rightSlot); + List leftSlotSet = equivalenceSlotMap.get(leftSlot); + List rightSlotSet = equivalenceSlotMap.get(rightSlot); if (leftSlotSet != null && rightSlotSet != null) { // Both present, we need to merge if (leftSlotSet.size() < rightSlotSet.size()) { // We swap them to merge - Set tmp = rightSlotSet; + List tmp = rightSlotSet; rightSlotSet = leftSlotSet; leftSlotSet = tmp; } @@ -80,7 +82,7 @@ public void addEquivalenceClass(SlotReference leftSlot, SlotReference rightSlot) equivalenceSlotMap.put(leftSlot, rightSlotSet); } else { // None are present, add to same equivalence class - Set equivalenceClass = new LinkedHashSet<>(); + List equivalenceClass = new ArrayList<>(); equivalenceClass.add(leftSlot); equivalenceClass.add(rightSlot); equivalenceSlotMap.put(leftSlot, equivalenceClass); @@ -88,7 +90,7 @@ public void addEquivalenceClass(SlotReference leftSlot, SlotReference rightSlot) } } - public Map> getEquivalenceSlotMap() { + public Map> getEquivalenceSlotMap() { return equivalenceSlotMap; } @@ -101,15 +103,15 @@ public boolean isEmpty() { */ public EquivalenceClass permute(Map mapping) { - Map> permutedEquivalenceSlotMap = new HashMap<>(); - for (Map.Entry> slotReferenceSetEntry : equivalenceSlotMap.entrySet()) { + Map> permutedEquivalenceSlotMap = new HashMap<>(); + for (Map.Entry> slotReferenceSetEntry : equivalenceSlotMap.entrySet()) { SlotReference mappedSlotReferenceKey = mapping.get(slotReferenceSetEntry.getKey()); if (mappedSlotReferenceKey == null) { // can not permute then need to return null return null; } - Set equivalenceValueSet = slotReferenceSetEntry.getValue(); - final Set mappedSlotReferenceSet = new HashSet<>(); + List equivalenceValueSet = slotReferenceSetEntry.getValue(); + final List mappedSlotReferenceSet = new ArrayList<>(); for (SlotReference target : equivalenceValueSet) { SlotReference mappedSlotReferenceValue = mapping.get(target); if (mappedSlotReferenceValue == null) { @@ -123,15 +125,14 @@ public EquivalenceClass permute(Map mapping) { } /** - * Return the list of equivalence set, remove duplicate + * Return the list of equivalence list, remove duplicate */ - public List> getEquivalenceSetList() { - + public List> getEquivalenceSetList() { if (equivalenceSlotList != null) { return equivalenceSlotList; } - List> equivalenceSets = new ArrayList<>(); - Set> visited = new HashSet<>(); + List> equivalenceSets = new ArrayList<>(); + List> visited = new ArrayList<>(); equivalenceSlotMap.values().forEach(slotSet -> { if (!visited.contains(slotSet)) { equivalenceSets.add(slotSet); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java index 8475bc6b46a1570..20ded0415ade075 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/Predicates.java @@ -18,7 +18,7 @@ package org.apache.doris.nereids.rules.exploration.mv; import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassSetMapping; +import org.apache.doris.nereids.rules.exploration.mv.mapping.EquivalenceClassMapping; import org.apache.doris.nereids.rules.exploration.mv.mapping.SlotMapping; import org.apache.doris.nereids.rules.expression.ExpressionNormalization; import org.apache.doris.nereids.rules.expression.ExpressionOptimization; @@ -33,6 +33,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.Sets; +import java.util.ArrayList; import java.util.Collection; import java.util.HashSet; import java.util.Iterator; @@ -98,15 +99,15 @@ public static Set compensateEquivalence(StructInfo queryStructInfo, if (queryEquivalenceClass.isEmpty() && !viewEquivalenceClass.isEmpty()) { return null; } - EquivalenceClassSetMapping queryToViewEquivalenceMapping = - EquivalenceClassSetMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased); + EquivalenceClassMapping queryToViewEquivalenceMapping = + EquivalenceClassMapping.generate(queryEquivalenceClass, viewEquivalenceClassQueryBased); // can not map all target equivalence class, can not compensate if (queryToViewEquivalenceMapping.getEquivalenceClassSetMap().size() < viewEquivalenceClass.getEquivalenceSetList().size()) { return null; } // do equal compensate - Set> mappedQueryEquivalenceSet = + Set> mappedQueryEquivalenceSet = queryToViewEquivalenceMapping.getEquivalenceClassSetMap().keySet(); queryEquivalenceClass.getEquivalenceSetList().forEach( queryEquivalenceSet -> { @@ -120,9 +121,9 @@ public static Set compensateEquivalence(StructInfo queryStructInfo, } } else { // compensate the equivalence both in query and view, but query has more equivalence - Set viewEquivalenceSet = + List viewEquivalenceSet = queryToViewEquivalenceMapping.getEquivalenceClassSetMap().get(queryEquivalenceSet); - Set copiedQueryEquivalenceSet = new HashSet<>(queryEquivalenceSet); + List copiedQueryEquivalenceSet = new ArrayList<>(queryEquivalenceSet); copiedQueryEquivalenceSet.removeAll(viewEquivalenceSet); SlotReference first = viewEquivalenceSet.iterator().next(); for (SlotReference slotReference : copiedQueryEquivalenceSet) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassSetMapping.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassMapping.java similarity index 54% rename from fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassSetMapping.java rename to fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassMapping.java index d4ec09c24a4b8c2..03ad43b182b6a8a 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassSetMapping.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/mapping/EquivalenceClassMapping.java @@ -21,6 +21,7 @@ import org.apache.doris.nereids.trees.expressions.SlotReference; import java.util.HashMap; +import java.util.HashSet; import java.util.List; import java.util.Map; import java.util.Set; @@ -30,39 +31,41 @@ * This will extract the equivalence class set in EquivalenceClass and mapping set in * two different EquivalenceClass. */ -public class EquivalenceClassSetMapping extends Mapping { +public class EquivalenceClassMapping extends Mapping { - private final Map, Set> equivalenceClassSetMap; + private final Map, List> equivalenceClassSetMap; - public EquivalenceClassSetMapping(Map, - Set> equivalenceClassSetMap) { + public EquivalenceClassMapping(Map, + List> equivalenceClassSetMap) { this.equivalenceClassSetMap = equivalenceClassSetMap; } - public static EquivalenceClassSetMapping of(Map, Set> equivalenceClassSetMap) { - return new EquivalenceClassSetMapping(equivalenceClassSetMap); + public static EquivalenceClassMapping of(Map, List> equivalenceClassSetMap) { + return new EquivalenceClassMapping(equivalenceClassSetMap); } /** * Generate source equivalence set map to target equivalence set */ - public static EquivalenceClassSetMapping generate(EquivalenceClass source, EquivalenceClass target) { + public static EquivalenceClassMapping generate(EquivalenceClass source, EquivalenceClass target) { - Map, Set> equivalenceClassSetMap = new HashMap<>(); - List> sourceSets = source.getEquivalenceSetList(); - List> targetSets = target.getEquivalenceSetList(); + Map, List> equivalenceClassSetMap = new HashMap<>(); + List> sourceSets = source.getEquivalenceSetList(); + List> targetSets = target.getEquivalenceSetList(); - for (Set sourceSet : sourceSets) { - for (Set targetSet : targetSets) { + for (List sourceList : sourceSets) { + Set sourceSet = new HashSet<>(sourceList); + for (List targetList : targetSets) { + Set targetSet = new HashSet<>(targetList); if (sourceSet.containsAll(targetSet)) { - equivalenceClassSetMap.put(sourceSet, targetSet); + equivalenceClassSetMap.put(sourceList, targetList); } } } - return EquivalenceClassSetMapping.of(equivalenceClassSetMap); + return EquivalenceClassMapping.of(equivalenceClassSetMap); } - public Map, Set> getEquivalenceClassSetMap() { + public Map, List> getEquivalenceClassSetMap() { return equivalenceClassSetMap; } } diff --git a/regression-test/data/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.out b/regression-test/data/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.out new file mode 100644 index 000000000000000..874bfea0ea0b1aa --- /dev/null +++ b/regression-test/data/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.out @@ -0,0 +1,11 @@ +-- This file is automatically generated. You should know what you did if you want to edit this +-- !query1_0_before -- +1 o mm +2 o mi +4 o yy + +-- !query1_0_after -- +1 o mm +2 o mi +4 o yy + diff --git a/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy b/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy new file mode 100644 index 000000000000000..b210dd44850f19e --- /dev/null +++ b/regression-test/suites/nereids_rules_p0/mv/unsafe_equals/null_un_safe_equals.groovy @@ -0,0 +1,85 @@ +package mv.unsafe_equals +// Licensed to the Apache Software Foundation (ASF) under one +// or more contributor license agreements. See the NOTICE file +// distributed with this work for additional information +// regarding copyright ownership. The ASF licenses this file +// to you under the Apache License, Version 2.0 (the +// "License"); you may not use this file except in compliance +// with the License. You may obtain a copy of the License at +// +// http://www.apache.org/licenses/LICENSE-2.0 +// +// Unless required by applicable law or agreed to in writing, +// software distributed under the License is distributed on an +// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// KIND, either express or implied. See the License for the +// specific language governing permissions and limitations +// under the License. + +suite("null_unsafe_equals") { + String db = context.config.getDbNameByFile(context.file) + sql "use ${db}" + sql "set runtime_filter_mode=OFF"; + sql "SET ignore_shape_nodes='PhysicalDistribute,PhysicalProject'" + + sql """ + drop table if exists orders + """ + + sql """ + CREATE TABLE IF NOT EXISTS orders ( + o_orderkey INTEGER NULL, + o_custkey INTEGER NULL, + o_orderstatus CHAR(1) NULL, + o_totalprice DECIMALV3(15,2) NULL, + o_orderdate DATE NULL, + o_orderpriority CHAR(15) NULL, + o_clerk CHAR(15) NULL, + o_shippriority INTEGER NULL, + O_COMMENT VARCHAR(79) NULL + ) + DUPLICATE KEY(o_orderkey, o_custkey) + PARTITION BY RANGE(o_orderdate) ( + PARTITION `day_2` VALUES LESS THAN ('2023-12-9'), + PARTITION `day_3` VALUES LESS THAN ("2023-12-11"), + PARTITION `day_4` VALUES LESS THAN ("2023-12-30") + ) + DISTRIBUTED BY HASH(o_orderkey) BUCKETS 3 + PROPERTIES ( + "replication_num" = "1" + ); + """ + + sql """ + insert into orders values + (null, 1, 'o', 9.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (1, null, 'o', 10.5, '2023-12-08', 'a', 'b', 1, 'yy'), + (2, 1, null, 11.5, '2023-12-09', 'a', 'b', 1, 'yy'), + (3, 1, 'o', null, '2023-12-10', 'a', 'b', 1, 'yy'), + (3, 1, 'o', 33.5, null, 'a', 'b', 1, 'yy'), + (4, 2, 'o', 43.2, '2023-12-11', null,'d',2, 'mm'), + (5, 2, 'o', 56.2, '2023-12-12', 'c',null, 2, 'mi'), + (5, 2, 'o', 1.2, '2023-12-12', 'c','d', null, 'mi'); + """ + + def mv1_0 = + """ + select count(*), o_orderstatus, o_comment + from orders + group by + o_orderstatus, o_comment; + """ + // query contains the filter which is 'o_orderstatus = o_orderstatus' should reject null + def query1_0 = + """ + select count(*), o_orderstatus, o_comment + from orders + where o_orderstatus = o_orderstatus + group by + o_orderstatus, o_comment; + """ + order_qt_query1_0_before "${query1_0}" + async_mv_rewrite_success(db, mv1_0, query1_0, "mv1_0") + order_qt_query1_0_after "${query1_0}" + sql """ DROP MATERIALIZED VIEW IF EXISTS mv1_0""" +}