[fix](mtmv) Fix some pr to 21, prs are (#39041)(#38958)(#39541)

fe/fe-core/src/main/java/org/apache/doris/catalog/MTMV.java

@@ -190,7 +190,7 @@ public void addTaskResult(MTMVTask task, MTMVRelation relation,
                 this.relation = relation;
                 if (!Env.isCheckpointThread()) {
                     try {
-                        this.cache = MTMVCache.from(this, MTMVPlanUtil.createMTMVContext(this));
+                        this.cache = MTMVCache.from(this, MTMVPlanUtil.createMTMVContext(this), true);
                     } catch (Throwable e) {
                         this.cache = null;
                         LOG.warn("generate cache failed", e);
@@ -277,7 +277,7 @@ public MTMVCache getOrGenerateCache(ConnectContext connectionContext) throws Ana
             writeMvLock();
             try {
                 if (cache == null) {
-                    this.cache = MTMVCache.from(this, connectionContext);
+                    this.cache = MTMVCache.from(this, connectionContext, true);
                 }
             } finally {
                 writeMvUnlock();

fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVCache.java

@@ -79,7 +79,7 @@ public StructInfo getStructInfo() {
         return structInfo;
     }
 
-    public static MTMVCache from(MTMV mtmv, ConnectContext connectContext) {
+    public static MTMVCache from(MTMV mtmv, ConnectContext connectContext, boolean needCost) {
         LogicalPlan unboundMvPlan = new NereidsParser().parseSingle(mtmv.getQuerySql());
         StatementContext mvSqlStatementContext = new StatementContext(connectContext,
                 new OriginStatement(mtmv.getQuerySql(), 0));
@@ -89,7 +89,13 @@ public static MTMVCache from(MTMV mtmv, ConnectContext connectContext) {
         }
         // Can not convert to table sink, because use the same column from different table when self join
         // the out slot is wrong
-        planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN);
+        if (needCost) {
+            // Only in mv rewrite, we need plan with eliminated cost which is used for mv chosen
+            planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.ALL_PLAN);
+        } else {
+            // No need cost for performance
+            planner.planWithLock(unboundMvPlan, PhysicalProperties.ANY, ExplainLevel.REWRITTEN_PLAN);
+        }
         Plan originPlan = planner.getCascadesContext().getRewritePlan();
         // Eliminate result sink because sink operator is useless in query rewrite by materialized view
         // and the top sort can also be removed
@@ -111,7 +117,8 @@ public Plan visitLogicalResultSink(LogicalResultSink<? extends Plan> logicalResu
         Optional<StructInfo> structInfoOptional = MaterializationContext.constructStructInfo(mvPlan, originPlan,
                 planner.getCascadesContext(),
                 new BitSet());
-        return new MTMVCache(mvPlan, originPlan, planner.getCascadesContext().getMemo().getRoot().getStatistics(),
+        return new MTMVCache(mvPlan, originPlan, needCost
+                ? planner.getCascadesContext().getMemo().getRoot().getStatistics() : null,
                 structInfoOptional.orElseGet(() -> null));
     }
 }

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/AbstractMaterializedViewAggregateRule.java

@@ -214,7 +214,7 @@ protected LogicalAggregate<Plan> doRewriteQueryByView(
         LogicalAggregate<Plan> queryAggregate = queryTopPlanAndAggPair.value();
         List<Expression> queryGroupByExpressions = queryAggregate.getGroupByExpressions();
         // handle the scene that query top plan not use the group by in query bottom aggregate
-        if (queryGroupByExpressions.size() != queryTopPlanGroupBySet.size()) {
+        if (needCompensateGroupBy(queryTopPlanGroupBySet, queryGroupByExpressions)) {
             for (Expression expression : queryGroupByExpressions) {
                 if (queryTopPlanGroupBySet.contains(expression)) {
                     continue;
@@ -263,6 +263,42 @@ protected LogicalAggregate<Plan> doRewriteQueryByView(
         return new LogicalAggregate<>(finalGroupExpressions, finalOutputExpressions, tempRewritedPlan);
     }
 
+    /**
+     * handle the scene that query top plan not use the group by in query bottom aggregate
+     * If mv is select o_orderdate from  orders group by o_orderdate;
+     * query is select 1 from orders group by o_orderdate.
+     * Or mv is select o_orderdate from orders group by o_orderdate
+     * query is select o_orderdate from  orders group by o_orderdate, o_orderkey;
+     * if the slot which query top project use can not cover the slot which query bottom aggregate group by slot
+     * should compensate group by to make sure the data is right.
+     * For example:
+     * mv is select o_orderdate from orders group by o_orderdate;
+     * query is select o_orderdate from  orders group by o_orderdate, o_orderkey;
+     *
+     * @param queryGroupByExpressions query bottom aggregate group by is o_orderdate, o_orderkey
+     * @param queryTopProject query top project is o_orderdate
+     * @return need to compensate group by if true or not need
+     *
+     */
+    private static boolean needCompensateGroupBy(Set<? extends Expression> queryTopProject,
+            List<Expression> queryGroupByExpressions) {
+        Set<Expression> queryGroupByExpressionSet = new HashSet<>(queryGroupByExpressions);
+        if (queryGroupByExpressionSet.size() != queryTopProject.size()) {
+            return true;
+        }
+        Set<NamedExpression> queryTopPlanGroupByUseNamedExpressions = new HashSet<>();
+        Set<NamedExpression> queryGroupByUseNamedExpressions = new HashSet<>();
+        for (Expression expr : queryTopProject) {
+            queryTopPlanGroupByUseNamedExpressions.addAll(expr.collect(NamedExpression.class::isInstance));
+        }
+        for (Expression expr : queryGroupByExpressionSet) {
+            queryGroupByUseNamedExpressions.addAll(expr.collect(NamedExpression.class::isInstance));
+        }
+        // if the slots query top project use can not cover the slots which query bottom aggregate use
+        // Should compensate.
+        return !queryTopPlanGroupByUseNamedExpressions.containsAll(queryGroupByUseNamedExpressions);
+    }
+
     /**
      * Try to rewrite query expression by view, contains both group by dimension and aggregate function
      */

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterProjectScanRule.java

@@ -19,8 +19,9 @@
 
 import org.apache.doris.nereids.rules.Rule;
 import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
 
 import com.google.common.collect.ImmutableList;
@@ -37,9 +38,11 @@ public class MaterializedViewFilterProjectScanRule extends MaterializedViewScanR
     @Override
     public List<Rule> buildRules() {
         return ImmutableList.of(
-                logicalFilter(logicalProject(logicalOlapScan())).thenApplyMultiNoThrow(ctx -> {
-                    LogicalFilter<LogicalProject<LogicalOlapScan>> root = ctx.root;
-                    return rewrite(root, ctx.cascadesContext);
-                }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_SCAN));
+                logicalFilter(logicalProject(any().when(LogicalCatalogRelation.class::isInstance)))
+                        .thenApplyMultiNoThrow(
+                                ctx -> {
+                                    LogicalFilter<LogicalProject<Plan>> root = ctx.root;
+                                    return rewrite(root, ctx.cascadesContext);
+                                }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_PROJECT_SCAN));
     }
 }

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewFilterScanRule.java

@@ -19,8 +19,9 @@
 
 import org.apache.doris.nereids.rules.Rule;
 import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
 
 import com.google.common.collect.ImmutableList;
 
@@ -36,8 +37,8 @@ public class MaterializedViewFilterScanRule extends MaterializedViewScanRule {
     @Override
     public List<Rule> buildRules() {
         return ImmutableList.of(
-                logicalFilter(logicalOlapScan()).thenApplyMultiNoThrow(ctx -> {
-                    LogicalFilter<LogicalOlapScan> root = ctx.root;
+                logicalFilter(any().when(LogicalCatalogRelation.class::isInstance)).thenApplyMultiNoThrow(ctx -> {
+                    LogicalFilter<Plan> root = ctx.root;
                     return rewrite(root, ctx.cascadesContext);
                 }).toRule(RuleType.MATERIALIZED_VIEW_FILTER_SCAN));
     }

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectFilterScanRule.java

@@ -19,8 +19,9 @@
 
 import org.apache.doris.nereids.rules.Rule;
 import org.apache.doris.nereids.rules.RuleType;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalFilter;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
 
 import com.google.common.collect.ImmutableList;
@@ -37,9 +38,11 @@ public class MaterializedViewProjectFilterScanRule extends MaterializedViewScanR
     @Override
     public List<Rule> buildRules() {
         return ImmutableList.of(
-                logicalProject(logicalFilter(logicalOlapScan())).thenApplyMultiNoThrow(ctx -> {
-                    LogicalProject<LogicalFilter<LogicalOlapScan>> root = ctx.root;
-                    return rewrite(root, ctx.cascadesContext);
-                }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_SCAN));
+                logicalProject(logicalFilter(any().when(LogicalCatalogRelation.class::isInstance)))
+                        .thenApplyMultiNoThrow(
+                                ctx -> {
+                                    LogicalProject<LogicalFilter<Plan>> root = ctx.root;
+                                    return rewrite(root, ctx.cascadesContext);
+                                }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_FILTER_SCAN));
     }
 }

fe/fe-core/src/main/java/org/apache/doris/nereids/rules/exploration/mv/MaterializedViewProjectScanRule.java

@@ -19,7 +19,8 @@
 
 import org.apache.doris.nereids.rules.Rule;
 import org.apache.doris.nereids.rules.RuleType;
-import org.apache.doris.nereids.trees.plans.logical.LogicalOlapScan;
+import org.apache.doris.nereids.trees.plans.Plan;
+import org.apache.doris.nereids.trees.plans.logical.LogicalCatalogRelation;
 import org.apache.doris.nereids.trees.plans.logical.LogicalProject;
 
 import com.google.common.collect.ImmutableList;
@@ -36,8 +37,8 @@ public class MaterializedViewProjectScanRule extends MaterializedViewScanRule {
     @Override
     public List<Rule> buildRules() {
         return ImmutableList.of(
-                logicalProject(logicalOlapScan()).thenApplyMultiNoThrow(ctx -> {
-                    LogicalProject<LogicalOlapScan> root = ctx.root;
+                logicalProject(any().when(LogicalCatalogRelation.class::isInstance)).thenApplyMultiNoThrow(ctx -> {
+                    LogicalProject<Plan> root = ctx.root;
                     return rewrite(root, ctx.cascadesContext);
                 }).toRule(RuleType.MATERIALIZED_VIEW_PROJECT_SCAN));
     }

fe/fe-core/src/main/java/org/apache/doris/nereids/trees/plans/visitor/TableCollector.java

@@ -75,7 +75,7 @@ private void expandMvAndCollect(MTMV mtmv, TableCollectorContext context) {
         }
         // Make sure use only one connection context when in query to avoid ConnectionContext.get() wrong
         MTMVCache expandedMv = MTMVCache.from(mtmv, context.getConnectContext() == null
-                ? MTMVPlanUtil.createMTMVContext(mtmv) : context.getConnectContext());
+                ? MTMVPlanUtil.createMTMVContext(mtmv) : context.getConnectContext(), false);
         expandedMv.getLogicalPlan().accept(this, context);
     }