diff --git a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java index b50d755951f9df..0207301c2d2f64 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java +++ b/fe/fe-core/src/main/java/org/apache/doris/job/extensions/mtmv/MTMVTask.java @@ -196,7 +196,7 @@ public void run() throws JobException { .subList(start, end > needRefreshPartitions.size() ? needRefreshPartitions.size() : end)); // need get names before exec Map execPartitionSnapshots = MTMVPartitionUtil - .generatePartitionSnapshots(mtmv, relation.getBaseTables(), execPartitionNames, + .generatePartitionSnapshots(mtmv, relation.getBaseTablesOneLevel(), execPartitionNames, partitionMappings); exec(ctx, execPartitionNames, tableWithPartKey); completedPartitions.addAll(execPartitionNames); @@ -288,7 +288,7 @@ public void before() throws JobException { * @throws DdlException */ private void refreshHmsTable() throws AnalysisException, DdlException { - for (BaseTableInfo tableInfo : relation.getBaseTables()) { + for (BaseTableInfo tableInfo : relation.getBaseTablesOneLevel()) { TableIf tableIf = MTMVUtil.getTable(tableInfo); if (tableIf instanceof HMSExternalTable) { HMSExternalTable hmsTable = (HMSExternalTable) tableIf; @@ -450,7 +450,8 @@ public List calculateNeedRefreshPartitions(Map> part // check if data is fresh // We need to use a newly generated relationship and cannot retrieve it using mtmv.getRelation() // to avoid rebuilding the baseTable and causing a change in the tableId - boolean fresh = MTMVPartitionUtil.isMTMVSync(mtmv, relation.getBaseTables(), mtmv.getExcludedTriggerTables(), + boolean fresh = MTMVPartitionUtil.isMTMVSync(mtmv, relation.getBaseTablesOneLevel(), + mtmv.getExcludedTriggerTables(), partitionMappings); if (fresh) { return Lists.newArrayList(); @@ -461,7 +462,8 @@ public List calculateNeedRefreshPartitions(Map> part } // We need to use a newly generated relationship and cannot retrieve it using mtmv.getRelation() // to avoid rebuilding the baseTable and causing a change in the tableId - return MTMVPartitionUtil.getMTMVNeedRefreshPartitions(mtmv, relation.getBaseTables(), partitionMappings); + return MTMVPartitionUtil.getMTMVNeedRefreshPartitions(mtmv, relation.getBaseTablesOneLevel(), + partitionMappings); } public MTMVTaskContext getTaskContext() { diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java index 3d77f42a1ccb40..61fc3e91651efa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVPartitionUtil.java @@ -201,7 +201,8 @@ public static boolean isMTMVSync(MTMV mtmv) { return false; } try { - return isMTMVSync(mtmv, mtmvRelation.getBaseTables(), Sets.newHashSet(), mtmv.calculatePartitionMappings()); + return isMTMVSync(mtmv, mtmvRelation.getBaseTablesOneLevel(), Sets.newHashSet(), + mtmv.calculatePartitionMappings()); } catch (AnalysisException e) { LOG.warn("isMTMVSync failed: ", e); return false; @@ -254,7 +255,7 @@ private static List getPartitionUnSyncTables(MTMV mtmv, String partition Set relatedPartitionNames) throws AnalysisException { List res = Lists.newArrayList(); - for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTables()) { + for (BaseTableInfo baseTableInfo : mtmv.getRelation().getBaseTablesOneLevel()) { TableIf table = MTMVUtil.getTable(baseTableInfo); if (!(table instanceof MTMVRelatedTableIf)) { continue; diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java index 3516e75427e8ea..5392313ba62a19 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVRewriteUtil.java @@ -71,7 +71,7 @@ public static Collection getMTMVCanRewritePartitions(MTMV mtmv, Conne partitionMappings = mtmv.calculatePartitionMappings(); } if (MTMVPartitionUtil.isMTMVPartitionSync(mtmv, partition.getName(), - partitionMappings.get(partition.getName()), mtmvRelation.getBaseTables(), + partitionMappings.get(partition.getName()), mtmvRelation.getBaseTablesOneLevel(), Sets.newHashSet())) { res.add(partition); } diff --git a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java index 1fd56d7b6e406b..4868ef94a1b570 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java +++ b/fe/fe-core/src/main/java/org/apache/doris/mtmv/MTMVUtil.java @@ -85,7 +85,7 @@ public static MTMV getMTMV(long dbId, long mtmvId) throws DdlException, MetaNotF * @return */ public static boolean mtmvContainsExternalTable(MTMV mtmv) { - Set baseTables = mtmv.getRelation().getBaseTables(); + Set baseTables = mtmv.getRelation().getBaseTablesOneLevel(); for (BaseTableInfo baseTableInfo : baseTables) { if (baseTableInfo.getCtlId() != InternalCatalog.INTERNAL_CATALOG_ID) { return true; diff --git a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java index c125e548d06ac6..d40e6db5d44757 100644 --- a/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java +++ b/fe/fe-core/src/test/java/org/apache/doris/mtmv/MTMVPartitionUtilTest.java @@ -111,7 +111,7 @@ public void setUp() throws NoSuchMethodException, SecurityException, AnalysisExc minTimes = 0; result = true; - relation.getBaseTables(); + relation.getBaseTablesOneLevel(); minTimes = 0; result = baseTables; diff --git a/regression-test/data/mtmv_p0/test_multi_level_mtmv.out b/regression-test/data/mtmv_p0/test_multi_level_mtmv.out index 3d8a1e1b43fbab..7543b21ffa7bbc 100644 --- a/regression-test/data/mtmv_p0/test_multi_level_mtmv.out +++ b/regression-test/data/mtmv_p0/test_multi_level_mtmv.out @@ -5,6 +5,12 @@ -- !mv2 -- 1 1 +-- !mv1_should_one_partition -- +["p_2"] + +-- !mv2_should_one_partition -- +["p_2"] + -- !status1 -- multi_level_mtmv1 SCHEMA_CHANGE SUCCESS diff --git a/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy b/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy index 81d4b78a3d7dc4..55689b741489eb 100644 --- a/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy +++ b/regression-test/suites/mtmv_p0/test_multi_level_mtmv.groovy @@ -29,6 +29,11 @@ suite("test_multi_level_mtmv") { k1 int, k2 int ) + PARTITION BY LIST(`k1`) + ( + PARTITION `p1` VALUES IN ('1'), + PARTITION `p2` VALUES IN ('2') + ) DISTRIBUTED BY HASH(k1) BUCKETS 10 PROPERTIES ( "replication_num" = "1" @@ -40,34 +45,48 @@ suite("test_multi_level_mtmv") { sql """ CREATE MATERIALIZED VIEW ${mv1} - BUILD DEFERRED REFRESH COMPLETE ON MANUAL + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(k1) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS SELECT * FROM ${tableName}; """ - def jobName1 = getJobName("regression_test_mtmv_p0", mv1); sql """ REFRESH MATERIALIZED VIEW ${mv1} AUTO """ - waitingMTMVTaskFinished(jobName1) + waitingMTMVTaskFinishedByMvName(mv1) order_qt_mv1 "select * from ${mv1}" sql """ CREATE MATERIALIZED VIEW ${mv2} - BUILD DEFERRED REFRESH COMPLETE ON MANUAL + BUILD DEFERRED REFRESH AUTO ON MANUAL + partition by(k1) DISTRIBUTED BY RANDOM BUCKETS 2 PROPERTIES ('replication_num' = '1') AS SELECT * FROM ${mv1}; """ - def jobName2 = getJobName("regression_test_mtmv_p0", mv2); sql """ REFRESH MATERIALIZED VIEW ${mv2} AUTO """ - waitingMTMVTaskFinished(jobName2) + waitingMTMVTaskFinishedByMvName(mv2) order_qt_mv2 "select * from ${mv2}" + sql """ + INSERT INTO ${tableName} VALUES(2,2); + """ + sql """ + REFRESH MATERIALIZED VIEW ${mv1} AUTO + """ + waitingMTMVTaskFinishedByMvName(mv1) + order_qt_mv1_should_one_partition "select NeedRefreshPartitions from tasks('type'='mv') where MvName = '${mv1}' order by CreateTime desc limit 1" + sql """ + REFRESH MATERIALIZED VIEW ${mv2} AUTO + """ + waitingMTMVTaskFinishedByMvName(mv2) + order_qt_mv2_should_one_partition "select NeedRefreshPartitions from tasks('type'='mv') where MvName = '${mv2}' order by CreateTime desc limit 1" + // drop table sql """ drop table ${tableName}