From 9d9428cba7b1a1b7e3022e07bea1385556f2479b Mon Sep 17 00:00:00 2001 From: starocean999 <40539150+starocean999@users.noreply.github.com> Date: Wed, 13 Dec 2023 13:35:35 +0800 Subject: [PATCH] Revert "[opt](Nereids) partition predicate large or to min-max (#19)" (#30) --- .../apache/doris/analysis/DateLiteral.java | 4 - .../expression/rules/PartitionPruner.java | 87 ------------------- .../rules/RewriteRangePartitionPredicate.java | 82 ----------------- .../org/apache/doris/qe/SessionVariable.java | 3 - 4 files changed, 176 deletions(-) delete mode 100644 fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteRangePartitionPredicate.java diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java index 8cb6a34dc05c93..dfd21c027a6064 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/DateLiteral.java @@ -1861,8 +1861,4 @@ private static int getTimeZoneSplitPos(String arg) { } return split + 1; } - - public double getDouble() { - return (year * 10000 + month * 100 + day) * 1000000L + hour * 10000 + minute * 100 + second; - } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java index 48b4fc99377218..a0064a35e18a57 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java @@ -17,11 +17,9 @@ package org.apache.doris.nereids.rules.expression.rules; -import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.catalog.ListPartitionItem; import org.apache.doris.catalog.PartitionInfo; import org.apache.doris.catalog.PartitionItem; -import org.apache.doris.catalog.PartitionKey; import org.apache.doris.catalog.RangePartitionItem; import org.apache.doris.nereids.CascadesContext; import org.apache.doris.nereids.trees.expressions.Cast; @@ -118,12 +116,6 @@ public static List prune(List partitionSlots, Expression partitionPr PartitionTableType partitionTableType) { partitionPredicate = TryEliminateUninterestedPredicates.rewrite( partitionPredicate, ImmutableSet.copyOf(partitionSlots), cascadesContext); - if (cascadesContext.getConnectContext().getSessionVariable().inToMinmaxParitionRewriteThreshold > 0 - && canRewriteToMinMax(idToPartitions)) { - double rangeLength = totalRangeLength(idToPartitions); - partitionPredicate = rewritePartitionPredicateForRangePartition( - partitionPredicate, rangeLength, cascadesContext); - } List evaluators = idToPartitions.entrySet() .stream() @@ -136,85 +128,6 @@ && canRewriteToMinMax(idToPartitions)) { return partitionPruner.prune(); } - private static boolean isRangePartition(Map idToPartitions) { - if (idToPartitions.isEmpty()) { - return false; - } - PartitionItem item = idToPartitions.values().iterator().next(); - return item instanceof RangePartitionItem; - } - - private static boolean canRewriteToMinMax(Map idToPartitions) { - if (isRangePartition(idToPartitions)) { - RangePartitionItem item = (RangePartitionItem) idToPartitions.values().iterator().next(); - if (item == null) { - return false; - } - PartitionKey k1 = item.getItems().upperEndpoint(); - if (k1.getKeys().size() != 1) { - return false; // multi keys or hive keys - } - LiteralExpr bound = k1.getKeys().get(0); - if (bound instanceof org.apache.doris.analysis.DateLiteral) { - return true; - } - } - return false; - } - - private static double getUpperBoundFromRangePartitionItem(RangePartitionItem item) { - org.apache.doris.analysis.DateLiteral d1 = - (org.apache.doris.analysis.DateLiteral) item.getItems().upperEndpoint().getKeys().get(0); - return d1.getDouble(); - } - - private static double getLowerBoundFromRangePartitionItem(RangePartitionItem item) { - org.apache.doris.analysis.DateLiteral d1 = - (org.apache.doris.analysis.DateLiteral) item.getItems().lowerEndpoint().getKeys().get(0); - return d1.getDouble(); - } - - // for range partition, total range is from the end of the first partition to the beginning of the last partition - private static double totalRangeLength(Map idToPartitions) { - if (idToPartitions.size() == 1) { - Long id = idToPartitions.keySet().iterator().next(); - RangePartitionItem item = (RangePartitionItem) idToPartitions.get(id); - double d1 = getUpperBoundFromRangePartitionItem(item); - double d2 = getLowerBoundFromRangePartitionItem(item); - return Math.max(1, d1 - d2); - } - - Long id = idToPartitions.keySet().iterator().next(); - RangePartitionItem firstItem = (RangePartitionItem) idToPartitions.get(id); - RangePartitionItem lastItem = firstItem; - double min = getLowerBoundFromRangePartitionItem(firstItem); - double max = getUpperBoundFromRangePartitionItem(lastItem); - for (Long pid : idToPartitions.keySet()) { - RangePartitionItem item = (RangePartitionItem) idToPartitions.get(pid); - double lower = getLowerBoundFromRangePartitionItem(item); - if (lower < min) { - min = lower; - firstItem = item; - } else { - double upper = getUpperBoundFromRangePartitionItem(item); - if (upper > max) { - max = upper; - lastItem = item; - } - } - } - double endOfFirstItem = getUpperBoundFromRangePartitionItem(firstItem); - double beginOfLastItem = getLowerBoundFromRangePartitionItem(lastItem); - return Math.max(1, beginOfLastItem - endOfFirstItem); - } - - private static Expression rewritePartitionPredicateForRangePartition(Expression partitionPredicate, double range, - CascadesContext ctx) { - partitionPredicate = OrToIn.INSTANCE.rewrite(partitionPredicate, null); - RewriteRangePartitionPredicate rewriter = new RewriteRangePartitionPredicate(range); - return partitionPredicate.accept(rewriter, ctx); - } - /** * convert partition item to partition evaluator */ diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteRangePartitionPredicate.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteRangePartitionPredicate.java deleted file mode 100644 index c464a8300481a8..00000000000000 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/RewriteRangePartitionPredicate.java +++ /dev/null @@ -1,82 +0,0 @@ -// Licensed to the Apache Software Foundation (ASF) under one -// or more contributor license agreements. See the NOTICE file -// distributed with this work for additional information -// regarding copyright ownership. The ASF licenses this file -// to you under the Apache License, Version 2.0 (the -// "License"); you may not use this file except in compliance -// with the License. You may obtain a copy of the License at -// -// http://www.apache.org/licenses/LICENSE-2.0 -// -// Unless required by applicable law or agreed to in writing, -// software distributed under the License is distributed on an -// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// KIND, either express or implied. See the License for the -// specific language governing permissions and limitations -// under the License. - -package org.apache.doris.nereids.rules.expression.rules; - -import org.apache.doris.nereids.CascadesContext; -import org.apache.doris.nereids.trees.expressions.And; -import org.apache.doris.nereids.trees.expressions.Expression; -import org.apache.doris.nereids.trees.expressions.GreaterThanEqual; -import org.apache.doris.nereids.trees.expressions.InPredicate; -import org.apache.doris.nereids.trees.expressions.LessThanEqual; -import org.apache.doris.nereids.trees.expressions.literal.DateLiteral; -import org.apache.doris.nereids.trees.expressions.literal.DateTimeLiteral; -import org.apache.doris.nereids.trees.expressions.literal.Literal; -import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; - -/** - * F = '2022-01-01 00:01:01' or F = 'date2' or ... - * => - * min_date <= F <= max_date - */ -public class RewriteRangePartitionPredicate extends - DefaultExpressionRewriter { - private final double rangeLength; - - public RewriteRangePartitionPredicate(double rangeLength) { - this.rangeLength = rangeLength; - } - - public static Expression rewrite(Expression expression, double rangeLength, CascadesContext ctx) { - return expression.accept(new RewriteRangePartitionPredicate(rangeLength), ctx); - } - - @Override - public Expression visitInPredicate(InPredicate inPredicate, CascadesContext ctx) { - if (inPredicate.getOptions().size() > 10) { - Expression opt0 = inPredicate.getOptions().get(0); - if (opt0 instanceof DateLiteral || opt0 instanceof DateTimeLiteral) { - Literal minOpt = (Literal) inPredicate.getOptions().get(0); - Double minVal = minOpt.getDouble(); - Literal maxOpt = (Literal) inPredicate.getOptions().get(0); - Double maxVal = maxOpt.getDouble(); - for (int i = 1; i < inPredicate.getOptions().size(); i++) { - Expression opt = inPredicate.getOptions().get(i); - if (!(opt instanceof Literal)) { - return inPredicate; - } - double optValue = ((Literal) opt).getDouble(); - if (optValue < minVal) { - minVal = optValue; - minOpt = (Literal) opt; - } else if (optValue > maxVal) { - maxVal = optValue; - maxOpt = (Literal) opt; - } - } - double inToMinMaxThreshold = ctx.getConnectContext() - .getSessionVariable().inToMinmaxParitionRewriteThreshold; - if (((maxVal - minVal) / rangeLength) < inToMinMaxThreshold) { - return new And( - new GreaterThanEqual(inPredicate.getCompareExpr(), minOpt), - new LessThanEqual(inPredicate.getCompareExpr(), maxOpt)); - } - } - } - return inPredicate; - } -} diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index 9f04193438ff84..fc6b972e1b8793 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -1136,9 +1136,6 @@ public void setMaxJoinNumberOfReorder(int maxJoinNumberOfReorder) { @VariableMgr.VarAttr(name = ENABLE_UNIQUE_KEY_PARTIAL_UPDATE, needForward = true) public boolean enableUniqueKeyPartialUpdate = false; - @VariableMgr.VarAttr(name = "in_to_minmax_parition_rewrite_threshold", needForward = true) - public double inToMinmaxParitionRewriteThreshold = 0.2; - // If this fe is in fuzzy mode, then will use initFuzzyModeVariables to generate some variables, // not the default value set in the code. public void initFuzzyModeVariables() {