From 0a24a1e820f7f957c7d694e22016839ca383f5ad Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 22 Jul 2024 13:25:02 +0800 Subject: [PATCH 1/3] speedup partition pruner --- .../main/java/org/apache/doris/analysis/Expr.java | 10 ++++++---- .../java/org/apache/doris/catalog/OlapTable.java | 4 +--- .../main/java/org/apache/doris/common/TreeNode.java | 2 +- .../expression/rules/OneRangePartitionEvaluator.java | 4 ++++ .../rules/expression/rules/PartitionPruner.java | 12 +++++++++--- .../trees/expressions/literal/DateV2Literal.java | 11 ++++++++--- .../java/org/apache/doris/qe/ConnectContext.java | 3 ++- .../java/org/apache/doris/qe/SessionVariable.java | 11 ++++++----- 8 files changed, 37 insertions(+), 20 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java index 89932e6ca08b21..6e9e713df9df9b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java +++ b/fe/fe-core/src/main/java/org/apache/doris/analysis/Expr.java @@ -57,6 +57,7 @@ import com.google.common.base.Objects; import com.google.common.base.Preconditions; import com.google.common.base.Strings; +import com.google.common.base.Suppliers; import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.gson.annotations.SerializedName; @@ -74,6 +75,7 @@ import java.util.Map; import java.util.Optional; import java.util.Set; +import java.util.function.Supplier; import java.util.stream.Collectors; /** @@ -277,7 +279,7 @@ public void setSelectivity() { protected Function fn; // Cached value of IsConstant(), set during analyze() and valid if isAnalyzed_ is true. - private boolean isConstant; + private Supplier isConstant = Suppliers.memoize(() -> false); // Flag to indicate whether to wrap this expr's toSql() in parenthesis. Set by parser. // Needed for properly capturing expr precedences in the SQL string. @@ -460,7 +462,7 @@ protected void analysisDone() { Preconditions.checkState(!isAnalyzed); // We need to compute the const-ness as the last step, since analysis may change // the result, e.g. by resolving function. - isConstant = isConstantImpl(); + isConstant = Suppliers.memoize(this::isConstantImpl); isAnalyzed = true; } @@ -1353,7 +1355,7 @@ public boolean isLiteral() { */ public final boolean isConstant() { if (isAnalyzed) { - return isConstant; + return isConstant.get(); } return isConstantImpl(); } @@ -2540,7 +2542,7 @@ public boolean refToCountStar() { // In this case, agg output must be materialized whether outer query block required or not. if (f.getFunctionName().getFunction().equals("count")) { for (Expr expr : funcExpr.children) { - if (expr.isConstant && !(expr instanceof LiteralExpr)) { + if (expr.isConstant() && !(expr instanceof LiteralExpr)) { return true; } } diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 92b6aa236ade81..63f4cee2c39cbc 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1989,9 +1989,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return Objects.hash(state, indexIdToMeta, indexNameToId, keysType, partitionInfo, idToPartition, - nameToPartition, defaultDistributionInfo, tempPartitions, bfColumns, bfFpp, colocateGroup, - hasSequenceCol, sequenceType, indexes, baseIndexId, tableProperty); + return (int) id; } public Column getBaseColumn(String columnName) { diff --git a/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java b/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java index b254dd9a1d0aa1..8e14d6e10ae0e7 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java +++ b/fe/fe-core/src/main/java/org/apache/doris/common/TreeNode.java @@ -33,7 +33,7 @@ */ public class TreeNode> { @SerializedName("children") - protected ArrayList children = Lists.newArrayList(); + protected ArrayList children = Lists.newArrayListWithCapacity(2); public NodeType getChild(int i) { return hasChild(i) ? children.get(i) : null; diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java index 4b7b940c9f68f8..8b01d2b8c672d2 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/OneRangePartitionEvaluator.java @@ -571,6 +571,10 @@ private EvaluateRangeResult mergeRanges( Map leftRanges = left.columnRanges; Map rightRanges = right.columnRanges; + if (leftRanges.equals(rightRanges)) { + return new EvaluateRangeResult(originResult, leftRanges, ImmutableList.of(left, right)); + } + Set slots = ImmutableSet.builder() .addAll(leftRanges.keySet()) .addAll(rightRanges.keySet()) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java index b65c0d2ec55990..b0b45077dcc21e 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/rules/expression/rules/PartitionPruner.java @@ -33,6 +33,7 @@ import org.apache.doris.nereids.trees.expressions.literal.NullLiteral; import org.apache.doris.nereids.trees.expressions.visitor.DefaultExpressionRewriter; import org.apache.doris.nereids.types.DateTimeType; +import org.apache.doris.nereids.util.Utils; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableList.Builder; @@ -125,14 +126,19 @@ public static List prune(List partitionSlots, Expression partitionPr "partitionPruningExpandThreshold", 10, sessionVariable -> sessionVariable.partitionPruningExpandThreshold); + partitionPredicate = OrToIn.INSTANCE.rewriteTree( + partitionPredicate, new ExpressionRewriteContext(cascadesContext)); + if (BooleanLiteral.TRUE.equals(partitionPredicate)) { + return Utils.fastToImmutableList(idToPartitions.keySet()); + } else if (Boolean.FALSE.equals(partitionPredicate) || partitionPredicate.isNullLiteral()) { + return ImmutableList.of(); + } + List evaluators = Lists.newArrayListWithCapacity(idToPartitions.size()); for (Entry kv : idToPartitions.entrySet()) { evaluators.add(toPartitionEvaluator( kv.getKey(), kv.getValue(), partitionSlots, cascadesContext, expandThreshold)); } - - partitionPredicate = OrToIn.INSTANCE.rewriteTree( - partitionPredicate, new ExpressionRewriteContext(cascadesContext)); PartitionPruner partitionPruner = new PartitionPruner(evaluators, partitionPredicate); //TODO: we keep default partition because it's too hard to prune it, we return false in canPrune(). return partitionPruner.prune(); diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java index d51a0eccc82cb7..873110e7779b90 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java @@ -17,7 +17,6 @@ package org.apache.doris.nereids.trees.expressions.literal; -import org.apache.doris.analysis.LiteralExpr; import org.apache.doris.catalog.Type; import org.apache.doris.nereids.exceptions.AnalysisException; import org.apache.doris.nereids.trees.expressions.Expression; @@ -25,12 +24,18 @@ import org.apache.doris.nereids.types.DateTimeV2Type; import org.apache.doris.nereids.types.DateV2Type; +import com.google.common.base.Suppliers; + import java.time.LocalDateTime; +import java.util.function.Supplier; /** * date v2 literal for nereids */ public class DateV2Literal extends DateLiteral { + private final Supplier LEGACY_LITERAL = Suppliers.memoize(() -> + new org.apache.doris.analysis.DateLiteral(year, month, day, Type.DATEV2) + ); public DateV2Literal(String s) throws AnalysisException { super(DateV2Type.INSTANCE, s); @@ -41,8 +46,8 @@ public DateV2Literal(long year, long month, long day) { } @Override - public LiteralExpr toLegacyLiteral() { - return new org.apache.doris.analysis.DateLiteral(year, month, day, Type.DATEV2); + public org.apache.doris.analysis.DateLiteral toLegacyLiteral() { + return LEGACY_LITERAL.get(); } @Override diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java index 71a10d2117165d..15e1926999057f 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/ConnectContext.java @@ -76,6 +76,7 @@ import com.google.common.collect.Lists; import com.google.common.collect.Maps; import com.google.common.collect.Sets; +import io.netty.util.concurrent.FastThreadLocal; import org.apache.logging.log4j.LogManager; import org.apache.logging.log4j.Logger; import org.json.JSONObject; @@ -99,7 +100,7 @@ // Use `volatile` to make the reference change atomic. public class ConnectContext { private static final Logger LOG = LogManager.getLogger(ConnectContext.class); - protected static ThreadLocal threadLocalInfo = new ThreadLocal<>(); + protected static FastThreadLocal threadLocalInfo = new FastThreadLocal<>(); private static final String SSL_PROTOCOL = "TLS"; diff --git a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java index b49c8a27bc6b5c..7b56e3611bf948 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/qe/SessionVariable.java @@ -3233,20 +3233,21 @@ public int getNthOptimizedPlan() { public Set getDisableNereidsRuleNames() { String checkPrivilege = RuleType.CHECK_PRIVILEGES.name(); String checkRowPolicy = RuleType.CHECK_ROW_POLICY.name(); - return Arrays.stream(disableNereidsRules.split(",[\\s]*")) - .map(rule -> rule.toUpperCase(Locale.ROOT)) - .filter(rule -> !StringUtils.equalsIgnoreCase(rule, checkPrivilege) + return Arrays.stream(disableNereidsRules.split(",")) + .map(rule -> rule.trim().toUpperCase(Locale.ROOT)) + .filter(rule -> !rule.isEmpty() + && !StringUtils.equalsIgnoreCase(rule, checkPrivilege) && !StringUtils.equalsIgnoreCase(rule, checkRowPolicy)) .collect(ImmutableSet.toImmutableSet()); } public BitSet getDisableNereidsRules() { BitSet bitSet = new BitSet(); - for (String ruleName : disableNereidsRules.split(",[\\s]*")) { + for (String ruleName : disableNereidsRules.split(",")) { + ruleName = ruleName.trim().toUpperCase(Locale.ROOT); if (ruleName.isEmpty()) { continue; } - ruleName = ruleName.toUpperCase(Locale.ROOT); RuleType ruleType = RuleType.valueOf(ruleName); if (ruleType == RuleType.CHECK_PRIVILEGES || ruleType == RuleType.CHECK_ROW_POLICY) { continue; From 2ad4f7822858d34e05cbc5a44b6dc1f166d59ff3 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 22 Jul 2024 13:38:45 +0800 Subject: [PATCH 2/3] speedup partition pruner --- .../nereids/trees/expressions/literal/DateV2Literal.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java index 873110e7779b90..1534d5518508fa 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java +++ b/fe/fe-core/src/main/java/org/apache/doris/nereids/trees/expressions/literal/DateV2Literal.java @@ -33,7 +33,7 @@ * date v2 literal for nereids */ public class DateV2Literal extends DateLiteral { - private final Supplier LEGACY_LITERAL = Suppliers.memoize(() -> + private final Supplier legacyLiteral = Suppliers.memoize(() -> new org.apache.doris.analysis.DateLiteral(year, month, day, Type.DATEV2) ); @@ -47,7 +47,7 @@ public DateV2Literal(long year, long month, long day) { @Override public org.apache.doris.analysis.DateLiteral toLegacyLiteral() { - return LEGACY_LITERAL.get(); + return legacyLiteral.get(); } @Override From 725b6137bb7ebe158f74f97b88637951f92327f1 Mon Sep 17 00:00:00 2001 From: 924060929 <924060929@qq.com> Date: Mon, 22 Jul 2024 16:38:47 +0800 Subject: [PATCH 3/3] speedup partition pruner --- .../src/main/java/org/apache/doris/catalog/OlapTable.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java index 63f4cee2c39cbc..a054eebfff4a1b 100644 --- a/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java +++ b/fe/fe-core/src/main/java/org/apache/doris/catalog/OlapTable.java @@ -1989,7 +1989,7 @@ public boolean equals(Object o) { @Override public int hashCode() { - return (int) id; + return (int) baseIndexId; } public Column getBaseColumn(String columnName) {