From 1792656535541837b4cc8d0ef40ed0bffded57bc Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Mon, 30 Sep 2024 14:21:45 +0000 Subject: [PATCH 001/149] Support UNNEST in decoupled mode --- pp | 42 +++++ .../druid/query/FilteredDataSource.java | 1 - qq | 81 +++++++++ .../calcite/planner/CalciteRulesManager.java | 5 +- .../planner/querygen/DruidQueryGenerator.java | 34 +++- .../calcite/rel/DruidCorrelateUnnestRel.java | 3 +- .../sql/calcite/rel/DruidJoinQueryRel.java | 4 +- .../druid/sql/calcite/rel/DruidQuery.java | 10 +- .../druid/sql/calcite/rel/DruidRel.java | 4 +- .../rule/logical/DruidLogicalRules.java | 3 +- .../sql/calcite/rule/logical/DruidUnnest.java | 128 +++++++++++++ .../calcite/rule/logical/DruidUnnestRule.java | 61 +++++++ .../calcite/rule/logical/DruidValuesRule.java | 1 - .../calcite/rule/logical/LogicalUnnest.java | 41 +++++ .../rule/logical/LogicalUnnestRule.java | 160 ++++++++++++++++ .../sql/calcite/rule/logical/Unnest.java | 66 +++++++ .../rule/logical/UnnestInputCleanupRule.java | 143 +++++++++++++++ .../sql/calcite/BaseCalciteQueryTest.java | 19 ++ .../sql/calcite/CalciteArraysQueryTest.java | 171 +++++++++--------- .../calcite/CalciteCorrelatedQueryTest.java | 5 + .../sql/calcite/CalciteJoinQueryTest.java | 18 +- .../calcite/CalciteNestedDataQueryTest.java | 2 + .../druid/sql/calcite/DecoupledExtension.java | 15 ++ ...coupledPlanningCalciteArraysQueryTest.java | 37 ++++ ...ledPlanningCalciteCorrelatedQueryTest.java | 37 ++++ ...ledPlanningCalciteNestedDataQueryTest.java | 37 ++++ .../sql/calcite/DecoupledTestConfig.java | 45 ++++- .../druid/sql/calcite/NotYetSupported.java | 6 +- .../unnest.iq | 89 +++++++++ ...UnnestExtractionFn@NullHandling=default.iq | 81 +++++++++ ...testUnnestExtractionFn@NullHandling=sql.iq | 72 ++++++++ ...tUnnestWithFilters@NullHandling=default.iq | 92 ++++++++++ .../testUnnestWithFilters@NullHandling=sql.iq | 94 ++++++++++ ...hFiltersInnerLimit@NullHandling=default.iq | 95 ++++++++++ ...tWithFiltersInnerLimit@NullHandling=sql.iq | 97 ++++++++++ ...rsInsideAndOutside@NullHandling=default.iq | 112 ++++++++++++ ...iltersInsideAndOutside@NullHandling=sql.iq | 116 ++++++++++++ ...essionInInnerQuery@NullHandling=default.iq | 92 ++++++++++ ...ExpressionInInnerQuery@NullHandling=sql.iq | 94 ++++++++++ .../testUnnestWithGroupByOnExpression.iq | 98 ++++++++++ ...essionInInnerQuery@NullHandling=default.iq | 92 ++++++++++ ...ExpressionInInnerQuery@NullHandling=sql.iq | 94 ++++++++++ ...sOfUnnestedColumns@NullHandling=default.iq | 75 ++++++++ ...tionsOfUnnestedColumns@NullHandling=sql.iq | 77 ++++++++ ...rsOnUnnestedColumn@NullHandling=default.iq | 71 ++++++++ ...iltersOnUnnestedColumn@NullHandling=sql.iq | 73 ++++++++ ...ilter@all_disabled@NullHandling=default.iq | 36 ++-- ...ithFilter@all_disabled@NullHandling=sql.iq | 34 ++-- ...Filter@all_enabled@NullHandling=default.iq | 36 ++-- ...WithFilter@all_enabled@NullHandling=sql.iq | 34 ++-- ...WithFilter@default@NullHandling=default.iq | 36 ++-- ...urceWithFilter@default@NullHandling=sql.iq | 34 ++-- ...ue-column_disabled@NullHandling=default.iq | 36 ++-- ...-value-column_disabled@NullHandling=sql.iq | 34 ++-- ...-rewrites-disabled@NullHandling=default.iq | 36 ++-- ...lter-rewrites-disabled@NullHandling=sql.iq | 34 ++-- ...er@filter-rewrites@NullHandling=default.iq | 36 ++-- ...Filter@filter-rewrites@NullHandling=sql.iq | 34 ++-- ...ter@join-to-filter@NullHandling=default.iq | 36 ++-- ...hFilter@join-to-filter@NullHandling=sql.iq | 34 ++-- ...estGroupByWithLiteralInSubqueryGrouping.iq | 2 +- 61 files changed, 2878 insertions(+), 407 deletions(-) create mode 100644 pp create mode 100644 qq create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidUnnest.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidUnnestRule.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/LogicalUnnest.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/LogicalUnnestRule.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/Unnest.java create mode 100644 sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/UnnestInputCleanupRule.java create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteArraysQueryTest.java create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteCorrelatedQueryTest.java create mode 100644 sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteNestedDataQueryTest.java create mode 100644 sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/unnest.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=default.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=sql.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFilters@NullHandling=default.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFilters@NullHandling=sql.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInnerLimit@NullHandling=default.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInnerLimit@NullHandling=sql.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInsideAndOutside@NullHandling=default.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInsideAndOutside@NullHandling=sql.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersWithExpressionInInnerQuery@NullHandling=default.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersWithExpressionInInnerQuery@NullHandling=sql.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithGroupByOnExpression.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithInFiltersWithExpressionInInnerQuery@NullHandling=default.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithInFiltersWithExpressionInInnerQuery@NullHandling=sql.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns@NullHandling=default.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns@NullHandling=sql.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithNotFiltersOnUnnestedColumn@NullHandling=default.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithNotFiltersOnUnnestedColumn@NullHandling=sql.iq diff --git a/pp b/pp new file mode 100644 index 000000000000..8f7b02aa06b6 --- /dev/null +++ b/pp @@ -0,0 +1,42 @@ +dec +{ + "queryType" : "scan", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + } ], + "resultFormat" : "compactedList", + "filter" : { + "type" : "in", + "dimension" : "m1", + "values" : [ "4.0", "5.0" ] + }, + "columns" : [ "dim3", "v0" ], + "context" : { + "debug" : true, + "defaultTimeout" : 300000, + "maxScatterGatherBytes" : 9223372036854775807, + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false, + "vectorSize" : 2, + "vectorize" : "force", + "vectorizeVirtualColumns" : "force" + }, + "columnTypes" : [ "STRING", "ARRAY" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} diff --git a/processing/src/main/java/org/apache/druid/query/FilteredDataSource.java b/processing/src/main/java/org/apache/druid/query/FilteredDataSource.java index 2d3104a0041f..1644b3218d38 100644 --- a/processing/src/main/java/org/apache/druid/query/FilteredDataSource.java +++ b/processing/src/main/java/org/apache/druid/query/FilteredDataSource.java @@ -51,7 +51,6 @@ */ public class FilteredDataSource implements DataSource { - private final DataSource base; private final DimFilter filter; diff --git a/qq b/qq new file mode 100644 index 000000000000..902733c947eb --- /dev/null +++ b/qq @@ -0,0 +1,81 @@ +deec +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "query", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "'a'", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "filter" : { + "type" : "selector", + "dimension" : "dim2", + "value" : "a" + }, + "columns" : [ "__time", "cnt", "d1", "d2", "dim1", "dim3", "dim4", "dim5", "dim6", "f1", "f2", "l1", "l2", "m1", "m2", "unique_dim1", "v0" ], + "context" : { + "debug" : true, + "defaultTimeout" : 300000, + "maxScatterGatherBytes" : 9223372036854775807, + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false, + "vectorSize" : 2, + "vectorize" : "force", + "vectorizeVirtualColumns" : "force" + }, + "columnTypes" : [ "LONG", "LONG", "DOUBLE", "DOUBLE", "STRING", "STRING", "STRING", "STRING", "STRING", "FLOAT", "FLOAT", "LONG", "LONG", "FLOAT", "DOUBLE", "COMPLEX", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : null + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest" ], + "context" : { + "debug" : true, + "defaultTimeout" : 300000, + "maxScatterGatherBytes" : 9223372036854775807, + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false, + "vectorSize" : 2, + "vectorize" : "force", + "vectorizeVirtualColumns" : "force" + }, + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java index a7b4ba212039..d6dd1310e6c5 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/CalciteRulesManager.java @@ -71,6 +71,8 @@ import org.apache.druid.sql.calcite.rule.logical.DruidAggregateRemoveRedundancyRule; import org.apache.druid.sql.calcite.rule.logical.DruidJoinRule; import org.apache.druid.sql.calcite.rule.logical.DruidLogicalRules; +import org.apache.druid.sql.calcite.rule.logical.LogicalUnnestRule; +import org.apache.druid.sql.calcite.rule.logical.UnnestInputCleanupRule; import org.apache.druid.sql.calcite.run.EngineFeature; import org.apache.druid.sql.hook.DruidHook; @@ -280,7 +282,8 @@ private Program buildDecoupledLogicalOptimizationProgram(PlannerContext plannerC builder.addRuleInstance(CoreRules.UNION_MERGE); builder.addRuleInstance(JoinExtractFilterRule.Config.DEFAULT.toRule()); builder.addRuleInstance(FilterIntoJoinRuleConfig.DEFAULT.withPredicate(DruidJoinRule::isSupportedPredicate).toRule()); - + builder.addRuleInstance(new LogicalUnnestRule()); + builder.addRuleInstance(new UnnestInputCleanupRule()); return Programs.of(builder.build(), true, DefaultRelMetadataProvider.INSTANCE); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/planner/querygen/DruidQueryGenerator.java b/sql/src/main/java/org/apache/druid/sql/calcite/planner/querygen/DruidQueryGenerator.java index 28de4e8c2554..511065b2c4f4 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/planner/querygen/DruidQueryGenerator.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/planner/querygen/DruidQueryGenerator.java @@ -28,7 +28,11 @@ import org.apache.calcite.rel.core.Window; import org.apache.calcite.rex.RexBuilder; import org.apache.druid.error.DruidException; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.FilteredDataSource; import org.apache.druid.query.QueryDataSource; +import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.planner.querygen.DruidQueryGenerator.PDQVertexFactory.PDQVertex; import org.apache.druid.sql.calcite.planner.querygen.SourceDescProducer.SourceDesc; @@ -232,6 +236,11 @@ boolean forceSubQuery(SourceDesc sourceDesc) } return this == RIGHT; } + + boolean filteredDatasourceAllowed() + { + return this == NONE; + } } /** @@ -401,7 +410,13 @@ public SourceDesc unwrapSourceDesc() if (canUnwrapSourceDesc()) { DruidQuery q = buildQuery(false); SourceDesc origInput = getSource(); - return new SourceDesc(origInput.dataSource, q.getOutputRowSignature()); + DataSource dataSource; + if (q.getFilter() == null) { + dataSource = origInput.dataSource; + } else { + dataSource = makeFilteredDataSource(origInput, q.getFilter()); + } + return new SourceDesc(dataSource, q.getOutputRowSignature()); } throw DruidException.defensive("Can't unwrap source of vertex[%s]", partialDruidQuery); } @@ -415,14 +430,29 @@ public boolean canUnwrapSourceDesc() if (partialDruidQuery.stage() == Stage.SCAN) { return true; } + if (jst.filteredDatasourceAllowed() && partialDruidQuery.stage() == PartialDruidQuery.Stage.WHERE_FILTER) { + return true; + } if (partialDruidQuery.stage() == PartialDruidQuery.Stage.SELECT_PROJECT && - partialDruidQuery.getWhereFilter() == null && + (jst.filteredDatasourceAllowed() || partialDruidQuery.getWhereFilter() == null) && partialDruidQuery.getSelectProject().isMapping()) { return true; } return false; } } + } + + /** + * This method should not live here. + * + * The fact that {@link Filtration} have to be run on the filter is out-of scope here. + */ + public static FilteredDataSource makeFilteredDataSource(SourceDesc sd, DimFilter filter) + { + Filtration filtration = Filtration.create(filter).optimizeFilterOnly(sd.rowSignature); + DimFilter newFilter = filtration.getDimFilter(); + return FilteredDataSource.create(sd.dataSource, newFilter); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidCorrelateUnnestRel.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidCorrelateUnnestRel.java index e5c403f1e1de..f3b45506a2f5 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidCorrelateUnnestRel.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidCorrelateUnnestRel.java @@ -577,7 +577,7 @@ public void visitList( /** * Shuttle that replaces correlating variables with regular field accesses to the left-hand side. */ - private static class CorrelatedFieldAccessToInputRef extends RexShuttle + public static class CorrelatedFieldAccessToInputRef extends RexShuttle { private final CorrelationId correlationId; @@ -595,7 +595,6 @@ public RexNode visitFieldAccess(final RexFieldAccess fieldAccess) return new RexInputRef(fieldAccess.getField().getIndex(), fieldAccess.getType()); } } - return super.visitFieldAccess(fieldAccess); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidJoinQueryRel.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidJoinQueryRel.java index e62b38696109..5615da3344fe 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidJoinQueryRel.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidJoinQueryRel.java @@ -423,7 +423,7 @@ public static boolean computeRightRequiresSubquery(final PlannerContext plannerC && DruidRels.druidTableIfLeafRel(right).filter(table -> table.getDataSource().isGlobal()).isPresent()); } - static Set findExistingJoinPrefixes(DataSource... dataSources) + public static Set findExistingJoinPrefixes(DataSource... dataSources) { final ArrayList copy = new ArrayList<>(Arrays.asList(dataSources)); @@ -442,7 +442,7 @@ static Set findExistingJoinPrefixes(DataSource... dataSources) * Returns a Pair of "rightPrefix" (for JoinDataSource) and the signature of rows that will result from * applying that prefix. */ - static Pair computeJoinRowSignature( + public static Pair computeJoinRowSignature( final RowSignature leftSignature, final RowSignature rightSignature, final Set prefixes diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java index 3ce33e722452..ce08e8860689 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidQuery.java @@ -792,7 +792,10 @@ VirtualColumns getVirtualColumns(final boolean includeDimensions) public static List getAllFiltersUnderDataSource(DataSource d, List dimFilterList) { if (d instanceof FilteredDataSource) { - dimFilterList.add(((FilteredDataSource) d).getFilter()); + DimFilter filter = ((FilteredDataSource) d).getFilter(); + if (filter != null) { + dimFilterList.add(filter); + } } for (DataSource ds : d.getChildren()) { dimFilterList.addAll(getAllFiltersUnderDataSource(ds, dimFilterList)); @@ -1741,4 +1744,9 @@ private RowSignature buildRowSignature(final VirtualColumns virtualColumns, fina } return builder.build(); } + + public DimFilter getFilter() + { + return filter; + } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidRel.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidRel.java index 8db108fbb940..3deff6865331 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidRel.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/DruidRel.java @@ -148,7 +148,9 @@ private RelNode visitNode(RelNode other) } if (other instanceof DruidRel) { DruidRel druidRel = (DruidRel) other; - return druidRel.getPartialDruidQuery().leafRel(); + if (druidRel.getPartialDruidQuery() != null && druidRel.getPartialDruidQuery().leafRel() != null) { + return druidRel.getPartialDruidQuery().leafRel(); + } } return other; } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidLogicalRules.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidLogicalRules.java index a7d1a2c8c689..7d53aeb6fe35 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidLogicalRules.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidLogicalRules.java @@ -105,7 +105,8 @@ public List rules() Convention.NONE, DruidLogicalConvention.instance(), DruidJoinRule.class.getSimpleName() - ) + ), + DruidUnnestRule.INSTANCE ) ); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidUnnest.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidUnnest.java new file mode 100644 index 000000000000..67170ffcd61e --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidUnnest.java @@ -0,0 +1,128 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.rule.logical; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.UnnestDataSource; +import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.segment.VirtualColumn; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.sql.calcite.expression.DruidExpression; +import org.apache.druid.sql.calcite.expression.Expressions; +import org.apache.druid.sql.calcite.filtration.Filtration; +import org.apache.druid.sql.calcite.planner.Calcites; +import org.apache.druid.sql.calcite.planner.PlannerContext; +import org.apache.druid.sql.calcite.planner.querygen.SourceDescProducer; +import org.apache.druid.sql.calcite.rel.DruidJoinQueryRel; +import org.apache.druid.sql.calcite.rel.logical.DruidLogicalNode; + +import java.util.List; + +public class DruidUnnest extends Unnest implements DruidLogicalNode, SourceDescProducer +{ + protected DruidUnnest(RelOptCluster cluster, RelTraitSet traits, RelNode input, RexNode unnestExpr, + RelDataType rowType, RexNode condition) + { + super(cluster, traits, input, unnestExpr, rowType, condition); + } + + @Override + protected RelNode copy(RelTraitSet traitSet, RelNode input) + { + return new DruidUnnest(getCluster(), traitSet, input, unnestExpr, rowType, filter); + } + + @Override + public SourceDesc getSourceDesc(PlannerContext plannerContext, List sources) + { + SourceDesc inputDesc = sources.get(0); + + RowSignature outputRowSignature = computeRowOutputSignature(inputDesc); + + RowSignature filterRowSignature = RowSignature.builder().add( + outputRowSignature.getColumnName(outputRowSignature.size() - 1), + outputRowSignature.getColumnType(outputRowSignature.size() - 1).get() + ).build(); + + VirtualColumn virtualColumn = buildUnnestVirtualColumn( + plannerContext, + inputDesc, + filterRowSignature.getColumnName(0) + ); + + DimFilter dimFilter = buildDimFilter(plannerContext, filterRowSignature); + DataSource dataSource = UnnestDataSource.create(inputDesc.dataSource, virtualColumn, dimFilter); + return new SourceDesc(dataSource, outputRowSignature); + } + + private DimFilter buildDimFilter(PlannerContext plannerContext, RowSignature filterRowSignature) + { + if (filter == null) { + return null; + } + DimFilter dimFilter = Expressions.toFilter( + plannerContext, + filterRowSignature, + null, + filter + ); + return Filtration.create(dimFilter).optimizeFilterOnly(filterRowSignature).getDimFilter(); + } + + private VirtualColumn buildUnnestVirtualColumn(PlannerContext plannerContext, SourceDesc inputDesc, String columnName) + { + final DruidExpression expressionToUnnest = Expressions.toDruidExpression( + plannerContext, + inputDesc.rowSignature, + unnestExpr + ); + + VirtualColumn virtualColumn = expressionToUnnest.toVirtualColumn( + columnName, + Calcites.getColumnTypeForRelDataType( + unnestExpr.getType() + ), + plannerContext.getExpressionParser() + ); + return virtualColumn; + } + + private RowSignature computeRowOutputSignature(SourceDesc inputDesc) + { + return DruidJoinQueryRel.computeJoinRowSignature( + inputDesc.rowSignature, + RowSignature.builder().add( + "unnest", + Calcites.getColumnTypeForRelDataType(getUnnestedType()) + ).build(), + DruidJoinQueryRel.findExistingJoinPrefixes(inputDesc.dataSource) + ).rhs; + } + + private RelDataType getUnnestedType() + { + return rowType.getFieldList().get(rowType.getFieldCount() - 1).getType(); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidUnnestRule.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidUnnestRule.java new file mode 100644 index 000000000000..257bb7ca38f5 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidUnnestRule.java @@ -0,0 +1,61 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.rule.logical; + +import org.apache.calcite.plan.Convention; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.convert.ConverterRule; +import org.apache.druid.sql.calcite.rel.logical.DruidLogicalConvention; + +public class DruidUnnestRule extends ConverterRule +{ + private static Config CONFIG = Config.INSTANCE.withConversion( + LogicalUnnest.class, + Convention.NONE, + DruidLogicalConvention.instance(), + DruidUnnestRule.class.getSimpleName() + ); + + public static final DruidUnnestRule INSTANCE = new DruidUnnestRule(CONFIG); + + private DruidUnnestRule(Config config) + { + super(config); + } + + @Override + public RelNode convert(RelNode rel) + { + LogicalUnnest unnest = (LogicalUnnest) rel; + RelTraitSet newTrait = unnest.getTraitSet().replace(DruidLogicalConvention.instance()); + return new DruidUnnest( + rel.getCluster(), + newTrait, + convert( + unnest.getInput(), + DruidLogicalConvention.instance() + ), + unnest.getUnnestExpr(), + unnest.getRowType(), + unnest.filter + ); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidValuesRule.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidValuesRule.java index 5fca4a229670..4c0080558a50 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidValuesRule.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/DruidValuesRule.java @@ -32,7 +32,6 @@ */ public class DruidValuesRule extends ConverterRule { - public DruidValuesRule( Class clazz, RelTrait in, diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/LogicalUnnest.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/LogicalUnnest.java new file mode 100644 index 000000000000..e23d4c6cd3ba --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/LogicalUnnest.java @@ -0,0 +1,41 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.rule.logical; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; + +public class LogicalUnnest extends Unnest +{ + protected LogicalUnnest(RelOptCluster cluster, RelTraitSet traits, RelNode input, RexNode unnestExpr, + RelDataType rowType, RexNode condition) + { + super(cluster, traits, input, unnestExpr, rowType, condition); + } + + @Override + protected RelNode copy(RelTraitSet traitSet, RelNode input) + { + return new LogicalUnnest(getCluster(), traitSet, input, unnestExpr, rowType, filter); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/LogicalUnnestRule.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/LogicalUnnestRule.java new file mode 100644 index 000000000000..662cf76f5ac0 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/LogicalUnnestRule.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.rule.logical; + +import com.google.common.collect.Iterables; +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Filter; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.core.Uncollect; +import org.apache.calcite.rel.logical.LogicalCorrelate; +import org.apache.calcite.rel.logical.LogicalValues; +import org.apache.calcite.rel.rules.SubstitutionRule; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.tools.RelBuilder; +import org.apache.druid.error.DruidException; +import org.apache.druid.sql.calcite.rel.DruidCorrelateUnnestRel; + +/** + * Recognizes a LogicalUnnest operation in the plan. + * + * Matches on the layout: + * + * 
+ *   LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{4}])
+ *     RelNodeSubtree
+ *     Uncollect
+ *       LogicalProject(arrayLongNulls=[$cor0.arrayLongNulls])
+ *         LogicalValues(tuples=[[{ 0 }]])
+ *
+ * + * Translates it to use a {@link LogicalUnnest} like: + * + * 
+ *   LogicalUnnest(unnestExpr=[$cor0.arrayLongNulls])
+ *     RelNodeSubtree
+ *
+ * + * It raises an error for cases when {@link LogicalCorrelate} can't be + * translated as those are currently unsupported in Druid. + */ +public class LogicalUnnestRule extends RelOptRule implements SubstitutionRule +{ + public LogicalUnnestRule() + { + super(operand(LogicalCorrelate.class, any())); + } + + @Override + public boolean autoPruneOld() + { + return true; + } + + @Override + public void onMatch(RelOptRuleCall call) + { + LogicalCorrelate cor = call.rel(0); + UnnestConfiguration unnestConfig = unwrapUnnestConfigurationExpression(cor.getRight().stripped()); + + if (unnestConfig == null) { + throw DruidException.defensive("Couldn't process possible unnest for reltree: \n%s", RelOptUtil.toString(cor)); + } + + unnestConfig.expr = new DruidCorrelateUnnestRel.CorrelatedFieldAccessToInputRef(cor.getCorrelationId()) + .apply(unnestConfig.expr); + + RelBuilder builder = call.builder(); + builder.push(cor.getLeft()); + RelNode newNode = builder.push( + new LogicalUnnest( + cor.getCluster(), + cor.getTraitSet(), + builder.build(), + unnestConfig.expr, + cor.getRowType(), + unnestConfig.condition + ) + ).build(); + call.transformTo(newNode); + } + + private static class UnnestConfiguration + { + public RexNode expr; + private RexNode condition; + + public UnnestConfiguration(RexNode unnestExpression, RexNode condition) + { + this.expr = unnestExpression; + this.condition = condition; + } + + public static UnnestConfiguration ofExpression(RexNode unnestExpression) + { + return new UnnestConfiguration(unnestExpression, null); + } + + public UnnestConfiguration withFilter(RexNode condition) + { + return new UnnestConfiguration(expr, condition); + } + } + + private UnnestConfiguration unwrapUnnestConfigurationExpression(RelNode rel) + { + rel = rel.stripped(); + if (rel instanceof Filter) { + Filter filter = (Filter) rel; + UnnestConfiguration conf = unwrapUnnestConfigurationExpression(filter.getInput()); + if (conf != null) { + return conf.withFilter(filter.getCondition()); + } + } + if (rel instanceof Uncollect) { + Uncollect uncollect = (Uncollect) rel; + if (!uncollect.withOrdinality) { + return unwrapProjectExpression(uncollect.getInput()); + } + } + return null; + } + + private UnnestConfiguration unwrapProjectExpression(RelNode rel) + { + rel = rel.stripped(); + if (rel instanceof Project) { + Project project = (Project) rel; + if (isValues(project.getInput().stripped())) { + return UnnestConfiguration.ofExpression(Iterables.getOnlyElement(project.getProjects())); + } + } + return null; + } + + private boolean isValues(RelNode rel) + { + rel = rel.stripped(); + return (rel instanceof LogicalValues); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/Unnest.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/Unnest.java new file mode 100644 index 000000000000..58680b053e83 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/Unnest.java @@ -0,0 +1,66 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.rule.logical; + +import org.apache.calcite.plan.RelOptCluster; +import org.apache.calcite.plan.RelTraitSet; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.RelWriter; +import org.apache.calcite.rel.SingleRel; +import org.apache.calcite.rel.type.RelDataType; +import org.apache.calcite.rex.RexNode; + +import java.util.List; + +public abstract class Unnest extends SingleRel +{ + protected final RexNode unnestExpr; + protected final RexNode filter; + + protected Unnest(RelOptCluster cluster, RelTraitSet traits, RelNode input, RexNode unnestExpr, + RelDataType rowType, RexNode condition) + { + super(cluster, traits, input); + this.unnestExpr = unnestExpr; + this.rowType = rowType; + this.filter = condition; + } + + public final RexNode getUnnestExpr() + { + return unnestExpr; + } + + @Override + public RelWriter explainTerms(RelWriter pw) + { + return super.explainTerms(pw) + .item("unnestExpr", unnestExpr) + .itemIf("filter", filter, filter != null); + } + + @Override + public final RelNode copy(RelTraitSet traitSet, List inputs) + { + return copy(traitSet, inputs.get(0)); + } + + protected abstract RelNode copy(RelTraitSet traitSet, RelNode input); +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/UnnestInputCleanupRule.java b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/UnnestInputCleanupRule.java new file mode 100644 index 000000000000..baae4c4c9f79 --- /dev/null +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rule/logical/UnnestInputCleanupRule.java @@ -0,0 +1,143 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite.rule.logical; + +import org.apache.calcite.plan.RelOptRule; +import org.apache.calcite.plan.RelOptRuleCall; +import org.apache.calcite.plan.RelOptUtil.InputFinder; +import org.apache.calcite.rel.RelNode; +import org.apache.calcite.rel.core.Project; +import org.apache.calcite.rel.rules.SubstitutionRule; +import org.apache.calcite.rex.RexInputRef; +import org.apache.calcite.rex.RexNode; +import org.apache.calcite.rex.RexShuttle; +import org.apache.calcite.util.ImmutableBitSet; +import org.apache.druid.error.DruidException; +import java.util.ArrayList; +import java.util.List; + +/** + * Makes tweaks to LogicalUnnest input. + * + * Removes any MV_TO_ARRAY call if its present for the input of the + * {@link LogicalUnnest}. + * + */ +public class UnnestInputCleanupRule extends RelOptRule implements SubstitutionRule +{ + public UnnestInputCleanupRule() + { + super( + operand( + LogicalUnnest.class, + operand(Project.class, any()) + ) + ); + } + + @Override + public void onMatch(RelOptRuleCall call) + { + LogicalUnnest unnest = call.rel(0); + Project project = call.rel(1); + + ImmutableBitSet input = InputFinder.analyze(unnest.unnestExpr).build(); + if (input.isEmpty()) { + throw DruidException.defensive("Found an unbound unnest expression."); + } + + if (!(unnest.unnestExpr instanceof RexInputRef)) { + // could be supported; but is there a need? + return; + } + if (input.cardinality() != 1) { + return; + } + + int inputIndex = input.nextSetBit(0); + + List projects = new ArrayList<>(project.getProjects()); + RexNode unnestInput = projects.get(inputIndex); + + projects.set( + inputIndex, + call.builder().getRexBuilder().makeInputRef(project.getInput(), 0) + ); + + RexNode newUnnestExpr = unnestInput.accept(new ExpressionPullerRexShuttle(projects, inputIndex)); + + if (projects.size() != project.getProjects().size()) { + // lets leave this for later + return; + } + + + RelNode newInputRel = call.builder() + .push(project.getInput()) + .project(projects) + .build(); + + + RelNode newUnnest = new LogicalUnnest( + unnest.getCluster(), unnest.getTraitSet(), newInputRel, newUnnestExpr, + unnest.getRowType(), unnest.filter + ); + call.transformTo(newUnnest); + call.getPlanner().prune(unnest); + } + + /** + * Pulls an expression thru a {@link Project}. + * + * May add new projections to the passed mutable list. + */ + private static class ExpressionPullerRexShuttle extends RexShuttle + { + private final List projects; + private int replaceableIndex; + + private ExpressionPullerRexShuttle(List projects, int replaceableIndex) + { + this.projects = projects; + this.replaceableIndex = replaceableIndex; + } + + @Override + public RexNode visitInputRef(RexInputRef inputRef) + { + int newIndex = projects.indexOf(inputRef); + if (newIndex < 0) { + if (replaceableIndex >= 0) { + newIndex = replaceableIndex; + projects.set(replaceableIndex, inputRef); + replaceableIndex = -1; + } else { + newIndex = projects.size(); + projects.add(inputRef); + } + } + if (newIndex == inputRef.getIndex()) { + return inputRef; + } else { + return new RexInputRef(newIndex, inputRef.getType()); + } + } + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 4b26af38adb5..42cf41ff32a2 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -560,6 +560,25 @@ public static ExpressionVirtualColumn expressionVirtualColumn( return new ExpressionVirtualColumn(name, expression, outputType, CalciteTests.createExprMacroTable()); } + /** + * Optionally updates the VC defintion for the one planned by the decoupled planner. + * + * Compared to original plans; decoupled planner: + * * moves the mv_to_array into the VC + * * the type is an ARRAY + */ + public ExpressionVirtualColumn nestedExpressionVirtualColumn( + String name, + String expression, + ColumnType outputType) + { + if (testBuilder().isDecoupledMode()) { + expression = StringUtils.format("mv_to_array(%s)", expression); + outputType = ColumnType.ofArray(outputType); + } + return expressionVirtualColumn(name, expression, outputType); + } + public static JoinDataSource join( DataSource left, DataSource right, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index b2f87b3d6335..531da0f8d66b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -68,6 +68,9 @@ import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinType; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; +import org.apache.druid.sql.calcite.DecoupledTestConfig.IgnoreQueriesReason; +import org.apache.druid.sql.calcite.DecoupledTestConfig.QuidemTestCaseReason; +import org.apache.druid.sql.calcite.NotYetSupported.Modes; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.CalciteTests; import org.apache.druid.sql.calcite.util.SqlTestFramework.StandardComponentSupplier; @@ -3885,6 +3888,7 @@ public void testArrayAggGroupByArrayContainsSubquery() } + @NotYetSupported(Modes.UNNEST_INLINED) @Test public void testUnnestInline() { @@ -3919,6 +3923,7 @@ public void testUnnestInline() ); } + @NotYetSupported(Modes.UNNEST_INLINED) @Test public void testUnnestInlineWithCount() { @@ -3949,6 +3954,7 @@ public void testUnnestInlineWithCount() ); } + @NotYetSupported(Modes.UNNEST_RESULT_MISMATCH) @Test public void testUnnest() { @@ -3959,7 +3965,7 @@ public void testUnnest() Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -4333,6 +4339,7 @@ public void testUnnestArrayColumnsDoubleNulls() ); } + @NotYetSupported(Modes.UNNEST_RESULT_MISMATCH) @Test public void testUnnestTwice() { @@ -4356,11 +4363,7 @@ public void testUnnestTwice() ), null ), - expressionVirtualColumn( - "_j0.unnest", - "\"dim3\"", - ColumnType.STRING - ), + nestedExpressionVirtualColumn("_j0.unnest", "\"dim3\"", ColumnType.STRING), null ) ) @@ -4513,11 +4516,7 @@ public void testUnnestTwiceWithFiltersAndExpressions() ), in("j0.unnest", ImmutableList.of("1", "2")) ), - expressionVirtualColumn( - "_j0.unnest", - "\"dim3\"", - ColumnType.STRING - ), + nestedExpressionVirtualColumn("_j0.unnest", "\"dim3\"", ColumnType.STRING), new LikeDimFilter("_j0.unnest", "_", null, null) ) ) @@ -4553,6 +4552,7 @@ public void testUnnestTwiceWithFiltersAndExpressions() } + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @Test public void testUnnestThriceWithFiltersOnDimAndUnnestCol() { @@ -4570,11 +4570,7 @@ public void testUnnestThriceWithFiltersOnDimAndUnnestCol() FilteredDataSource.create( UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE5), - expressionVirtualColumn( - "j0.unnest", - "\"dimMultivalEnumerated\"", - ColumnType.STRING - ), + nestedExpressionVirtualColumn("j0.unnest", "\"dimMultivalEnumerated\"", ColumnType.STRING), null ), and( @@ -4584,17 +4580,9 @@ public void testUnnestThriceWithFiltersOnDimAndUnnestCol() equality("j0.unnest", "Baz", ColumnType.STRING) ) ), - expressionVirtualColumn( - "_j0.unnest", - "\"dimMultivalEnumerated\"", - ColumnType.STRING - ), null - ), - expressionVirtualColumn( - "__j0.unnest", - "\"dimMultivalEnumerated\"", - ColumnType.STRING + nestedExpressionVirtualColumn("_j0.unnest", "\"dimMultivalEnumerated\"", ColumnType.STRING), null ), + nestedExpressionVirtualColumn("__j0.unnest", "\"dimMultivalEnumerated\"", ColumnType.STRING), null ) ) @@ -4649,6 +4637,8 @@ public void testUnnestThriceWithFiltersOnDimAndUnnestCol() ) ); } + + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @Test public void testUnnestThriceWithFiltersOnDimAndAllUnnestColumns() { @@ -4800,11 +4790,7 @@ public void testUnnestThriceWithFiltersOnDimAndUnnestColumnsORCombinations() FilteredDataSource.create( UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE5), - expressionVirtualColumn( - "j0.unnest", - "\"dimMultivalEnumerated\"", - ColumnType.STRING - ), + nestedExpressionVirtualColumn("j0.unnest", "\"dimMultivalEnumerated\"", ColumnType.STRING), null ), NullHandling.sqlCompatible() ? equality("dimZipf", "27", ColumnType.LONG) : range( @@ -4816,11 +4802,7 @@ public void testUnnestThriceWithFiltersOnDimAndUnnestColumnsORCombinations() false ) ), - expressionVirtualColumn( - "_j0.unnest", - "\"dimMultivalEnumerated\"", - ColumnType.STRING - ), + nestedExpressionVirtualColumn("_j0.unnest", "\"dimMultivalEnumerated\"", ColumnType.STRING), null ), or( @@ -4828,11 +4810,7 @@ public void testUnnestThriceWithFiltersOnDimAndUnnestColumnsORCombinations() equality("_j0.unnest", "Hello", ColumnType.STRING) ) // (j0.unnest = Baz || _j0.unnest = Hello) ), - expressionVirtualColumn( - "__j0.unnest", - "\"dimMultivalEnumerated\"", - ColumnType.STRING - ), + nestedExpressionVirtualColumn("__j0.unnest", "\"dimMultivalEnumerated\"", ColumnType.STRING), equality("__j0.unnest", "World", ColumnType.STRING) ) ) @@ -4932,6 +4910,7 @@ public void testUnnestThriceWithFiltersOnDimAndAllUnnestColumnsArrayColumnsOrFil ); } + @NotYetSupported(Modes.UNNEST_RESULT_MISMATCH) @Test public void testUnnestWithGroupBy() { @@ -4943,7 +4922,7 @@ public void testUnnestWithGroupBy() GroupByQuery.builder() .setDataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .setInterval(querySegmentSpec(Filtration.eternity())) @@ -5002,6 +4981,7 @@ public void testUnnestWithGroupByArrayColumn() ); } + @NotYetSupported(Modes.UNNEST_RESULT_MISMATCH) @Test public void testUnnestWithGroupByOrderBy() { @@ -5013,7 +4993,7 @@ public void testUnnestWithGroupByOrderBy() GroupByQuery.builder() .setDataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .setInterval(querySegmentSpec(Filtration.eternity())) @@ -5053,6 +5033,7 @@ public void testUnnestWithGroupByOrderBy() ); } + @NotYetSupported(Modes.UNNEST_RESULT_MISMATCH) @Test public void testUnnestWithGroupByOrderByWithLimit() { @@ -5063,7 +5044,7 @@ public void testUnnestWithGroupByOrderByWithLimit() new TopNQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5101,7 +5082,7 @@ public void testUnnestWithGroupByHaving() GroupByQuery.builder() .setDataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .setInterval(querySegmentSpec(Filtration.eternity())) @@ -5138,7 +5119,7 @@ public void testUnnestWithLimit() Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5156,6 +5137,7 @@ public void testUnnestWithLimit() ); } + @NotYetSupported(Modes.UNNEST_RESULT_MISMATCH) @Test public void testUnnestFirstQueryOnSelect() { @@ -5166,7 +5148,7 @@ public void testUnnestFirstQueryOnSelect() Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5208,11 +5190,7 @@ public void testUnnestVirtualWithColumns1() ImmutableList.of(Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn( - "j0.unnest", - "\"dim3\"", - ColumnType.STRING - ), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), equality("j0.unnest", "a", ColumnType.STRING) )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5247,11 +5225,7 @@ public void testUnnestVirtualWithColumns2() ImmutableList.of(Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn( - "j0.unnest", - "\"dim3\"", - ColumnType.STRING - ), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5275,6 +5249,8 @@ public void testUnnestVirtualWithColumns2() ) ); } + + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNNEST_EXTRA_SCAN, separateDefaultModeTest = true) @Test public void testUnnestWithFilters() { @@ -5288,7 +5264,7 @@ public void testUnnestWithFilters() new TableDataSource(CalciteTests.DATASOURCE3), equality("dim2", "a", ColumnType.STRING) ), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5305,6 +5281,7 @@ public void testUnnestWithFilters() ); } + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNNEST_EXTRA_SCAN, separateDefaultModeTest = true) @Test public void testUnnestWithFiltersWithExpressionInInnerQuery() { @@ -5318,7 +5295,7 @@ public void testUnnestWithFiltersWithExpressionInInnerQuery() new TableDataSource(CalciteTests.DATASOURCE3), equality("dim2", "a", ColumnType.STRING) ), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5340,6 +5317,7 @@ public void testUnnestWithFiltersWithExpressionInInnerQuery() ); } + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNNEST_EXTRA_SCAN, separateDefaultModeTest = true) @Test public void testUnnestWithInFiltersWithExpressionInInnerQuery() { @@ -5353,7 +5331,7 @@ public void testUnnestWithInFiltersWithExpressionInInnerQuery() new TableDataSource(CalciteTests.DATASOURCE3), in("dim2", ImmutableList.of("a", "b")) ), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5373,6 +5351,7 @@ public void testUnnestWithInFiltersWithExpressionInInnerQuery() ); } + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNUSED_VIRTUALCOLUMN, separateDefaultModeTest = true) @Test public void testUnnestWithFiltersInnerLimit() { @@ -5395,7 +5374,7 @@ public void testUnnestWithFiltersInnerLimit() .context(QUERY_CONTEXT_UNNEST) .build() ), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5417,6 +5396,7 @@ public void testUnnestWithFiltersInnerLimit() ); } + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNNEST_EXTRA_SCAN, separateDefaultModeTest = true) @Test public void testUnnestWithFiltersInsideAndOutside() { @@ -5453,6 +5433,7 @@ public void testUnnestWithFiltersInsideAndOutside() ); } + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @Test public void testUnnestWithFiltersInsideAndOutside1() { @@ -5492,6 +5473,7 @@ public void testUnnestWithFiltersInsideAndOutside1() ); } + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @Test public void testUnnestWithFiltersOutside() { @@ -5532,6 +5514,7 @@ public void testUnnestWithFiltersOutside() ); } + @NotYetSupported(Modes.UNNEST_RESULT_MISMATCH) @Test public void testUnnestWithInFilters() { @@ -5545,7 +5528,7 @@ public void testUnnestWithInFilters() new TableDataSource(CalciteTests.DATASOURCE3), in("dim2", ImmutableList.of("a", "b", "ab", "abc")) ), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5655,6 +5638,8 @@ public void testUnnestWithGroupByOrderByOnVirtualColumn() ); } + @NotYetSupported(Modes.UNNEST_RESULT_MISMATCH) + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @Test public void testUnnestWithJoinOnTheLeft() { @@ -5713,6 +5698,7 @@ public void testUnnestWithJoinOnTheLeft() ); } + @NotYetSupported(Modes.UNNEST_INLINED) @Test public void testUnnestWithConstant() { @@ -5769,6 +5755,7 @@ public void testUnnestWithConstant() ); } + @NotYetSupported(Modes.UNNEST_RESULT_MISMATCH) @Test public void testUnnestWithSQLFunctionOnUnnestedColumn() { @@ -5779,7 +5766,7 @@ public void testUnnestWithSQLFunctionOnUnnestedColumn() Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5826,7 +5813,7 @@ public void testUnnestWithINFiltersWithLeftRewrite() new TableDataSource(CalciteTests.DATASOURCE3), range("m1", ColumnType.LONG, null, 10L, false, true) ), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), in("j0.unnest", ImmutableSet.of("a", "b")) )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5883,7 +5870,7 @@ public void testUnnestWithInvalidINFiltersOnUnnestedColumn() Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), in("j0.unnest", ImmutableSet.of("foo", "bar")) )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5896,6 +5883,7 @@ public void testUnnestWithInvalidINFiltersOnUnnestedColumn() ); } + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNNEST_DIFFERENT_RESULTSET, separateDefaultModeTest = true) @Test public void testUnnestWithNotFiltersOnUnnestedColumn() { @@ -5906,7 +5894,7 @@ public void testUnnestWithNotFiltersOnUnnestedColumn() Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), not(equality("j0.unnest", "d", ColumnType.STRING)) )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -5945,7 +5933,7 @@ public void testUnnestWithSelectorFiltersOnSelectedColumn() Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), equality("j0.unnest", "b", ColumnType.STRING) )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -6034,7 +6022,7 @@ public void testUnnestWithMultipleAndFiltersOnSelectedColumns() range("m2", ColumnType.LONG, null, 10L, false, true) ) ), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), equality("j0.unnest", "b", ColumnType.STRING) )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -6060,7 +6048,7 @@ public void testUnnestWithMultipleOrFiltersOnSelectedColumns() Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -6093,7 +6081,7 @@ public void testUnnestWithMultipleAndFiltersOnSelectedUnnestedColumns() Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), in("j0.unnest", ImmutableSet.of("a", "b")) )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -6120,7 +6108,7 @@ public void testUnnestWithMultipleOrFiltersOnUnnestedColumns() Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), in("j0.unnest", ImmutableSet.of("b", "d")) )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -6137,6 +6125,7 @@ public void testUnnestWithMultipleOrFiltersOnUnnestedColumns() ); } + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNNEST_DIFFERENT_RESULTSET, separateDefaultModeTest = true) @Test public void testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns() { @@ -6147,7 +6136,7 @@ public void testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns() Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), or( expressionFilter("(strlen(\"j0.unnest\") < 2)"), equality("j0.unnest", "d", ColumnType.STRING) @@ -6197,7 +6186,7 @@ public void testUnnestWithMultipleOrFiltersOnSelectedNonUnnestedColumns() range("m2", ColumnType.LONG, null, 2L, false, true) ) ), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -6258,7 +6247,7 @@ public void testUnnestWithMultipleOrFiltersOnUnnestedColumnsAndOnOriginalColumn( Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -6291,7 +6280,7 @@ public void testUnnestWithMultipleOrFiltersOnUnnestedColumnsAndOnOriginalColumnD Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -6315,6 +6304,7 @@ public void testUnnestWithMultipleOrFiltersOnUnnestedColumnsAndOnOriginalColumnD ); } + @NotYetSupported(Modes.UNNEST_RESULT_MISMATCH) @Test public void testUnnestWithCountOnColumn() { @@ -6326,7 +6316,7 @@ public void testUnnestWithCountOnColumn() Druids.newTimeseriesQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -6351,7 +6341,7 @@ public void testUnnestWithGroupByHavingSelector() GroupByQuery.builder() .setDataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .setInterval(querySegmentSpec(Filtration.eternity())) @@ -6405,7 +6395,7 @@ public void testUnnestWithSumOnUnnestedColumn() Druids.newTimeseriesQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -6460,7 +6450,7 @@ public void testUnnestWithGroupByHavingWithWhereOnAggCol() GroupByQuery.builder() .setDataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), in("j0.unnest", ImmutableSet.of("a", "c")) )) .setInterval(querySegmentSpec(Filtration.eternity())) @@ -6490,7 +6480,7 @@ public void testUnnestWithGroupByHavingWithWhereOnUnnestCol() GroupByQuery.builder() .setDataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), in("j0.unnest", ImmutableSet.of("a", "c")) )) .setInterval(querySegmentSpec(Filtration.eternity())) @@ -6629,7 +6619,7 @@ public void testUnnestWithTimeFilterOnly() new TableDataSource(CalciteTests.DATASOURCE1), range("__time", ColumnType.LONG, 946771200000L, 946858200000L, false, false) ), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Intervals.of("2000-01-02T00:00:00.000Z/2000-01-03T00:10:00.001Z"))) @@ -6684,6 +6674,7 @@ public void testUnnestWithTimeFilterOnlyArrayColumn() ); } + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @Test public void testUnnestWithTimeFilterAndAnotherFilter() { @@ -6702,7 +6693,7 @@ public void testUnnestWithTimeFilterAndAnotherFilter() range("__time", ColumnType.LONG, 946771200000L, 946858200000L, false, false) ) ), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Intervals.of("2000-01-02T00:00:00.000Z/2000-01-03T00:10:00.001Z"))) @@ -6736,7 +6727,7 @@ public void testUnnestWithTimeFilterOrAnotherFilter() range("__time", ColumnType.LONG, 946771200000L, 946858200000L, false, false) ) ), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Filtration.eternity())) @@ -6771,7 +6762,7 @@ public void testUnnestWithTimeFilterOnlyNested() expressionVirtualColumn("j0.unnest", "array(\"m1\",\"m2\")", ColumnType.FLOAT_ARRAY), null ), - expressionVirtualColumn("_j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("_j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Intervals.of("2000-01-02T00:00:00.000Z/2000-01-03T00:10:00.001Z"))) @@ -6814,7 +6805,7 @@ public void testUnnestWithTimeFilterOnlyNestedAndNestedAgain() expressionVirtualColumn("_j0.unnest", "array(\"dim1\",\"dim2\")", ColumnType.STRING_ARRAY), null ), - expressionVirtualColumn("__j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("__j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .intervals(querySegmentSpec(Intervals.of("2000-01-02T00:00:00.000Z/2000-01-03T00:10:00.001Z"))) @@ -6840,6 +6831,7 @@ public void testUnnestWithTimeFilterOnlyNestedAndNestedAgain() ); } + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @Test public void testUnnestWithTimeFilterInsideSubquery() { @@ -6890,6 +6882,7 @@ public void testUnnestWithTimeFilterInsideSubquery() ); } + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @Test public void testUnnestWithTimeFilterInsideSubqueryArrayColumns() { @@ -6938,6 +6931,7 @@ public void testUnnestWithTimeFilterInsideSubqueryArrayColumns() ); } + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @Test public void testUnnestWithFilterAndUnnestNestedBackToBack() { @@ -7018,6 +7012,7 @@ public void testUnnestWithFilterAndUnnestNestedBackToBack() ); } + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @Test public void testUnnestWithLookup() { @@ -7047,6 +7042,7 @@ public void testUnnestWithLookup() ); } + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNNEST_VC_USES_PROJECTED_CONSTANT) @Test public void testUnnestWithGroupByOnExpression() { @@ -7197,6 +7193,7 @@ public void testArrayToMvPostaggInline() ); } + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNNEST_DIFFERENT_RESULTSET, separateDefaultModeTest = true) @Test public void testUnnestExtractionFn() { @@ -7207,7 +7204,7 @@ public void testUnnestExtractionFn() Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), NullHandling.sqlCompatible() ? expressionFilter("(substring(\"j0.unnest\", 0, -1) != 'b')") : not(selector("j0.unnest", "b", new SubstringDimExtractionFn(0, null))) @@ -7246,7 +7243,7 @@ public void testUnnestExtractionFnNull() Druids.newScanQueryBuilder() .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE3), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), NullHandling.sqlCompatible() ? expressionFilter("notnull(substring(\"j0.unnest\", 0, -1))") : not(selector("j0.unnest", null, new SubstringDimExtractionFn(0, null))) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCorrelatedQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCorrelatedQueryTest.java index e0d71b6cfbaf..ef442ecac43c 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCorrelatedQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteCorrelatedQueryTest.java @@ -42,6 +42,7 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.join.JoinType; import org.apache.druid.segment.virtual.ExpressionVirtualColumn; +import org.apache.druid.sql.calcite.DecoupledTestConfig.IgnoreQueriesReason; import org.apache.druid.sql.calcite.util.CalciteTests; import org.junit.jupiter.params.ParameterizedTest; import org.junit.jupiter.params.provider.MethodSource; @@ -168,6 +169,7 @@ public void testCorrelatedSubquery(Map queryContext) ); } + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @MethodSource("provideQueryContexts") @ParameterizedTest(name = "{0}") public void testCorrelatedSubqueryWithLeftFilter(Map queryContext) @@ -257,6 +259,7 @@ public void testCorrelatedSubqueryWithLeftFilter(Map queryContex ); } + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @MethodSource("provideQueryContexts") @ParameterizedTest(name = "{0}") public void testCorrelatedSubqueryWithLeftFilter_leftDirectAccessDisabled(Map queryContext) @@ -352,6 +355,7 @@ public void testCorrelatedSubqueryWithLeftFilter_leftDirectAccessDisabled(Map queryContext) @@ -446,6 +450,7 @@ public void testCorrelatedSubqueryWithCorrelatedQueryFilter(Map ); } + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @MethodSource("provideQueryContexts") @ParameterizedTest(name = "{0}") public void testCorrelatedSubqueryWithCorrelatedQueryFilter_Scan(Map queryContext) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java index 026261b566b5..4ab78beb945e 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java @@ -84,6 +84,7 @@ import org.apache.druid.segment.virtual.ListFilteredVirtualColumn; import org.apache.druid.server.QueryLifecycle; import org.apache.druid.server.security.Access; +import org.apache.druid.sql.calcite.DecoupledTestConfig.IgnoreQueriesReason; import org.apache.druid.sql.calcite.DecoupledTestConfig.QuidemTestCaseReason; import org.apache.druid.sql.calcite.NotYetSupported.Modes; import org.apache.druid.sql.calcite.SqlTestFrameworkConfig.MinTopNThreshold; @@ -4948,7 +4949,7 @@ public void testUsingSubqueryAsPartOfOrFilter(Map queryContext) ); } - @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.IRRELEVANT_SCANQUERY, separateDefaultModeTest = true) + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING, separateDefaultModeTest = true) @MethodSource("provideQueryContexts") @ParameterizedTest(name = "{0}") public void testNestedGroupByOnInlineDataSourceWithFilter(Map queryContext) @@ -6139,7 +6140,6 @@ public void testJoinWithInputRefCondition() } @Test - @NotYetSupported(Modes.UNNEST_NOT_SUPPORTED_CORRELATE_CONVERSION) public void testJoinsWithUnnestOnLeft() { // Segment map function of MSQ needs some work @@ -6160,7 +6160,7 @@ public void testJoinsWithUnnestOnLeft() join( UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE1), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null ), new QueryDataSource( @@ -6193,8 +6193,8 @@ public void testJoinsWithUnnestOnLeft() ); } + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @Test - @NotYetSupported(Modes.UNNEST_NOT_SUPPORTED_CORRELATE_CONVERSION) public void testJoinsWithUnnestOverFilteredDSOnLeft() { // Segment map function of MSQ needs some work @@ -6252,7 +6252,6 @@ public void testJoinsWithUnnestOverFilteredDSOnLeft() } @Test - @NotYetSupported(Modes.UNNEST_NOT_SUPPORTED_CORRELATE_CONVERSION) public void testJoinsWithUnnestOverJoin() { // Segment map function of MSQ needs some work @@ -6288,7 +6287,7 @@ public void testJoinsWithUnnestOverJoin() "(\"dim2\" == \"j0.dim2\")", JoinType.INNER ), - expressionVirtualColumn("_j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("_j0.unnest", "\"dim3\"", ColumnType.STRING), null ), new QueryDataSource( @@ -6337,7 +6336,6 @@ public void testJoinsWithUnnestOverJoin() } @Test - @NotYetSupported(Modes.UNNEST_NOT_SUPPORTED_CORRELATE_CONVERSION) public void testSelfJoinsWithUnnestOnLeftAndRight() { // Segment map function of MSQ needs some work @@ -6358,7 +6356,7 @@ public void testSelfJoinsWithUnnestOnLeftAndRight() join( UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE1), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null ), new QueryDataSource( @@ -6366,7 +6364,7 @@ public void testSelfJoinsWithUnnestOnLeftAndRight() .intervals(querySegmentSpec(Filtration.eternity())) .dataSource(UnnestDataSource.create( new TableDataSource(CalciteTests.DATASOURCE1), - expressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), + nestedExpressionVirtualColumn("j0.unnest", "\"dim3\"", ColumnType.STRING), null )) .columns("dim2", "j0.unnest") @@ -6405,8 +6403,8 @@ public void testSelfJoinsWithUnnestOnLeftAndRight() ); } + @DecoupledTestConfig(ignoreExpectedQueriesReason = IgnoreQueriesReason.UNNEST_EXTRA_SCANQUERY) @Test - @NotYetSupported(Modes.UNNEST_NOT_SUPPORTED_CORRELATE_CONVERSION) public void testJoinsOverUnnestOverFilterDSOverJoin() { // Segment map function of MSQ needs some work diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index b03b6698b1df..013753a02fcc 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -75,6 +75,7 @@ import org.apache.druid.segment.writeout.OffHeapMemorySegmentWriteOutMediumFactory; import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.calcite.CalciteNestedDataQueryTest.NestedComponentSupplier; +import org.apache.druid.sql.calcite.NotYetSupported.Modes; import org.apache.druid.sql.calcite.filtration.Filtration; import org.apache.druid.sql.calcite.util.SqlTestFramework.StandardComponentSupplier; import org.apache.druid.sql.calcite.util.TestDataBuilder; @@ -7347,6 +7348,7 @@ public void testNvlJsonValueDoubleSometimesMissingRangeFilter() ); } + @NotYetSupported(Modes.ERROR_CANNOT_TRANSLATE_COUNT_DISTINCT) @Test public void testApproxCountDistinctOnUnsupportedComplexColumn() { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledExtension.java b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledExtension.java index 094a23cb1d67..e6311f8d8345 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledExtension.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledExtension.java @@ -21,6 +21,7 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.query.Query; import org.apache.druid.query.QueryContexts; import org.apache.druid.quidem.DruidQTestInfo; import org.apache.druid.quidem.ProjectPathUtils; @@ -32,6 +33,7 @@ import org.junit.jupiter.api.extension.ExtensionContext; import java.io.File; +import java.util.List; public class DecoupledExtension implements BeforeEachCallback { @@ -64,6 +66,8 @@ public QueryTestBuilder testBuilder() boolean runQuidem = (decTestConfig != null && decTestConfig.quidemReason().isPresent()); + boolean ignoreQueries = (decTestConfig != null && decTestConfig.ignoreExpectedQueriesReason().isPresent()); + CalciteTestConfig testConfig = baseTest.new CalciteTestConfig(CONTEXT_OVERRIDES) { @@ -101,6 +105,17 @@ public DruidQTestInfo getQTestInfo() }; QueryTestBuilder builder = new QueryTestBuilder(testConfig) + { + @Override + public QueryTestBuilder expectedQueries(List> expectedQueries) + { + if (ignoreQueries) { + return this; + } else { + return super.expectedQueries(expectedQueries); + } + } + } .cannotVectorize(baseTest.cannotVectorize) .skipVectorize(baseTest.skipVectorize); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteArraysQueryTest.java new file mode 100644 index 000000000000..d1a64e8e19fe --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteArraysQueryTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite; + +import org.apache.druid.sql.calcite.NotYetSupported.NotYetSupportedProcessor; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; + +@ExtendWith(NotYetSupportedProcessor.class) +public class DecoupledPlanningCalciteArraysQueryTest extends CalciteArraysQueryTest +{ + @RegisterExtension + DecoupledExtension decoupledExtension = new DecoupledExtension(this); + + @Override + protected QueryTestBuilder testBuilder() + { + return decoupledExtension.testBuilder(); + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteCorrelatedQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteCorrelatedQueryTest.java new file mode 100644 index 000000000000..031167cdcc57 --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteCorrelatedQueryTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite; + +import org.apache.druid.sql.calcite.NotYetSupported.NotYetSupportedProcessor; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; + +@ExtendWith(NotYetSupportedProcessor.class) +public class DecoupledPlanningCalciteCorrelatedQueryTest extends CalciteCorrelatedQueryTest +{ + @RegisterExtension + DecoupledExtension decoupledExtension = new DecoupledExtension(this); + + @Override + protected QueryTestBuilder testBuilder() + { + return decoupledExtension.testBuilder(); + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteNestedDataQueryTest.java new file mode 100644 index 000000000000..b4e246de9536 --- /dev/null +++ b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledPlanningCalciteNestedDataQueryTest.java @@ -0,0 +1,37 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.sql.calcite; + +import org.apache.druid.sql.calcite.NotYetSupported.NotYetSupportedProcessor; +import org.junit.jupiter.api.extension.ExtendWith; +import org.junit.jupiter.api.extension.RegisterExtension; + +@ExtendWith(NotYetSupportedProcessor.class) +public class DecoupledPlanningCalciteNestedDataQueryTest extends CalciteNestedDataQueryTest +{ + @RegisterExtension + DecoupledExtension decoupledExtension = new DecoupledExtension(this); + + @Override + protected QueryTestBuilder testBuilder() + { + return decoupledExtension.testBuilder(); + } +} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledTestConfig.java b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledTestConfig.java index 9bc4fe0fee47..10d631b665c0 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledTestConfig.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledTestConfig.java @@ -21,6 +21,8 @@ import org.apache.calcite.rel.rules.CoreRules; import org.apache.druid.query.QueryContexts; +import org.apache.druid.query.UnnestDataSource; + import java.lang.annotation.ElementType; import java.lang.annotation.Retention; import java.lang.annotation.RetentionPolicy; @@ -40,6 +42,25 @@ */ QuidemTestCaseReason quidemReason() default QuidemTestCaseReason.NONE; + /** + * Run the tests normally; however disable the native plan checks. + */ + IgnoreQueriesReason ignoreExpectedQueriesReason() default IgnoreQueriesReason.NONE; + + enum IgnoreQueriesReason + { + NONE, + /** + * An extra ScanQuery to service a Project and/or Filter was added. + */ + UNNEST_EXTRA_SCANQUERY; + + public boolean isPresent() + { + return this != NONE; + } + } + enum QuidemTestCaseReason { NONE, @@ -114,7 +135,29 @@ enum QuidemTestCaseReason /** * Strange things; needs more investigation */ - IRRELEVANT_SCANQUERY; + IRRELEVANT_SCANQUERY, + /** + * Extra scan query under {@link UnnestDataSource}. + */ + UNNEST_EXTRA_SCAN, + /** + * Extra virtualcolumn appeared; seemingly unused + */ + UNUSED_VIRTUALCOLUMN, + /** + * Unnest uses a VC to access a constant like array(1,2,3). + */ + UNNEST_VC_USES_PROJECTED_CONSTANT, + /** + * This should need some investigation. + * + * Its not invalid; just strange. + */ + SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING, + /** + * New plan UNNEST-s a different resultset. + */ + UNNEST_DIFFERENT_RESULTSET; public boolean isPresent() { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java index 1a83a708e210..f049e1541834 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java @@ -92,10 +92,12 @@ enum Modes UNSUPPORTED_DATASOURCE(DruidException.class, "WindowOperatorQuery must run on top of a query or inline data source"), UNION_WITH_COMPLEX_OPERAND(DruidException.class, "Only Table and Values are supported as inputs for Union"), UNION_MORE_STRICT_ROWTYPE_CHECK(DruidException.class, "Row signature mismatch in Union inputs"), - UNNEST_NOT_SUPPORTED_CORRELATE_CONVERSION(DruidException.class, "Missing conversion( is|s are) LogicalCorrelate"), SORT_REMOVE_TROUBLE(DruidException.class, "Calcite assertion violated.*Sort\\."), SORT_REMOVE_CONSTANT_KEYS_CONFLICT(DruidException.class, "not enough rules"), - REQUIRE_TIME_CONDITION(CannotBuildQueryException.class, "requireTimeCondition is enabled"); + REQUIRE_TIME_CONDITION(CannotBuildQueryException.class, "requireTimeCondition is enabled"), + ERROR_CANNOT_TRANSLATE_COUNT_DISTINCT(AssertionError.class, "Cannot translate aggregator.COUNT.DISTINCT"), + UNNEST_INLINED(Exception.class, "Missing conversion is Uncollect"), + UNNEST_RESULT_MISMATCH(AssertionError.class, "(Result count mismatch|column content mismatch)"); // @formatter:on public Class throwableClass; diff --git a/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/unnest.iq b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/unnest.iq new file mode 100644 index 000000000000..aac89b7f3c7c --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.quidem.SqlQuidemTest/unnest.iq @@ -0,0 +1,89 @@ +#!set plannerStrategy DECOUPLED +!use druidtest://?numMergeBuffers=3 +!set outputformat mysql + +SELECT d3 FROM (select * from druid.numfoo where dim2='a'), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3); +LogicalProject(d3=[$17]) + LogicalProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], d3=[$18]) + LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{17}]) + LogicalProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], $f17=[MV_TO_ARRAY($3)]) + LogicalFilter(condition=[=($2, 'a')]) + LogicalTableScan(table=[[druid, numfoo]]) + LogicalProject(d3=[$0]) + Uncollect + LogicalProject(EXPR$0=[$cor0.$f17]) + LogicalValues(tuples=[[{ 0 }]]) + +!convertedPlan + + +LogicalProject(d3=[$18]) + LogicalCorrelate(correlation=[$cor0], joinType=[inner], requiredColumns=[{17}]) + LogicalProject(__time=[$0], dim1=[$1], dim2=[CAST('a':VARCHAR):VARCHAR], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], $f17=[MV_TO_ARRAY($3)]) + LogicalFilter(condition=[=($2, 'a')]) + LogicalTableScan(table=[[druid, numfoo]]) + Uncollect + LogicalProject(EXPR$0=[$cor0.$f17]) + LogicalValues(tuples=[[{ 0 }]]) + +!logicalPlan + +LogicalProject(d3=[$17]) + LogicalCorrelate(correlation=[$cor2], joinType=[inner], requiredColumns=[{3}]) + LogicalFilter(condition=[=($2, 'a')]) + LogicalTableScan(table=[[druid, numfoo]]) + DruidUnnestRel(expr=[MV_TO_ARRAY($cor2.dim3)], filter=[null]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "filter", + "base" : { + "type" : "table", + "name" : "numfoo" + }, + "filter" : { + "type" : "equals", + "column" : "dim2", + "matchValueType" : "STRING", + "matchValue" : "a" + } + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "\"dim3\"", + "outputType" : "STRING" + }, + "unnestFilter" : null + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest" ], + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan ++----+ +| d3 | ++----+ +| | +| a | +| b | ++----+ +(3 rows) + +!ok + +# LogicalPlan plan = calcite.plan(); +# PhysicalQuery pq = DruidQueryGenerator.generate(plan); + diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=default.iq new file mode 100644 index 000000000000..2acefa4270fd --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=default.iq @@ -0,0 +1,81 @@ +# testUnnestExtractionFn@NullHandling=default case-crc:404dc668 +# quidem testcase reason: UNNEST_DIFFERENT_RESULTSET +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT substring(d3,1) FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) WHERE substring(d3,1) <> 'b'; ++--------+ +| EXPR$0 | ++--------+ +| a | +| c | +| d | ++--------+ +(3 rows) + +!ok +LogicalProject(EXPR$0=[SUBSTRING($18, 1)]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[<>(SUBSTRING($0, 1), 'b')]) + LogicalProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(EXPR$0=[SUBSTRING($18, 1)], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[<>(SUBSTRING($0, 1), 'b')]) + DruidProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0], druid=[logical]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "table", + "name" : "numfoo" + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : { + "type" : "not", + "field" : { + "type" : "selector", + "dimension" : "j0.unnest", + "value" : "b", + "extractionFn" : { + "type" : "substring", + "index" : 0, + "length" : null + } + } + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "substring(\"j0.unnest\", 0, -1)", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "columns" : [ "v0" ], + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=sql.iq new file mode 100644 index 000000000000..ca41ee34e6a6 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=sql.iq @@ -0,0 +1,72 @@ +# testUnnestExtractionFn@NullHandling=sql case-crc:404dc668 +# quidem testcase reason: UNNEST_DIFFERENT_RESULTSET +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT substring(d3,1) FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) WHERE substring(d3,1) <> 'b'; ++--------+ +| EXPR$0 | ++--------+ +| a | +| c | +| d | ++--------+ +(3 rows) + +!ok +LogicalProject(EXPR$0=[SUBSTRING($18, 1)]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[<>(SUBSTRING($0, 1), 'b')]) + LogicalProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(EXPR$0=[SUBSTRING($18, 1)], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[<>(SUBSTRING($0, 1), 'b')]) + DruidProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0], druid=[logical]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "table", + "name" : "numfoo" + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : { + "type" : "expression", + "expression" : "(substring(\"j0.unnest\", 0, -1) != 'b')" + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "substring(\"j0.unnest\", 0, -1)", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "columns" : [ "v0" ], + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFilters@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFilters@NullHandling=default.iq new file mode 100644 index 000000000000..64963f6fc241 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFilters@NullHandling=default.iq @@ -0,0 +1,92 @@ +# testUnnestWithFilters@NullHandling=default case-crc:810b3e0d +# quidem testcase reason: UNNEST_EXTRA_SCAN +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT d3 FROM (select * from druid.numfoo where dim2='a'), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3); ++----+ +| d3 | ++----+ +| a | +| b | ++----+ +(2 rows) + +!ok +LogicalProject(d3=[$18]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($3)]) + LogicalProject(__time=[$0], dim1=[$1], $f2=[CAST('a':VARCHAR):VARCHAR], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0]) + LogicalFilter(condition=[=($2, 'a')]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(d3=[$18], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($3)]) + DruidProject(__time=[$0], dim1=[$1], $f2=[CAST('a':VARCHAR):VARCHAR], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0], druid=[logical]) + DruidFilter(condition=[=($2, 'a')]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "query", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "'a'", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "filter" : { + "type" : "selector", + "dimension" : "dim2", + "value" : "a" + }, + "columns" : [ "__time", "cnt", "d1", "d2", "dim1", "dim3", "dim4", "dim5", "dim6", "f1", "f2", "l1", "l2", "m1", "m2", "unique_dim1", "v0" ], + "columnTypes" : [ "LONG", "LONG", "DOUBLE", "DOUBLE", "STRING", "STRING", "STRING", "STRING", "STRING", "FLOAT", "FLOAT", "LONG", "LONG", "FLOAT", "DOUBLE", "COMPLEX", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : null + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest" ], + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFilters@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFilters@NullHandling=sql.iq new file mode 100644 index 000000000000..f032fc5c14d8 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFilters@NullHandling=sql.iq @@ -0,0 +1,94 @@ +# testUnnestWithFilters@NullHandling=sql case-crc:810b3e0d +# quidem testcase reason: UNNEST_EXTRA_SCAN +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT d3 FROM (select * from druid.numfoo where dim2='a'), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3); ++----+ +| d3 | ++----+ +| | +| a | +| b | ++----+ +(3 rows) + +!ok +LogicalProject(d3=[$18]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($3)]) + LogicalProject(__time=[$0], dim1=[$1], $f2=[CAST('a':VARCHAR):VARCHAR], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0]) + LogicalFilter(condition=[=($2, 'a')]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(d3=[$18], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($3)]) + DruidProject(__time=[$0], dim1=[$1], $f2=[CAST('a':VARCHAR):VARCHAR], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0], druid=[logical]) + DruidFilter(condition=[=($2, 'a')]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "query", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "'a'", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "filter" : { + "type" : "equals", + "column" : "dim2", + "matchValueType" : "STRING", + "matchValue" : "a" + }, + "columns" : [ "__time", "cnt", "d1", "d2", "dim1", "dim3", "dim4", "dim5", "dim6", "f1", "f2", "l1", "l2", "m1", "m2", "unique_dim1", "v0" ], + "columnTypes" : [ "LONG", "LONG", "DOUBLE", "DOUBLE", "STRING", "STRING", "STRING", "STRING", "STRING", "FLOAT", "FLOAT", "LONG", "LONG", "FLOAT", "DOUBLE", "COMPLEX", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : null + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest" ], + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInnerLimit@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInnerLimit@NullHandling=default.iq new file mode 100644 index 000000000000..0d98641f3893 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInnerLimit@NullHandling=default.iq @@ -0,0 +1,95 @@ +# testUnnestWithFiltersInnerLimit@NullHandling=default case-crc:1f5acfc8 +# quidem testcase reason: UNUSED_VIRTUALCOLUMN +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT d3 FROM (select dim2,dim3 from druid.numfoo where dim2='a' LIMIT 2), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3); ++----+ +| d3 | ++----+ +| a | +| b | ++----+ +(2 rows) + +!ok +LogicalProject(d3=[$3]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($1)]) + LogicalProject($f0=[CAST('a':VARCHAR):VARCHAR], dim3=[$3], __time=[$0]) + LogicalSort(fetch=[2]) + LogicalFilter(condition=[=($2, 'a')]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(d3=[$3], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($1)]) + DruidProject($f0=[CAST('a':VARCHAR):VARCHAR], dim3=[$3], __time=[$0], druid=[logical]) + DruidSort(fetch=[2], druid=[logical]) + DruidFilter(condition=[=($2, 'a')]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "query", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "'a'", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "limit" : 2, + "filter" : { + "type" : "selector", + "dimension" : "dim2", + "value" : "a" + }, + "columns" : [ "__time", "dim3", "v0" ], + "columnTypes" : [ "LONG", "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : null + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest" ], + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInnerLimit@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInnerLimit@NullHandling=sql.iq new file mode 100644 index 000000000000..878106d9ac9d --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInnerLimit@NullHandling=sql.iq @@ -0,0 +1,97 @@ +# testUnnestWithFiltersInnerLimit@NullHandling=sql case-crc:1f5acfc8 +# quidem testcase reason: UNUSED_VIRTUALCOLUMN +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT d3 FROM (select dim2,dim3 from druid.numfoo where dim2='a' LIMIT 2), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3); ++----+ +| d3 | ++----+ +| | +| a | +| b | ++----+ +(3 rows) + +!ok +LogicalProject(d3=[$3]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($1)]) + LogicalProject($f0=[CAST('a':VARCHAR):VARCHAR], dim3=[$3], __time=[$0]) + LogicalSort(fetch=[2]) + LogicalFilter(condition=[=($2, 'a')]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(d3=[$3], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($1)]) + DruidProject($f0=[CAST('a':VARCHAR):VARCHAR], dim3=[$3], __time=[$0], druid=[logical]) + DruidSort(fetch=[2], druid=[logical]) + DruidFilter(condition=[=($2, 'a')]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "query", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "'a'", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "limit" : 2, + "filter" : { + "type" : "equals", + "column" : "dim2", + "matchValueType" : "STRING", + "matchValue" : "a" + }, + "columns" : [ "__time", "dim3", "v0" ], + "columnTypes" : [ "LONG", "STRING", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : null + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest" ], + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInsideAndOutside@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInsideAndOutside@NullHandling=default.iq new file mode 100644 index 000000000000..ca22c18ff45f --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInsideAndOutside@NullHandling=default.iq @@ -0,0 +1,112 @@ +# testUnnestWithFiltersInsideAndOutside@NullHandling=default case-crc:10c26262 +# quidem testcase reason: UNNEST_EXTRA_SCAN +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT d3 FROM + (select * from druid.numfoo where dim2='a') as t, + UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) +WHERE t.dim1 <> 'foo' +AND unnested.d3 <> 'b'; ++----+ +| d3 | ++----+ +| a | ++----+ +(1 row) + +!ok +LogicalProject(d3=[$18]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[<>($0, 'b')]) + LogicalProject(__time=[$0], dim1=[$1], $f2=[CAST('a':VARCHAR):VARCHAR], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0]) + LogicalFilter(condition=[AND(=($2, 'a'), <>($1, 'foo'))]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(d3=[$18], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[<>($0, 'b')]) + DruidProject(__time=[$0], dim1=[$1], $f2=[CAST('a':VARCHAR):VARCHAR], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0], druid=[logical]) + DruidFilter(condition=[AND(=($2, 'a'), <>($1, 'foo'))]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "query", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "'a'", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "filter" : { + "type" : "and", + "fields" : [ { + "type" : "selector", + "dimension" : "dim2", + "value" : "a" + }, { + "type" : "not", + "field" : { + "type" : "selector", + "dimension" : "dim1", + "value" : "foo" + } + } ] + }, + "columns" : [ "__time", "cnt", "d1", "d2", "dim1", "dim3", "dim4", "dim5", "dim6", "f1", "f2", "l1", "l2", "m1", "m2", "unique_dim1", "v0" ], + "columnTypes" : [ "LONG", "LONG", "DOUBLE", "DOUBLE", "STRING", "STRING", "STRING", "STRING", "STRING", "FLOAT", "FLOAT", "LONG", "LONG", "FLOAT", "DOUBLE", "COMPLEX", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : { + "type" : "not", + "field" : { + "type" : "selector", + "dimension" : "j0.unnest", + "value" : "b" + } + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest" ], + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInsideAndOutside@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInsideAndOutside@NullHandling=sql.iq new file mode 100644 index 000000000000..45c21b71c3f4 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersInsideAndOutside@NullHandling=sql.iq @@ -0,0 +1,116 @@ +# testUnnestWithFiltersInsideAndOutside@NullHandling=sql case-crc:10c26262 +# quidem testcase reason: UNNEST_EXTRA_SCAN +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT d3 FROM + (select * from druid.numfoo where dim2='a') as t, + UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) +WHERE t.dim1 <> 'foo' +AND unnested.d3 <> 'b'; ++----+ +| d3 | ++----+ +| | +| a | ++----+ +(2 rows) + +!ok +LogicalProject(d3=[$18]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[<>($0, 'b')]) + LogicalProject(__time=[$0], dim1=[$1], $f2=[CAST('a':VARCHAR):VARCHAR], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0]) + LogicalFilter(condition=[AND(=($2, 'a'), <>($1, 'foo'))]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(d3=[$18], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[<>($0, 'b')]) + DruidProject(__time=[$0], dim1=[$1], $f2=[CAST('a':VARCHAR):VARCHAR], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0], druid=[logical]) + DruidFilter(condition=[AND(=($2, 'a'), <>($1, 'foo'))]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "query", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "'a'", + "outputType" : "STRING" + } ], + "resultFormat" : "compactedList", + "filter" : { + "type" : "and", + "fields" : [ { + "type" : "equals", + "column" : "dim2", + "matchValueType" : "STRING", + "matchValue" : "a" + }, { + "type" : "not", + "field" : { + "type" : "equals", + "column" : "dim1", + "matchValueType" : "STRING", + "matchValue" : "foo" + } + } ] + }, + "columns" : [ "__time", "cnt", "d1", "d2", "dim1", "dim3", "dim4", "dim5", "dim6", "f1", "f2", "l1", "l2", "m1", "m2", "unique_dim1", "v0" ], + "columnTypes" : [ "LONG", "LONG", "DOUBLE", "DOUBLE", "STRING", "STRING", "STRING", "STRING", "STRING", "FLOAT", "FLOAT", "LONG", "LONG", "FLOAT", "DOUBLE", "COMPLEX", "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : { + "type" : "not", + "field" : { + "type" : "equals", + "column" : "j0.unnest", + "matchValueType" : "STRING", + "matchValue" : "b" + } + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest" ], + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersWithExpressionInInnerQuery@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersWithExpressionInInnerQuery@NullHandling=default.iq new file mode 100644 index 000000000000..6ca74e443390 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersWithExpressionInInnerQuery@NullHandling=default.iq @@ -0,0 +1,92 @@ +# testUnnestWithFiltersWithExpressionInInnerQuery@NullHandling=default case-crc:fa9be8db +# quidem testcase reason: UNNEST_EXTRA_SCAN +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT t,d3 FROM (select FLOOR(__time to hour) t, dim3 from druid.numfoo where dim2='a'), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3); ++-------------------------+----+ +| t | d3 | ++-------------------------+----+ +| 2000-01-01 00:00:00.000 | a | +| 2000-01-01 00:00:00.000 | b | ++-------------------------+----+ +(2 rows) + +!ok +LogicalProject(t=[$0], d3=[$3]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($1)]) + LogicalProject($f0=[FLOOR($0, FLAG(HOUR))], dim3=[$3], __time=[$0]) + LogicalFilter(condition=[=($2, 'a')]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(t=[$0], d3=[$3], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($1)]) + DruidProject($f0=[FLOOR($0, FLAG(HOUR))], dim3=[$3], __time=[$0], druid=[logical]) + DruidFilter(condition=[=($2, 'a')]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "query", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "timestamp_floor(\"__time\",'PT1H',null,'UTC')", + "outputType" : "LONG" + } ], + "resultFormat" : "compactedList", + "filter" : { + "type" : "selector", + "dimension" : "dim2", + "value" : "a" + }, + "columns" : [ "__time", "dim3", "v0" ], + "columnTypes" : [ "LONG", "STRING", "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : null + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest", "v0" ], + "columnTypes" : [ "STRING", "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersWithExpressionInInnerQuery@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersWithExpressionInInnerQuery@NullHandling=sql.iq new file mode 100644 index 000000000000..1f523a13de1f --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithFiltersWithExpressionInInnerQuery@NullHandling=sql.iq @@ -0,0 +1,94 @@ +# testUnnestWithFiltersWithExpressionInInnerQuery@NullHandling=sql case-crc:fa9be8db +# quidem testcase reason: UNNEST_EXTRA_SCAN +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT t,d3 FROM (select FLOOR(__time to hour) t, dim3 from druid.numfoo where dim2='a'), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3); ++-------------------------+----+ +| t | d3 | ++-------------------------+----+ +| 2000-01-01 00:00:00.000 | a | +| 2000-01-01 00:00:00.000 | b | +| 2001-01-01 00:00:00.000 | | ++-------------------------+----+ +(3 rows) + +!ok +LogicalProject(t=[$0], d3=[$3]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($1)]) + LogicalProject($f0=[FLOOR($0, FLAG(HOUR))], dim3=[$3], __time=[$0]) + LogicalFilter(condition=[=($2, 'a')]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(t=[$0], d3=[$3], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($1)]) + DruidProject($f0=[FLOOR($0, FLAG(HOUR))], dim3=[$3], __time=[$0], druid=[logical]) + DruidFilter(condition=[=($2, 'a')]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "query", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "timestamp_floor(\"__time\",'PT1H',null,'UTC')", + "outputType" : "LONG" + } ], + "resultFormat" : "compactedList", + "filter" : { + "type" : "equals", + "column" : "dim2", + "matchValueType" : "STRING", + "matchValue" : "a" + }, + "columns" : [ "__time", "dim3", "v0" ], + "columnTypes" : [ "LONG", "STRING", "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : null + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest", "v0" ], + "columnTypes" : [ "STRING", "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithGroupByOnExpression.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithGroupByOnExpression.iq new file mode 100644 index 000000000000..0816c2e37913 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithGroupByOnExpression.iq @@ -0,0 +1,98 @@ +# testUnnestWithGroupByOnExpression case-crc:d500d30b +# quidem testcase reason: UNNEST_VC_USES_PROJECTED_CONSTANT +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +WITH X as +( +SELECT +ARRAY[1,2,3] as allNums +FROM foo +GROUP BY 1 +) +select * from X CROSS JOIN UNNEST(X.allNums) as ud(num); ++-----------+-----+ +| allNums | num | ++-----------+-----+ +| [1, 2, 3] | 1 | +| [1, 2, 3] | 2 | +| [1, 2, 3] | 3 | ++-----------+-----+ +(3 rows) + +!ok +LogicalUnnest(unnestExpr=[$0]) + LogicalAggregate(group=[{0}]) + LogicalProject(allNums=[ARRAY(1, 2, 3)]) + LogicalTableScan(table=[[druid, foo]]) + +!logicalPlan +DruidUnnest(unnestExpr=[$0]) + DruidAggregate(group=[{0}], druid=[logical]) + DruidProject(allNums=[ARRAY(1, 2, 3)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "query", + "query" : { + "queryType" : "groupBy", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "array(1,2,3)", + "outputType" : "ARRAY" + } ], + "granularity" : { + "type" : "all" + }, + "dimensions" : [ { + "type" : "default", + "dimension" : "v0", + "outputName" : "d0", + "outputType" : "ARRAY" + } ], + "limitSpec" : { + "type" : "NoopLimitSpec" + } + } + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "\"d0\"", + "outputType" : "ARRAY" + }, + "unnestFilter" : null + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "d0", "j0.unnest" ], + "columnTypes" : [ "ARRAY", "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithInFiltersWithExpressionInInnerQuery@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithInFiltersWithExpressionInInnerQuery@NullHandling=default.iq new file mode 100644 index 000000000000..6c203130987d --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithInFiltersWithExpressionInInnerQuery@NullHandling=default.iq @@ -0,0 +1,92 @@ +# testUnnestWithInFiltersWithExpressionInInnerQuery@NullHandling=default case-crc:49354254 +# quidem testcase reason: UNNEST_EXTRA_SCAN +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT t,d3 FROM (select FLOOR(__time to hour) t, dim3 from druid.numfoo where dim2 IN ('a','b')), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3); ++-------------------------+----+ +| t | d3 | ++-------------------------+----+ +| 2000-01-01 00:00:00.000 | a | +| 2000-01-01 00:00:00.000 | b | ++-------------------------+----+ +(2 rows) + +!ok +LogicalProject(t=[$0], d3=[$3]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($1)]) + LogicalProject($f0=[FLOOR($0, FLAG(HOUR))], dim3=[$3], __time=[$0]) + LogicalFilter(condition=[SEARCH($2, Sarg['a':VARCHAR, 'b':VARCHAR]:VARCHAR)]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(t=[$0], d3=[$3], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($1)]) + DruidProject($f0=[FLOOR($0, FLAG(HOUR))], dim3=[$3], __time=[$0], druid=[logical]) + DruidFilter(condition=[SEARCH($2, Sarg['a':VARCHAR, 'b':VARCHAR]:VARCHAR)]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "query", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "timestamp_floor(\"__time\",'PT1H',null,'UTC')", + "outputType" : "LONG" + } ], + "resultFormat" : "compactedList", + "filter" : { + "type" : "in", + "dimension" : "dim2", + "values" : [ "a", "b" ] + }, + "columns" : [ "__time", "dim3", "v0" ], + "columnTypes" : [ "LONG", "STRING", "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : null + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest", "v0" ], + "columnTypes" : [ "STRING", "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithInFiltersWithExpressionInInnerQuery@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithInFiltersWithExpressionInInnerQuery@NullHandling=sql.iq new file mode 100644 index 000000000000..35e883c07845 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithInFiltersWithExpressionInInnerQuery@NullHandling=sql.iq @@ -0,0 +1,94 @@ +# testUnnestWithInFiltersWithExpressionInInnerQuery@NullHandling=sql case-crc:49354254 +# quidem testcase reason: UNNEST_EXTRA_SCAN +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT t,d3 FROM (select FLOOR(__time to hour) t, dim3 from druid.numfoo where dim2 IN ('a','b')), UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3); ++-------------------------+----+ +| t | d3 | ++-------------------------+----+ +| 2000-01-01 00:00:00.000 | a | +| 2000-01-01 00:00:00.000 | b | +| 2001-01-01 00:00:00.000 | | ++-------------------------+----+ +(3 rows) + +!ok +LogicalProject(t=[$0], d3=[$3]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($1)]) + LogicalProject($f0=[FLOOR($0, FLAG(HOUR))], dim3=[$3], __time=[$0]) + LogicalFilter(condition=[SEARCH($2, Sarg['a':VARCHAR, 'b':VARCHAR]:VARCHAR)]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(t=[$0], d3=[$3], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($1)]) + DruidProject($f0=[FLOOR($0, FLAG(HOUR))], dim3=[$3], __time=[$0], druid=[logical]) + DruidFilter(condition=[SEARCH($2, Sarg['a':VARCHAR, 'b':VARCHAR]:VARCHAR)]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "query", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "timestamp_floor(\"__time\",'PT1H',null,'UTC')", + "outputType" : "LONG" + } ], + "resultFormat" : "compactedList", + "filter" : { + "type" : "inType", + "column" : "dim2", + "matchValueType" : "STRING", + "sortedValues" : [ "a", "b" ] + }, + "columns" : [ "__time", "dim3", "v0" ], + "columnTypes" : [ "LONG", "STRING", "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : null + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest", "v0" ], + "columnTypes" : [ "STRING", "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns@NullHandling=default.iq new file mode 100644 index 000000000000..4f12b5750dca --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns@NullHandling=default.iq @@ -0,0 +1,75 @@ +# testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns@NullHandling=default case-crc:02630011 +# quidem testcase reason: UNNEST_DIFFERENT_RESULTSET +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where strlen(d3) < 2 or d3='d' ; ++----+ +| d3 | ++----+ +| a | +| b | +| b | +| c | +| d | ++----+ +(5 rows) + +!ok +LogicalProject(d3=[$18]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[OR(<(STRLEN($0), 2), =($0, 'd'))]) + LogicalProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(d3=[$18], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[OR(<(STRLEN($0), 2), =($0, 'd'))]) + DruidProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0], druid=[logical]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "table", + "name" : "numfoo" + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : { + "type" : "or", + "fields" : [ { + "type" : "expression", + "expression" : "(strlen(\"j0.unnest\") < 2)" + }, { + "type" : "selector", + "dimension" : "j0.unnest", + "value" : "d" + } ] + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest" ], + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns@NullHandling=sql.iq new file mode 100644 index 000000000000..e5d8a68baf69 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns@NullHandling=sql.iq @@ -0,0 +1,77 @@ +# testUnnestWithMultipleOrFiltersOnVariationsOfUnnestedColumns@NullHandling=sql case-crc:02630011 +# quidem testcase reason: UNNEST_DIFFERENT_RESULTSET +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where strlen(d3) < 2 or d3='d' ; ++----+ +| d3 | ++----+ +| | +| a | +| b | +| b | +| c | +| d | ++----+ +(6 rows) + +!ok +LogicalProject(d3=[$18]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[OR(<(STRLEN($0), 2), =($0, 'd'))]) + LogicalProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(d3=[$18], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[OR(<(STRLEN($0), 2), =($0, 'd'))]) + DruidProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0], druid=[logical]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "table", + "name" : "numfoo" + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : { + "type" : "or", + "fields" : [ { + "type" : "expression", + "expression" : "(strlen(\"j0.unnest\") < 2)" + }, { + "type" : "equals", + "column" : "j0.unnest", + "matchValueType" : "STRING", + "matchValue" : "d" + } ] + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest" ], + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithNotFiltersOnUnnestedColumn@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithNotFiltersOnUnnestedColumn@NullHandling=default.iq new file mode 100644 index 000000000000..84d0c9b478e7 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithNotFiltersOnUnnestedColumn@NullHandling=default.iq @@ -0,0 +1,71 @@ +# testUnnestWithNotFiltersOnUnnestedColumn@NullHandling=default case-crc:5fc83255 +# quidem testcase reason: UNNEST_DIFFERENT_RESULTSET +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where d3!='d' ; ++----+ +| d3 | ++----+ +| a | +| b | +| b | +| c | ++----+ +(4 rows) + +!ok +LogicalProject(d3=[$18]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[<>($0, 'd')]) + LogicalProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(d3=[$18], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[<>($0, 'd')]) + DruidProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0], druid=[logical]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "table", + "name" : "numfoo" + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : { + "type" : "not", + "field" : { + "type" : "selector", + "dimension" : "j0.unnest", + "value" : "d" + } + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest" ], + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithNotFiltersOnUnnestedColumn@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithNotFiltersOnUnnestedColumn@NullHandling=sql.iq new file mode 100644 index 000000000000..bc027187606b --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestWithNotFiltersOnUnnestedColumn@NullHandling=sql.iq @@ -0,0 +1,73 @@ +# testUnnestWithNotFiltersOnUnnestedColumn@NullHandling=sql case-crc:5fc83255 +# quidem testcase reason: UNNEST_DIFFERENT_RESULTSET +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set sqlStringifyArrays false +!set outputformat mysql +!use druidtest:/// +SELECT d3 FROM druid.numfoo, UNNEST(MV_TO_ARRAY(dim3)) as unnested (d3) where d3!='d' ; ++----+ +| d3 | ++----+ +| | +| a | +| b | +| b | +| c | ++----+ +(5 rows) + +!ok +LogicalProject(d3=[$18]) + LogicalUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[<>($0, 'd')]) + LogicalProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0]) + LogicalTableScan(table=[[druid, numfoo]]) + +!logicalPlan +DruidProject(d3=[$18], druid=[logical]) + DruidUnnest(unnestExpr=[MV_TO_ARRAY($3)], filter=[<>($0, 'd')]) + DruidProject(__time=[$0], dim1=[$1], dim2=[$2], dim3=[$3], dim4=[$4], dim5=[$5], dim6=[$6], d1=[$7], d2=[$8], f1=[$9], f2=[$10], l1=[$11], l2=[$12], cnt=[$13], m1=[$14], m2=[$15], unique_dim1=[$16], __time0=[$0], druid=[logical]) + DruidTableScan(table=[[druid, numfoo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "unnest", + "base" : { + "type" : "table", + "name" : "numfoo" + }, + "virtualColumn" : { + "type" : "expression", + "name" : "j0.unnest", + "expression" : "mv_to_array(\"dim3\")", + "outputType" : "ARRAY" + }, + "unnestFilter" : { + "type" : "not", + "field" : { + "type" : "equals", + "column" : "j0.unnest", + "matchValueType" : "STRING", + "matchValue" : "d" + } + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "j0.unnest" ], + "columnTypes" : [ "STRING" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_disabled@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_disabled@NullHandling=default.iq index d7654db2683f..20ec2ea73667 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_disabled@NullHandling=default.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_disabled@NullHandling=default.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@all_disabled@NullHandling=default case-crc:4c1847a0 -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@all_disabled@NullHandling=default case-crc:cac513c5 +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set enableJoinFilterRewrite false @@ -59,29 +59,23 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "bound", + "dimension" : "__time", + "lower" : "978393600000", + "ordering" : { + "type" : "numeric" + } } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_disabled@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_disabled@NullHandling=sql.iq index 099cd78426f8..f10f8b5da377 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_disabled@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_disabled@NullHandling=sql.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@all_disabled@NullHandling=sql case-crc:4c1847a0 -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@all_disabled@NullHandling=sql case-crc:cac513c5 +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set enableJoinFilterRewrite false @@ -59,29 +59,21 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "range", + "column" : "__time", + "matchValueType" : "LONG", + "lower" : 978393600000 } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_enabled@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_enabled@NullHandling=default.iq index 7b9e1e8c7b46..0b39259a86c4 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_enabled@NullHandling=default.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_enabled@NullHandling=default.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@all_enabled@NullHandling=default case-crc:d0070228 -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@all_enabled@NullHandling=default case-crc:5eacffa7 +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set enableJoinFilterRewrite true @@ -59,29 +59,23 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "bound", + "dimension" : "__time", + "lower" : "978393600000", + "ordering" : { + "type" : "numeric" + } } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_enabled@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_enabled@NullHandling=sql.iq index bb208e492bf7..a908f72f7db9 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_enabled@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@all_enabled@NullHandling=sql.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@all_enabled@NullHandling=sql case-crc:d0070228 -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@all_enabled@NullHandling=sql case-crc:5eacffa7 +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set enableJoinFilterRewrite true @@ -59,29 +59,21 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "range", + "column" : "__time", + "matchValueType" : "LONG", + "lower" : 978393600000 } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@default@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@default@NullHandling=default.iq index 1d931c684cf9..d9d6187ebef8 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@default@NullHandling=default.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@default@NullHandling=default.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@default@NullHandling=default case-crc:af596bbc -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@default@NullHandling=default case-crc:35936266 +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set maxScatterGatherBytes 9223372036854775807 @@ -56,29 +56,23 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "bound", + "dimension" : "__time", + "lower" : "978393600000", + "ordering" : { + "type" : "numeric" + } } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@default@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@default@NullHandling=sql.iq index 3e0dd5aa5345..90488aa3e3bf 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@default@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@default@NullHandling=sql.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@default@NullHandling=sql case-crc:af596bbc -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@default@NullHandling=sql case-crc:35936266 +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set maxScatterGatherBytes 9223372036854775807 @@ -56,29 +56,21 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "range", + "column" : "__time", + "matchValueType" : "LONG", + "lower" : 978393600000 } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-on-value-column_disabled@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-on-value-column_disabled@NullHandling=default.iq index 8a3ac331ad5a..a5fb9afeabcb 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-on-value-column_disabled@NullHandling=default.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-on-value-column_disabled@NullHandling=default.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@filter-on-value-column_disabled@NullHandling=default case-crc:f03f6096 -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@filter-on-value-column_disabled@NullHandling=default case-crc:7c6cef9c +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set enableJoinFilterRewrite true @@ -59,29 +59,23 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "bound", + "dimension" : "__time", + "lower" : "978393600000", + "ordering" : { + "type" : "numeric" + } } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-on-value-column_disabled@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-on-value-column_disabled@NullHandling=sql.iq index a5fcb7974bf2..d904ef913db7 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-on-value-column_disabled@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-on-value-column_disabled@NullHandling=sql.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@filter-on-value-column_disabled@NullHandling=sql case-crc:f03f6096 -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@filter-on-value-column_disabled@NullHandling=sql case-crc:7c6cef9c +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set enableJoinFilterRewrite true @@ -59,29 +59,21 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "range", + "column" : "__time", + "matchValueType" : "LONG", + "lower" : 978393600000 } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites-disabled@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites-disabled@NullHandling=default.iq index 72706fda5ea6..121c2724a62a 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites-disabled@NullHandling=default.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites-disabled@NullHandling=default.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites-disabled@NullHandling=default case-crc:d155bb65 -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites-disabled@NullHandling=default case-crc:5d06346f +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set enableJoinFilterRewrite false @@ -59,29 +59,23 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "bound", + "dimension" : "__time", + "lower" : "978393600000", + "ordering" : { + "type" : "numeric" + } } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites-disabled@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites-disabled@NullHandling=sql.iq index 2b1f2d942429..4090856cccaa 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites-disabled@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites-disabled@NullHandling=sql.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites-disabled@NullHandling=sql case-crc:d155bb65 -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites-disabled@NullHandling=sql case-crc:5d06346f +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set enableJoinFilterRewrite false @@ -59,29 +59,21 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "range", + "column" : "__time", + "matchValueType" : "LONG", + "lower" : 978393600000 } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites@NullHandling=default.iq index 38fa8f556802..dede5bf4ed0a 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites@NullHandling=default.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites@NullHandling=default.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites@NullHandling=default case-crc:9c16e907 -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites@NullHandling=default case-crc:1045660d +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set enableJoinFilterRewrite true @@ -59,29 +59,23 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "bound", + "dimension" : "__time", + "lower" : "978393600000", + "ordering" : { + "type" : "numeric" + } } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites@NullHandling=sql.iq index e253673fb063..d9b008e0915f 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites@NullHandling=sql.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites@NullHandling=sql case-crc:9c16e907 -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@filter-rewrites@NullHandling=sql case-crc:1045660d +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set enableJoinFilterRewrite true @@ -59,29 +59,21 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "range", + "column" : "__time", + "matchValueType" : "LONG", + "lower" : 978393600000 } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@join-to-filter@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@join-to-filter@NullHandling=default.iq index 0e6bb9c108f8..220d7f4545f2 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@join-to-filter@NullHandling=default.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@join-to-filter@NullHandling=default.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@join-to-filter@NullHandling=default case-crc:34b62ae1 -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@join-to-filter@NullHandling=default case-crc:6cf62e05 +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set enableJoinFilterRewrite false @@ -59,29 +59,23 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "bound", + "dimension" : "__time", + "lower" : "978393600000", + "ordering" : { + "type" : "numeric" + } } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@join-to-filter@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@join-to-filter@NullHandling=sql.iq index 043c4d0a48f2..8297fc814d97 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@join-to-filter@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testNestedGroupByOnInlineDataSourceWithFilter@join-to-filter@NullHandling=sql.iq @@ -1,5 +1,5 @@ -# testNestedGroupByOnInlineDataSourceWithFilter@join-to-filter@NullHandling=sql case-crc:34b62ae1 -# quidem testcase reason: IRRELEVANT_SCANQUERY +# testNestedGroupByOnInlineDataSourceWithFilter@join-to-filter@NullHandling=sql case-crc:6cf62e05 +# quidem testcase reason: SCAN_QUERY_ON_FILTERED_DS_DOING_FILTERING !set debug true !set defaultTimeout 300000 !set enableJoinFilterRewrite false @@ -59,29 +59,21 @@ DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) "query" : { "queryType" : "scan", "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "dim1" ], - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false + "type" : "filter", + "base" : { + "type" : "table", + "name" : "foo" + }, + "filter" : { + "type" : "range", + "column" : "__time", + "matchValueType" : "LONG", + "lower" : 978393600000 } }, "intervals" : { "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + "intervals" : [ "2001-01-02T00:00:00.000Z/146140482-04-24T15:36:27.903Z" ] }, "resultFormat" : "compactedList", "filter" : { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq index b5d2c682643b..85c1b392c49a 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq @@ -30,8 +30,8 @@ SELECT +-------+----+ | t1 | t2 | +-------+----+ -| dummy | | | dummy | b | +| dummy | | +-------+----+ (2 rows) From be298b77ff754acb7fd84a96092fa180823ef093 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Mon, 30 Sep 2024 14:24:44 +0000 Subject: [PATCH 002/149] cleanup --- pp | 42 ---------- qq | 81 ------------------- ...estGroupByWithLiteralInSubqueryGrouping.iq | 2 +- 3 files changed, 1 insertion(+), 124 deletions(-) delete mode 100644 pp delete mode 100644 qq diff --git a/pp b/pp deleted file mode 100644 index 8f7b02aa06b6..000000000000 --- a/pp +++ /dev/null @@ -1,42 +0,0 @@ -dec -{ - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "numfoo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "virtualColumns" : [ { - "type" : "expression", - "name" : "v0", - "expression" : "mv_to_array(\"dim3\")", - "outputType" : "ARRAY" - } ], - "resultFormat" : "compactedList", - "filter" : { - "type" : "in", - "dimension" : "m1", - "values" : [ "4.0", "5.0" ] - }, - "columns" : [ "dim3", "v0" ], - "context" : { - "debug" : true, - "defaultTimeout" : 300000, - "maxScatterGatherBytes" : 9223372036854775807, - "plannerStrategy" : "DECOUPLED", - "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", - "sqlQueryId" : "dummy", - "sqlStringifyArrays" : false, - "vectorSize" : 2, - "vectorize" : "force", - "vectorizeVirtualColumns" : "force" - }, - "columnTypes" : [ "STRING", "ARRAY" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false -} diff --git a/qq b/qq deleted file mode 100644 index 902733c947eb..000000000000 --- a/qq +++ /dev/null @@ -1,81 +0,0 @@ -deec -{ - "queryType" : "scan", - "dataSource" : { - "type" : "unnest", - "base" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "numfoo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "virtualColumns" : [ { - "type" : "expression", - "name" : "v0", - "expression" : "'a'", - "outputType" : "STRING" - } ], - "resultFormat" : "compactedList", - "filter" : { - "type" : "selector", - "dimension" : "dim2", - "value" : "a" - }, - "columns" : [ "__time", "cnt", "d1", "d2", "dim1", "dim3", "dim4", "dim5", "dim6", "f1", "f2", "l1", "l2", "m1", "m2", "unique_dim1", "v0" ], - "context" : { - "debug" : true, - "defaultTimeout" : 300000, - "maxScatterGatherBytes" : 9223372036854775807, - "plannerStrategy" : "DECOUPLED", - "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", - "sqlQueryId" : "dummy", - "sqlStringifyArrays" : false, - "vectorSize" : 2, - "vectorize" : "force", - "vectorizeVirtualColumns" : "force" - }, - "columnTypes" : [ "LONG", "LONG", "DOUBLE", "DOUBLE", "STRING", "STRING", "STRING", "STRING", "STRING", "FLOAT", "FLOAT", "LONG", "LONG", "FLOAT", "DOUBLE", "COMPLEX", "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false - } - }, - "virtualColumn" : { - "type" : "expression", - "name" : "j0.unnest", - "expression" : "mv_to_array(\"dim3\")", - "outputType" : "ARRAY" - }, - "unnestFilter" : null - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "j0.unnest" ], - "context" : { - "debug" : true, - "defaultTimeout" : 300000, - "maxScatterGatherBytes" : 9223372036854775807, - "plannerStrategy" : "DECOUPLED", - "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", - "sqlQueryId" : "dummy", - "sqlStringifyArrays" : false, - "vectorSize" : 2, - "vectorize" : "force", - "vectorizeVirtualColumns" : "force" - }, - "columnTypes" : [ "STRING" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false -} diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq index 85c1b392c49a..b5d2c682643b 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq @@ -30,8 +30,8 @@ SELECT +-------+----+ | t1 | t2 | +-------+----+ -| dummy | b | | dummy | | +| dummy | b | +-------+----+ (2 rows) From 726eec63f9370b348799d8a58756d5c0490444a0 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 1 Oct 2024 09:16:11 +0000 Subject: [PATCH 003/149] some stuff --- .../java/org/apache/druid/query/Query.java | 6 + .../apache/druid/query/QueryDataSource.java | 27 ++- .../query/union/RealUnionQueryRunner.java | 50 +++++ .../apache/druid/query/union/UnionQuery.java | 180 ++++++++++++++++++ .../sql/calcite/rel/logical/DruidUnion.java | 82 ++++++-- .../druid/sql/calcite/CalciteQueryTest.java | 1 - ...estGroupByWithLiteralInSubqueryGrouping.iq | 2 +- 7 files changed, 322 insertions(+), 26 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java create mode 100644 processing/src/main/java/org/apache/druid/query/union/UnionQuery.java diff --git a/processing/src/main/java/org/apache/druid/query/Query.java b/processing/src/main/java/org/apache/druid/query/Query.java index 4795ecbb5611..67f190567000 100644 --- a/processing/src/main/java/org/apache/druid/query/Query.java +++ b/processing/src/main/java/org/apache/druid/query/Query.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.base.Preconditions; +import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; import org.apache.druid.error.DruidException; @@ -82,6 +83,11 @@ public interface Query DataSource getDataSource(); + default List getDataSources() + { + return ImmutableList.of(getDataSource()); + } + boolean hasFilters(); DimFilter getFilter(); diff --git a/processing/src/main/java/org/apache/druid/query/QueryDataSource.java b/processing/src/main/java/org/apache/druid/query/QueryDataSource.java index 3d889472ea9f..3cdda56aa9db 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryDataSource.java +++ b/processing/src/main/java/org/apache/druid/query/QueryDataSource.java @@ -26,9 +26,11 @@ import com.google.common.collect.Iterables; import org.apache.druid.java.util.common.IAE; import org.apache.druid.query.planning.DataSourceAnalysis; +import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.SegmentReference; import java.util.Collections; +import java.util.HashSet; import java.util.List; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; @@ -49,7 +51,11 @@ public QueryDataSource(@JsonProperty("query") Query query) @Override public Set getTableNames() { - return query.getDataSource().getTableNames(); + Set names = new HashSet<>(); + for (DataSource ds : query.getDataSources()) { + names.addAll(ds.getTableNames()); + } + return names; } @JsonProperty @@ -61,7 +67,7 @@ public Query getQuery() @Override public List getChildren() { - return Collections.singletonList(query.getDataSource()); + return query.getDataSources(); } @Override @@ -118,14 +124,17 @@ public byte[] getCacheKey() public DataSourceAnalysis getAnalysis() { final Query subQuery = this.getQuery(); - if (!(subQuery instanceof BaseQuery)) { - // We must verify that the subQuery is a BaseQuery, because it is required to make - // "DataSourceAnalysis.getBaseQuerySegmentSpec" work properly. - // All built-in query types are BaseQuery, so we only expect this with funky extension queries. - throw new IAE("Cannot analyze subquery of class[%s]", subQuery.getClass().getName()); + if (subQuery instanceof BaseQuery) { + final DataSource current = subQuery.getDataSource(); + return current.getAnalysis().maybeWithBaseQuery(subQuery); + } + if(subQuery instanceof UnionQuery) { + return new DataSourceAnalysis(this, null, null, Collections.emptyList()); } - final DataSource current = subQuery.getDataSource(); - return current.getAnalysis().maybeWithBaseQuery(subQuery); + // We must verify that the subQuery is a BaseQuery, because it is required to make + // "DataSourceAnalysis.getBaseQuerySegmentSpec" work properly. + // All built-in query types are BaseQuery, so we only expect this with funky extension queries. + throw new IAE("Cannot analyze subquery of class[%s]", subQuery.getClass().getName()); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java new file mode 100644 index 000000000000..1e454acb25de --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java @@ -0,0 +1,50 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.union; + +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.query.QueryPlus; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.context.ResponseContext; + +public class RealUnionQueryRunner implements QueryRunner +{ + + public RealUnionQueryRunner(UnionQuery unionQuery, QuerySegmentWalker walker) + { + if(true) + { + throw new RuntimeException("FIXME: Unimplemented!"); + } + + } + + @Override + public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) + { + if (true) { + throw new RuntimeException("FIXME: Unimplemented!"); + } + return null; + + } + +} diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java new file mode 100644 index 000000000000..8c6536a794bf --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -0,0 +1,180 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.union; + +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Ordering; +import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.DataSource; +import org.apache.druid.query.Query; +import org.apache.druid.query.QueryContexts; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.query.spec.QuerySegmentSpec; +import org.joda.time.DateTimeZone; +import org.joda.time.Duration; +import org.joda.time.Interval; + +import java.util.ArrayList; +import java.util.List; +import java.util.Map; + +public class UnionQuery implements Query +{ + protected final Map context; + protected final List> queries; + + public UnionQuery(List> queries2) + { + this(queries2, queries2.get(0).getContext()); + } + + public UnionQuery(List> queries, Map context) + { + this.queries = queries; + this.context = context; + } + + @Override + public DataSource getDataSource() + { + throw new RuntimeException("This is not supported"); + } + + @Override + public List getDataSources() + { + + List dataSources = new ArrayList<>(); + for (Query query : queries) { + dataSources.add(query.getDataSource()); + } + return dataSources; + } + + @Override + public boolean hasFilters() + { + return false; + + } + + @Override + public DimFilter getFilter() + { + return null; + } + + @Override + public String getType() + { + return getClass().getSimpleName(); + } + + @Override + public QueryRunner getRunner(QuerySegmentWalker walker) + { + return new RealUnionQueryRunner(this, walker); + } + + @Override + public List getIntervals() + { + throw new RuntimeException("FIXME: Unimplemented!"); + } + + @Override + public Duration getDuration() + { + throw new RuntimeException("FIXME: Unimplemented!"); + } + + @Override + public Granularity getGranularity() + { + throw new RuntimeException("FIXME: Unimplemented!"); + } + + @Override + public DateTimeZone getTimezone() + { + throw new RuntimeException("FIXME: Unimplemented!"); + } + + @Override + public Map getContext() + { + return context; + } + + @Override + public Ordering getResultOrdering() + { + if (true) { + throw new RuntimeException("FIXME: Unimplemented!"); + } + return null; + + } + + @Override + public Query withOverriddenContext(Map contextOverrides) + { + return new UnionQuery(queries, QueryContexts.override(getContext(), contextOverrides)); + } + + @Override + public Query withQuerySegmentSpec(QuerySegmentSpec spec) + { + throw new RuntimeException("FIXME: Unimplemented!"); + } + + @Override + public Query withId(String id) + { + return withOverriddenContext(ImmutableMap.of(BaseQuery.QUERY_ID, id)); + } + + @Override + public String getId() + { + return context().getString(BaseQuery.QUERY_ID); + } + + @Override + public Query withSubQueryId(String subQueryId) + { + return withOverriddenContext(ImmutableMap.of(BaseQuery.SUB_QUERY_ID, subQueryId)); + } + + @Override + public String getSubQueryId() + { + return context().getString(BaseQuery.SUB_QUERY_ID); + } + + @Override + public Query withDataSource(DataSource dataSource) + { + throw new RuntimeException("FIXME: Unimplemented!"); + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/logical/DruidUnion.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/logical/DruidUnion.java index 96981a751e64..a737e369aeec 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/logical/DruidUnion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/logical/DruidUnion.java @@ -31,11 +31,15 @@ import org.apache.druid.error.DruidException; import org.apache.druid.query.DataSource; import org.apache.druid.query.InlineDataSource; +import org.apache.druid.query.Query; +import org.apache.druid.query.QueryDataSource; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.UnionDataSource; +import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.sql.calcite.planner.PlannerContext; import org.apache.druid.sql.calcite.planner.querygen.SourceDescProducer; + import java.util.ArrayList; import java.util.List; @@ -66,24 +70,72 @@ public RelOptCost computeSelfCost(RelOptPlanner planner, RelMetadataQuery mq) @Override public SourceDesc getSourceDesc(PlannerContext plannerContext, List sources) { - List dataSources = new ArrayList<>(); - RowSignature signature = null; - for (SourceDesc sourceDesc : sources) { - checkDataSourceSupported(sourceDesc.dataSource); - dataSources.add(sourceDesc.dataSource); - if (signature == null) { - signature = sourceDesc.rowSignature; - } else { - if (!signature.equals(sourceDesc.rowSignature)) { - throw DruidException.defensive( - "Row signature mismatch in Union inputs [%s] and [%s]", - signature, - sourceDesc.rowSignature - ); + if (mayUseUnionDataSource(sources)) { + List dataSources = new ArrayList<>(); + RowSignature signature = null; + for (SourceDesc sourceDesc : sources) { + checkDataSourceSupported(sourceDesc.dataSource); + dataSources.add(sourceDesc.dataSource); + if (signature == null) { + signature = sourceDesc.rowSignature; + } else { + if (!signature.equals(sourceDesc.rowSignature)) { + throw DruidException.defensive( + "Row signature mismatch in Union inputs [%s] and [%s]", + signature, + sourceDesc.rowSignature + ); + } + } + } + return new SourceDesc(new UnionDataSource(dataSources), signature); + } + if (mayUseUnionQuery(sources)) { + RowSignature signature = null; + List> queries = new ArrayList<>(); + for (SourceDesc sourceDesc : sources) { + QueryDataSource qds = (QueryDataSource) sourceDesc.dataSource; + queries.add(qds.getQuery()); + if (signature == null) { + signature = sourceDesc.rowSignature; + } else { + if (!signature.equals(sourceDesc.rowSignature)) { + throw DruidException.defensive( + "Row signature mismatch in Union inputs [%s] and [%s]", + signature, + sourceDesc.rowSignature + ); + } } } + return new SourceDesc(new QueryDataSource(new UnionQuery(queries)), signature); + } + + throw DruidException.defensive("XXXOnly Table and Values are supported as inputs for Union [%s]", sources); + } + + private boolean mayUseUnionQuery(List sources) + { + for (SourceDesc sourceDesc : sources) { + DataSource dataSource = sourceDesc.dataSource; + if (dataSource instanceof QueryDataSource) { + continue; + } + return false; + } + return true; + } + + private boolean mayUseUnionDataSource(List sources) + { + for (SourceDesc sourceDesc : sources) { + DataSource dataSource = sourceDesc.dataSource; + if (dataSource instanceof TableDataSource || dataSource instanceof InlineDataSource) { + continue; + } + return false; } - return new SourceDesc(new UnionDataSource(dataSources), signature); + return true; } private void checkDataSourceSupported(DataSource dataSource) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 5af5ec3097c5..344b1f7bbe72 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -3108,7 +3108,6 @@ public void testTopNWithSelectAndOrderByProjections() ); } - @NotYetSupported(Modes.UNION_WITH_COMPLEX_OPERAND) @Test public void testUnionAllQueries() { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq index b5d2c682643b..85c1b392c49a 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq @@ -30,8 +30,8 @@ SELECT +-------+----+ | t1 | t2 | +-------+----+ -| dummy | | | dummy | b | +| dummy | | +-------+----+ (2 rows) From fadaa32af77814a89f82eaf92ab0f7d49bd4fe57 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 1 Oct 2024 13:06:22 +0000 Subject: [PATCH 004/149] add stuff --- .../java/org/apache/druid/query/Query.java | 10 + .../apache/druid/query/QueryDataSource.java | 7 +- .../org/apache/druid/query/QueryPlus.java | 11 +- .../druid/query/groupby/GroupByQuery.java | 2 +- .../query/groupby/SupportRowSignature.java | 28 ++ .../query/timeseries/TimeseriesQuery.java | 10 +- .../union/RealUnionQueryQueryToolChest.java | 241 ++++++++++++++++++ .../query/union/RealUnionQueryRunner.java | 32 ++- .../union/RealUnionQueryRunnerFactory.java | 65 +++++ .../apache/druid/query/union/UnionQuery.java | 35 ++- .../druid/guice/QueryRunnerFactoryModule.java | 3 + .../server/ClientQuerySegmentWalker.java | 12 +- .../apache/druid/server/QueryStackTests.java | 3 + 13 files changed, 429 insertions(+), 30 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java create mode 100644 processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java create mode 100644 processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java diff --git a/processing/src/main/java/org/apache/druid/query/Query.java b/processing/src/main/java/org/apache/druid/query/Query.java index 67f190567000..f8305f4b03ae 100644 --- a/processing/src/main/java/org/apache/druid/query/Query.java +++ b/processing/src/main/java/org/apache/druid/query/Query.java @@ -24,10 +24,12 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Iterables; import com.google.common.collect.Ordering; import org.apache.druid.error.DruidException; import org.apache.druid.guice.annotations.ExtensionPoint; import org.apache.druid.java.util.common.HumanReadableBytes; +import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.datasourcemetadata.DataSourceMetadataQuery; import org.apache.druid.query.filter.DimFilter; @@ -291,4 +293,12 @@ default Interval getSingleInterval() ) ); } + + default Query withDataSources(List children) + { + if (children.size() != 1) { + throw new IAE("Must have exactly one child"); + } + return withDataSource(Iterables.getOnlyElement(children)); + } } diff --git a/processing/src/main/java/org/apache/druid/query/QueryDataSource.java b/processing/src/main/java/org/apache/druid/query/QueryDataSource.java index 3cdda56aa9db..75170ff09e56 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryDataSource.java +++ b/processing/src/main/java/org/apache/druid/query/QueryDataSource.java @@ -23,7 +23,6 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; -import com.google.common.collect.Iterables; import org.apache.druid.java.util.common.IAE; import org.apache.druid.query.planning.DataSourceAnalysis; import org.apache.druid.query.union.UnionQuery; @@ -73,11 +72,7 @@ public List getChildren() @Override public DataSource withChildren(List children) { - if (children.size() != 1) { - throw new IAE("Must have exactly one child"); - } - - return new QueryDataSource(query.withDataSource(Iterables.getOnlyElement(children))); + return new QueryDataSource(query.withDataSources(children)); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/QueryPlus.java b/processing/src/main/java/org/apache/druid/query/QueryPlus.java index 1b18e9439099..75599dae7106 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryPlus.java +++ b/processing/src/main/java/org/apache/druid/query/QueryPlus.java @@ -21,10 +21,11 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; +import org.apache.druid.error.DruidException; import org.apache.druid.guice.annotations.PublicApi; +import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.query.context.ResponseContext; - import javax.annotation.Nullable; /** @@ -59,6 +60,14 @@ public Query getQuery() return query; } + public > T unwrapQuery(Class clazz) + { + if(clazz.isInstance(query)) { + return (T) query; + } + throw DruidException.defensive("Encountered unexpected query type [%s] instead of [%s]", query.getClass(), clazz); + } + @Nullable public QueryMetrics getQueryMetrics() { diff --git a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java index 35510d7c5eea..68c071060633 100644 --- a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java @@ -85,7 +85,7 @@ /** * */ -public class GroupByQuery extends BaseQuery +public class GroupByQuery extends BaseQuery implements SupportRowSignature { public static final String CTX_KEY_SORT_BY_DIMS_FIRST = "sortByDimsFirst"; public static final String CTX_TIMESTAMP_RESULT_FIELD = "timestampResultField"; diff --git a/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java b/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java new file mode 100644 index 000000000000..9698c3b9bf26 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java @@ -0,0 +1,28 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.groupby; + +import org.apache.druid.segment.column.RowSignature; + +public interface SupportRowSignature +{ + public RowSignature getResultRowSignature(final RowSignature.Finalization finalization); + +} diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java index 88d488f85b9c..d08b05c35dc7 100644 --- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java +++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java @@ -38,11 +38,13 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.query.groupby.SupportRowSignature; import org.apache.druid.query.groupby.orderby.DefaultLimitSpec.LimitJsonIncludeFilter; import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.RowSignature.Finalization; import javax.annotation.Nullable; import java.util.ArrayList; @@ -55,7 +57,7 @@ /** */ @JsonTypeName("timeseries") -public class TimeseriesQuery extends BaseQuery> +public class TimeseriesQuery extends BaseQuery> implements SupportRowSignature { public static final String CTX_GRAND_TOTAL = "grandTotal"; public static final String SKIP_EMPTY_BUCKETS = "skipEmptyBuckets"; @@ -181,6 +183,12 @@ public boolean isSkipEmptyBuckets() return context().getBoolean(SKIP_EMPTY_BUCKETS, false); } + @Override + public RowSignature getResultRowSignature(Finalization finalization) { + return getResultSignature(finalization); + } + + @Deprecated public RowSignature getResultSignature(final RowSignature.Finalization finalization) { final RowSignature.Builder builder = RowSignature.builder(); diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java new file mode 100644 index 000000000000..c9ee0785f0a4 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java @@ -0,0 +1,241 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.union; + +import com.fasterxml.jackson.core.type.TypeReference; +import com.google.common.base.Function; +import com.google.common.base.Functions; +import com.google.common.collect.ImmutableMap; +import org.apache.druid.error.DruidException; +import org.apache.druid.frame.allocation.MemoryAllocatorFactory; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.query.DefaultQueryMetrics; +import org.apache.druid.query.FrameSignaturePair; +import org.apache.druid.query.Query; +import org.apache.druid.query.QueryMetrics; +import org.apache.druid.query.QueryPlus; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryToolChest; +import org.apache.druid.query.ResultSerializationMode; +import org.apache.druid.query.aggregation.MetricManipulationFn; +import org.apache.druid.query.context.ResponseContext; +import org.apache.druid.query.groupby.SupportRowSignature; +import org.apache.druid.query.operator.WindowOperatorQuery; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.rowsandcols.column.Column; +import org.apache.druid.query.rowsandcols.column.ColumnAccessor; +import org.apache.druid.query.rowsandcols.column.NullColumn; +import org.apache.druid.query.rowsandcols.semantic.FrameMaker; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.RowSignature.Finalization; + +import java.util.ArrayList; +import java.util.List; +import java.util.Optional; + +public class RealUnionQueryQueryToolChest extends QueryToolChest +{ + + @Override + @SuppressWarnings("unchecked") + public QueryRunner mergeResults(QueryRunner runner) + { + return new RowsAndColumnsSerializingQueryRunner( + (queryPlus, responseContext) -> { + return runner.run(queryPlus, responseContext); + } + ); + } + + @Override + public QueryMetrics makeMetrics(UnionQuery query) + { + return new DefaultQueryMetrics<>(); + } + + @Override + public Function makePreComputeManipulatorFn( + UnionQuery query, + MetricManipulationFn fn + ) + { + return Functions.identity(); + } + + @Override + public TypeReference getResultTypeReference() + { + return new TypeReference() + { + }; + } + + @Override + public RowSignature resultArraySignature(UnionQuery query) + { + Query q0 = query.queries.get(0); + if (q0 instanceof SupportRowSignature) { + return ((SupportRowSignature) q0).getResultRowSignature(Finalization.UNKNOWN); + } + throw new UnsupportedOperationException("Not supported"); + } + + @Override + @SuppressWarnings({"unchecked", "rawtypes"}) + public Sequence resultsAsArrays( + UnionQuery query, + Sequence resultSequence + ) + { + // Dark magic; see RowsAndColumnsSerializingQueryRunner. + return (Sequence) resultSequence; + } + + @Override + @SuppressWarnings({"unchecked", "rawtypes"}) + public Optional> resultsAsFrames( + UnionQuery query, + Sequence resultSequence, + MemoryAllocatorFactory memoryAllocatorFactory, + boolean useNestedForUnknownTypes + ) + { + // see RowsAndColumnsSerializingQueryRunner + return Optional.of((Sequence) resultSequence); + } + + /** + * This class exists to serialize the RowsAndColumns that are used in this query and make it the return Sequence + * actually be a Sequence of rows or frames, as the query requires. + * This is relatively broken in a number of regards, the most obvious of which is that it is going to run counter to the stated class on the Generic of the QueryToolChest. + * That is, the code makes it look like you are getting a Sequence of RowsAndColumns, but, by using this, the query will + * actually ultimately produce a Sequence of Object[] or Frames. This works because of type Erasure in Java (it's all Object + * at the end of the day). + *

+ * While it might seem like this will break all sorts of things, the Generic type is actually there more as a type + * "hint" to make the writing of the ToolChest and Factory and stuff a bit more simple. Any caller of this cannot + * truly depend on the type anyway other than to just throw it across the wire, so this should just magically work + * even though it looks like it shouldn't even compile. + *

+ * Not our proudest moment, but we use the tools available to us. + */ + @SuppressWarnings({"unchecked", "rawtypes"}) + private static class RowsAndColumnsSerializingQueryRunner implements QueryRunner + { + + private final QueryRunner baseQueryRunner; + + private RowsAndColumnsSerializingQueryRunner( + QueryRunner baseQueryRunner + ) + { + this.baseQueryRunner = baseQueryRunner; + } + + @Override + public Sequence run( + QueryPlus queryPlus, + ResponseContext responseContext + ) + { + // We only want to do this operation once at the very, very top of the execution tree. So we check and set + // a context parameter so that if this merge code runs anywhere else, it will skip this part. + final WindowOperatorQuery query = (WindowOperatorQuery) queryPlus.getQuery(); + if (query.context().getBoolean("unravel", true)) { + final Sequence baseSequence = baseQueryRunner.run( + queryPlus.withQuery(query.withOverriddenContext(ImmutableMap.of("unravel", false))), + responseContext + ); + final ResultSerializationMode serializationMode = query.context().getEnum( + ResultSerializationMode.CTX_SERIALIZATION_PARAMETER, + ResultSerializationMode.class, + ResultSerializationMode.ROWS + ); + switch (serializationMode) { + case ROWS: + return asRows(baseSequence, query); + case FRAMES: + return asFrames(baseSequence); + default: + throw DruidException.defensive("Serialization mode[%s] not supported", serializationMode); + } + } + + return baseQueryRunner.run(queryPlus, responseContext); + } + + /** + * Translates Sequence of RACs to a Sequence of Object[] + */ + private static Sequence asRows(final Sequence baseSequence, final WindowOperatorQuery query) + { + final RowSignature rowSignature = query.getRowSignature(); + return baseSequence.flatMap( + rac -> { + List results = new ArrayList<>(rac.numRows()); + + ColumnAccessor[] accessors = new ColumnAccessor[rowSignature.size()]; + int index = 0; + for (String columnName : rowSignature.getColumnNames()) { + final Column column = rac.findColumn(columnName); + if (column == null) { + final ColumnType columnType = rowSignature + .getColumnType(columnName) + .orElse(ColumnType.UNKNOWN_COMPLEX); + + accessors[index] = new NullColumn.Accessor(columnType, rac.numRows()); + } else { + accessors[index] = column.toAccessor(); + } + ++index; + } + + for (int i = 0; i < rac.numRows(); ++i) { + Object[] objArr = new Object[accessors.length]; + for (int j = 0; j < accessors.length; j++) { + objArr[j] = accessors[j].getObject(i); + } + results.add(objArr); + } + + return Sequences.simple(results); + } + ); + } + + /** + * Translates a sequence of RACs to a Sequence of Frames + */ + private static Sequence asFrames(final Sequence baseSequence) + { + return baseSequence.map( + rac -> { + FrameMaker frameMaker = FrameMaker.fromRAC(rac); + return new FrameSignaturePair( + frameMaker.toColumnBasedFrame(), + frameMaker.computeSignature() + ); + } + ); + } + } +} diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java index 1e454acb25de..077ee4b897ff 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java @@ -24,27 +24,39 @@ import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.context.ResponseContext; +import org.apache.druid.query.rowsandcols.RowsAndColumns; -public class RealUnionQueryRunner implements QueryRunner +public class RealUnionQueryRunner implements QueryRunner { + private UnionQuery unionQuery; + private QuerySegmentWalker walker; - public RealUnionQueryRunner(UnionQuery unionQuery, QuerySegmentWalker walker) + public RealUnionQueryRunner(UnionQuery unionQuery, QuerySegmentWalker walker) { - if(true) - { - throw new RuntimeException("FIXME: Unimplemented!"); - } + this.unionQuery = unionQuery; + this.walker = walker; } @Override - public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) + public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) { - if (true) { - throw new RuntimeException("FIXME: Unimplemented!"); + queryPlus.unwrap(UnionQuery.class); + UnionQuery query = (UnionQuery) queryPlus.getQuery(); + if (!(query instanceof ScanQuery)) { + throw new ISE("Got a [%s] which isn't a %s", query.getClass(), ScanQuery.class); } - return null; + ScanQuery.verifyOrderByForNativeExecution((ScanQuery) query); + + // it happens in unit tests + final Long timeoutAt = responseContext.getTimeoutTime(); + if (timeoutAt == null || timeoutAt == 0L) { + responseContext.putTimeoutTime(JodaUtils.MAX_INSTANT); + } + return engine.process((ScanQuery) query, segment, responseContext, queryPlus.getQueryMetrics()); } + + } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java new file mode 100644 index 000000000000..91f6531cd88d --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java @@ -0,0 +1,65 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.union; + +import org.apache.druid.query.QueryProcessingPool; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryRunnerFactory; +import org.apache.druid.query.QueryToolChest; +import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.segment.Segment; + +public class RealUnionQueryRunnerFactory implements QueryRunnerFactory +{ + + public RealUnionQueryRunnerFactory(String string) + { + } + + @Override + public QueryRunner createRunner(Segment segment) + { + if(true) + { + throw new RuntimeException("FIXME: Unimplemented!"); + } + return null; + + } + + @Override + public QueryRunner mergeRunners(QueryProcessingPool queryProcessingPool, + Iterable> queryRunners) + { + if(true) + { + throw new RuntimeException("FIXME: Unimplemented!"); + } + return null; + + } + + @Override + public QueryToolChest getToolchest() + { + return new RealUnionQueryQueryToolChest(); + } + +} diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index 8c6536a794bf..f6bd6768d8b3 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -19,6 +19,7 @@ package org.apache.druid.query.union; +import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; import org.apache.druid.java.util.common.granularity.Granularity; @@ -29,6 +30,7 @@ import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.query.rowsandcols.RowsAndColumns; import org.apache.druid.query.spec.QuerySegmentSpec; import org.joda.time.DateTimeZone; import org.joda.time.Duration; @@ -38,8 +40,9 @@ import java.util.List; import java.util.Map; -public class UnionQuery implements Query +public class UnionQuery implements Query { + RealUnionQueryRunnerFactory a; protected final Map context; protected final List> queries; @@ -50,6 +53,7 @@ public UnionQuery(List> queries2) public UnionQuery(List> queries, Map context) { + Preconditions.checkArgument(queries.size() > 1, "union with fewer than 2 queries makes no sense"); this.queries = queries; this.context = context; } @@ -91,9 +95,9 @@ public String getType() } @Override - public QueryRunner getRunner(QuerySegmentWalker walker) + public QueryRunner getRunner(QuerySegmentWalker walker) { - return new RealUnionQueryRunner(this, walker); + return new RealUnionQueryRunner(this, walker); } @Override @@ -127,7 +131,7 @@ public Map getContext() } @Override - public Ordering getResultOrdering() + public Ordering getResultOrdering() { if (true) { throw new RuntimeException("FIXME: Unimplemented!"); @@ -137,19 +141,19 @@ public Ordering getResultOrdering() } @Override - public Query withOverriddenContext(Map contextOverrides) + public Query withOverriddenContext(Map contextOverrides) { - return new UnionQuery(queries, QueryContexts.override(getContext(), contextOverrides)); + return new UnionQuery(queries, QueryContexts.override(getContext(), contextOverrides)); } @Override - public Query withQuerySegmentSpec(QuerySegmentSpec spec) + public Query withQuerySegmentSpec(QuerySegmentSpec spec) { throw new RuntimeException("FIXME: Unimplemented!"); } @Override - public Query withId(String id) + public Query withId(String id) { return withOverriddenContext(ImmutableMap.of(BaseQuery.QUERY_ID, id)); } @@ -161,7 +165,7 @@ public String getId() } @Override - public Query withSubQueryId(String subQueryId) + public Query withSubQueryId(String subQueryId) { return withOverriddenContext(ImmutableMap.of(BaseQuery.SUB_QUERY_ID, subQueryId)); } @@ -173,8 +177,19 @@ public String getSubQueryId() } @Override - public Query withDataSource(DataSource dataSource) + public Query withDataSource(DataSource dataSource) { throw new RuntimeException("FIXME: Unimplemented!"); } + + @Override + public Query withDataSources(List children) + { + Preconditions.checkArgument(queries.size() == children.size(), "Number of children must match number of queries"); + List> newQueries= new ArrayList<>(); + for (int i = 0; i < queries.size(); i++) { + newQueries.add(queries.get(i).withDataSource(children.get(i))); + } + return new UnionQuery(newQueries, context); + } } diff --git a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java index 5e71a0d1f082..f71228a3fbee 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java @@ -46,6 +46,8 @@ import org.apache.druid.query.timeseries.TimeseriesQueryRunnerFactory; import org.apache.druid.query.topn.TopNQuery; import org.apache.druid.query.topn.TopNQueryRunnerFactory; +import org.apache.druid.query.union.RealUnionQueryRunnerFactory; +import org.apache.druid.query.union.UnionQuery; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QuerySchedulerProvider; @@ -66,6 +68,7 @@ public class QueryRunnerFactoryModule extends QueryToolChestModule .put(TimeseriesQuery.class, TimeseriesQueryRunnerFactory.class) .put(TopNQuery.class, TopNQueryRunnerFactory.class) .put(WindowOperatorQuery.class, WindowOperatorQueryQueryRunnerFactory.class) + .put(UnionQuery.class, RealUnionQueryRunnerFactory.class) .build(); @Override diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 37ae14f56c30..1e54f78652a6 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -62,6 +62,7 @@ import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.planning.DataSourceAnalysis; +import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.server.initialization.ServerConfig; @@ -289,6 +290,10 @@ public QueryRunner getQueryRunnerForSegments(final Query query, final */ private boolean canRunQueryUsingLocalWalker(Query query) { + if(query instanceof UnionQuery) { + // FIXME ?? + return false; + } final DataSource dataSourceFromQuery = query.getDataSource(); final DataSourceAnalysis analysis = dataSourceFromQuery.getAnalysis(); final QueryToolChest> toolChest = warehouse.getToolChest(query); @@ -308,6 +313,11 @@ private boolean canRunQueryUsingLocalWalker(Query query) */ private boolean canRunQueryUsingClusterWalker(Query query) { + if(query instanceof UnionQuery) { + // FIXME ?? + return true; + } + final DataSource dataSourceFromQuery = query.getDataSource(); final DataSourceAnalysis analysis = dataSourceFromQuery.getAnalysis(); final QueryToolChest> toolChest = warehouse.getToolChest(query); @@ -623,7 +633,7 @@ private DataSource insertSubqueryIds( ) { if (currentDataSource instanceof QueryDataSource - && queryDataSourceToSubqueryIds.containsKey((QueryDataSource) currentDataSource)) { + && queryDataSourceToSubqueryIds.containsKey(currentDataSource)) { QueryDataSource queryDataSource = (QueryDataSource) currentDataSource; Pair nestingInfo = queryDataSourceToSubqueryIds.get(queryDataSource); String subQueryId = nestingInfo.lhs + "." + nestingInfo.rhs; diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index 041c4654d92e..cb4667130849 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -77,6 +77,8 @@ import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.query.topn.TopNQueryQueryToolChest; import org.apache.druid.query.topn.TopNQueryRunnerFactory; +import org.apache.druid.query.union.RealUnionQueryRunnerFactory; +import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.ReferenceCountingSegment; import org.apache.druid.segment.SegmentWrangler; import org.apache.druid.segment.TestHelper; @@ -371,6 +373,7 @@ public int getMinTopNThreshold() QueryRunnerTestHelper.NOOP_QUERYWATCHER ) ) + .put(UnionQuery.class, new RealUnionQueryRunnerFactory("asd")) .put(GroupByQuery.class, groupByQueryRunnerFactory) .put(TimeBoundaryQuery.class, new TimeBoundaryQueryRunnerFactory(QueryRunnerTestHelper.NOOP_QUERYWATCHER)) .put(WindowOperatorQuery.class, new WindowOperatorQueryQueryRunnerFactory()) From 6daa7d439d83e750fdd1b2e9e78b1fc9f800cbda Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 1 Oct 2024 13:38:18 +0000 Subject: [PATCH 005/149] updates --- .../org/apache/druid/query/QueryPlus.java | 4 +- .../query/union/RealUnionQueryRunner.java | 41 ++++++++++++------- .../apache/druid/query/union/UnionQuery.java | 17 ++++++-- 3 files changed, 42 insertions(+), 20 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryPlus.java b/processing/src/main/java/org/apache/druid/query/QueryPlus.java index 75599dae7106..f862fd7b825a 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryPlus.java +++ b/processing/src/main/java/org/apache/druid/query/QueryPlus.java @@ -60,10 +60,10 @@ public Query getQuery() return query; } - public > T unwrapQuery(Class clazz) + public > C unwrapQuery(Class clazz) { if(clazz.isInstance(query)) { - return (T) query; + return (C) query; } throw DruidException.defensive("Encountered unexpected query type [%s] instead of [%s]", query.getClass(), clazz); } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java index 077ee4b897ff..7c7e2eb4076a 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java @@ -20,41 +20,52 @@ package org.apache.druid.query.union; import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.query.Query; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.Result; import org.apache.druid.query.context.ResponseContext; import org.apache.druid.query.rowsandcols.RowsAndColumns; +import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; +import org.apache.druid.query.timeseries.TimeseriesResultValue; + +import java.util.List; public class RealUnionQueryRunner implements QueryRunner { - private UnionQuery unionQuery; private QuerySegmentWalker walker; - public RealUnionQueryRunner(UnionQuery unionQuery, QuerySegmentWalker walker) + public RealUnionQueryRunner(QuerySegmentWalker walker) { - this.unionQuery = unionQuery; this.walker = walker; - } @Override public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) { - queryPlus.unwrap(UnionQuery.class); - UnionQuery query = (UnionQuery) queryPlus.getQuery(); - if (!(query instanceof ScanQuery)) { - throw new ISE("Got a [%s] which isn't a %s", query.getClass(), ScanQuery.class); + UnionQuery unionQuery = queryPlus.unwrapQuery(UnionQuery.class); + + for (Query q: unionQuery.queries) { + if(q instanceof TimeseriesQuery) { + runTsQuery(queryPlus,(TimeseriesQuery)q, responseContext); + } } + return Sequences.empty(); + } - ScanQuery.verifyOrderByForNativeExecution((ScanQuery) query); + private void runTsQuery(QueryPlus queryPlus, TimeseriesQuery q, ResponseContext responseContext) + { + QueryRunner> runner = q.getRunner(walker); + Sequence> res = runner.run(queryPlus.withQuery(q), responseContext); + + TimeseriesQueryQueryToolChest tsToolChest = new TimeseriesQueryQueryToolChest(); + Sequence res1 = tsToolChest.resultsAsArrays(q, res); + List li = res1.toList(); +// tsToolChest.mergeResults() - // it happens in unit tests - final Long timeoutAt = responseContext.getTimeoutTime(); - if (timeoutAt == null || timeoutAt == 0L) { - responseContext.putTimeoutTime(JodaUtils.MAX_INSTANT); - } - return engine.process((ScanQuery) query, segment, responseContext, queryPlus.getQueryMetrics()); } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index f6bd6768d8b3..bfa7269b102b 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -39,6 +39,7 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.function.Function; public class UnionQuery implements Query { @@ -97,7 +98,7 @@ public String getType() @Override public QueryRunner getRunner(QuerySegmentWalker walker) { - return new RealUnionQueryRunner(this, walker); + return new RealUnionQueryRunner(walker); } @Override @@ -137,13 +138,13 @@ public Ordering getResultOrdering() throw new RuntimeException("FIXME: Unimplemented!"); } return null; - } @Override public Query withOverriddenContext(Map contextOverrides) { - return new UnionQuery(queries, QueryContexts.override(getContext(), contextOverrides)); + List> newQueries = mapQueries(q -> q.withOverriddenContext(contextOverrides)); + return new UnionQuery(newQueries, QueryContexts.override(getContext(), contextOverrides)); } @Override @@ -192,4 +193,14 @@ public Query withDataSources(List children) } return new UnionQuery(newQueries, context); } + + List> mapQueries(Function mapFn) + { + List> newQueries = new ArrayList<>(); + for (Query query : queries) { + newQueries.add(mapFn.apply(query)); + } + return newQueries; + } + } From 1dc39ea4880a8ac97538fea8cb4b53ef54340a35 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 2 Oct 2024 12:23:40 +0000 Subject: [PATCH 006/149] add debug result --- .../org/apache/druid/sql/calcite/CalciteArraysQueryTest.java | 5 +++-- 1 file changed, 3 insertions(+), 2 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index 531da0f8d66b..1cd61275421a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -7193,7 +7193,7 @@ public void testArrayToMvPostaggInline() ); } - @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNNEST_DIFFERENT_RESULTSET, separateDefaultModeTest = true) + // this is due to substring('',1') is null @Test public void testUnnestExtractionFn() { @@ -7228,7 +7228,8 @@ public void testUnnestExtractionFn() ImmutableList.of( new Object[]{"a"}, new Object[]{"c"}, - new Object[]{"d"} + new Object[]{"d"}, + new Object[]{""} ) ); } From df44cb51747773c24174dc41150a452f6a8f4ec6 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 2 Oct 2024 13:16:41 +0000 Subject: [PATCH 007/149] add cast --- .../apache/druid/messages/client/MessageRelayClientImpl.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java b/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java index 140bd45e1af4..c1adeefd5167 100644 --- a/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java +++ b/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java @@ -26,6 +26,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.java.util.http.client.response.BytesFullResponseHandler; +import org.apache.druid.java.util.http.client.response.BytesFullResponseHolder; import org.apache.druid.messages.MessageBatch; import org.apache.druid.rpc.RequestBuilder; import org.apache.druid.rpc.ServiceClient; @@ -69,7 +70,7 @@ public ListenableFuture> getMessages( ); return FutureUtils.transform( - serviceClient.asyncRequest( + (ListenableFuture) serviceClient.asyncRequest( new RequestBuilder(HttpMethod.GET, path), new BytesFullResponseHandler() ), From 8f9b80895de5c3f765a3af81f9cfea96c20a4b5c Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 2 Oct 2024 16:20:56 +0000 Subject: [PATCH 008/149] x --- .../java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 42cf41ff32a2..77865984023e 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -728,7 +728,7 @@ public void testQuery( { testBuilder() .sql(sql) - .expectedQueries(expectedQueries) +// .expectedQueries(expectedQueries) .expectedResults(expectedResults) .run(); } From 932d61d320dc512d39f2b9bdb2576f44927d87a9 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 05:55:19 +0000 Subject: [PATCH 009/149] return emptylist --- .../org/apache/druid/query/union/RealUnionQueryRunner.java | 1 + .../main/java/org/apache/druid/segment/DimensionSelector.java | 3 ++- 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java index 7c7e2eb4076a..fa0c64c5eaff 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java @@ -58,6 +58,7 @@ public Sequence run(QueryPlus queryPlus, Respons private void runTsQuery(QueryPlus queryPlus, TimeseriesQuery q, ResponseContext responseContext) { + // FIXME: is passing the walker here correct? QueryRunner> runner = q.getRunner(walker); Sequence> res = runner.run(queryPlus.withQuery(q), responseContext); diff --git a/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java b/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java index e4cebe9d46b1..8818e21d4674 100644 --- a/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java +++ b/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java @@ -36,6 +36,7 @@ import javax.annotation.Nullable; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -143,7 +144,7 @@ static Object rowToObject(IndexedInts row, DimensionDictionarySelector selector) { int rowSize = row.size(); if (rowSize == 0) { - return null; + return Collections.emptyList(); } else if (rowSize == 1) { return selector.lookupName(row.get(0)); } else { From d62e9936e8b5cec809897f88d438c91ab06772da Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 05:55:27 +0000 Subject: [PATCH 010/149] Revert "return emptylist" This reverts commit 932d61d320dc512d39f2b9bdb2576f44927d87a9. --- .../org/apache/druid/query/union/RealUnionQueryRunner.java | 1 - .../main/java/org/apache/druid/segment/DimensionSelector.java | 3 +-- 2 files changed, 1 insertion(+), 3 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java index fa0c64c5eaff..7c7e2eb4076a 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java @@ -58,7 +58,6 @@ public Sequence run(QueryPlus queryPlus, Respons private void runTsQuery(QueryPlus queryPlus, TimeseriesQuery q, ResponseContext responseContext) { - // FIXME: is passing the walker here correct? QueryRunner> runner = q.getRunner(walker); Sequence> res = runner.run(queryPlus.withQuery(q), responseContext); diff --git a/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java b/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java index 8818e21d4674..e4cebe9d46b1 100644 --- a/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java +++ b/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java @@ -36,7 +36,6 @@ import javax.annotation.Nullable; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -144,7 +143,7 @@ static Object rowToObject(IndexedInts row, DimensionDictionarySelector selector) { int rowSize = row.size(); if (rowSize == 0) { - return Collections.emptyList(); + return null; } else if (rowSize == 1) { return selector.lookupName(row.get(0)); } else { From d5ecc0c09c4c2721ffdb8e3ef7f0d7047f9fa86f Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 05:55:41 +0000 Subject: [PATCH 011/149] undo empty --- .../main/java/org/apache/druid/segment/DimensionSelector.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java b/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java index e4cebe9d46b1..8818e21d4674 100644 --- a/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java +++ b/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java @@ -36,6 +36,7 @@ import javax.annotation.Nullable; import java.util.Arrays; +import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -143,7 +144,7 @@ static Object rowToObject(IndexedInts row, DimensionDictionarySelector selector) { int rowSize = row.size(); if (rowSize == 0) { - return null; + return Collections.emptyList(); } else if (rowSize == 1) { return selector.lookupName(row.get(0)); } else { From 4f382e5e303462451c0614318c625aeacd3405ed Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 09:19:46 +0000 Subject: [PATCH 012/149] use UnionResult for now --- .../union/RealUnionQueryQueryToolChest.java | 116 ++++-------------- .../query/union/RealUnionQueryRunner.java | 15 ++- .../union/RealUnionQueryRunnerFactory.java | 11 +- .../druid/query/union/RealUnionResult.java | 25 ++++ .../apache/druid/query/union/UnionQuery.java | 19 ++- 5 files changed, 67 insertions(+), 119 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java index c9ee0785f0a4..5aeeb3399f6c 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java @@ -22,11 +22,8 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; import com.google.common.base.Functions; -import com.google.common.collect.ImmutableMap; -import org.apache.druid.error.DruidException; import org.apache.druid.frame.allocation.MemoryAllocatorFactory; import org.apache.druid.java.util.common.guava.Sequence; -import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.query.DefaultQueryMetrics; import org.apache.druid.query.FrameSignaturePair; import org.apache.druid.query.Query; @@ -34,32 +31,23 @@ import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QueryToolChest; -import org.apache.druid.query.ResultSerializationMode; import org.apache.druid.query.aggregation.MetricManipulationFn; import org.apache.druid.query.context.ResponseContext; import org.apache.druid.query.groupby.SupportRowSignature; import org.apache.druid.query.operator.WindowOperatorQuery; -import org.apache.druid.query.rowsandcols.RowsAndColumns; -import org.apache.druid.query.rowsandcols.column.Column; -import org.apache.druid.query.rowsandcols.column.ColumnAccessor; -import org.apache.druid.query.rowsandcols.column.NullColumn; -import org.apache.druid.query.rowsandcols.semantic.FrameMaker; -import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature.Finalization; -import java.util.ArrayList; -import java.util.List; import java.util.Optional; -public class RealUnionQueryQueryToolChest extends QueryToolChest +public class RealUnionQueryQueryToolChest extends QueryToolChest { @Override @SuppressWarnings("unchecked") - public QueryRunner mergeResults(QueryRunner runner) + public QueryRunner mergeResults(QueryRunner runner) { - return new RowsAndColumnsSerializingQueryRunner( + return new RealUnionResultSerializingQueryRunner( (queryPlus, responseContext) -> { return runner.run(queryPlus, responseContext); } @@ -73,7 +61,7 @@ public QueryMetrics makeMetrics(UnionQuery query) } @Override - public Function makePreComputeManipulatorFn( + public Function makePreComputeManipulatorFn( UnionQuery query, MetricManipulationFn fn ) @@ -82,9 +70,9 @@ public Function makePreComputeManipulatorFn( } @Override - public TypeReference getResultTypeReference() + public TypeReference getResultTypeReference() { - return new TypeReference() + return new TypeReference() { }; } @@ -103,10 +91,10 @@ public RowSignature resultArraySignature(UnionQuery query) @SuppressWarnings({"unchecked", "rawtypes"}) public Sequence resultsAsArrays( UnionQuery query, - Sequence resultSequence + Sequence resultSequence ) { - // Dark magic; see RowsAndColumnsSerializingQueryRunner. + // Dark magic; see RealUnionResultSerializingQueryRunner. return (Sequence) resultSequence; } @@ -114,20 +102,20 @@ public Sequence resultsAsArrays( @SuppressWarnings({"unchecked", "rawtypes"}) public Optional> resultsAsFrames( UnionQuery query, - Sequence resultSequence, + Sequence resultSequence, MemoryAllocatorFactory memoryAllocatorFactory, boolean useNestedForUnknownTypes ) { - // see RowsAndColumnsSerializingQueryRunner + // see RealUnionResultSerializingQueryRunner return Optional.of((Sequence) resultSequence); } /** - * This class exists to serialize the RowsAndColumns that are used in this query and make it the return Sequence + * This class exists to serialize the RealUnionResult that are used in this query and make it the return Sequence * actually be a Sequence of rows or frames, as the query requires. * This is relatively broken in a number of regards, the most obvious of which is that it is going to run counter to the stated class on the Generic of the QueryToolChest. - * That is, the code makes it look like you are getting a Sequence of RowsAndColumns, but, by using this, the query will + * That is, the code makes it look like you are getting a Sequence of RealUnionResult, but, by using this, the query will * actually ultimately produce a Sequence of Object[] or Frames. This works because of type Erasure in Java (it's all Object * at the end of the day). *

@@ -139,13 +127,13 @@ public Optional> resultsAsFrames( * Not our proudest moment, but we use the tools available to us. */ @SuppressWarnings({"unchecked", "rawtypes"}) - private static class RowsAndColumnsSerializingQueryRunner implements QueryRunner + private static class RealUnionResultSerializingQueryRunner implements QueryRunner { - private final QueryRunner baseQueryRunner; + private final QueryRunner baseQueryRunner; - private RowsAndColumnsSerializingQueryRunner( - QueryRunner baseQueryRunner + private RealUnionResultSerializingQueryRunner( + QueryRunner baseQueryRunner ) { this.baseQueryRunner = baseQueryRunner; @@ -157,85 +145,23 @@ public Sequence run( ResponseContext responseContext ) { - // We only want to do this operation once at the very, very top of the execution tree. So we check and set - // a context parameter so that if this merge code runs anywhere else, it will skip this part. - final WindowOperatorQuery query = (WindowOperatorQuery) queryPlus.getQuery(); - if (query.context().getBoolean("unravel", true)) { - final Sequence baseSequence = baseQueryRunner.run( - queryPlus.withQuery(query.withOverriddenContext(ImmutableMap.of("unravel", false))), - responseContext - ); - final ResultSerializationMode serializationMode = query.context().getEnum( - ResultSerializationMode.CTX_SERIALIZATION_PARAMETER, - ResultSerializationMode.class, - ResultSerializationMode.ROWS - ); - switch (serializationMode) { - case ROWS: - return asRows(baseSequence, query); - case FRAMES: - return asFrames(baseSequence); - default: - throw DruidException.defensive("Serialization mode[%s] not supported", serializationMode); - } - } - - return baseQueryRunner.run(queryPlus, responseContext); + throw new UnsupportedOperationException("Not supported"); } /** * Translates Sequence of RACs to a Sequence of Object[] */ - private static Sequence asRows(final Sequence baseSequence, final WindowOperatorQuery query) + private static Sequence asRows(final Sequence baseSequence, final WindowOperatorQuery query) { - final RowSignature rowSignature = query.getRowSignature(); - return baseSequence.flatMap( - rac -> { - List results = new ArrayList<>(rac.numRows()); - - ColumnAccessor[] accessors = new ColumnAccessor[rowSignature.size()]; - int index = 0; - for (String columnName : rowSignature.getColumnNames()) { - final Column column = rac.findColumn(columnName); - if (column == null) { - final ColumnType columnType = rowSignature - .getColumnType(columnName) - .orElse(ColumnType.UNKNOWN_COMPLEX); - - accessors[index] = new NullColumn.Accessor(columnType, rac.numRows()); - } else { - accessors[index] = column.toAccessor(); - } - ++index; - } - - for (int i = 0; i < rac.numRows(); ++i) { - Object[] objArr = new Object[accessors.length]; - for (int j = 0; j < accessors.length; j++) { - objArr[j] = accessors[j].getObject(i); - } - results.add(objArr); - } - - return Sequences.simple(results); - } - ); + throw new UnsupportedOperationException("Not supported"); } /** * Translates a sequence of RACs to a Sequence of Frames */ - private static Sequence asFrames(final Sequence baseSequence) + private static Sequence asFrames(final Sequence baseSequence) { - return baseSequence.map( - rac -> { - FrameMaker frameMaker = FrameMaker.fromRAC(rac); - return new FrameSignaturePair( - frameMaker.toColumnBasedFrame(), - frameMaker.computeSignature() - ); - } - ); + throw new UnsupportedOperationException("Not supported"); } } } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java index 7c7e2eb4076a..be4a89857ef8 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java @@ -21,7 +21,6 @@ import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; -import org.apache.druid.query.Query; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; @@ -34,7 +33,7 @@ import java.util.List; -public class RealUnionQueryRunner implements QueryRunner +public class RealUnionQueryRunner implements QueryRunner { private QuerySegmentWalker walker; @@ -44,15 +43,15 @@ public RealUnionQueryRunner(QuerySegmentWalker walker) } @Override - public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) + public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) { UnionQuery unionQuery = queryPlus.unwrapQuery(UnionQuery.class); - for (Query q: unionQuery.queries) { - if(q instanceof TimeseriesQuery) { - runTsQuery(queryPlus,(TimeseriesQuery)q, responseContext); - } - } +// for (Query q: unionQuery.queries) { +// if(q instanceof TimeseriesQuery) { +// runTsQuery(queryPlus,(TimeseriesQuery)q, responseContext); +// } +// } return Sequences.empty(); } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java index 91f6531cd88d..5a68fb1b19b5 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java @@ -23,10 +23,9 @@ import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QueryRunnerFactory; import org.apache.druid.query.QueryToolChest; -import org.apache.druid.query.rowsandcols.RowsAndColumns; import org.apache.druid.segment.Segment; -public class RealUnionQueryRunnerFactory implements QueryRunnerFactory +public class RealUnionQueryRunnerFactory implements QueryRunnerFactory { public RealUnionQueryRunnerFactory(String string) @@ -34,7 +33,7 @@ public RealUnionQueryRunnerFactory(String string) } @Override - public QueryRunner createRunner(Segment segment) + public QueryRunner createRunner(Segment segment) { if(true) { @@ -45,8 +44,8 @@ public QueryRunner createRunner(Segment segment) } @Override - public QueryRunner mergeRunners(QueryProcessingPool queryProcessingPool, - Iterable> queryRunners) + public QueryRunner mergeRunners(QueryProcessingPool queryProcessingPool, + Iterable> queryRunners) { if(true) { @@ -57,7 +56,7 @@ public QueryRunner mergeRunners(QueryProcessingPool queryProcess } @Override - public QueryToolChest getToolchest() + public QueryToolChest getToolchest() { return new RealUnionQueryQueryToolChest(); } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java new file mode 100644 index 000000000000..ae18636faf7b --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java @@ -0,0 +1,25 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.union; + +public class RealUnionResult +{ + +} diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index bfa7269b102b..3972cfd4efd9 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -30,7 +30,6 @@ import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.filter.DimFilter; -import org.apache.druid.query.rowsandcols.RowsAndColumns; import org.apache.druid.query.spec.QuerySegmentSpec; import org.joda.time.DateTimeZone; import org.joda.time.Duration; @@ -41,7 +40,7 @@ import java.util.Map; import java.util.function.Function; -public class UnionQuery implements Query +public class UnionQuery implements Query { RealUnionQueryRunnerFactory a; protected final Map context; @@ -96,7 +95,7 @@ public String getType() } @Override - public QueryRunner getRunner(QuerySegmentWalker walker) + public QueryRunner getRunner(QuerySegmentWalker walker) { return new RealUnionQueryRunner(walker); } @@ -132,7 +131,7 @@ public Map getContext() } @Override - public Ordering getResultOrdering() + public Ordering getResultOrdering() { if (true) { throw new RuntimeException("FIXME: Unimplemented!"); @@ -141,20 +140,20 @@ public Ordering getResultOrdering() } @Override - public Query withOverriddenContext(Map contextOverrides) + public Query withOverriddenContext(Map contextOverrides) { List> newQueries = mapQueries(q -> q.withOverriddenContext(contextOverrides)); return new UnionQuery(newQueries, QueryContexts.override(getContext(), contextOverrides)); } @Override - public Query withQuerySegmentSpec(QuerySegmentSpec spec) + public Query withQuerySegmentSpec(QuerySegmentSpec spec) { throw new RuntimeException("FIXME: Unimplemented!"); } @Override - public Query withId(String id) + public Query withId(String id) { return withOverriddenContext(ImmutableMap.of(BaseQuery.QUERY_ID, id)); } @@ -166,7 +165,7 @@ public String getId() } @Override - public Query withSubQueryId(String subQueryId) + public Query withSubQueryId(String subQueryId) { return withOverriddenContext(ImmutableMap.of(BaseQuery.SUB_QUERY_ID, subQueryId)); } @@ -178,13 +177,13 @@ public String getSubQueryId() } @Override - public Query withDataSource(DataSource dataSource) + public Query withDataSource(DataSource dataSource) { throw new RuntimeException("FIXME: Unimplemented!"); } @Override - public Query withDataSources(List children) + public Query withDataSources(List children) { Preconditions.checkArgument(queries.size() == children.size(), "Number of children must match number of queries"); List> newQueries= new ArrayList<>(); From 44ea85ea924f3fcbba407c5e332a3bae7d318420 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 09:46:47 +0000 Subject: [PATCH 013/149] add some stuff --- .../union/RealUnionQueryQueryToolChest.java | 13 +++++-- .../query/union/RealUnionQueryRunner.java | 36 ++++++++++++++----- .../union/RealUnionQueryRunnerFactory.java | 1 + .../druid/query/union/RealUnionResult.java | 14 ++++++++ 4 files changed, 52 insertions(+), 12 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java index 5aeeb3399f6c..bbb0ca5677a1 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java @@ -43,10 +43,18 @@ public class RealUnionQueryQueryToolChest extends QueryToolChest { + public void RealUnionQueryQueryToolChest() + { + int asd=1; + } @Override @SuppressWarnings("unchecked") public QueryRunner mergeResults(QueryRunner runner) { + if(true) { + throw new UnsupportedOperationException("Not supported"); + } + return new RealUnionResultSerializingQueryRunner( (queryPlus, responseContext) -> { return runner.run(queryPlus, responseContext); @@ -94,7 +102,7 @@ public Sequence resultsAsArrays( Sequence resultSequence ) { - // Dark magic; see RealUnionResultSerializingQueryRunner. + return (Sequence) resultSequence; } @@ -107,8 +115,7 @@ public Optional> resultsAsFrames( boolean useNestedForUnknownTypes ) { - // see RealUnionResultSerializingQueryRunner - return Optional.of((Sequence) resultSequence); + throw new UnsupportedOperationException("Not supported"); } /** diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java index be4a89857ef8..d270aa577816 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java @@ -21,6 +21,7 @@ import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.query.Query; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; @@ -31,6 +32,7 @@ import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; import org.apache.druid.query.timeseries.TimeseriesResultValue; +import java.util.ArrayList; import java.util.List; public class RealUnionQueryRunner implements QueryRunner @@ -47,12 +49,30 @@ public Sequence run(QueryPlus queryPlus, Respo { UnionQuery unionQuery = queryPlus.unwrapQuery(UnionQuery.class); -// for (Query q: unionQuery.queries) { -// if(q instanceof TimeseriesQuery) { -// runTsQuery(queryPlus,(TimeseriesQuery)q, responseContext); -// } -// } - return Sequences.empty(); + List seqs = new ArrayList(); + for (Query query : unionQuery.queries) { + seqs.add(makeUnionResult(queryPlus.withQuery(query), responseContext)); + } + return Sequences.simple(seqs); + + // return Sequences.map( + // Sequences.simple(unionQuery.queries), + // this::makeUnionResult + // ); + } + + // private RealUnionResult makeUnionResult(Query query) + // { + // QueryRunner runner = query.getRunner(walker); + // return new RealUnionResult( + // runner.run + // } + + private RealUnionResult makeUnionResult(QueryPlus withQuery, ResponseContext responseContext) + { + QueryRunner runner = withQuery.getQuery().getRunner(walker); + Sequence seq = runner.run(withQuery, responseContext); + return new RealUnionResult(seq); } private void runTsQuery(QueryPlus queryPlus, TimeseriesQuery q, ResponseContext responseContext) @@ -63,10 +83,8 @@ private void runTsQuery(QueryPlus queryPlus, TimeseriesQuery q, TimeseriesQueryQueryToolChest tsToolChest = new TimeseriesQueryQueryToolChest(); Sequence res1 = tsToolChest.resultsAsArrays(q, res); List li = res1.toList(); -// tsToolChest.mergeResults() + // tsToolChest.mergeResults() } - - } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java index 5a68fb1b19b5..838c32379328 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java @@ -30,6 +30,7 @@ public class RealUnionQueryRunnerFactory implements QueryRunnerFactory seq; + + public RealUnionResult(Sequence seq) + { + this.seq = seq; + } } From 24ae9128481e5efba3097987d6d1020779fb5aaa Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 09:48:24 +0000 Subject: [PATCH 014/149] make method non-static --- .../druid/server/ClientQuerySegmentWalker.java | 14 +++++--------- 1 file changed, 5 insertions(+), 9 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 2e0f715a7d4b..2c6d4c45dace 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -468,9 +468,7 @@ private DataSource inlineIfNecessary( maxSubqueryRows, maxSubqueryMemory, useNestedForUnknownTypeInSubquery, - subqueryStatsProvider, - !dryRun, - emitter + !dryRun ); } else { // Cannot inline subquery. Attempt to inline one level deeper, and then try again. @@ -664,6 +662,8 @@ private DataSource insertSubqueryIds( .collect(Collectors.toList())); } + + /** * * Convert the results of a particular query into a materialized (List-based) InlineDataSource. @@ -676,14 +676,12 @@ private DataSource insertSubqueryIds( * If zero, this method will throw an error immediately. * @param memoryLimit User configured byte limit. * @param useNestedForUnknownTypeInSubquery Uses nested json for unknown types when materializing subquery results - * @param subqueryStatsProvider Statistics about the subquery materialization * @param emitMetrics Flag to control if the metrics need to be emitted while materializing. The metrics are omitted * when we are performing a dry run of the query to avoid double reporting the same metric incorrectly - * @param emitter Metrics emitter * @return Inlined datasource represented by the provided results * @throws ResourceLimitExceededException if the limit is exceeded */ - private static > DataSource toInlineDataSource( + private > DataSource toInlineDataSource( final QueryType query, final Sequence queryResults, final QueryToolChest toolChest, @@ -693,9 +691,7 @@ private static > DataSource toInlineDataSource( final int limit, long memoryLimit, final boolean useNestedForUnknownTypeInSubquery, - final SubqueryCountStatsProvider subqueryStatsProvider, - final boolean emitMetrics, - final ServiceEmitter emitter + final boolean emitMetrics ) { final int rowLimitToUse = limit < 0 ? Integer.MAX_VALUE : limit; From 6035164281be5a867b0e392a10f95cdf494a512d Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 09:49:47 +0000 Subject: [PATCH 015/149] make non-static more --- .../server/ClientQuerySegmentWalker.java | 24 ++++++------------- 1 file changed, 7 insertions(+), 17 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 2c6d4c45dace..d8a91d0378f0 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -711,9 +711,7 @@ private > DataSource toInlineDataSource( toolChest, limitAccumulator, limit, - subqueryStatsProvider, - emitMetrics, - emitter + emitMetrics ); break; case MEMORY_LIMIT: @@ -729,9 +727,7 @@ private > DataSource toInlineDataSource( memoryLimitAccumulator, memoryLimit, useNestedForUnknownTypeInSubquery, - subqueryStatsProvider, - emitMetrics, - emitter + emitMetrics ); if (!maybeDataSource.isPresent()) { cannotMaterializeToFrames.set(true); @@ -748,9 +744,7 @@ private > DataSource toInlineDataSource( toolChest, limitAccumulator, limit, - subqueryStatsProvider, - emitMetrics, - emitter + emitMetrics ); } else { subqueryStatsProvider.incrementSubqueriesWithByteLimit(); @@ -767,7 +761,7 @@ private > DataSource toInlineDataSource( * This method materializes the query results as Frames. The method defaults back to materializing as rows in case * one cannot materialize the results as frames */ - private static > Optional materializeResultsAsFrames( + private > Optional materializeResultsAsFrames( final QueryType query, final Sequence results, final QueryToolChest toolChest, @@ -775,9 +769,7 @@ private static > Optional materializeR final AtomicLong memoryLimitAccumulator, final long memoryLimit, final boolean useNestedForUnknownTypeInSubquery, - final SubqueryCountStatsProvider subqueryStatsProvider, - final boolean emitMetrics, - final ServiceEmitter emitter + final boolean emitMetrics ) { boolean startedAccumulating = false; @@ -856,15 +848,13 @@ private static > Optional materializeR /** * This method materializes the query results as {@code List} */ - private static > DataSource materializeResultsAsArray( + private > DataSource materializeResultsAsArray( final QueryType query, final Sequence results, final QueryToolChest toolChest, final AtomicInteger limitAccumulator, final int limit, - final SubqueryCountStatsProvider subqueryStatsProvider, - boolean emitMetrics, - final ServiceEmitter emitter + boolean emitMetrics ) { final int rowLimitToUse = limit < 0 ? Integer.MAX_VALUE : limit; From 9ef426f63a8fc2ca499a5edfa68108703b31459d Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 09:58:46 +0000 Subject: [PATCH 016/149] make getToolChest method --- .../druid/server/ClientQuerySegmentWalker.java | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index d8a91d0378f0..ccedddf2ce91 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -461,7 +461,6 @@ private DataSource inlineIfNecessary( return toInlineDataSource( subQuery, queryResults, - warehouse.getToolChest(subQuery), subqueryRowLimitAccumulator, subqueryMemoryLimitAccumulator, cannotMaterializeToFrames, @@ -684,7 +683,6 @@ private DataSource insertSubqueryIds( private > DataSource toInlineDataSource( final QueryType query, final Sequence queryResults, - final QueryToolChest toolChest, final AtomicInteger limitAccumulator, final AtomicLong memoryLimitAccumulator, final AtomicBoolean cannotMaterializeToFrames, @@ -708,7 +706,6 @@ private > DataSource toInlineDataSource( dataSource = materializeResultsAsArray( query, queryResults, - toolChest, limitAccumulator, limit, emitMetrics @@ -722,7 +719,6 @@ private > DataSource toInlineDataSource( Optional maybeDataSource = materializeResultsAsFrames( query, queryResults, - toolChest, limitAccumulator, memoryLimitAccumulator, memoryLimit, @@ -741,7 +737,6 @@ private > DataSource toInlineDataSource( dataSource = materializeResultsAsArray( query, queryResults, - toolChest, limitAccumulator, limit, emitMetrics @@ -764,7 +759,6 @@ private > DataSource toInlineDataSource( private > Optional materializeResultsAsFrames( final QueryType query, final Sequence results, - final QueryToolChest toolChest, final AtomicInteger limitAccumulator, final AtomicLong memoryLimitAccumulator, final long memoryLimit, @@ -772,6 +766,7 @@ private > Optional materializeResultsA final boolean emitMetrics ) { + QueryToolChest toolChest = getToolChest(query); boolean startedAccumulating = false; try { Optional> framesOptional = toolChest.resultsAsFrames( @@ -845,18 +840,23 @@ private > Optional materializeResultsA } } + private > QueryToolChest getToolChest(final QueryType query) + { + return warehouse.getToolChest(query); + } + /** * This method materializes the query results as {@code List} */ private > DataSource materializeResultsAsArray( final QueryType query, final Sequence results, - final QueryToolChest toolChest, final AtomicInteger limitAccumulator, final int limit, boolean emitMetrics ) { + QueryToolChest toolChest = warehouse.getToolChest(query); final int rowLimitToUse = limit < 0 ? Integer.MAX_VALUE : limit; final RowSignature signature = toolChest.resultArraySignature(query); From 2f78fd3aa123ef62139cd226fa5556545312c8d1 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 13:04:53 +0000 Subject: [PATCH 017/149] setwarehouse --- .../main/java/org/apache/druid/query/QueryToolChest.java | 6 ++++++ .../org/apache/druid/server/ClientQuerySegmentWalker.java | 4 +++- 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java index 978b49226154..4f6afbc14aef 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java @@ -47,6 +47,7 @@ public abstract class QueryToolChest> resultsAsFrames( { return Optional.empty(); } + + public void setWarehouse(QueryToolChestWarehouse warehouse) + { + this.warehouse=warehouse; + } } diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index ccedddf2ce91..ff7079516995 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -842,7 +842,9 @@ private > Optional materializeResultsA private > QueryToolChest getToolChest(final QueryType query) { - return warehouse.getToolChest(query); + QueryToolChest toolChest = warehouse.getToolChest(query); + toolChest.setWarehouse(warehouse); + return toolChest; } /** From cae132a7f570096a3632d59008b9cda75d1d45a4 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 13:23:35 +0000 Subject: [PATCH 018/149] make union work --- .../java/org/apache/druid/query/Query.java | 3 + .../apache/druid/query/QueryToolChest.java | 2 +- .../union/RealUnionQueryQueryToolChest.java | 28 ++++++++- .../druid/query/union/RealUnionResult.java | 6 ++ .../apache/druid/query/union/UnionQuery.java | 5 ++ .../server/ClientQuerySegmentWalker.java | 1 + ...imitPushdownExtraction@NullHandling=sql.iq | 51 +--------------- .../testGroupBySortPushDown.iq | 44 +------------- ...estGroupByWithLiteralInSubqueryGrouping.iq | 60 +------------------ ...eatedIdenticalVirtualExpressionGrouping.iq | 38 +----------- 10 files changed, 47 insertions(+), 191 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/Query.java b/processing/src/main/java/org/apache/druid/query/Query.java index f8305f4b03ae..988e3fd3f0d2 100644 --- a/processing/src/main/java/org/apache/druid/query/Query.java +++ b/processing/src/main/java/org/apache/druid/query/Query.java @@ -43,6 +43,7 @@ import org.apache.druid.query.timeboundary.TimeBoundaryQuery; import org.apache.druid.query.timeseries.TimeseriesQuery; import org.apache.druid.query.topn.TopNQuery; +import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.Segment; import org.apache.druid.segment.VirtualColumns; import org.apache.druid.utils.CollectionUtils; @@ -69,6 +70,7 @@ @JsonSubTypes.Type(name = Query.TIMESERIES, value = TimeseriesQuery.class), @JsonSubTypes.Type(name = Query.TOPN, value = TopNQuery.class), @JsonSubTypes.Type(name = Query.WINDOW_OPERATOR, value = WindowOperatorQuery.class), + @JsonSubTypes.Type(name = Query.UNION_QUERY, value = UnionQuery.class), }) public interface Query { @@ -82,6 +84,7 @@ public interface Query String TIMESERIES = "timeseries"; String TOPN = "topN"; String WINDOW_OPERATOR = "windowOperator"; + String UNION_QUERY = "union"; DataSource getDataSource(); diff --git a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java index 4f6afbc14aef..9152fdddac84 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java @@ -47,7 +47,7 @@ public abstract class QueryToolChest @@ -102,8 +105,31 @@ public Sequence resultsAsArrays( Sequence resultSequence ) { + List results = resultSequence.toList(); - return (Sequence) resultSequence; + List> resultSeqs = new ArrayList>(); + + for (int i = 0; i < results.size(); i++) { + Query q = query.queries.get(i); + RealUnionResult realUnionResult = results.get(i); + + resultSeqs.add(resultsAsArrays(q, realUnionResult)); + } + + return Sequences.concat(resultSeqs); + } + + private > Sequence resultsAsArrays(QueryType q, RealUnionResult realUnionResult) + { + QueryToolChest toolChest = warehouse.getToolChest(q); + return toolChest.resultsAsArrays(q, realUnionResult.getResults()); + } + + @SuppressWarnings("unused") + private Sequence resultsAsArrays1(Query q, RealUnionResult realUnionResult) + { + warehouse.getToolChest(q); + return null; } @Override diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java index aec6bfe7b1d0..3bac27e122e5 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java @@ -36,4 +36,10 @@ public RealUnionResult(Sequence seq) this.seq = seq; } + public Sequence getResults() + { + return (Sequence) seq; + + } + } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index 3972cfd4efd9..b4d6f4544136 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -19,6 +19,7 @@ package org.apache.druid.query.union; +import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; @@ -43,7 +44,11 @@ public class UnionQuery implements Query { RealUnionQueryRunnerFactory a; + + @JsonProperty protected final Map context; + + @JsonProperty protected final List> queries; public UnionQuery(List> queries2) diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index ff7079516995..94b4a3a612f3 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -865,6 +865,7 @@ private > DataSource materializeResultsAsArray( final ArrayList resultList = new ArrayList<>(); final int initialSubqueryRows = limitAccumulator.get(); + toolChest.setWarehouse(warehouse); toolChest.resultsAsArrays(query, results).accumulate( resultList, (acc, in) -> { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByLimitPushdownExtraction@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByLimitPushdownExtraction@NullHandling=sql.iq index e6c589194f76..934c9e982255 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByLimitPushdownExtraction@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByLimitPushdownExtraction@NullHandling=sql.iq @@ -1,4 +1,4 @@ -# testGroupByLimitPushdownExtraction@NullHandling=sql case-crc:006ffd2b +# testGroupByLimitPushdownExtraction@NullHandling=sql case-crc:0e3d9e6e # quidem testcase reason: IMPROVED_PLAN !set debug true !set defaultTimeout 300000 @@ -34,52 +34,3 @@ DruidProject(dim4=[CAST('a':VARCHAR):VARCHAR], EXPR$1=[$0], EXPR$2=[$1], druid=[ DruidTableScan(table=[[druid, numfoo]], druid=[logical]) !druidPlan -{ - "queryType" : "topN", - "dataSource" : { - "type" : "table", - "name" : "numfoo" - }, - "dimension" : { - "type" : "extraction", - "dimension" : "dim5", - "outputName" : "_d0", - "outputType" : "STRING", - "extractionFn" : { - "type" : "substring", - "index" : 0, - "length" : 1 - } - }, - "metric" : { - "type" : "dimension", - "ordering" : { - "type" : "lexicographic" - } - }, - "threshold" : 2, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "filter" : { - "type" : "equals", - "column" : "dim4", - "matchValueType" : "STRING", - "matchValue" : "a" - }, - "granularity" : { - "type" : "all" - }, - "aggregations" : [ { - "type" : "count", - "name" : "a0" - } ], - "postAggregations" : [ { - "type" : "expression", - "name" : "s0", - "expression" : "'a'", - "outputType" : "STRING" - } ] -} -!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupBySortPushDown.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupBySortPushDown.iq index ce70e1c2ba2e..50bcf1fd9838 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupBySortPushDown.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupBySortPushDown.iq @@ -1,4 +1,4 @@ -# testGroupBySortPushDown case-crc:0f7a48bb +# testGroupBySortPushDown case-crc:19417c11 # quidem testcase reason: AGG_COL_EXCHANGE !set debug true !set defaultTimeout 300000 @@ -32,45 +32,3 @@ DruidProject(dim2=[$1], dim1=[$0], EXPR$2=[$2], druid=[logical]) DruidTableScan(table=[[druid, foo]], druid=[logical]) !druidPlan -{ - "queryType" : "groupBy", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "granularity" : { - "type" : "all" - }, - "dimensions" : [ { - "type" : "default", - "dimension" : "dim1", - "outputName" : "d0", - "outputType" : "STRING" - }, { - "type" : "default", - "dimension" : "dim2", - "outputName" : "d1", - "outputType" : "STRING" - } ], - "aggregations" : [ { - "type" : "longSum", - "name" : "a0", - "fieldName" : "cnt" - } ], - "limitSpec" : { - "type" : "default", - "columns" : [ { - "dimension" : "d0", - "direction" : "ascending", - "dimensionOrder" : { - "type" : "lexicographic" - } - } ], - "limit" : 4 - } -} -!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq index 85c1b392c49a..946bd07613d9 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq @@ -1,4 +1,4 @@ -# testGroupByWithLiteralInSubqueryGrouping case-crc:a63c5a2f +# testGroupByWithLiteralInSubqueryGrouping case-crc:1924083f # quidem testcase reason: IMPROVED_PLAN !set debug true !set defaultTimeout 300000 @@ -50,61 +50,3 @@ DruidProject(t1=['dummy'], t2=[$0], druid=[logical]) DruidTableScan(table=[[druid, numfoo]], druid=[logical]) !druidPlan -{ - "queryType" : "groupBy", - "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "groupBy", - "dataSource" : { - "type" : "table", - "name" : "numfoo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "granularity" : { - "type" : "all" - }, - "dimensions" : [ { - "type" : "default", - "dimension" : "dim4", - "outputName" : "_d0", - "outputType" : "STRING" - } ], - "limitSpec" : { - "type" : "NoopLimitSpec" - } - } - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "virtualColumns" : [ { - "type" : "expression", - "name" : "v0", - "expression" : "case_searched((\"_d0\" == 'b'),\"_d0\",null)", - "outputType" : "STRING" - } ], - "granularity" : { - "type" : "all" - }, - "dimensions" : [ { - "type" : "default", - "dimension" : "v0", - "outputName" : "d0", - "outputType" : "STRING" - } ], - "postAggregations" : [ { - "type" : "expression", - "name" : "p0", - "expression" : "'dummy'", - "outputType" : "STRING" - } ], - "limitSpec" : { - "type" : "NoopLimitSpec" - } -} -!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testRepeatedIdenticalVirtualExpressionGrouping.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testRepeatedIdenticalVirtualExpressionGrouping.iq index f87b98802c6c..4b9502748c0d 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testRepeatedIdenticalVirtualExpressionGrouping.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testRepeatedIdenticalVirtualExpressionGrouping.iq @@ -1,4 +1,4 @@ -# testRepeatedIdenticalVirtualExpressionGrouping case-crc:c6ee44b6 +# testRepeatedIdenticalVirtualExpressionGrouping case-crc:c0459668 # quidem testcase reason: IMPROVED_PLAN !set debug true !set defaultTimeout 300000 @@ -33,39 +33,3 @@ DruidProject(col_a=[$0], col_b=[true], druid=[logical]) DruidTableScan(table=[[druid, foo]], druid=[logical]) !druidPlan -{ - "queryType" : "groupBy", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "virtualColumns" : [ { - "type" : "expression", - "name" : "v0", - "expression" : "1", - "outputType" : "LONG" - } ], - "granularity" : { - "type" : "all" - }, - "dimensions" : [ { - "type" : "default", - "dimension" : "v0", - "outputName" : "d0", - "outputType" : "LONG" - } ], - "postAggregations" : [ { - "type" : "expression", - "name" : "p0", - "expression" : "1", - "outputType" : "LONG" - } ], - "limitSpec" : { - "type" : "NoopLimitSpec" - } -} -!nativePlan From 63caf06a857f276032f9a97907e3c8ebdb1a2e3e Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 14:25:21 +0000 Subject: [PATCH 019/149] foxes --- .../union/RealUnionQueryQueryToolChest.java | 1 - .../sql/calcite/BaseCalciteQueryTest.java | 12 +- .../sql/calcite/CalciteJoinQueryTest.java | 2 - .../druid/sql/calcite/CalciteQueryTest.java | 2 +- ...GroupByTimeFloorAndDim@NullHandling=sql.iq | 104 +----------------- ...stTimeFilterOnSubquery@NullHandling=sql.iq | 57 +--------- 6 files changed, 14 insertions(+), 164 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java index 5d3451e89e2a..e0663b263b88 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java @@ -112,7 +112,6 @@ public Sequence resultsAsArrays( for (int i = 0; i < results.size(); i++) { Query q = query.queries.get(i); RealUnionResult realUnionResult = results.get(i); - resultSeqs.add(resultsAsArrays(q, realUnionResult)); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 77865984023e..8c808050f4f0 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -1338,7 +1338,7 @@ protected static boolean isRewriteJoinToFilter(final Map queryCo public static Query recursivelyClearContext(final Query query, ObjectMapper queryJsonMapper) { try { - Query newQuery = query.withDataSource(recursivelyClearContext(query.getDataSource(), queryJsonMapper)); + Query newQuery = query.withDataSources(recursivelyClearContext2(query.getDataSources(), queryJsonMapper)); final JsonNode newQueryNode = queryJsonMapper.valueToTree(newQuery); ((ObjectNode) newQueryNode).remove("context"); return queryJsonMapper.treeToValue(newQueryNode, Query.class); @@ -1348,6 +1348,16 @@ public static Query recursivelyClearContext(final Query query, ObjectM } } + private static List recursivelyClearContext2(final List dataSource, ObjectMapper queryJsonMapper) + { + + List ret=new ArrayList(); + for (DataSource dataSource2 : dataSource) { + ret.add(recursivelyClearContext(dataSource2, queryJsonMapper)); + } + return ret; + } + /** * Override the contexts of all subqueries of a particular datasource. */ diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java index 4ab78beb945e..ba5d964bc95f 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java @@ -4325,7 +4325,6 @@ public void testJoinOnMultiValuedColumnShouldThrowException(Map }); } - @NotYetSupported(Modes.UNION_WITH_COMPLEX_OPERAND) @MethodSource("provideQueryContexts") @ParameterizedTest(name = "{0}") public void testUnionAllTwoQueriesLeftQueryIsJoin(Map queryContext) @@ -4415,7 +4414,6 @@ public void testUnionAllTwoQueriesRightQueryIsJoin(Map queryCont ); } - @NotYetSupported(Modes.UNION_WITH_COMPLEX_OPERAND) @Test public void testUnionAllTwoQueriesBothQueriesAreJoin() { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 1543c3226da6..76bdcd2587e6 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -3113,6 +3113,7 @@ public void testTopNWithSelectAndOrderByProjections() public void testUnionAllQueries() { msqIncompatible(); + skipVectorize(); testQuery( "SELECT COUNT(*) FROM foo UNION ALL SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo", ImmutableList.of( @@ -3142,7 +3143,6 @@ public void testUnionAllQueries() ); } - @NotYetSupported(Modes.UNION_WITH_COMPLEX_OPERAND) @Test public void testUnionAllQueriesWithLimit() { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim@NullHandling=sql.iq index 5105e214557d..e9ebb14afb6c 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim@NullHandling=sql.iq @@ -1,4 +1,4 @@ -# testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim@NullHandling=sql case-crc:d9fe7bd2 +# testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim@NullHandling=sql case-crc:de79d440 # quidem testcase reason: IMPROVED_PLAN !set debug true !set defaultTimeout 300000 @@ -42,105 +42,3 @@ DruidSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[DESC], druid=[logical]) DruidTableScan(table=[[druid, foo]], druid=[logical]) !druidPlan -{ - "queryType" : "groupBy", - "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "groupBy", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "virtualColumns" : [ { - "type" : "expression", - "name" : "v0", - "expression" : "timestamp_floor(\"__time\",'P1D',null,'UTC')", - "outputType" : "LONG" - } ], - "granularity" : { - "type" : "all" - }, - "dimensions" : [ { - "type" : "default", - "dimension" : "v0", - "outputName" : "d0", - "outputType" : "LONG" - }, { - "type" : "default", - "dimension" : "dim2", - "outputName" : "d1", - "outputType" : "STRING" - } ], - "aggregations" : [ { - "type" : "doubleSum", - "name" : "a0", - "fieldName" : "m1" - } ], - "having" : { - "type" : "filter", - "filter" : { - "type" : "range", - "column" : "a0", - "matchValueType" : "LONG", - "lower" : 1, - "lowerOpen" : true - }, - "finalize" : true - }, - "limitSpec" : { - "type" : "NoopLimitSpec" - } - } - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "virtualColumns" : [ { - "type" : "expression", - "name" : "v0", - "expression" : "timestamp_floor(\"d0\",'P1M',null,'UTC')", - "outputType" : "LONG" - } ], - "granularity" : { - "type" : "all" - }, - "dimensions" : [ { - "type" : "default", - "dimension" : "d1", - "outputName" : "_d0", - "outputType" : "STRING" - }, { - "type" : "default", - "dimension" : "v0", - "outputName" : "_d1", - "outputType" : "LONG" - } ], - "aggregations" : [ { - "type" : "doubleSum", - "name" : "_a0", - "fieldName" : "a0" - } ], - "limitSpec" : { - "type" : "default", - "columns" : [ { - "dimension" : "_d0", - "direction" : "ascending", - "dimensionOrder" : { - "type" : "lexicographic" - } - }, { - "dimension" : "_d1", - "direction" : "descending", - "dimensionOrder" : { - "type" : "numeric" - } - } ] - } -} -!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testTimeFilterOnSubquery@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testTimeFilterOnSubquery@NullHandling=sql.iq index 40b9d8747773..03a2dc138649 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testTimeFilterOnSubquery@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testTimeFilterOnSubquery@NullHandling=sql.iq @@ -1,4 +1,4 @@ -# testTimeFilterOnSubquery@NullHandling=sql case-crc:73448efc +# testTimeFilterOnSubquery@NullHandling=sql case-crc:5f2fb110 # quidem testcase reason: EQUIV_PLAN_EXTRA_COLUMNS !set debug true !set defaultTimeout 300000 @@ -31,58 +31,3 @@ DruidProject(__time=[$0], m1=[$5], druid=[logical]) DruidTableScan(table=[[druid, foo]], druid=[logical]) !druidPlan -{ - "queryType" : "scan", - "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "scan", - "dataSource" : { - "type" : "table", - "name" : "foo" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "limit" : 100, - "columns" : [ "__time", "cnt", "dim1", "dim2", "dim3", "m1", "m2", "unique_dim1" ], - "columnTypes" : [ "LONG", "LONG", "STRING", "STRING", "STRING", "FLOAT", "DOUBLE", "COMPLEX" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false - } - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "filter" : { - "type" : "or", - "fields" : [ { - "type" : "range", - "column" : "__time", - "matchValueType" : "LONG", - "lower" : 946684800000, - "upper" : 946771200000, - "upperOpen" : true - }, { - "type" : "range", - "column" : "__time", - "matchValueType" : "LONG", - "lower" : 978307200000, - "upper" : 978393600000, - "upperOpen" : true - } ] - }, - "columns" : [ "__time", "m1" ], - "columnTypes" : [ "LONG", "FLOAT" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false -} -!nativePlan From dc25d2fad5d7278dbd883d93ea92addc3276409d Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 14:34:10 +0000 Subject: [PATCH 020/149] fix serialization --- .../apache/druid/query/union/UnionQuery.java | 21 +++++-- .../druid/guice/QueryToolChestModule.java | 3 + .../sql/calcite/BaseCalciteQueryTest.java | 2 +- .../druid/sql/calcite/CalciteQueryTest.java | 54 ++++++++++------- .../testGroupBySortPushDown.iq | 44 +++++++++++++- ...estGroupByWithLiteralInSubqueryGrouping.iq | 60 ++++++++++++++++++- 6 files changed, 153 insertions(+), 31 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index b4d6f4544136..6019244c8b12 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -19,6 +19,7 @@ package org.apache.druid.query.union; +import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; @@ -43,12 +44,10 @@ public class UnionQuery implements Query { - RealUnionQueryRunnerFactory a; - - @JsonProperty + @JsonProperty("context") protected final Map context; - @JsonProperty + @JsonProperty("queries") protected final List> queries; public UnionQuery(List> queries2) @@ -56,7 +55,10 @@ public UnionQuery(List> queries2) this(queries2, queries2.get(0).getContext()); } - public UnionQuery(List> queries, Map context) + @JsonCreator + public UnionQuery( + @JsonProperty("queries") List> queries, + @JsonProperty("context") Map context) { Preconditions.checkArgument(queries.size() > 1, "union with fewer than 2 queries makes no sense"); this.queries = queries; @@ -207,4 +209,13 @@ List> mapQueries(Function mapFn) return newQueries; } + @Override + public String toString() + { + return "UnionQuery [context=" + context + ", queries=" + queries + "]"; + } + + + + } diff --git a/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java b/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java index da9ab41e299c..0e0750a45dd9 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java @@ -62,6 +62,8 @@ import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.query.topn.TopNQueryMetricsFactory; import org.apache.druid.query.topn.TopNQueryQueryToolChest; +import org.apache.druid.query.union.RealUnionQueryQueryToolChest; +import org.apache.druid.query.union.UnionQuery; import java.util.Map; @@ -86,6 +88,7 @@ public class QueryToolChestModule implements Module .put(TimeseriesQuery.class, TimeseriesQueryQueryToolChest.class) .put(TopNQuery.class, TopNQueryQueryToolChest.class) .put(WindowOperatorQuery.class, WindowOperatorQueryQueryToolChest.class) + .put(UnionQuery.class, RealUnionQueryQueryToolChest.class) .build(); @Override diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 8c808050f4f0..fa5ad7380ec1 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -728,7 +728,7 @@ public void testQuery( { testBuilder() .sql(sql) -// .expectedQueries(expectedQueries) + .expectedQueries(expectedQueries) .expectedResults(expectedResults) .run(); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 76bdcd2587e6..28059f7f6bdc 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -114,6 +114,7 @@ import org.apache.druid.query.topn.InvertedTopNMetricSpec; import org.apache.druid.query.topn.NumericTopNMetricSpec; import org.apache.druid.query.topn.TopNQueryBuilder; +import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnHolder; @@ -3114,31 +3115,38 @@ public void testUnionAllQueries() { msqIncompatible(); skipVectorize(); + ImmutableList> queries = ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators(aggregators(new CountAggregatorFactory("a0"))) + .context(QUERY_CONTEXT_DEFAULT) + .build(), + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .context(QUERY_CONTEXT_DEFAULT) + .build(), + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators(aggregators(new CountAggregatorFactory("a0"))) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ); + if (testBuilder().isDecoupledMode()) { + queries = ImmutableList.of( + new UnionQuery(queries) + ); + } + testQuery( "SELECT COUNT(*) FROM foo UNION ALL SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo", - ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .aggregators(aggregators(new CountAggregatorFactory("a0"))) - .context(QUERY_CONTEXT_DEFAULT) - .build(), - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .aggregators(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .context(QUERY_CONTEXT_DEFAULT) - .build(), - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .aggregators(aggregators(new CountAggregatorFactory("a0"))) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ), + queries, ImmutableList.of(new Object[]{6L}, new Object[]{6L}, new Object[]{6L}) ); } diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupBySortPushDown.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupBySortPushDown.iq index 50bcf1fd9838..ce70e1c2ba2e 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupBySortPushDown.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupBySortPushDown.iq @@ -1,4 +1,4 @@ -# testGroupBySortPushDown case-crc:19417c11 +# testGroupBySortPushDown case-crc:0f7a48bb # quidem testcase reason: AGG_COL_EXCHANGE !set debug true !set defaultTimeout 300000 @@ -32,3 +32,45 @@ DruidProject(dim2=[$1], dim1=[$0], EXPR$2=[$2], druid=[logical]) DruidTableScan(table=[[druid, foo]], druid=[logical]) !druidPlan +{ + "queryType" : "groupBy", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "dimensions" : [ { + "type" : "default", + "dimension" : "dim1", + "outputName" : "d0", + "outputType" : "STRING" + }, { + "type" : "default", + "dimension" : "dim2", + "outputName" : "d1", + "outputType" : "STRING" + } ], + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "limitSpec" : { + "type" : "default", + "columns" : [ { + "dimension" : "d0", + "direction" : "ascending", + "dimensionOrder" : { + "type" : "lexicographic" + } + } ], + "limit" : 4 + } +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq index 946bd07613d9..85c1b392c49a 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq @@ -1,4 +1,4 @@ -# testGroupByWithLiteralInSubqueryGrouping case-crc:1924083f +# testGroupByWithLiteralInSubqueryGrouping case-crc:a63c5a2f # quidem testcase reason: IMPROVED_PLAN !set debug true !set defaultTimeout 300000 @@ -50,3 +50,61 @@ DruidProject(t1=['dummy'], t2=[$0], druid=[logical]) DruidTableScan(table=[[druid, numfoo]], druid=[logical]) !druidPlan +{ + "queryType" : "groupBy", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "groupBy", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "dimensions" : [ { + "type" : "default", + "dimension" : "dim4", + "outputName" : "_d0", + "outputType" : "STRING" + } ], + "limitSpec" : { + "type" : "NoopLimitSpec" + } + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "case_searched((\"_d0\" == 'b'),\"_d0\",null)", + "outputType" : "STRING" + } ], + "granularity" : { + "type" : "all" + }, + "dimensions" : [ { + "type" : "default", + "dimension" : "v0", + "outputName" : "d0", + "outputType" : "STRING" + } ], + "postAggregations" : [ { + "type" : "expression", + "name" : "p0", + "expression" : "'dummy'", + "outputType" : "STRING" + } ], + "limitSpec" : { + "type" : "NoopLimitSpec" + } +} +!nativePlan From 60d04083c4d325151cef5023db902950bea26655 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 14:47:17 +0000 Subject: [PATCH 021/149] re-add --- ...imitPushdownExtraction@NullHandling=sql.iq | 51 ++++++++- ...GroupByTimeFloorAndDim@NullHandling=sql.iq | 104 +++++++++++++++++- ...eatedIdenticalVirtualExpressionGrouping.iq | 38 ++++++- ...stTimeFilterOnSubquery@NullHandling=sql.iq | 57 +++++++++- 4 files changed, 246 insertions(+), 4 deletions(-) diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByLimitPushdownExtraction@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByLimitPushdownExtraction@NullHandling=sql.iq index 934c9e982255..e6c589194f76 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByLimitPushdownExtraction@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByLimitPushdownExtraction@NullHandling=sql.iq @@ -1,4 +1,4 @@ -# testGroupByLimitPushdownExtraction@NullHandling=sql case-crc:0e3d9e6e +# testGroupByLimitPushdownExtraction@NullHandling=sql case-crc:006ffd2b # quidem testcase reason: IMPROVED_PLAN !set debug true !set defaultTimeout 300000 @@ -34,3 +34,52 @@ DruidProject(dim4=[CAST('a':VARCHAR):VARCHAR], EXPR$1=[$0], EXPR$2=[$1], druid=[ DruidTableScan(table=[[druid, numfoo]], druid=[logical]) !druidPlan +{ + "queryType" : "topN", + "dataSource" : { + "type" : "table", + "name" : "numfoo" + }, + "dimension" : { + "type" : "extraction", + "dimension" : "dim5", + "outputName" : "_d0", + "outputType" : "STRING", + "extractionFn" : { + "type" : "substring", + "index" : 0, + "length" : 1 + } + }, + "metric" : { + "type" : "dimension", + "ordering" : { + "type" : "lexicographic" + } + }, + "threshold" : 2, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "filter" : { + "type" : "equals", + "column" : "dim4", + "matchValueType" : "STRING", + "matchValue" : "a" + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "postAggregations" : [ { + "type" : "expression", + "name" : "s0", + "expression" : "'a'", + "outputType" : "STRING" + } ] +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim@NullHandling=sql.iq index e9ebb14afb6c..5105e214557d 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim@NullHandling=sql.iq @@ -1,4 +1,4 @@ -# testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim@NullHandling=sql case-crc:de79d440 +# testGroupByTimeFloorAndDimOnGroupByTimeFloorAndDim@NullHandling=sql case-crc:d9fe7bd2 # quidem testcase reason: IMPROVED_PLAN !set debug true !set defaultTimeout 300000 @@ -42,3 +42,105 @@ DruidSort(sort0=[$0], sort1=[$1], dir0=[ASC], dir1=[DESC], druid=[logical]) DruidTableScan(table=[[druid, foo]], druid=[logical]) !druidPlan +{ + "queryType" : "groupBy", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "groupBy", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "timestamp_floor(\"__time\",'P1D',null,'UTC')", + "outputType" : "LONG" + } ], + "granularity" : { + "type" : "all" + }, + "dimensions" : [ { + "type" : "default", + "dimension" : "v0", + "outputName" : "d0", + "outputType" : "LONG" + }, { + "type" : "default", + "dimension" : "dim2", + "outputName" : "d1", + "outputType" : "STRING" + } ], + "aggregations" : [ { + "type" : "doubleSum", + "name" : "a0", + "fieldName" : "m1" + } ], + "having" : { + "type" : "filter", + "filter" : { + "type" : "range", + "column" : "a0", + "matchValueType" : "LONG", + "lower" : 1, + "lowerOpen" : true + }, + "finalize" : true + }, + "limitSpec" : { + "type" : "NoopLimitSpec" + } + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "timestamp_floor(\"d0\",'P1M',null,'UTC')", + "outputType" : "LONG" + } ], + "granularity" : { + "type" : "all" + }, + "dimensions" : [ { + "type" : "default", + "dimension" : "d1", + "outputName" : "_d0", + "outputType" : "STRING" + }, { + "type" : "default", + "dimension" : "v0", + "outputName" : "_d1", + "outputType" : "LONG" + } ], + "aggregations" : [ { + "type" : "doubleSum", + "name" : "_a0", + "fieldName" : "a0" + } ], + "limitSpec" : { + "type" : "default", + "columns" : [ { + "dimension" : "_d0", + "direction" : "ascending", + "dimensionOrder" : { + "type" : "lexicographic" + } + }, { + "dimension" : "_d1", + "direction" : "descending", + "dimensionOrder" : { + "type" : "numeric" + } + } ] + } +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testRepeatedIdenticalVirtualExpressionGrouping.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testRepeatedIdenticalVirtualExpressionGrouping.iq index 4b9502748c0d..f87b98802c6c 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testRepeatedIdenticalVirtualExpressionGrouping.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testRepeatedIdenticalVirtualExpressionGrouping.iq @@ -1,4 +1,4 @@ -# testRepeatedIdenticalVirtualExpressionGrouping case-crc:c0459668 +# testRepeatedIdenticalVirtualExpressionGrouping case-crc:c6ee44b6 # quidem testcase reason: IMPROVED_PLAN !set debug true !set defaultTimeout 300000 @@ -33,3 +33,39 @@ DruidProject(col_a=[$0], col_b=[true], druid=[logical]) DruidTableScan(table=[[druid, foo]], druid=[logical]) !druidPlan +{ + "queryType" : "groupBy", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "virtualColumns" : [ { + "type" : "expression", + "name" : "v0", + "expression" : "1", + "outputType" : "LONG" + } ], + "granularity" : { + "type" : "all" + }, + "dimensions" : [ { + "type" : "default", + "dimension" : "v0", + "outputName" : "d0", + "outputType" : "LONG" + } ], + "postAggregations" : [ { + "type" : "expression", + "name" : "p0", + "expression" : "1", + "outputType" : "LONG" + } ], + "limitSpec" : { + "type" : "NoopLimitSpec" + } +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testTimeFilterOnSubquery@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testTimeFilterOnSubquery@NullHandling=sql.iq index 03a2dc138649..40b9d8747773 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testTimeFilterOnSubquery@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testTimeFilterOnSubquery@NullHandling=sql.iq @@ -1,4 +1,4 @@ -# testTimeFilterOnSubquery@NullHandling=sql case-crc:5f2fb110 +# testTimeFilterOnSubquery@NullHandling=sql case-crc:73448efc # quidem testcase reason: EQUIV_PLAN_EXTRA_COLUMNS !set debug true !set defaultTimeout 300000 @@ -31,3 +31,58 @@ DruidProject(__time=[$0], m1=[$5], druid=[logical]) DruidTableScan(table=[[druid, foo]], druid=[logical]) !druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "scan", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "limit" : 100, + "columns" : [ "__time", "cnt", "dim1", "dim2", "dim3", "m1", "m2", "unique_dim1" ], + "columnTypes" : [ "LONG", "LONG", "STRING", "STRING", "STRING", "FLOAT", "DOUBLE", "COMPLEX" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "filter" : { + "type" : "or", + "fields" : [ { + "type" : "range", + "column" : "__time", + "matchValueType" : "LONG", + "lower" : 946684800000, + "upper" : 946771200000, + "upperOpen" : true + }, { + "type" : "range", + "column" : "__time", + "matchValueType" : "LONG", + "lower" : 978307200000, + "upper" : 978393600000, + "upperOpen" : true + } ] + }, + "columns" : [ "__time", "m1" ], + "columnTypes" : [ "LONG", "FLOAT" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan From 43ae60d91ec3516fc32ef8a3ee8bc162a88bae46 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 14:48:18 +0000 Subject: [PATCH 022/149] undo emptylit --- .../main/java/org/apache/druid/segment/DimensionSelector.java | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java b/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java index 8818e21d4674..e4cebe9d46b1 100644 --- a/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java +++ b/processing/src/main/java/org/apache/druid/segment/DimensionSelector.java @@ -36,7 +36,6 @@ import javax.annotation.Nullable; import java.util.Arrays; -import java.util.Collections; import java.util.List; import java.util.stream.Collectors; @@ -144,7 +143,7 @@ static Object rowToObject(IndexedInts row, DimensionDictionarySelector selector) { int rowSize = row.size(); if (rowSize == 0) { - return Collections.emptyList(); + return null; } else if (rowSize == 1) { return selector.lookupName(row.get(0)); } else { From 8f58d490c8d109877d5023562d33ca87e106bf22 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 14:49:49 +0000 Subject: [PATCH 023/149] x --- .../src/main/java/org/apache/druid/query/union/UnionQuery.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index 6019244c8b12..87684a4efcbd 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -200,7 +200,7 @@ public Query withDataSources(List children) return new UnionQuery(newQueries, context); } - List> mapQueries(Function mapFn) + List> mapQueries(Function, Query> mapFn) { List> newQueries = new ArrayList<>(); for (Query query : queries) { From 2a4ab819de329479e6f8299ce880b778ac5c9ed9 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 3 Oct 2024 15:22:13 +0000 Subject: [PATCH 024/149] cant be fixed? --- .../apache/druid/query/union/UnionQuery.java | 11 ++++++ .../server/ClientQuerySegmentWalker.java | 37 ++++++++++++------- 2 files changed, 35 insertions(+), 13 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index 87684a4efcbd..985323464870 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -32,6 +32,7 @@ import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.query.planning.DataSourceAnalysis; import org.apache.druid.query.spec.QuerySegmentSpec; import org.joda.time.DateTimeZone; import org.joda.time.Duration; @@ -215,6 +216,16 @@ public String toString() return "UnionQuery [context=" + context + ", queries=" + queries + "]"; } + public DataSourceAnalysis getDataSourceAnalysis() + { + if(true) + { + throw new RuntimeException("FIXME: Unimplemented!"); + } + return null; + + } + diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 94b4a3a612f3..8060dcf4b5d8 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -291,20 +291,29 @@ public QueryRunner getQueryRunnerForSegments(final Query query, final private boolean canRunQueryUsingLocalWalker(Query query) { if(query instanceof UnionQuery) { + + UnionQuery unionQuery = (UnionQuery) query; + final DataSourceAnalysis analysis = unionQuery.getDataSourceAnalysis(); +// return analysis.isConcreteBased() && !analysis.isConcreteAndTableBased() && unionQuery.dataSource_isGlobal() +// && (!(dataSourceFromQuery instanceof QueryDataSource) +// || toolChest.canPerformSubquery(((QueryDataSource) dataSourceFromQuery).getQuery())); + // FIXME ?? return false; - } - final DataSource dataSourceFromQuery = query.getDataSource(); - final DataSourceAnalysis analysis = dataSourceFromQuery.getAnalysis(); - final QueryToolChest> toolChest = warehouse.getToolChest(query); + } else { - // 1) Must be based on a concrete datasource that is not a table. - // 2) Must be based on globally available data (so we have a copy here on the Broker). - // 3) If there is an outer query, it must be handleable by the query toolchest (the local walker does not handle - // subqueries on its own). - return analysis.isConcreteBased() && !analysis.isConcreteAndTableBased() && dataSourceFromQuery.isGlobal() - && (!(dataSourceFromQuery instanceof QueryDataSource) - || toolChest.canPerformSubquery(((QueryDataSource) dataSourceFromQuery).getQuery())); + final DataSource dataSourceFromQuery = query.getDataSource(); + final DataSourceAnalysis analysis = dataSourceFromQuery.getAnalysis(); + final QueryToolChest> toolChest = warehouse.getToolChest(query); + + // 1) Must be based on a concrete datasource that is not a table. + // 2) Must be based on globally available data (so we have a copy here on the Broker). + // 3) If there is an outer query, it must be handleable by the query toolchest (the local walker does not handle + // subqueries on its own). + return analysis.isConcreteBased() && !analysis.isConcreteAndTableBased() && dataSourceFromQuery.isGlobal() + && (!(dataSourceFromQuery instanceof QueryDataSource) + || toolChest.canPerformSubquery(((QueryDataSource) dataSourceFromQuery).getQuery())); + } } /** @@ -314,8 +323,10 @@ private boolean canRunQueryUsingLocalWalker(Query query) private boolean canRunQueryUsingClusterWalker(Query query) { if(query instanceof UnionQuery) { - // FIXME ?? - return true; + UnionQuery unionQuery = (UnionQuery) query; + final DataSourceAnalysis analysis = unionQuery.getDataSourceAnalysis(); + + return analysis.isConcreteAndTableBased(); } final DataSource dataSourceFromQuery = query.getDataSource(); From f96969dbccea0a017eb954b0372c6f10c3585dc2 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 9 Oct 2024 21:17:39 +0000 Subject: [PATCH 025/149] mods --- .../druid/query/QuerySegmentWalker.java | 5 + .../apache/druid/query/QueryToolChest.java | 9 ++ .../union/RealUnionQueryQueryToolChest.java | 148 +++++++++++++++--- .../query/union/RealUnionQueryRunner.java | 31 ---- .../server/ClientQuerySegmentWalker.java | 11 ++ 5 files changed, 147 insertions(+), 57 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/QuerySegmentWalker.java b/processing/src/main/java/org/apache/druid/query/QuerySegmentWalker.java index 7084a80935d1..8e57a1086d4f 100644 --- a/processing/src/main/java/org/apache/druid/query/QuerySegmentWalker.java +++ b/processing/src/main/java/org/apache/druid/query/QuerySegmentWalker.java @@ -54,4 +54,9 @@ public interface QuerySegmentWalker * @return the Queryable object with the given SegmentSpecs */ QueryRunner getQueryRunnerForSegments(Query query, Iterable specs); + + default QueryRunner executeQuery(Query query) + { + throw new UnsupportedOperationException("executeQuery is not implemented"); + } } diff --git a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java index 9152fdddac84..983a5c2645c7 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java @@ -428,4 +428,13 @@ public void setWarehouse(QueryToolChestWarehouse warehouse) { this.warehouse=warehouse; } + + public Optional> executeQuery( + // ideally; it should know about the warehouse + QueryToolChestWarehouse warehouse, + Query query, + QuerySegmentWalker clientQuerySegmentWalker) + { + return Optional.empty(); + } } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java index e0663b263b88..b66efabe85e4 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java @@ -31,7 +31,9 @@ import org.apache.druid.query.QueryMetrics; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.QueryToolChest; +import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.aggregation.MetricManipulationFn; import org.apache.druid.query.context.ResponseContext; import org.apache.druid.query.groupby.SupportRowSignature; @@ -45,17 +47,110 @@ public class RealUnionQueryQueryToolChest extends QueryToolChest { + public Optional> executeQuery(QueryToolChestWarehouse warehouse, + Query query, QuerySegmentWalker clientQuerySegmentWalker) + { + RealUnionQueryRunner2 runner = new RealUnionQueryRunner2(warehouse, (UnionQuery) query, clientQuerySegmentWalker); + return Optional.of(runner); + } + + public Optional> executeQuery1(QueryToolChestWarehouse warehouse, + Query query, QuerySegmentWalker clientQuerySegmentWalker) + { + UnionQuery unionQuery = (UnionQuery) query; + List queryRunners = new ArrayList<>(); + for (Query q : unionQuery.queries) { + QueryRunner subRunner = clientQuerySegmentWalker.executeQuery(q); + queryRunners.add(subRunner); + } + QueryRunner unionRunner = new LocalRealUnionQueryRunner( + queryRunners + ); + return Optional.of(unionRunner); + } + + private static class RealUnionQueryRunner2 implements QueryRunner + { + + private QueryToolChestWarehouse warehouse; + private QuerySegmentWalker walker; + private UnionQuery query; + private List runners; + + public RealUnionQueryRunner2(QueryToolChestWarehouse warehouse, UnionQuery query, + QuerySegmentWalker walker) + { + this.warehouse = warehouse; + this.query = query; + this.walker = walker; + + this.runners = makeSubQueryRunners(query); + } + + private List makeSubQueryRunners(UnionQuery unionQuery) + { + List runners = new ArrayList<>(); + for (Query query : unionQuery.queries) { + runners.add(query.getRunner(walker)); + } + return runners; + + } + + @Override + public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) + { + UnionQuery unionQuery = queryPlus.unwrapQuery(UnionQuery.class); + + List seqs = new ArrayList(); + for (int i = 0; i < runners.size(); i++) { + Query q = unionQuery.queries.get(i); + QueryRunner r = runners.get(i); + seqs.add(makeUnionResult(r, queryPlus.withQuery(q), responseContext)); + } + return Sequences.simple(seqs); + } + + private RealUnionResult makeUnionResult(QueryRunner runner, QueryPlus withQuery, + ResponseContext responseContext) + { + Sequence seq = runner.run(withQuery, responseContext); + return new RealUnionResult(seq); + } + } + + private static class LocalRealUnionQueryRunner implements QueryRunner + { + + public LocalRealUnionQueryRunner(List queryRunners) + { + + } + + @Override + public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) + { + return buildSequence(); + } + + Sequence buildSequence() + { + return null; + } + + } public void RealUnionQueryQueryToolChest() { - int asd=1; + int asd = 1; } + @Override @SuppressWarnings("unchecked") public QueryRunner mergeResults(QueryRunner runner) { - if(true) { - throw new UnsupportedOperationException("Not supported"); + if (true) { + throw new UnsupportedOperationException("Not supported"); } return new RealUnionResultSerializingQueryRunner( @@ -74,8 +169,7 @@ public QueryMetrics makeMetrics(UnionQuery query) @Override public Function makePreComputeManipulatorFn( UnionQuery query, - MetricManipulationFn fn - ) + MetricManipulationFn fn) { return Functions.identity(); } @@ -102,8 +196,7 @@ public RowSignature resultArraySignature(UnionQuery query) @SuppressWarnings({"unchecked", "rawtypes"}) public Sequence resultsAsArrays( UnionQuery query, - Sequence resultSequence - ) + Sequence resultSequence) { List results = resultSequence.toList(); @@ -118,14 +211,15 @@ public Sequence resultsAsArrays( return Sequences.concat(resultSeqs); } - private > Sequence resultsAsArrays(QueryType q, RealUnionResult realUnionResult) + private > Sequence resultsAsArrays(QueryType q, + RealUnionResult realUnionResult) { QueryToolChest toolChest = warehouse.getToolChest(q); return toolChest.resultsAsArrays(q, realUnionResult.getResults()); } @SuppressWarnings("unused") - private Sequence resultsAsArrays1(Query q, RealUnionResult realUnionResult) + private Sequence resultsAsArrays1(Query q, RealUnionResult realUnionResult) { warehouse.getToolChest(q); return null; @@ -137,24 +231,28 @@ public Optional> resultsAsFrames( UnionQuery query, Sequence resultSequence, MemoryAllocatorFactory memoryAllocatorFactory, - boolean useNestedForUnknownTypes - ) + boolean useNestedForUnknownTypes) { - throw new UnsupportedOperationException("Not supported"); + throw new UnsupportedOperationException("Not supported"); } /** - * This class exists to serialize the RealUnionResult that are used in this query and make it the return Sequence - * actually be a Sequence of rows or frames, as the query requires. - * This is relatively broken in a number of regards, the most obvious of which is that it is going to run counter to the stated class on the Generic of the QueryToolChest. - * That is, the code makes it look like you are getting a Sequence of RealUnionResult, but, by using this, the query will - * actually ultimately produce a Sequence of Object[] or Frames. This works because of type Erasure in Java (it's all Object - * at the end of the day). + * This class exists to serialize the RealUnionResult that are used in this + * query and make it the return Sequence actually be a Sequence of rows or + * frames, as the query requires. This is relatively broken in a number of + * regards, the most obvious of which is that it is going to run counter to + * the stated class on the Generic of the QueryToolChest. That is, the code + * makes it look like you are getting a Sequence of RealUnionResult, but, by + * using this, the query will actually ultimately produce a Sequence of + * Object[] or Frames. This works because of type Erasure in Java (it's all + * Object at the end of the day). *

- * While it might seem like this will break all sorts of things, the Generic type is actually there more as a type - * "hint" to make the writing of the ToolChest and Factory and stuff a bit more simple. Any caller of this cannot - * truly depend on the type anyway other than to just throw it across the wire, so this should just magically work - * even though it looks like it shouldn't even compile. + * While it might seem like this will break all sorts of things, the Generic + * type is actually there more as a type "hint" to make the writing of the + * ToolChest and Factory and stuff a bit more simple. Any caller of this + * cannot truly depend on the type anyway other than to just throw it across + * the wire, so this should just magically work even though it looks like it + * shouldn't even compile. *

* Not our proudest moment, but we use the tools available to us. */ @@ -165,8 +263,7 @@ private static class RealUnionResultSerializingQueryRunner implements QueryRunne private final QueryRunner baseQueryRunner; private RealUnionResultSerializingQueryRunner( - QueryRunner baseQueryRunner - ) + QueryRunner baseQueryRunner) { this.baseQueryRunner = baseQueryRunner; } @@ -174,8 +271,7 @@ private RealUnionResultSerializingQueryRunner( @Override public Sequence run( QueryPlus queryPlus, - ResponseContext responseContext - ) + ResponseContext responseContext) { throw new UnsupportedOperationException("Not supported"); } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java index d270aa577816..39c00250730e 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java @@ -25,13 +25,7 @@ import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; -import org.apache.druid.query.Result; import org.apache.druid.query.context.ResponseContext; -import org.apache.druid.query.rowsandcols.RowsAndColumns; -import org.apache.druid.query.timeseries.TimeseriesQuery; -import org.apache.druid.query.timeseries.TimeseriesQueryQueryToolChest; -import org.apache.druid.query.timeseries.TimeseriesResultValue; - import java.util.ArrayList; import java.util.List; @@ -54,37 +48,12 @@ public Sequence run(QueryPlus queryPlus, Respo seqs.add(makeUnionResult(queryPlus.withQuery(query), responseContext)); } return Sequences.simple(seqs); - - // return Sequences.map( - // Sequences.simple(unionQuery.queries), - // this::makeUnionResult - // ); } - // private RealUnionResult makeUnionResult(Query query) - // { - // QueryRunner runner = query.getRunner(walker); - // return new RealUnionResult( - // runner.run - // } - private RealUnionResult makeUnionResult(QueryPlus withQuery, ResponseContext responseContext) { QueryRunner runner = withQuery.getQuery().getRunner(walker); Sequence seq = runner.run(withQuery, responseContext); return new RealUnionResult(seq); } - - private void runTsQuery(QueryPlus queryPlus, TimeseriesQuery q, ResponseContext responseContext) - { - QueryRunner> runner = q.getRunner(walker); - Sequence> res = runner.run(queryPlus.withQuery(q), responseContext); - - TimeseriesQueryQueryToolChest tsToolChest = new TimeseriesQueryQueryToolChest(); - Sequence res1 = tsToolChest.resultsAsArrays(q, res); - List li = res1.toList(); - // tsToolChest.mergeResults() - - } - } diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 8060dcf4b5d8..57e1f9723e88 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -175,11 +175,22 @@ public ClientQuerySegmentWalker( ); } + @Override + public QueryRunner executeQuery(Query query) + { + return getQueryRunnerForIntervals(query, query.getIntervals()); + } + @Override public QueryRunner getQueryRunnerForIntervals(final Query query, final Iterable intervals) { final QueryToolChest> toolChest = warehouse.getToolChest(query); + Optional> toolchestExecResult = toolChest.executeQuery(warehouse, query, this); + if (toolchestExecResult.isPresent()) { + return toolchestExecResult.get(); + } + // transform TableDataSource to GlobalTableDataSource when eligible // before further transformation to potentially inline From b36e9eb79c7227309c47a3f998663e8528e98254 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 9 Oct 2024 21:19:11 +0000 Subject: [PATCH 026/149] minimalize change --- .../server/ClientQuerySegmentWalker.java | 102 ++++++++---------- 1 file changed, 43 insertions(+), 59 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 57e1f9723e88..1075b472ee36 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -62,7 +62,6 @@ import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.planning.DataSourceAnalysis; -import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.server.initialization.ServerConfig; @@ -175,12 +174,6 @@ public ClientQuerySegmentWalker( ); } - @Override - public QueryRunner executeQuery(Query query) - { - return getQueryRunnerForIntervals(query, query.getIntervals()); - } - @Override public QueryRunner getQueryRunnerForIntervals(final Query query, final Iterable intervals) { @@ -301,30 +294,17 @@ public QueryRunner getQueryRunnerForSegments(final Query query, final */ private boolean canRunQueryUsingLocalWalker(Query query) { - if(query instanceof UnionQuery) { - - UnionQuery unionQuery = (UnionQuery) query; - final DataSourceAnalysis analysis = unionQuery.getDataSourceAnalysis(); -// return analysis.isConcreteBased() && !analysis.isConcreteAndTableBased() && unionQuery.dataSource_isGlobal() -// && (!(dataSourceFromQuery instanceof QueryDataSource) -// || toolChest.canPerformSubquery(((QueryDataSource) dataSourceFromQuery).getQuery())); - - // FIXME ?? - return false; - } else { + final DataSource dataSourceFromQuery = query.getDataSource(); + final DataSourceAnalysis analysis = dataSourceFromQuery.getAnalysis(); + final QueryToolChest> toolChest = warehouse.getToolChest(query); - final DataSource dataSourceFromQuery = query.getDataSource(); - final DataSourceAnalysis analysis = dataSourceFromQuery.getAnalysis(); - final QueryToolChest> toolChest = warehouse.getToolChest(query); - - // 1) Must be based on a concrete datasource that is not a table. - // 2) Must be based on globally available data (so we have a copy here on the Broker). - // 3) If there is an outer query, it must be handleable by the query toolchest (the local walker does not handle - // subqueries on its own). - return analysis.isConcreteBased() && !analysis.isConcreteAndTableBased() && dataSourceFromQuery.isGlobal() - && (!(dataSourceFromQuery instanceof QueryDataSource) - || toolChest.canPerformSubquery(((QueryDataSource) dataSourceFromQuery).getQuery())); - } + // 1) Must be based on a concrete datasource that is not a table. + // 2) Must be based on globally available data (so we have a copy here on the Broker). + // 3) If there is an outer query, it must be handleable by the query toolchest (the local walker does not handle + // subqueries on its own). + return analysis.isConcreteBased() && !analysis.isConcreteAndTableBased() && dataSourceFromQuery.isGlobal() + && (!(dataSourceFromQuery instanceof QueryDataSource) + || toolChest.canPerformSubquery(((QueryDataSource) dataSourceFromQuery).getQuery())); } /** @@ -333,13 +313,6 @@ private boolean canRunQueryUsingLocalWalker(Query query) */ private boolean canRunQueryUsingClusterWalker(Query query) { - if(query instanceof UnionQuery) { - UnionQuery unionQuery = (UnionQuery) query; - final DataSourceAnalysis analysis = unionQuery.getDataSourceAnalysis(); - - return analysis.isConcreteAndTableBased(); - } - final DataSource dataSourceFromQuery = query.getDataSource(); final DataSourceAnalysis analysis = dataSourceFromQuery.getAnalysis(); final QueryToolChest> toolChest = warehouse.getToolChest(query); @@ -483,13 +456,16 @@ private DataSource inlineIfNecessary( return toInlineDataSource( subQuery, queryResults, + warehouse.getToolChest(subQuery), subqueryRowLimitAccumulator, subqueryMemoryLimitAccumulator, cannotMaterializeToFrames, maxSubqueryRows, maxSubqueryMemory, useNestedForUnknownTypeInSubquery, - !dryRun + subqueryStatsProvider, + !dryRun, + emitter ); } else { // Cannot inline subquery. Attempt to inline one level deeper, and then try again. @@ -683,8 +659,6 @@ private DataSource insertSubqueryIds( .collect(Collectors.toList())); } - - /** * * Convert the results of a particular query into a materialized (List-based) InlineDataSource. @@ -697,21 +671,26 @@ private DataSource insertSubqueryIds( * If zero, this method will throw an error immediately. * @param memoryLimit User configured byte limit. * @param useNestedForUnknownTypeInSubquery Uses nested json for unknown types when materializing subquery results + * @param subqueryStatsProvider Statistics about the subquery materialization * @param emitMetrics Flag to control if the metrics need to be emitted while materializing. The metrics are omitted * when we are performing a dry run of the query to avoid double reporting the same metric incorrectly + * @param emitter Metrics emitter * @return Inlined datasource represented by the provided results * @throws ResourceLimitExceededException if the limit is exceeded */ - private > DataSource toInlineDataSource( + private static > DataSource toInlineDataSource( final QueryType query, final Sequence queryResults, + final QueryToolChest toolChest, final AtomicInteger limitAccumulator, final AtomicLong memoryLimitAccumulator, final AtomicBoolean cannotMaterializeToFrames, final int limit, long memoryLimit, final boolean useNestedForUnknownTypeInSubquery, - final boolean emitMetrics + final SubqueryCountStatsProvider subqueryStatsProvider, + final boolean emitMetrics, + final ServiceEmitter emitter ) { final int rowLimitToUse = limit < 0 ? Integer.MAX_VALUE : limit; @@ -728,9 +707,12 @@ private > DataSource toInlineDataSource( dataSource = materializeResultsAsArray( query, queryResults, + toolChest, limitAccumulator, limit, - emitMetrics + subqueryStatsProvider, + emitMetrics, + emitter ); break; case MEMORY_LIMIT: @@ -741,11 +723,14 @@ private > DataSource toInlineDataSource( Optional maybeDataSource = materializeResultsAsFrames( query, queryResults, + toolChest, limitAccumulator, memoryLimitAccumulator, memoryLimit, useNestedForUnknownTypeInSubquery, - emitMetrics + subqueryStatsProvider, + emitMetrics, + emitter ); if (!maybeDataSource.isPresent()) { cannotMaterializeToFrames.set(true); @@ -759,9 +744,12 @@ private > DataSource toInlineDataSource( dataSource = materializeResultsAsArray( query, queryResults, + toolChest, limitAccumulator, limit, - emitMetrics + subqueryStatsProvider, + emitMetrics, + emitter ); } else { subqueryStatsProvider.incrementSubqueriesWithByteLimit(); @@ -778,17 +766,19 @@ private > DataSource toInlineDataSource( * This method materializes the query results as Frames. The method defaults back to materializing as rows in case * one cannot materialize the results as frames */ - private > Optional materializeResultsAsFrames( + private static > Optional materializeResultsAsFrames( final QueryType query, final Sequence results, + final QueryToolChest toolChest, final AtomicInteger limitAccumulator, final AtomicLong memoryLimitAccumulator, final long memoryLimit, final boolean useNestedForUnknownTypeInSubquery, - final boolean emitMetrics + final SubqueryCountStatsProvider subqueryStatsProvider, + final boolean emitMetrics, + final ServiceEmitter emitter ) { - QueryToolChest toolChest = getToolChest(query); boolean startedAccumulating = false; try { Optional> framesOptional = toolChest.resultsAsFrames( @@ -862,32 +852,26 @@ private > Optional materializeResultsA } } - private > QueryToolChest getToolChest(final QueryType query) - { - QueryToolChest toolChest = warehouse.getToolChest(query); - toolChest.setWarehouse(warehouse); - return toolChest; - } - /** * This method materializes the query results as {@code List} */ - private > DataSource materializeResultsAsArray( + private static > DataSource materializeResultsAsArray( final QueryType query, final Sequence results, + final QueryToolChest toolChest, final AtomicInteger limitAccumulator, final int limit, - boolean emitMetrics + final SubqueryCountStatsProvider subqueryStatsProvider, + boolean emitMetrics, + final ServiceEmitter emitter ) { - QueryToolChest toolChest = warehouse.getToolChest(query); final int rowLimitToUse = limit < 0 ? Integer.MAX_VALUE : limit; final RowSignature signature = toolChest.resultArraySignature(query); final ArrayList resultList = new ArrayList<>(); final int initialSubqueryRows = limitAccumulator.get(); - toolChest.setWarehouse(warehouse); toolChest.resultsAsArrays(query, results).accumulate( resultList, (acc, in) -> { From 27c80e88c03c3b7de66a9b7707871f79091e851c Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 10 Oct 2024 08:01:33 +0000 Subject: [PATCH 027/149] canMaterializeQuery --- .../server/ClientQuerySegmentWalker.java | 22 ++++++++++++++++++- .../sql/calcite/CalciteJoinQueryTest.java | 1 - 2 files changed, 21 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 1075b472ee36..497514930369 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -432,7 +432,7 @@ private DataSource inlineIfNecessary( dryRun ); } - } else if (canRunQueryUsingLocalWalker(subQuery) || canRunQueryUsingClusterWalker(subQuery)) { + } else if (canMaterializeQuery(subQuery)) { // Subquery needs to be inlined. Assign it a subquery id and run it. final Sequence queryResults; @@ -516,6 +516,26 @@ private DataSource inlineIfNecessary( } } + private boolean canMaterializeQuery(final Query query) + { + final DataSource dataSourceFromQuery = query.getDataSource(); + final DataSourceAnalysis analysis = dataSourceFromQuery.getAnalysis(); + final QueryToolChest> toolChest = warehouse.getToolChest(query); + + // 1) Must be based on a concrete datasource that is not a table. + // 2) Must be based on globally available data (so we have a copy here on the Broker). + // 3) If there is an outer query, it must be handleable by the query toolchest (the local walker does not handle + // subqueries on its own). + // 1) Must be based on a concrete table (the only shape the Druid cluster can handle). + // 2) If there is an outer query, it must be handleable by the query toolchest (the cluster walker does not handle + // subqueries on its own). + boolean toolchestCanPerform = !(dataSourceFromQuery instanceof QueryDataSource) + || toolChest.canPerformSubquery(((QueryDataSource) dataSourceFromQuery).getQuery()); + return toolchestCanPerform && analysis.isConcreteBased() + && (dataSourceFromQuery.isGlobal() || analysis.isTableBased()); + + } + /** * Decorate query runners created by {@link #clusterClient}, adding result caching, result merging, metric * emission, etc. Not to be used on runners from {@link #localClient}, since we expect it to do this kind diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java index ba5d964bc95f..12218b0b2966 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java @@ -4369,7 +4369,6 @@ public void testUnionAllTwoQueriesLeftQueryIsJoin(Map queryConte ); } - @NotYetSupported(Modes.UNION_WITH_COMPLEX_OPERAND) @MethodSource("provideQueryContexts") @ParameterizedTest(name = "{0}") public void testUnionAllTwoQueriesRightQueryIsJoin(Map queryContext) From 5284ad85b5ff06e258f0668d124c1d1cf6607249 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 10 Oct 2024 08:01:35 +0000 Subject: [PATCH 028/149] Revert "canMaterializeQuery" This reverts commit 27c80e88c03c3b7de66a9b7707871f79091e851c. --- .../server/ClientQuerySegmentWalker.java | 22 +------------------ .../sql/calcite/CalciteJoinQueryTest.java | 1 + 2 files changed, 2 insertions(+), 21 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 497514930369..1075b472ee36 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -432,7 +432,7 @@ private DataSource inlineIfNecessary( dryRun ); } - } else if (canMaterializeQuery(subQuery)) { + } else if (canRunQueryUsingLocalWalker(subQuery) || canRunQueryUsingClusterWalker(subQuery)) { // Subquery needs to be inlined. Assign it a subquery id and run it. final Sequence queryResults; @@ -516,26 +516,6 @@ private DataSource inlineIfNecessary( } } - private boolean canMaterializeQuery(final Query query) - { - final DataSource dataSourceFromQuery = query.getDataSource(); - final DataSourceAnalysis analysis = dataSourceFromQuery.getAnalysis(); - final QueryToolChest> toolChest = warehouse.getToolChest(query); - - // 1) Must be based on a concrete datasource that is not a table. - // 2) Must be based on globally available data (so we have a copy here on the Broker). - // 3) If there is an outer query, it must be handleable by the query toolchest (the local walker does not handle - // subqueries on its own). - // 1) Must be based on a concrete table (the only shape the Druid cluster can handle). - // 2) If there is an outer query, it must be handleable by the query toolchest (the cluster walker does not handle - // subqueries on its own). - boolean toolchestCanPerform = !(dataSourceFromQuery instanceof QueryDataSource) - || toolChest.canPerformSubquery(((QueryDataSource) dataSourceFromQuery).getQuery()); - return toolchestCanPerform && analysis.isConcreteBased() - && (dataSourceFromQuery.isGlobal() || analysis.isTableBased()); - - } - /** * Decorate query runners created by {@link #clusterClient}, adding result caching, result merging, metric * emission, etc. Not to be used on runners from {@link #localClient}, since we expect it to do this kind diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java index 12218b0b2966..ba5d964bc95f 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java @@ -4369,6 +4369,7 @@ public void testUnionAllTwoQueriesLeftQueryIsJoin(Map queryConte ); } + @NotYetSupported(Modes.UNION_WITH_COMPLEX_OPERAND) @MethodSource("provideQueryContexts") @ParameterizedTest(name = "{0}") public void testUnionAllTwoQueriesRightQueryIsJoin(Map queryContext) From ec5a3501f0829aec565e51354784948b8470b033 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 10 Oct 2024 10:12:54 +0000 Subject: [PATCH 029/149] mx --- .../main/java/org/apache/druid/query/Query.java | 6 ++++++ .../org/apache/druid/query/QueryToolChest.java | 7 +++++++ .../druid/query/planning/DataSourceAnalysis.java | 9 +++++++++ .../org/apache/druid/query/union/UnionQuery.java | 15 +++------------ .../druid/server/ClientQuerySegmentWalker.java | 15 +++++---------- 5 files changed, 30 insertions(+), 22 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/Query.java b/processing/src/main/java/org/apache/druid/query/Query.java index 988e3fd3f0d2..55fc0dce5ad4 100644 --- a/processing/src/main/java/org/apache/druid/query/Query.java +++ b/processing/src/main/java/org/apache/druid/query/Query.java @@ -36,6 +36,7 @@ import org.apache.druid.query.groupby.GroupByQuery; import org.apache.druid.query.metadata.metadata.SegmentMetadataQuery; import org.apache.druid.query.operator.WindowOperatorQuery; +import org.apache.druid.query.planning.DataSourceAnalysis; import org.apache.druid.query.scan.ScanQuery; import org.apache.druid.query.search.SearchQuery; import org.apache.druid.query.select.SelectQuery; @@ -304,4 +305,9 @@ default Query withDataSources(List children) } return withDataSource(Iterables.getOnlyElement(children)); } + + default DataSourceAnalysis getDataSourceAnalysis() + { + return getDataSource().getAnalysis(); + } } diff --git a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java index 983a5c2645c7..0f016169d0c0 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java @@ -437,4 +437,11 @@ public Optional> executeQuery( { return Optional.empty(); } + + public boolean canExecuteFully(Query query) { + + + + return canPerformSubquery(query); + } } diff --git a/processing/src/main/java/org/apache/druid/query/planning/DataSourceAnalysis.java b/processing/src/main/java/org/apache/druid/query/planning/DataSourceAnalysis.java index f5c600b96ffb..14a5192d0f26 100644 --- a/processing/src/main/java/org/apache/druid/query/planning/DataSourceAnalysis.java +++ b/processing/src/main/java/org/apache/druid/query/planning/DataSourceAnalysis.java @@ -32,6 +32,7 @@ import org.apache.druid.segment.join.JoinPrefixUtils; import javax.annotation.Nullable; + import java.util.List; import java.util.Objects; import java.util.Optional; @@ -294,4 +295,12 @@ public String toString() ", preJoinableClauses=" + preJoinableClauses + '}'; } + + /** + * {@link DataSource#isGlobal()}. + */ + public boolean isGlobal() + { + return baseDataSource.isGlobal(); + } } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index 985323464870..35997d60f585 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -58,7 +58,7 @@ public UnionQuery(List> queries2) @JsonCreator public UnionQuery( - @JsonProperty("queries") List> queries, + @JsonProperty("queries") List> queries, @JsonProperty("context") Map context) { Preconditions.checkArgument(queries.size() > 1, "union with fewer than 2 queries makes no sense"); @@ -194,7 +194,7 @@ public Query withDataSource(DataSource dataSource) public Query withDataSources(List children) { Preconditions.checkArgument(queries.size() == children.size(), "Number of children must match number of queries"); - List> newQueries= new ArrayList<>(); + List> newQueries = new ArrayList<>(); for (int i = 0; i < queries.size(); i++) { newQueries.add(queries.get(i).withDataSource(children.get(i))); } @@ -218,15 +218,6 @@ public String toString() public DataSourceAnalysis getDataSourceAnalysis() { - if(true) - { - throw new RuntimeException("FIXME: Unimplemented!"); - } - return null; - + throw new RuntimeException("FIXME: Unimplemented!"); } - - - - } diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 1075b472ee36..dbe756978099 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -294,17 +294,15 @@ public QueryRunner getQueryRunnerForSegments(final Query query, final */ private boolean canRunQueryUsingLocalWalker(Query query) { - final DataSource dataSourceFromQuery = query.getDataSource(); - final DataSourceAnalysis analysis = dataSourceFromQuery.getAnalysis(); final QueryToolChest> toolChest = warehouse.getToolChest(query); + DataSourceAnalysis analysis = query.getDataSourceAnalysis(); // 1) Must be based on a concrete datasource that is not a table. // 2) Must be based on globally available data (so we have a copy here on the Broker). // 3) If there is an outer query, it must be handleable by the query toolchest (the local walker does not handle // subqueries on its own). - return analysis.isConcreteBased() && !analysis.isConcreteAndTableBased() && dataSourceFromQuery.isGlobal() - && (!(dataSourceFromQuery instanceof QueryDataSource) - || toolChest.canPerformSubquery(((QueryDataSource) dataSourceFromQuery).getQuery())); + return analysis.isConcreteBased() && !analysis.isConcreteAndTableBased() && analysis.isGlobal() + && toolChest.canExecuteFully(query); } /** @@ -313,16 +311,13 @@ private boolean canRunQueryUsingLocalWalker(Query query) */ private boolean canRunQueryUsingClusterWalker(Query query) { - final DataSource dataSourceFromQuery = query.getDataSource(); - final DataSourceAnalysis analysis = dataSourceFromQuery.getAnalysis(); final QueryToolChest> toolChest = warehouse.getToolChest(query); + DataSourceAnalysis analysis = query.getDataSourceAnalysis(); // 1) Must be based on a concrete table (the only shape the Druid cluster can handle). // 2) If there is an outer query, it must be handleable by the query toolchest (the cluster walker does not handle // subqueries on its own). - return analysis.isConcreteAndTableBased() - && (!(dataSourceFromQuery instanceof QueryDataSource) - || toolChest.canPerformSubquery(((QueryDataSource) dataSourceFromQuery).getQuery())); + return analysis.isConcreteAndTableBased() && toolChest.canExecuteFully(query); } From b5471e01fec5a55f58c81f85d8991fbf491fd1bd Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 10 Oct 2024 16:12:47 +0000 Subject: [PATCH 030/149] fix --- .../apache/druid/query/QueryToolChest.java | 10 +-- .../union/RealUnionQueryQueryToolChest.java | 7 ++ .../union/RealUnionQueryRunnerFactory.java | 7 +- .../apache/druid/query/union/UnionQuery.java | 78 ++++++++++++++++++- .../server/ClientQuerySegmentWalker.java | 46 +++++++---- 5 files changed, 124 insertions(+), 24 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java index 0f016169d0c0..3ea27149d53c 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java @@ -438,10 +438,10 @@ public Optional> executeQuery( return Optional.empty(); } - public boolean canExecuteFully(Query query) { - - - - return canPerformSubquery(query); + public boolean canExecuteFully(Query query) + { + DataSource dataSourceFromQuery = query.getDataSource(); + return (!(dataSourceFromQuery instanceof QueryDataSource) + || canPerformSubquery(((QueryDataSource) dataSourceFromQuery).getQuery())); } } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java index b66efabe85e4..dbbb52177f4d 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java @@ -47,10 +47,17 @@ public class RealUnionQueryQueryToolChest extends QueryToolChest { + +public RealUnionQueryQueryToolChest() { +int asd=1; +} + + public Optional> executeQuery(QueryToolChestWarehouse warehouse, Query query, QuerySegmentWalker clientQuerySegmentWalker) { RealUnionQueryRunner2 runner = new RealUnionQueryRunner2(warehouse, (UnionQuery) query, clientQuerySegmentWalker); + setWarehouse(warehouse); return Optional.of(runner); } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java index 838c32379328..3aae4fff541d 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java @@ -28,6 +28,8 @@ public class RealUnionQueryRunnerFactory implements QueryRunnerFactory { + private static RealUnionQueryQueryToolChest tt; + public RealUnionQueryRunnerFactory(String string) { int asd=1; @@ -59,7 +61,10 @@ public QueryRunner mergeRunners(QueryProcessingPool queryProces @Override public QueryToolChest getToolchest() { - return new RealUnionQueryQueryToolChest(); + if (tt == null) { + tt = new RealUnionQueryQueryToolChest(); + } + return tt; } } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index 35997d60f585..63af5f75e9dd 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -24,6 +24,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; +import org.apache.druid.error.DruidException; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; @@ -31,9 +32,11 @@ import org.apache.druid.query.QueryContexts; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.UnionDataSource; import org.apache.druid.query.filter.DimFilter; import org.apache.druid.query.planning.DataSourceAnalysis; import org.apache.druid.query.spec.QuerySegmentSpec; +import org.apache.druid.segment.SegmentReference; import org.joda.time.DateTimeZone; import org.joda.time.Duration; import org.joda.time.Interval; @@ -41,6 +44,8 @@ import java.util.ArrayList; import java.util.List; import java.util.Map; +import java.util.Set; +import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; public class UnionQuery implements Query @@ -218,6 +223,77 @@ public String toString() public DataSourceAnalysis getDataSourceAnalysis() { - throw new RuntimeException("FIXME: Unimplemented!"); + OpagueDataSourceCover ds = new OpagueDataSourceCover(new UnionDataSource(getDataSources())); + return new DataSourceAnalysis(ds, null, null, null); + } + + private static class OpagueDataSourceCover implements DataSource + { + private DataSource delegate; + + public OpagueDataSourceCover(DataSource delegate) + { + this.delegate = delegate; + } + + @Override + public Set getTableNames() + { + return delegate.getTableNames(); + } + + @Override + public List getChildren() + { + return delegate.getChildren(); + } + + @Override + public DataSource withChildren(List children) + { + throw DruidException.defensive().build("Not implemented"); + } + + @Override + public boolean isCacheable(boolean isBroker) + { + return delegate.isCacheable(isBroker); + } + + @Override + public boolean isGlobal() + { + return delegate.isGlobal(); + } + + @Override + public boolean isConcrete() + { + return delegate.isConcrete(); + } + + @Override + public Function createSegmentMapFunction(Query query, AtomicLong cpuTimeAcc) + { + throw DruidException.defensive().build("Not implemented"); + } + + @Override + public DataSource withUpdatedDataSource(DataSource newSource) + { + throw DruidException.defensive().build("Not implemented"); + } + + @Override + public byte[] getCacheKey() + { + return delegate.getCacheKey(); + } + + @Override + public DataSourceAnalysis getAnalysis() + { + throw DruidException.defensive().build("Not implemented"); + } } } diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index dbe756978099..f862c6c35a50 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -62,6 +62,7 @@ import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.planning.DataSourceAnalysis; +import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.server.initialization.ServerConfig; @@ -223,6 +224,7 @@ public QueryRunner getQueryRunnerForIntervals(final Query query, final // Dry run didn't go well. throw new ISE("Cannot handle subquery structure for dataSource: %s", query.getDataSource()); } + assert toolChest.canExecuteFully(query.withDataSource(inlineDryRun)); // Now that we know the structure is workable, actually do the inlining (if necessary). AtomicLong memoryLimitAcc = new AtomicLong(0); @@ -294,15 +296,15 @@ public QueryRunner getQueryRunnerForSegments(final Query query, final */ private boolean canRunQueryUsingLocalWalker(Query query) { + final DataSourceAnalysis analysis = query.getDataSourceAnalysis(); final QueryToolChest> toolChest = warehouse.getToolChest(query); - DataSourceAnalysis analysis = query.getDataSourceAnalysis(); // 1) Must be based on a concrete datasource that is not a table. // 2) Must be based on globally available data (so we have a copy here on the Broker). // 3) If there is an outer query, it must be handleable by the query toolchest (the local walker does not handle // subqueries on its own). return analysis.isConcreteBased() && !analysis.isConcreteAndTableBased() && analysis.isGlobal() - && toolChest.canExecuteFully(query); + && toolChest.canExecuteFully(query); } /** @@ -311,13 +313,17 @@ private boolean canRunQueryUsingLocalWalker(Query query) */ private boolean canRunQueryUsingClusterWalker(Query query) { + if (query instanceof UnionQuery) { + return true; + } + final DataSourceAnalysis analysis = query.getDataSourceAnalysis(); final QueryToolChest> toolChest = warehouse.getToolChest(query); - DataSourceAnalysis analysis = query.getDataSourceAnalysis(); // 1) Must be based on a concrete table (the only shape the Druid cluster can handle). // 2) If there is an outer query, it must be handleable by the query toolchest (the cluster walker does not handle // subqueries on its own). - return analysis.isConcreteAndTableBased() && toolChest.canExecuteFully(query); + return analysis.isConcreteAndTableBased() + && toolChest.canExecuteFully(query); } @@ -443,6 +449,7 @@ private DataSource inlineIfNecessary( .toString() ) ); + warehouse.getToolChest(subQuery).setWarehouse(warehouse); queryResults = subQueryWithSerialization .getRunner(this) .run(QueryPlus.wrap(subQueryWithSerialization), DirectDruidClient.makeResponseContextForQuery()); @@ -464,21 +471,26 @@ private DataSource inlineIfNecessary( ); } else { // Cannot inline subquery. Attempt to inline one level deeper, and then try again. + + List newDataSources = new ArrayList(); + for (DataSource ds : dataSource.getChildren()) { + newDataSources.add( + inlineIfNecessary( + ds, + null, + subqueryRowLimitAccumulator, + subqueryMemoryLimitAccumulator, + cannotMaterializeToFrames, + maxSubqueryRows, + maxSubqueryMemory, + useNestedForUnknownTypeInSubquery, + dryRun + ) + ); + } return inlineIfNecessary( dataSource.withChildren( - Collections.singletonList( - inlineIfNecessary( - Iterables.getOnlyElement(dataSource.getChildren()), - null, - subqueryRowLimitAccumulator, - subqueryMemoryLimitAccumulator, - cannotMaterializeToFrames, - maxSubqueryRows, - maxSubqueryMemory, - useNestedForUnknownTypeInSubquery, - dryRun - ) - ) + newDataSources ), toolChestIfOutermost, subqueryRowLimitAccumulator, From 94093b30aa1aded8ea036ad85a6599b8bf3563c7 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 03:58:50 +0000 Subject: [PATCH 031/149] add some interface --- .../druid/query/QueryExecSomething.java | 27 +++++++++++++++++++ .../union/RealUnionQueryQueryToolChest.java | 11 +++----- .../server/ClientQuerySegmentWalker.java | 16 ++++++----- 3 files changed, 40 insertions(+), 14 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/QueryExecSomething.java diff --git a/processing/src/main/java/org/apache/druid/query/QueryExecSomething.java b/processing/src/main/java/org/apache/druid/query/QueryExecSomething.java new file mode 100644 index 000000000000..2f18c40d2bd9 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/QueryExecSomething.java @@ -0,0 +1,27 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query; + +public interface QueryExecSomething +{ + public QueryRunner executeQuery2(QueryToolChestWarehouse warehouse, + Query query, QuerySegmentWalker clientQuerySegmentWalker); + +} diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java index dbbb52177f4d..57ecccf387c8 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java @@ -28,6 +28,7 @@ import org.apache.druid.query.DefaultQueryMetrics; import org.apache.druid.query.FrameSignaturePair; import org.apache.druid.query.Query; +import org.apache.druid.query.QueryExecSomething; import org.apache.druid.query.QueryMetrics; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; @@ -46,19 +47,15 @@ import java.util.Optional; public class RealUnionQueryQueryToolChest extends QueryToolChest + implements QueryExecSomething { -public RealUnionQueryQueryToolChest() { -int asd=1; -} - - - public Optional> executeQuery(QueryToolChestWarehouse warehouse, + public QueryRunner executeQuery2(QueryToolChestWarehouse warehouse, Query query, QuerySegmentWalker clientQuerySegmentWalker) { RealUnionQueryRunner2 runner = new RealUnionQueryRunner2(warehouse, (UnionQuery) query, clientQuerySegmentWalker); setWarehouse(warehouse); - return Optional.of(runner); + return runner; } public Optional> executeQuery1(QueryToolChestWarehouse warehouse, diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index f862c6c35a50..ee7398bf2c23 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -49,6 +49,7 @@ import org.apache.druid.query.Query; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.QueryDataSource; +import org.apache.druid.query.QueryExecSomething; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; @@ -62,7 +63,6 @@ import org.apache.druid.query.SegmentDescriptor; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.planning.DataSourceAnalysis; -import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.server.initialization.ServerConfig; @@ -180,9 +180,10 @@ public QueryRunner getQueryRunnerForIntervals(final Query query, final { final QueryToolChest> toolChest = warehouse.getToolChest(query); - Optional> toolchestExecResult = toolChest.executeQuery(warehouse, query, this); - if (toolchestExecResult.isPresent()) { - return toolchestExecResult.get(); +// Optional> toolchestExecResult = toolChest.executeQuery21(warehouse, query, this); + if (toolChest instanceof QueryExecSomething) { + QueryExecSomething t = (QueryExecSomething) toolChest; + return t.executeQuery2(warehouse, query, this); } // transform TableDataSource to GlobalTableDataSource when eligible @@ -313,12 +314,13 @@ private boolean canRunQueryUsingLocalWalker(Query query) */ private boolean canRunQueryUsingClusterWalker(Query query) { - if (query instanceof UnionQuery) { - return true; - } final DataSourceAnalysis analysis = query.getDataSourceAnalysis(); final QueryToolChest> toolChest = warehouse.getToolChest(query); + if (toolChest instanceof QueryExecSomething) { + return true; + } + // 1) Must be based on a concrete table (the only shape the Druid cluster can handle). // 2) If there is an outer query, it must be handleable by the query toolchest (the cluster walker does not handle // subqueries on its own). From ad00767f61e187c8fa4c482ecde1cb4b2ce65347 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 06:44:14 +0000 Subject: [PATCH 032/149] updates --- .../sql/calcite/CalciteJoinQueryTest.java | 5 +- .../sql/calcite/DecoupledTestConfig.java | 6 +- ...estUnionAllTwoQueriesBothQueriesAreJoin.iq | 147 ++++++++++++++++++ ...lTwoQueriesLeftQueryIsJoin@all_disabled.iq | 139 +++++++++++++++++ ...llTwoQueriesLeftQueryIsJoin@all_enabled.iq | 139 +++++++++++++++++ ...ionAllTwoQueriesLeftQueryIsJoin@default.iq | 130 ++++++++++++++++ ...yIsJoin@filter-on-value-column_disabled.iq | 139 +++++++++++++++++ ...eftQueryIsJoin@filter-rewrites-disabled.iq | 139 +++++++++++++++++ ...oQueriesLeftQueryIsJoin@filter-rewrites.iq | 139 +++++++++++++++++ ...woQueriesLeftQueryIsJoin@join-to-filter.iq | 139 +++++++++++++++++ ...TwoQueriesRightQueryIsJoin@all_disabled.iq | 139 +++++++++++++++++ ...lTwoQueriesRightQueryIsJoin@all_enabled.iq | 139 +++++++++++++++++ ...onAllTwoQueriesRightQueryIsJoin@default.iq | 130 ++++++++++++++++ ...yIsJoin@filter-on-value-column_disabled.iq | 139 +++++++++++++++++ ...ghtQueryIsJoin@filter-rewrites-disabled.iq | 139 +++++++++++++++++ ...QueriesRightQueryIsJoin@filter-rewrites.iq | 139 +++++++++++++++++ ...oQueriesRightQueryIsJoin@join-to-filter.iq | 139 +++++++++++++++++ 17 files changed, 2084 insertions(+), 2 deletions(-) create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@all_disabled.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@all_enabled.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@default.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@filter-on-value-column_disabled.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@filter-rewrites-disabled.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@filter-rewrites.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@join-to-filter.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@all_disabled.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@all_enabled.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@default.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@filter-on-value-column_disabled.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@filter-rewrites-disabled.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@filter-rewrites.iq create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@join-to-filter.iq diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java index ba5d964bc95f..7c551954cb26 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java @@ -4325,6 +4325,7 @@ public void testJoinOnMultiValuedColumnShouldThrowException(Map }); } + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNION_ALL_QUERY) @MethodSource("provideQueryContexts") @ParameterizedTest(name = "{0}") public void testUnionAllTwoQueriesLeftQueryIsJoin(Map queryContext) @@ -4369,7 +4370,7 @@ public void testUnionAllTwoQueriesLeftQueryIsJoin(Map queryConte ); } - @NotYetSupported(Modes.UNION_WITH_COMPLEX_OPERAND) + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNION_ALL_QUERY) @MethodSource("provideQueryContexts") @ParameterizedTest(name = "{0}") public void testUnionAllTwoQueriesRightQueryIsJoin(Map queryContext) @@ -4414,6 +4415,8 @@ public void testUnionAllTwoQueriesRightQueryIsJoin(Map queryCont ); } + + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNION_ALL_QUERY) @Test public void testUnionAllTwoQueriesBothQueriesAreJoin() { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledTestConfig.java b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledTestConfig.java index 10d631b665c0..60d261c33e0a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledTestConfig.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledTestConfig.java @@ -157,7 +157,11 @@ enum QuidemTestCaseReason /** * New plan UNNEST-s a different resultset. */ - UNNEST_DIFFERENT_RESULTSET; + UNNEST_DIFFERENT_RESULTSET, + /** + * Uses a UNION ALL query. + */ + UNION_ALL_QUERY; public boolean isPresent() { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq new file mode 100644 index 000000000000..65b1fac2fad7 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq @@ -0,0 +1,147 @@ +# testUnionAllTwoQueriesBothQueriesAreJoin case-crc:0924562a +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT COUNT(*) FROM foo LEFT JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[left]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[left]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "LEFT" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@all_disabled.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@all_disabled.iq new file mode 100644 index 000000000000..890e2d861616 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@all_disabled.iq @@ -0,0 +1,139 @@ +# testUnionAllTwoQueriesLeftQueryIsJoin@all_disabled case-crc:7ee1bcbd +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set enableJoinFilterRewrite false +!set enableJoinFilterRewriteValueColumnFilters false +!set enableRewriteJoinToFilter false +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) UNION ALL SELECT SUM(cnt) FROM foo; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "false", + "enableJoinFilterRewriteValueColumnFilters" : "false", + "enableRewriteJoinToFilter" : "false", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "false", + "enableJoinFilterRewriteValueColumnFilters" : "false", + "enableRewriteJoinToFilter" : "false", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@all_enabled.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@all_enabled.iq new file mode 100644 index 000000000000..f8fef5db5176 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@all_enabled.iq @@ -0,0 +1,139 @@ +# testUnionAllTwoQueriesLeftQueryIsJoin@all_enabled case-crc:6ac8a0fc +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set enableJoinFilterRewrite true +!set enableJoinFilterRewriteValueColumnFilters true +!set enableRewriteJoinToFilter true +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) UNION ALL SELECT SUM(cnt) FROM foo; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "true", + "enableJoinFilterRewriteValueColumnFilters" : "true", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "true", + "enableJoinFilterRewriteValueColumnFilters" : "true", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@default.iq new file mode 100644 index 000000000000..c215c664567c --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@default.iq @@ -0,0 +1,130 @@ +# testUnionAllTwoQueriesLeftQueryIsJoin@default case-crc:4327c7fa +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) UNION ALL SELECT SUM(cnt) FROM foo; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@filter-on-value-column_disabled.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@filter-on-value-column_disabled.iq new file mode 100644 index 000000000000..1e5ff447f876 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@filter-on-value-column_disabled.iq @@ -0,0 +1,139 @@ +# testUnionAllTwoQueriesLeftQueryIsJoin@filter-on-value-column_disabled case-crc:10dc98d8 +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set enableJoinFilterRewrite true +!set enableJoinFilterRewriteValueColumnFilters false +!set enableRewriteJoinToFilter true +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) UNION ALL SELECT SUM(cnt) FROM foo; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "true", + "enableJoinFilterRewriteValueColumnFilters" : "false", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "true", + "enableJoinFilterRewriteValueColumnFilters" : "false", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@filter-rewrites-disabled.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@filter-rewrites-disabled.iq new file mode 100644 index 000000000000..f5ee8b38c819 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@filter-rewrites-disabled.iq @@ -0,0 +1,139 @@ +# testUnionAllTwoQueriesLeftQueryIsJoin@filter-rewrites-disabled case-crc:08a9a1e5 +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set enableJoinFilterRewrite false +!set enableJoinFilterRewriteValueColumnFilters true +!set enableRewriteJoinToFilter true +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) UNION ALL SELECT SUM(cnt) FROM foo; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "false", + "enableJoinFilterRewriteValueColumnFilters" : "true", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "false", + "enableJoinFilterRewriteValueColumnFilters" : "true", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@filter-rewrites.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@filter-rewrites.iq new file mode 100644 index 000000000000..66213a432d60 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@filter-rewrites.iq @@ -0,0 +1,139 @@ +# testUnionAllTwoQueriesLeftQueryIsJoin@filter-rewrites case-crc:a2003660 +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set enableJoinFilterRewrite true +!set enableJoinFilterRewriteValueColumnFilters true +!set enableRewriteJoinToFilter false +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) UNION ALL SELECT SUM(cnt) FROM foo; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "true", + "enableJoinFilterRewriteValueColumnFilters" : "true", + "enableRewriteJoinToFilter" : "false", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "true", + "enableJoinFilterRewriteValueColumnFilters" : "true", + "enableRewriteJoinToFilter" : "false", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@join-to-filter.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@join-to-filter.iq new file mode 100644 index 000000000000..01202d21211b --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesLeftQueryIsJoin@join-to-filter.iq @@ -0,0 +1,139 @@ +# testUnionAllTwoQueriesLeftQueryIsJoin@join-to-filter case-crc:05dd3d7b +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set enableJoinFilterRewrite false +!set enableJoinFilterRewriteValueColumnFilters false +!set enableRewriteJoinToFilter true +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) UNION ALL SELECT SUM(cnt) FROM foo; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "false", + "enableJoinFilterRewriteValueColumnFilters" : "false", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "false", + "enableJoinFilterRewriteValueColumnFilters" : "false", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@all_disabled.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@all_disabled.iq new file mode 100644 index 000000000000..8000f8cc6df8 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@all_disabled.iq @@ -0,0 +1,139 @@ +# testUnionAllTwoQueriesRightQueryIsJoin@all_disabled case-crc:1181ba66 +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set enableJoinFilterRewrite false +!set enableJoinFilterRewriteValueColumnFilters false +!set enableRewriteJoinToFilter false +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "false", + "enableJoinFilterRewriteValueColumnFilters" : "false", + "enableRewriteJoinToFilter" : "false", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "false", + "enableJoinFilterRewriteValueColumnFilters" : "false", + "enableRewriteJoinToFilter" : "false", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@all_enabled.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@all_enabled.iq new file mode 100644 index 000000000000..0093d4e581da --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@all_enabled.iq @@ -0,0 +1,139 @@ +# testUnionAllTwoQueriesRightQueryIsJoin@all_enabled case-crc:05a8a627 +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set enableJoinFilterRewrite true +!set enableJoinFilterRewriteValueColumnFilters true +!set enableRewriteJoinToFilter true +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "true", + "enableJoinFilterRewriteValueColumnFilters" : "true", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "true", + "enableJoinFilterRewriteValueColumnFilters" : "true", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@default.iq new file mode 100644 index 000000000000..791603a22fe2 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@default.iq @@ -0,0 +1,130 @@ +# testUnionAllTwoQueriesRightQueryIsJoin@default case-crc:2c47c121 +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@filter-on-value-column_disabled.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@filter-on-value-column_disabled.iq new file mode 100644 index 000000000000..d17665f24acb --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@filter-on-value-column_disabled.iq @@ -0,0 +1,139 @@ +# testUnionAllTwoQueriesRightQueryIsJoin@filter-on-value-column_disabled case-crc:7fbc9e03 +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set enableJoinFilterRewrite true +!set enableJoinFilterRewriteValueColumnFilters false +!set enableRewriteJoinToFilter true +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "true", + "enableJoinFilterRewriteValueColumnFilters" : "false", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "true", + "enableJoinFilterRewriteValueColumnFilters" : "false", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@filter-rewrites-disabled.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@filter-rewrites-disabled.iq new file mode 100644 index 000000000000..1199eca6b1ee --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@filter-rewrites-disabled.iq @@ -0,0 +1,139 @@ +# testUnionAllTwoQueriesRightQueryIsJoin@filter-rewrites-disabled case-crc:67c9a73e +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set enableJoinFilterRewrite false +!set enableJoinFilterRewriteValueColumnFilters true +!set enableRewriteJoinToFilter true +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "false", + "enableJoinFilterRewriteValueColumnFilters" : "true", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "false", + "enableJoinFilterRewriteValueColumnFilters" : "true", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@filter-rewrites.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@filter-rewrites.iq new file mode 100644 index 000000000000..80bba8f87d08 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@filter-rewrites.iq @@ -0,0 +1,139 @@ +# testUnionAllTwoQueriesRightQueryIsJoin@filter-rewrites case-crc:cd6030bb +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set enableJoinFilterRewrite true +!set enableJoinFilterRewriteValueColumnFilters true +!set enableRewriteJoinToFilter false +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "true", + "enableJoinFilterRewriteValueColumnFilters" : "true", + "enableRewriteJoinToFilter" : "false", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "true", + "enableJoinFilterRewriteValueColumnFilters" : "true", + "enableRewriteJoinToFilter" : "false", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@join-to-filter.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@join-to-filter.iq new file mode 100644 index 000000000000..02a589ab2026 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesRightQueryIsJoin@join-to-filter.iq @@ -0,0 +1,139 @@ +# testUnionAllTwoQueriesRightQueryIsJoin@join-to-filter case-crc:6abd3ba0 +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set enableJoinFilterRewrite false +!set enableJoinFilterRewriteValueColumnFilters false +!set enableRewriteJoinToFilter true +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +(SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "false", + "enableJoinFilterRewriteValueColumnFilters" : "false", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "enableJoinFilterRewrite" : "false", + "enableJoinFilterRewriteValueColumnFilters" : "false", + "enableRewriteJoinToFilter" : "true", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan From 4dd587e4da65badc8576e6b52234f82a00fec18e Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 08:34:26 +0000 Subject: [PATCH 033/149] defaultconglomerate.of --- .../CachingClusteredClientBenchmark.java | 2 +- .../indexing/kafka/KafkaIndexTaskTest.java | 2 +- .../kinesis/KinesisIndexTaskTest.java | 2 +- ...DefaultQueryRunnerFactoryConglomerate.java | 14 +- .../druid/guice/QueryRunnerFactoryModule.java | 1 - .../StreamAppenderatorTester.java | 4 +- ...nifiedIndexerAppenderatorsManagerTest.java | 2 +- .../apache/druid/server/QueryStackTests.java | 2 +- ...estUnionAllTwoQueriesBothQueriesAreJoin.iq | 132 ------------------ 9 files changed, 18 insertions(+), 143 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/CachingClusteredClientBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/CachingClusteredClientBenchmark.java index 8e0715e0fe5c..6647be2e42cb 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/CachingClusteredClientBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/CachingClusteredClientBenchmark.java @@ -258,7 +258,7 @@ public int getNumThreads() } }; - conglomerate = new DefaultQueryRunnerFactoryConglomerate( + conglomerate = DefaultQueryRunnerFactoryConglomerate.of( ImmutableMap., QueryRunnerFactory>builder() .put( TimeseriesQuery.class, diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java index 23bdeb14acb8..6b955660e2cf 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java @@ -2891,7 +2891,7 @@ private static DataSchema cloneDataSchema(final DataSchema dataSchema) @Override protected QueryRunnerFactoryConglomerate makeQueryRunnerConglomerate() { - return new DefaultQueryRunnerFactoryConglomerate( + return DefaultQueryRunnerFactoryConglomerate.of( ImmutableMap., QueryRunnerFactory>builder() .put( TimeseriesQuery.class, diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java index 2ef391484008..d4cdfacb98ca 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java @@ -2418,7 +2418,7 @@ private static DataSchema cloneDataSchema(final DataSchema dataSchema) @Override protected QueryRunnerFactoryConglomerate makeQueryRunnerConglomerate() { - return new DefaultQueryRunnerFactoryConglomerate( + return DefaultQueryRunnerFactoryConglomerate.of( ImmutableMap.of( TimeseriesQuery.class, new TimeseriesQueryRunnerFactory( diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java index 7f004ffa6343..d3049de767d5 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java @@ -28,10 +28,10 @@ */ public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactoryConglomerate { - private final Map, QueryRunnerFactory> factories; + private Map, QueryRunnerFactory> factories; @Inject - public DefaultQueryRunnerFactoryConglomerate(Map, QueryRunnerFactory> factories) + public void setFactories(Map, QueryRunnerFactory> factories) { // Accesses to IdentityHashMap should be faster than to HashMap or ImmutableMap. // Class doesn't override Object.equals(). @@ -42,6 +42,14 @@ public DefaultQueryRunnerFactoryConglomerate(Map, QueryRu @SuppressWarnings("unchecked") public > QueryRunnerFactory findFactory(QueryType query) { - return (QueryRunnerFactory) factories.get(query.getClass()); + return factories.get(query.getClass()); + } + + + public static DefaultQueryRunnerFactoryConglomerate of(Map, QueryRunnerFactory> factories) + { + DefaultQueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate(); + conglomerate.setFactories(factories); + return conglomerate; } } diff --git a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java index f71228a3fbee..00cefa261d17 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java @@ -50,7 +50,6 @@ import org.apache.druid.query.union.UnionQuery; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QuerySchedulerProvider; - import java.util.Map; /** diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java index 29d758aaed02..db1949b58273 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java @@ -224,7 +224,7 @@ public Map makeLoadSpec(URI uri) objectMapper, indexIO, indexMerger, - new DefaultQueryRunnerFactoryConglomerate( + DefaultQueryRunnerFactoryConglomerate.of( ImmutableMap.of( TimeseriesQuery.class, new TimeseriesQueryRunnerFactory( new TimeseriesQueryQueryToolChest(), @@ -268,7 +268,7 @@ public int getDropSegmentDelayMillis() objectMapper, indexIO, indexMerger, - new DefaultQueryRunnerFactoryConglomerate( + DefaultQueryRunnerFactoryConglomerate.of( ImmutableMap.of( TimeseriesQuery.class, new TimeseriesQueryRunnerFactory( new TimeseriesQueryQueryToolChest(), diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java index 21f627baa085..ae87b4779f89 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java @@ -83,7 +83,7 @@ public class UnifiedIndexerAppenderatorsManagerTest extends InitializedNullHandl new CachePopulatorStats(), TestHelper.makeJsonMapper(), new NoopServiceEmitter(), - () -> new DefaultQueryRunnerFactoryConglomerate(ImmutableMap.of()) + () -> DefaultQueryRunnerFactoryConglomerate.of(ImmutableMap.of()) ); private AppenderatorConfig appenderatorConfig; diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index cb4667130849..9e16018f2c53 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -331,7 +331,7 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat processingConfig ); - final QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate( + final QueryRunnerFactoryConglomerate conglomerate = DefaultQueryRunnerFactoryConglomerate.of( ImmutableMap., QueryRunnerFactory>builder() .put( SegmentMetadataQuery.class, diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq index 65b1fac2fad7..08b51819b8b1 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq @@ -9,139 +9,7 @@ !set outputformat mysql !use druidtest:/// (SELECT COUNT(*) FROM foo LEFT JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; -+--------+ -| EXPR$0 | -+--------+ -| 1 | -| 6 | -+--------+ -(2 rows) - !ok -LogicalUnion(all=[true]) - LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) - LogicalJoin(condition=[=($0, $1)], joinType=[left]) - LogicalProject(dim1=[$1]) - LogicalTableScan(table=[[druid, foo]]) - LogicalProject(k=[$0]) - LogicalTableScan(table=[[lookup, lookyloo]]) - LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) - LogicalJoin(condition=[=($0, $1)], joinType=[inner]) - LogicalProject(dim1=[$1]) - LogicalTableScan(table=[[druid, foo]]) - LogicalProject(k=[$0]) - LogicalTableScan(table=[[lookup, lookyloo]]) - !logicalPlan -DruidUnion(all=[true]) - DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) - DruidJoin(condition=[=($0, $1)], joinType=[left]) - DruidProject(dim1=[$1], druid=[logical]) - DruidTableScan(table=[[druid, foo]], druid=[logical]) - DruidProject(k=[$0], druid=[logical]) - DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) - DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) - DruidJoin(condition=[=($0, $1)], joinType=[inner]) - DruidProject(dim1=[$1], druid=[logical]) - DruidTableScan(table=[[druid, foo]], druid=[logical]) - DruidProject(k=[$0], druid=[logical]) - DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) - !druidPlan -{ - "queryType" : "scan", - "dataSource" : { - "type" : "query", - "query" : { - "queryType" : "union", - "queries" : [ { - "queryType" : "timeseries", - "dataSource" : { - "type" : "join", - "left" : { - "type" : "table", - "name" : "foo" - }, - "right" : { - "type" : "lookup", - "lookup" : "lookyloo" - }, - "rightPrefix" : "j0.", - "condition" : "(\"dim1\" == \"j0.k\")", - "joinType" : "LEFT" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "granularity" : { - "type" : "all" - }, - "aggregations" : [ { - "type" : "count", - "name" : "a0" - } ], - "context" : { - "debug" : "true", - "defaultTimeout" : "300000", - "maxScatterGatherBytes" : "9223372036854775807", - "outputformat" : "MYSQL", - "plannerStrategy" : "DECOUPLED", - "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", - "sqlQueryId" : "dummy", - "sqlStringifyArrays" : false - } - }, { - "queryType" : "timeseries", - "dataSource" : { - "type" : "join", - "left" : { - "type" : "table", - "name" : "foo" - }, - "right" : { - "type" : "lookup", - "lookup" : "lookyloo" - }, - "rightPrefix" : "j0.", - "condition" : "(\"dim1\" == \"j0.k\")", - "joinType" : "INNER" - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "granularity" : { - "type" : "all" - }, - "aggregations" : [ { - "type" : "count", - "name" : "a0" - } ], - "context" : { - "debug" : "true", - "defaultTimeout" : "300000", - "maxScatterGatherBytes" : "9223372036854775807", - "outputformat" : "MYSQL", - "plannerStrategy" : "DECOUPLED", - "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", - "sqlQueryId" : "dummy", - "sqlStringifyArrays" : false - } - } ], - "context" : null - } - }, - "intervals" : { - "type" : "intervals", - "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] - }, - "resultFormat" : "compactedList", - "columns" : [ "a0" ], - "columnTypes" : [ "LONG" ], - "granularity" : { - "type" : "all" - }, - "legacy" : false -} !nativePlan From fa4e598b00e8f599c7d4898b7d6ea901bbafe548 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 07:54:28 +0000 Subject: [PATCH 034/149] ConglomerateBackedQueryToolChestWarehouse --- ...lomerateBackedQueryToolChestWarehouse.java | 36 +++++++++++++++++++ .../CachingClusteredClientPerfTest.java | 1 - ...yRunnerBasedOnClusteredClientTestBase.java | 10 +----- .../SegmentMetadataCacheTestBase.java | 12 ++----- .../apache/druid/server/QueryStackTests.java | 12 ++----- .../sql/calcite/util/QueryFrameworkUtils.java | 13 ++----- 6 files changed, 43 insertions(+), 41 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java diff --git a/processing/src/main/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java b/processing/src/main/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java new file mode 100644 index 000000000000..f80ac15987fd --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java @@ -0,0 +1,36 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query; + +public class ConglomerateBackedQueryToolChestWarehouse implements QueryToolChestWarehouse +{ + private QueryRunnerFactoryConglomerate conglomerate; + + public ConglomerateBackedQueryToolChestWarehouse(QueryRunnerFactoryConglomerate conglomerate) + { + this.conglomerate = conglomerate; + } + + @Override + public > QueryToolChest getToolChest(final QueryType query) + { + return conglomerate.findFactory(query).getToolchest(); + } +} diff --git a/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java b/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java index b31799cf8b4c..e6cc643fa7a1 100644 --- a/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java +++ b/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java @@ -172,7 +172,6 @@ private DataSegment makeDataSegment(String dataSource, Interval interval, String private static class MockQueryToolChestWareHouse implements QueryToolChestWarehouse { - @Override public > QueryToolChest getToolChest(QueryType query) { diff --git a/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java b/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java index 308427d40fa4..c79480d9ae7f 100644 --- a/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java +++ b/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java @@ -109,15 +109,7 @@ protected QueryRunnerBasedOnClusteredClientTestBase() CLOSER, () -> TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD ); - - toolChestWarehouse = new QueryToolChestWarehouse() - { - @Override - public > QueryToolChest getToolChest(final QueryType query) - { - return conglomerate.findFactory(query).getToolchest(); - } - }; + toolChestWarehouse = new ConglomerateBackedQueryToolChestWarehouse(conglomerate); } @AfterClass diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java index c62577e01256..f08d5942f146 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java @@ -29,12 +29,11 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.query.ConglomerateBackedQueryToolChestWarehouse; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; -import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; -import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; @@ -112,14 +111,7 @@ public void setUpCommon() { resourceCloser = Closer.create(); conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser); - queryToolChestWarehouse = new QueryToolChestWarehouse() - { - @Override - public > QueryToolChest getToolChest(final QueryType query) - { - return conglomerate.findFactory(query).getToolchest(); - } - }; + queryToolChestWarehouse = new ConglomerateBackedQueryToolChestWarehouse(conglomerate); } public void setUpData() throws Exception diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index 041c4654d92e..312baa8025b1 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -33,6 +33,7 @@ import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.query.BrokerParallelMergeConfig; +import org.apache.druid.query.ConglomerateBackedQueryToolChestWarehouse; import org.apache.druid.query.DataSource; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate; @@ -45,8 +46,6 @@ import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QueryRunnerTestHelper; import org.apache.druid.query.QuerySegmentWalker; -import org.apache.druid.query.QueryToolChest; -import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.RetryQueryRunnerConfig; import org.apache.druid.query.TestBufferPool; import org.apache.druid.query.expression.LookupEnabledTestExprMacroTable; @@ -137,14 +136,7 @@ public static ClientQuerySegmentWalker createClientQuerySegmentWalker( emitter, clusterWalker, localWalker, - new QueryToolChestWarehouse() - { - @Override - public > QueryToolChest getToolChest(final QueryType query) - { - return conglomerate.findFactory(query).getToolchest(); - } - }, + new ConglomerateBackedQueryToolChestWarehouse(conglomerate), joinableFactory, new RetryQueryRunnerConfig(), injector.getInstance(ObjectMapper.class), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index 5463f5623196..d46067639479 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -28,14 +28,12 @@ import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.java.util.emitter.core.NoopEmitter; import org.apache.druid.java.util.emitter.service.ServiceEmitter; +import org.apache.druid.query.ConglomerateBackedQueryToolChestWarehouse; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; import org.apache.druid.query.GlobalTableDataSource; -import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; -import org.apache.druid.query.QueryToolChest; -import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.loading.SegmentCacheManager; @@ -90,14 +88,7 @@ public static QueryLifecycleFactory createMockQueryLifecycleFactory( ) { return new QueryLifecycleFactory( - new QueryToolChestWarehouse() - { - @Override - public > QueryToolChest getToolChest(final QueryType query) - { - return conglomerate.findFactory(query).getToolchest(); - } - }, + new ConglomerateBackedQueryToolChestWarehouse(conglomerate), walker, new DefaultGenericQueryMetricsFactory(), new ServiceEmitter("dummy", "dummy", new NoopEmitter()), From 870de0017741138327903f46346d36caf8fd5b76 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 08:43:03 +0000 Subject: [PATCH 035/149] extract --- .../org/apache/druid/server/QueryStackTests.java | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index 312baa8025b1..0db83f78dfc7 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -311,6 +311,22 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat final TestGroupByBuffers groupByBuffers = closer.register(TestGroupByBuffers.createFromProcessingConfig(processingConfig)); + return createQueryRunnerFactoryConglomerate( + processingConfig, + minTopNThresholdSupplier, + jsonMapper, + testBufferPool, + groupByBuffers); + } + + public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( + final DruidProcessingConfig processingConfig, + final Supplier minTopNThresholdSupplier, + final ObjectMapper jsonMapper, + final TestBufferPool testBufferPool, + final TestGroupByBuffers groupByBuffers + ) + { final GroupByQueryRunnerFactory groupByQueryRunnerFactory = GroupByQueryRunnerTest.makeQueryRunnerFactory( jsonMapper, From e3465004489aae2ed05858ee61ab49c176061047 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 10:24:35 +0000 Subject: [PATCH 036/149] make mintopnthresholdsupplier history --- ...yRunnerBasedOnClusteredClientTestBase.java | 2 +- .../apache/druid/server/QueryStackTests.java | 27 +++++++++---------- .../sql/calcite/util/SqlTestFramework.java | 2 +- 3 files changed, 15 insertions(+), 16 deletions(-) diff --git a/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java b/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java index c79480d9ae7f..68da75759176 100644 --- a/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java +++ b/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java @@ -107,7 +107,7 @@ protected QueryRunnerBasedOnClusteredClientTestBase() { conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate( CLOSER, - () -> TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD + TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD ); toolChestWarehouse = new ConglomerateBackedQueryToolChestWarehouse(conglomerate); } diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index 0db83f78dfc7..09d61d7c189b 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -99,7 +99,6 @@ import java.util.Collections; import java.util.Map; import java.util.Set; -import java.util.function.Supplier; /** * Utilities for creating query-stack objects for tests. @@ -227,20 +226,20 @@ public int getNumMergeBuffers() */ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate(final Closer closer) { - return createQueryRunnerFactoryConglomerate(closer, () -> TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD); + return createQueryRunnerFactoryConglomerate(closer, TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD); } public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( final Closer closer, - final Supplier minTopNThresholdSupplier + final Integer minTopNThreshold ) { - return createQueryRunnerFactoryConglomerate(closer, minTopNThresholdSupplier, TestHelper.makeJsonMapper()); + return createQueryRunnerFactoryConglomerate(closer, minTopNThreshold, TestHelper.makeJsonMapper()); } public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( final Closer closer, - final Supplier minTopNThresholdSupplier, + final Integer minTopNThreshold, final ObjectMapper jsonMapper ) { @@ -249,7 +248,7 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat getProcessingConfig( DEFAULT_NUM_MERGE_BUFFERS ), - minTopNThresholdSupplier, + minTopNThreshold, jsonMapper ); } @@ -262,7 +261,7 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat return createQueryRunnerFactoryConglomerate( closer, processingConfig, - () -> TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD + TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD ); } @@ -275,7 +274,7 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat return createQueryRunnerFactoryConglomerate( closer, processingConfig, - () -> TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD, + TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD, jsonMapper ); } @@ -283,13 +282,13 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( final Closer closer, final DruidProcessingConfig processingConfig, - final Supplier minTopNThresholdSupplier + final Integer minTopNThreshold ) { return createQueryRunnerFactoryConglomerate( closer, processingConfig, - minTopNThresholdSupplier, + minTopNThreshold, TestHelper.makeJsonMapper() ); } @@ -298,7 +297,7 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( final Closer closer, final DruidProcessingConfig processingConfig, - final Supplier minTopNThresholdSupplier, + final Integer minTopNThreshold, final ObjectMapper jsonMapper ) { @@ -313,7 +312,7 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat return createQueryRunnerFactoryConglomerate( processingConfig, - minTopNThresholdSupplier, + minTopNThreshold, jsonMapper, testBufferPool, groupByBuffers); @@ -321,7 +320,7 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( final DruidProcessingConfig processingConfig, - final Supplier minTopNThresholdSupplier, + final Integer minTopNThreshold, final ObjectMapper jsonMapper, final TestBufferPool testBufferPool, final TestGroupByBuffers groupByBuffers @@ -373,7 +372,7 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat @Override public int getMinTopNThreshold() { - return minTopNThresholdSupplier.get(); + return minTopNThreshold; } }), QueryRunnerTestHelper.NOOP_QUERYWATCHER diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index d59078deff9d..41e5ece67dda 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -268,7 +268,7 @@ public QueryRunnerFactoryConglomerate createCongolmerate( if (builder.mergeBufferCount == 0) { return QueryStackTests.createQueryRunnerFactoryConglomerate( resourceCloser, - () -> builder.minTopNThreshold, + builder.minTopNThreshold, jsonMapper ); } else { From 3644b160bbd87b3f64c87c2dd61ecf6bbb88c6ce Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 11:48:50 +0000 Subject: [PATCH 037/149] inline twice --- .../apache/druid/server/QueryStackTests.java | 28 ------------------- .../sql/calcite/util/SqlTestFramework.java | 1 + 2 files changed, 1 insertion(+), 28 deletions(-) diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index 09d61d7c189b..a558a8835e72 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -257,43 +257,15 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat final Closer closer, final DruidProcessingConfig processingConfig ) - { - return createQueryRunnerFactoryConglomerate( - closer, - processingConfig, - TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD - ); - } - - public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( - final Closer closer, - final DruidProcessingConfig processingConfig, - final ObjectMapper jsonMapper - ) { return createQueryRunnerFactoryConglomerate( closer, processingConfig, TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD, - jsonMapper - ); - } - - public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( - final Closer closer, - final DruidProcessingConfig processingConfig, - final Integer minTopNThreshold - ) - { - return createQueryRunnerFactoryConglomerate( - closer, - processingConfig, - minTopNThreshold, TestHelper.makeJsonMapper() ); } - public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( final Closer closer, final DruidProcessingConfig processingConfig, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 41e5ece67dda..692803737630 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -275,6 +275,7 @@ public QueryRunnerFactoryConglomerate createCongolmerate( return QueryStackTests.createQueryRunnerFactoryConglomerate( resourceCloser, QueryStackTests.getProcessingConfig(builder.mergeBufferCount), + TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD, jsonMapper ); } From b8138b8204690bc34c05b6ffe74b2fc57e569cf9 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 11:59:46 +0000 Subject: [PATCH 038/149] cleanup method; mergebuffercount==0 no more make minTopNThreshold irrelevant --- .../sql/calcite/SqlTestFrameworkConfig.java | 3 ++- .../sql/calcite/util/SqlTestFramework.java | 20 ++++++------------- 2 files changed, 8 insertions(+), 15 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java b/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java index 1fc39d52ec51..73ad030d9701 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java @@ -31,6 +31,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.quidem.DruidAvaticaTestDriver; +import org.apache.druid.server.QueryStackTests; import org.apache.druid.sql.calcite.util.CacheTestHelperModule.ResultCacheMode; import org.apache.druid.sql.calcite.util.SqlTestFramework; import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplier; @@ -94,7 +95,7 @@ public class SqlTestFrameworkConfig { @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.METHOD, ElementType.TYPE}) - @NumMergeBuffers(0) + @NumMergeBuffers(QueryStackTests.DEFAULT_NUM_MERGE_BUFFERS) public @interface NumMergeBuffers { ConfigOptionProcessor PROCESSOR = new ConfigOptionProcessor(NumMergeBuffers.class) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 692803737630..f9cfae54005a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -265,20 +265,12 @@ public QueryRunnerFactoryConglomerate createCongolmerate( ObjectMapper jsonMapper ) { - if (builder.mergeBufferCount == 0) { - return QueryStackTests.createQueryRunnerFactoryConglomerate( - resourceCloser, - builder.minTopNThreshold, - jsonMapper - ); - } else { - return QueryStackTests.createQueryRunnerFactoryConglomerate( - resourceCloser, - QueryStackTests.getProcessingConfig(builder.mergeBufferCount), - TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD, - jsonMapper - ); - } + return QueryStackTests.createQueryRunnerFactoryConglomerate( + resourceCloser, + QueryStackTests.getProcessingConfig(builder.mergeBufferCount), + builder.minTopNThreshold, + jsonMapper + ); } @Override From 763e1a0e074f1d9bf20fb499c3f483cae90c7343 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 12:02:06 +0000 Subject: [PATCH 039/149] one less method --- .../java/org/apache/druid/server/QueryStackTests.java | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index a558a8835e72..fc45f119a539 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -233,15 +233,6 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat final Closer closer, final Integer minTopNThreshold ) - { - return createQueryRunnerFactoryConglomerate(closer, minTopNThreshold, TestHelper.makeJsonMapper()); - } - - public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( - final Closer closer, - final Integer minTopNThreshold, - final ObjectMapper jsonMapper - ) { return createQueryRunnerFactoryConglomerate( closer, @@ -249,7 +240,7 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat DEFAULT_NUM_MERGE_BUFFERS ), minTopNThreshold, - jsonMapper + TestHelper.makeJsonMapper() ); } From fc8a1f1675196792c2b988c6c68654756a0b0fa4 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 12:11:02 +0000 Subject: [PATCH 040/149] use non-closer related method --- .../apache/druid/server/QueryStackTests.java | 24 ++++++++++++++----- .../sql/calcite/util/SqlTestFramework.java | 15 +++++++++--- 2 files changed, 30 insertions(+), 9 deletions(-) diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index fc45f119a539..fb964af16f2d 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -257,21 +257,32 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat ); } - public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( - final Closer closer, - final DruidProcessingConfig processingConfig, - final Integer minTopNThreshold, - final ObjectMapper jsonMapper - ) + public static TestBufferPool makeTestBufferPool(final Closer closer) { final TestBufferPool testBufferPool = TestBufferPool.offHeap(COMPUTE_BUFFER_SIZE, Integer.MAX_VALUE); closer.register(() -> { // Verify that all objects have been returned to the pool. Assert.assertEquals(0, testBufferPool.getOutstandingObjectCount()); }); + return testBufferPool; + } + public static TestGroupByBuffers makeGroupByBuffers(final Closer closer, final DruidProcessingConfig processingConfig) + { final TestGroupByBuffers groupByBuffers = closer.register(TestGroupByBuffers.createFromProcessingConfig(processingConfig)); + return groupByBuffers; + } + + public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( + final Closer closer, + final DruidProcessingConfig processingConfig, + final Integer minTopNThreshold, + final ObjectMapper jsonMapper + ) + { + final TestBufferPool testBufferPool = makeTestBufferPool(closer); + final TestGroupByBuffers groupByBuffers = makeGroupByBuffers(closer, processingConfig); return createQueryRunnerFactoryConglomerate( processingConfig, @@ -281,6 +292,7 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat groupByBuffers); } + public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( final DruidProcessingConfig processingConfig, final Integer minTopNThreshold, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index f9cfae54005a..624e10e3fbd7 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -39,9 +39,12 @@ import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.DruidProcessingConfig; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.TestBufferPool; +import org.apache.druid.query.groupby.TestGroupByBuffers; import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.quidem.TestSqlModule; @@ -265,11 +268,17 @@ public QueryRunnerFactoryConglomerate createCongolmerate( ObjectMapper jsonMapper ) { + DruidProcessingConfig processingConfig = QueryStackTests.getProcessingConfig(builder.mergeBufferCount); + Closer closer = resourceCloser; + final TestBufferPool testBufferPool = QueryStackTests.makeTestBufferPool(closer); + final TestGroupByBuffers groupByBuffers = QueryStackTests.makeGroupByBuffers(closer, processingConfig); + return QueryStackTests.createQueryRunnerFactoryConglomerate( - resourceCloser, - QueryStackTests.getProcessingConfig(builder.mergeBufferCount), + processingConfig, builder.minTopNThreshold, - jsonMapper + jsonMapper, + testBufferPool, + groupByBuffers ); } From dcab034d20215fe4b8b212feb722e5105acabe54 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 12:18:12 +0000 Subject: [PATCH 041/149] push it out even more --- ...AsBrokerQueryComponentSupplierWrapper.java | 11 ++++++-- .../druid/quidem/DruidAvaticaTestDriver.java | 10 +++++-- .../sql/calcite/util/SqlTestFramework.java | 28 +++++++++++++------ 3 files changed, 35 insertions(+), 14 deletions(-) diff --git a/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java b/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java index 74149cefcb25..62f9e6109fe4 100644 --- a/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java +++ b/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java @@ -75,8 +75,11 @@ import org.apache.druid.initialization.TombstoneDataStorageModule; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.metadata.storage.derby.DerbyMetadataStorageDruidModule; +import org.apache.druid.query.DruidProcessingConfig; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.RetryQueryRunnerConfig; +import org.apache.druid.query.TestBufferPool; +import org.apache.druid.query.groupby.TestGroupByBuffers; import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.rpc.guice.ServiceClientModule; import org.apache.druid.segment.join.JoinableFactoryWrapper; @@ -155,11 +158,13 @@ public void configureGuice(CoreInjectorBuilder builder, List overrideMod } @Override - public QueryRunnerFactoryConglomerate createCongolmerate(Builder builder, Closer closer, ObjectMapper om) + public QueryRunnerFactoryConglomerate createCongolmerate(Builder builder, Closer resourceCloser, + ObjectMapper jsonMapper, TestBufferPool testBufferPool, TestGroupByBuffers groupByBuffers, + DruidProcessingConfig processingConfig) { - return delegate.createCongolmerate(builder, closer, om); + return delegate + .createCongolmerate(builder, resourceCloser, jsonMapper, testBufferPool, groupByBuffers, processingConfig); } - @Override public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(QueryRunnerFactoryConglomerate conglomerate, JoinableFactoryWrapper joinableFactory, Injector injector) diff --git a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java index debd835fc785..149ab47ff22e 100644 --- a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java +++ b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java @@ -32,7 +32,10 @@ import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.io.Closer; +import org.apache.druid.query.DruidProcessingConfig; import org.apache.druid.query.QueryRunnerFactoryConglomerate; +import org.apache.druid.query.TestBufferPool; +import org.apache.druid.query.groupby.TestGroupByBuffers; import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.server.DruidNode; @@ -238,9 +241,12 @@ public void configureGuice(DruidInjectorBuilder builder) } @Override - public QueryRunnerFactoryConglomerate createCongolmerate(Builder builder, Closer closer, ObjectMapper jsonMapper) + public QueryRunnerFactoryConglomerate createCongolmerate(Builder builder, Closer resourceCloser, + ObjectMapper jsonMapper, TestBufferPool testBufferPool, TestGroupByBuffers groupByBuffers, + DruidProcessingConfig processingConfig) { - return delegate.createCongolmerate(builder, closer, jsonMapper); + return delegate + .createCongolmerate(builder, resourceCloser, jsonMapper, testBufferPool, groupByBuffers, processingConfig); } @Override diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 624e10e3fbd7..17ed95858bf4 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -149,6 +149,7 @@ public interface QueryComponentSupplier extends Closeable */ void gatherProperties(Properties properties); + /** * Configure modules needed for tests. This is the preferred way to configure * Jackson: include the production module in this method that includes the @@ -158,8 +159,11 @@ public interface QueryComponentSupplier extends Closeable QueryRunnerFactoryConglomerate createCongolmerate( Builder builder, - Closer closer, - ObjectMapper jsonMapper + Closer resourceCloser, + ObjectMapper jsonMapper, + TestBufferPool testBufferPool, + TestGroupByBuffers groupByBuffers, + DruidProcessingConfig processingConfig ); SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker( @@ -265,14 +269,12 @@ public void configureGuice(DruidInjectorBuilder builder) public QueryRunnerFactoryConglomerate createCongolmerate( Builder builder, Closer resourceCloser, - ObjectMapper jsonMapper + ObjectMapper jsonMapper, + final TestBufferPool testBufferPool, + final TestGroupByBuffers groupByBuffers, + DruidProcessingConfig processingConfig ) { - DruidProcessingConfig processingConfig = QueryStackTests.getProcessingConfig(builder.mergeBufferCount); - Closer closer = resourceCloser; - final TestBufferPool testBufferPool = QueryStackTests.makeTestBufferPool(closer); - final TestGroupByBuffers groupByBuffers = QueryStackTests.makeGroupByBuffers(closer, processingConfig); - return QueryStackTests.createQueryRunnerFactoryConglomerate( processingConfig, builder.minTopNThreshold, @@ -598,7 +600,15 @@ public void configure(Binder binder) @LazySingleton public QueryRunnerFactoryConglomerate conglomerate() { - return componentSupplier.createCongolmerate(builder, resourceCloser, queryJsonMapper()); + DruidProcessingConfig processingConfig = QueryStackTests.getProcessingConfig(builder.mergeBufferCount); + Closer closer = resourceCloser; + final TestBufferPool testBufferPool = QueryStackTests.makeTestBufferPool(closer); + final TestGroupByBuffers groupByBuffers = QueryStackTests.makeGroupByBuffers(closer, processingConfig); + + return componentSupplier.createCongolmerate( + builder, resourceCloser, queryJsonMapper(), + testBufferPool, groupByBuffers, processingConfig + ); } @Provides From 98034b1766c004199a54a052c4ac5171a2f6e448 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 12:22:27 +0000 Subject: [PATCH 042/149] make testbuffersomethiung arrive via injector --- .../sql/calcite/util/SqlTestFramework.java | 35 ++++++++++++++++--- 1 file changed, 31 insertions(+), 4 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 17ed95858bf4..41e263bbf673 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -598,16 +598,43 @@ public void configure(Binder binder) @Provides @LazySingleton - public QueryRunnerFactoryConglomerate conglomerate() + public DruidProcessingConfig makeProcessingConfig() + { + return QueryStackTests.getProcessingConfig(builder.mergeBufferCount); + } + + @Provides + @LazySingleton + public TestBufferPool makeProcessingConfig1() { - DruidProcessingConfig processingConfig = QueryStackTests.getProcessingConfig(builder.mergeBufferCount); Closer closer = resourceCloser; final TestBufferPool testBufferPool = QueryStackTests.makeTestBufferPool(closer); + return testBufferPool; + } + + @Provides + @LazySingleton + public TestGroupByBuffers makeProcessingConfig1(DruidProcessingConfig processingConfig) + { + Closer closer = resourceCloser; final TestGroupByBuffers groupByBuffers = QueryStackTests.makeGroupByBuffers(closer, processingConfig); + return groupByBuffers; + } + @Provides + @LazySingleton + public QueryRunnerFactoryConglomerate conglomerate( + DruidProcessingConfig processingConfig, + TestBufferPool testBufferPool, + TestGroupByBuffers groupByBuffers) + { return componentSupplier.createCongolmerate( - builder, resourceCloser, queryJsonMapper(), - testBufferPool, groupByBuffers, processingConfig + builder, + resourceCloser, + queryJsonMapper(), + testBufferPool, + groupByBuffers, + processingConfig ); } From 7fc3331ce023f4bd25e56d18563a664d765c7fe4 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 12:24:00 +0000 Subject: [PATCH 043/149] inline/etc --- .../druid/sql/calcite/util/SqlTestFramework.java | 12 ++++-------- 1 file changed, 4 insertions(+), 8 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 41e263bbf673..71e163ea8c0e 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -605,20 +605,16 @@ public DruidProcessingConfig makeProcessingConfig() @Provides @LazySingleton - public TestBufferPool makeProcessingConfig1() + public TestBufferPool makeTestBufferPool() { - Closer closer = resourceCloser; - final TestBufferPool testBufferPool = QueryStackTests.makeTestBufferPool(closer); - return testBufferPool; + return QueryStackTests.makeTestBufferPool(resourceCloser); } @Provides @LazySingleton - public TestGroupByBuffers makeProcessingConfig1(DruidProcessingConfig processingConfig) + public TestGroupByBuffers makeTestGroupByBuffers(DruidProcessingConfig processingConfig) { - Closer closer = resourceCloser; - final TestGroupByBuffers groupByBuffers = QueryStackTests.makeGroupByBuffers(closer, processingConfig); - return groupByBuffers; + return QueryStackTests.makeGroupByBuffers(resourceCloser, processingConfig); } @Provides From 8fe033127f6e1640812eb8ee863c22f5b4cdf068 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 13:08:47 +0000 Subject: [PATCH 044/149] compile fix --- .../druid/messages/client/MessageRelayClientImpl.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java b/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java index 140bd45e1af4..b2de6581a720 100644 --- a/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java +++ b/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java @@ -26,6 +26,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.java.util.http.client.response.BytesFullResponseHandler; +import org.apache.druid.java.util.http.client.response.BytesFullResponseHolder; import org.apache.druid.messages.MessageBatch; import org.apache.druid.rpc.RequestBuilder; import org.apache.druid.rpc.ServiceClient; @@ -68,11 +69,12 @@ public ListenableFuture> getMessages( startWatermark ); + ListenableFuture asyncRequest = (ListenableFuture) serviceClient.asyncRequest( + new RequestBuilder(HttpMethod.GET, path), + new BytesFullResponseHandler() + ); return FutureUtils.transform( - serviceClient.asyncRequest( - new RequestBuilder(HttpMethod.GET, path), - new BytesFullResponseHandler() - ), + asyncRequest, holder -> { if (holder.getResponse().getStatus().getCode() == HttpStatus.NO_CONTENT_204) { return new MessageBatch<>(Collections.emptyList(), epoch, startWatermark); From 8628e6d8378ad54526f2f68acafdd8cde37d6945 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 13:43:35 +0000 Subject: [PATCH 045/149] Revert "defaultconglomerate.of" This reverts commit 4dd587e4da65badc8576e6b52234f82a00fec18e. --- .../CachingClusteredClientBenchmark.java | 2 +- .../indexing/kafka/KafkaIndexTaskTest.java | 2 +- .../kinesis/KinesisIndexTaskTest.java | 2 +- ...DefaultQueryRunnerFactoryConglomerate.java | 14 +- .../druid/guice/QueryRunnerFactoryModule.java | 1 + .../StreamAppenderatorTester.java | 4 +- ...nifiedIndexerAppenderatorsManagerTest.java | 2 +- .../apache/druid/server/QueryStackTests.java | 2 +- ...estUnionAllTwoQueriesBothQueriesAreJoin.iq | 132 ++++++++++++++++++ 9 files changed, 143 insertions(+), 18 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/CachingClusteredClientBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/CachingClusteredClientBenchmark.java index 6647be2e42cb..8e0715e0fe5c 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/CachingClusteredClientBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/CachingClusteredClientBenchmark.java @@ -258,7 +258,7 @@ public int getNumThreads() } }; - conglomerate = DefaultQueryRunnerFactoryConglomerate.of( + conglomerate = new DefaultQueryRunnerFactoryConglomerate( ImmutableMap., QueryRunnerFactory>builder() .put( TimeseriesQuery.class, diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java index 6b955660e2cf..23bdeb14acb8 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java @@ -2891,7 +2891,7 @@ private static DataSchema cloneDataSchema(final DataSchema dataSchema) @Override protected QueryRunnerFactoryConglomerate makeQueryRunnerConglomerate() { - return DefaultQueryRunnerFactoryConglomerate.of( + return new DefaultQueryRunnerFactoryConglomerate( ImmutableMap., QueryRunnerFactory>builder() .put( TimeseriesQuery.class, diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java index d4cdfacb98ca..2ef391484008 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java @@ -2418,7 +2418,7 @@ private static DataSchema cloneDataSchema(final DataSchema dataSchema) @Override protected QueryRunnerFactoryConglomerate makeQueryRunnerConglomerate() { - return DefaultQueryRunnerFactoryConglomerate.of( + return new DefaultQueryRunnerFactoryConglomerate( ImmutableMap.of( TimeseriesQuery.class, new TimeseriesQueryRunnerFactory( diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java index d3049de767d5..7f004ffa6343 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java @@ -28,10 +28,10 @@ */ public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactoryConglomerate { - private Map, QueryRunnerFactory> factories; + private final Map, QueryRunnerFactory> factories; @Inject - public void setFactories(Map, QueryRunnerFactory> factories) + public DefaultQueryRunnerFactoryConglomerate(Map, QueryRunnerFactory> factories) { // Accesses to IdentityHashMap should be faster than to HashMap or ImmutableMap. // Class doesn't override Object.equals(). @@ -42,14 +42,6 @@ public void setFactories(Map, QueryRunnerFactory> factori @SuppressWarnings("unchecked") public > QueryRunnerFactory findFactory(QueryType query) { - return factories.get(query.getClass()); - } - - - public static DefaultQueryRunnerFactoryConglomerate of(Map, QueryRunnerFactory> factories) - { - DefaultQueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate(); - conglomerate.setFactories(factories); - return conglomerate; + return (QueryRunnerFactory) factories.get(query.getClass()); } } diff --git a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java index 00cefa261d17..f71228a3fbee 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java @@ -50,6 +50,7 @@ import org.apache.druid.query.union.UnionQuery; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QuerySchedulerProvider; + import java.util.Map; /** diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java index db1949b58273..29d758aaed02 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java @@ -224,7 +224,7 @@ public Map makeLoadSpec(URI uri) objectMapper, indexIO, indexMerger, - DefaultQueryRunnerFactoryConglomerate.of( + new DefaultQueryRunnerFactoryConglomerate( ImmutableMap.of( TimeseriesQuery.class, new TimeseriesQueryRunnerFactory( new TimeseriesQueryQueryToolChest(), @@ -268,7 +268,7 @@ public int getDropSegmentDelayMillis() objectMapper, indexIO, indexMerger, - DefaultQueryRunnerFactoryConglomerate.of( + new DefaultQueryRunnerFactoryConglomerate( ImmutableMap.of( TimeseriesQuery.class, new TimeseriesQueryRunnerFactory( new TimeseriesQueryQueryToolChest(), diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java index ae87b4779f89..21f627baa085 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java @@ -83,7 +83,7 @@ public class UnifiedIndexerAppenderatorsManagerTest extends InitializedNullHandl new CachePopulatorStats(), TestHelper.makeJsonMapper(), new NoopServiceEmitter(), - () -> DefaultQueryRunnerFactoryConglomerate.of(ImmutableMap.of()) + () -> new DefaultQueryRunnerFactoryConglomerate(ImmutableMap.of()) ); private AppenderatorConfig appenderatorConfig; diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index 9e16018f2c53..cb4667130849 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -331,7 +331,7 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat processingConfig ); - final QueryRunnerFactoryConglomerate conglomerate = DefaultQueryRunnerFactoryConglomerate.of( + final QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate( ImmutableMap., QueryRunnerFactory>builder() .put( SegmentMetadataQuery.class, diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq index 08b51819b8b1..65b1fac2fad7 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq @@ -9,7 +9,139 @@ !set outputformat mysql !use druidtest:/// (SELECT COUNT(*) FROM foo LEFT JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) + !ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[left]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalJoin(condition=[=($0, $1)], joinType=[inner]) + LogicalProject(dim1=[$1]) + LogicalTableScan(table=[[druid, foo]]) + LogicalProject(k=[$0]) + LogicalTableScan(table=[[lookup, lookyloo]]) + !logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[left]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidJoin(condition=[=($0, $1)], joinType=[inner]) + DruidProject(dim1=[$1], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidProject(k=[$0], druid=[logical]) + DruidTableScan(table=[[lookup, lookyloo]], druid=[logical]) + !druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "LEFT" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "join", + "left" : { + "type" : "table", + "name" : "foo" + }, + "right" : { + "type" : "lookup", + "lookup" : "lookyloo" + }, + "rightPrefix" : "j0.", + "condition" : "(\"dim1\" == \"j0.k\")", + "joinType" : "INNER" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} !nativePlan From bc4c881863a4f451d3a004774c871db3a78afff2 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 15:27:37 +0000 Subject: [PATCH 046/149] rename/etc --- .../query/union/RealUnionQueryRunner2.java | 82 ++++++++++++++++ .../union/RealUnionQueryRunnerFactory.java | 4 +- .../druid/query/union/RealUnionResult.java | 2 +- .../apache/druid/query/union/UnionQuery.java | 17 ++-- ...est.java => UnionQueryQueryToolChest.java} | 93 +------------------ .../druid/guice/QueryToolChestModule.java | 4 +- .../server/ClientQuerySegmentWalker.java | 16 +++- 7 files changed, 108 insertions(+), 110 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner2.java rename processing/src/main/java/org/apache/druid/query/union/{RealUnionQueryQueryToolChest.java => UnionQueryQueryToolChest.java} (72%) diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner2.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner2.java new file mode 100644 index 000000000000..1868c1d36d2d --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner2.java @@ -0,0 +1,82 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.union; + +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.query.Query; +import org.apache.druid.query.QueryPlus; +import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.QueryToolChestWarehouse; +import org.apache.druid.query.context.ResponseContext; + +import java.util.ArrayList; +import java.util.List; + +class RealUnionQueryRunner2 implements QueryRunner +{ + + private QueryToolChestWarehouse warehouse; + private QuerySegmentWalker walker; + private UnionQuery query; + private List runners; + + public RealUnionQueryRunner2(QueryToolChestWarehouse warehouse, UnionQuery query, + QuerySegmentWalker walker) + { + this.warehouse = warehouse; + this.query = query; + this.walker = walker; + + this.runners = makeSubQueryRunners(query); + } + + private List makeSubQueryRunners(UnionQuery unionQuery) + { + List runners = new ArrayList<>(); + for (Query query : unionQuery.queries) { + runners.add(query.getRunner(walker)); + } + return runners; + + } + + @Override + public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) + { + UnionQuery unionQuery = queryPlus.unwrapQuery(UnionQuery.class); + + List seqs = new ArrayList(); + for (int i = 0; i < runners.size(); i++) { + Query q = unionQuery.queries.get(i); + QueryRunner r = runners.get(i); + seqs.add(makeUnionResult(r, queryPlus.withQuery(q), responseContext)); + } + return Sequences.simple(seqs); + } + + private RealUnionResult makeUnionResult(QueryRunner runner, QueryPlus withQuery, + ResponseContext responseContext) + { + Sequence seq = runner.run(withQuery, responseContext); + return new RealUnionResult(seq); + } +} \ No newline at end of file diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java index 3aae4fff541d..281c29bf9eb0 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java @@ -28,7 +28,7 @@ public class RealUnionQueryRunnerFactory implements QueryRunnerFactory { - private static RealUnionQueryQueryToolChest tt; + private static UnionQueryQueryToolChest tt; public RealUnionQueryRunnerFactory(String string) { @@ -62,7 +62,7 @@ public QueryRunner mergeRunners(QueryProcessingPool queryProces public QueryToolChest getToolchest() { if (tt == null) { - tt = new RealUnionQueryQueryToolChest(); + tt = new UnionQueryQueryToolChest(); } return tt; } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java index 3bac27e122e5..23ec20039d69 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java +++ b/processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java @@ -29,7 +29,7 @@ */ public class RealUnionResult { - private Sequence seq; + private final Sequence seq; public RealUnionResult(Sequence seq) { diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index 63af5f75e9dd..10705f1cd9ec 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -74,7 +74,7 @@ public UnionQuery( @Override public DataSource getDataSource() { - throw new RuntimeException("This is not supported"); + throw DruidException.defensive("This is not supported"); } @Override @@ -110,31 +110,31 @@ public String getType() @Override public QueryRunner getRunner(QuerySegmentWalker walker) { - return new RealUnionQueryRunner(walker); + throw DruidException.defensive("This is not supported"); } @Override public List getIntervals() { - throw new RuntimeException("FIXME: Unimplemented!"); + throw DruidException.defensive("This is not supported"); } @Override public Duration getDuration() { - throw new RuntimeException("FIXME: Unimplemented!"); + throw DruidException.defensive("This is not supported"); } @Override public Granularity getGranularity() { - throw new RuntimeException("FIXME: Unimplemented!"); + throw DruidException.defensive("This is not supported"); } @Override public DateTimeZone getTimezone() { - throw new RuntimeException("FIXME: Unimplemented!"); + throw DruidException.defensive("This is not supported"); } @Override @@ -146,10 +146,7 @@ public Map getContext() @Override public Ordering getResultOrdering() { - if (true) { - throw new RuntimeException("FIXME: Unimplemented!"); - } - return null; + throw new RuntimeException("FIXME: Unimplemented!"); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java similarity index 72% rename from processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java rename to processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 57ecccf387c8..be3c76b11717 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -46,7 +46,7 @@ import java.util.List; import java.util.Optional; -public class RealUnionQueryQueryToolChest extends QueryToolChest +public class UnionQueryQueryToolChest extends QueryToolChest implements QueryExecSomething { @@ -58,97 +58,6 @@ public QueryRunner executeQuery2(QueryToolChestWarehouse wareho return runner; } - public Optional> executeQuery1(QueryToolChestWarehouse warehouse, - Query query, QuerySegmentWalker clientQuerySegmentWalker) - { - UnionQuery unionQuery = (UnionQuery) query; - List queryRunners = new ArrayList<>(); - for (Query q : unionQuery.queries) { - QueryRunner subRunner = clientQuerySegmentWalker.executeQuery(q); - queryRunners.add(subRunner); - } - QueryRunner unionRunner = new LocalRealUnionQueryRunner( - queryRunners - ); - return Optional.of(unionRunner); - } - - private static class RealUnionQueryRunner2 implements QueryRunner - { - - private QueryToolChestWarehouse warehouse; - private QuerySegmentWalker walker; - private UnionQuery query; - private List runners; - - public RealUnionQueryRunner2(QueryToolChestWarehouse warehouse, UnionQuery query, - QuerySegmentWalker walker) - { - this.warehouse = warehouse; - this.query = query; - this.walker = walker; - - this.runners = makeSubQueryRunners(query); - } - - private List makeSubQueryRunners(UnionQuery unionQuery) - { - List runners = new ArrayList<>(); - for (Query query : unionQuery.queries) { - runners.add(query.getRunner(walker)); - } - return runners; - - } - - @Override - public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) - { - UnionQuery unionQuery = queryPlus.unwrapQuery(UnionQuery.class); - - List seqs = new ArrayList(); - for (int i = 0; i < runners.size(); i++) { - Query q = unionQuery.queries.get(i); - QueryRunner r = runners.get(i); - seqs.add(makeUnionResult(r, queryPlus.withQuery(q), responseContext)); - } - return Sequences.simple(seqs); - } - - private RealUnionResult makeUnionResult(QueryRunner runner, QueryPlus withQuery, - ResponseContext responseContext) - { - Sequence seq = runner.run(withQuery, responseContext); - return new RealUnionResult(seq); - } - } - - private static class LocalRealUnionQueryRunner implements QueryRunner - { - - public LocalRealUnionQueryRunner(List queryRunners) - { - - } - - @Override - public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) - { - return buildSequence(); - } - - Sequence buildSequence() - { - return null; - } - - } - - public void RealUnionQueryQueryToolChest() - { - int asd = 1; - } - @Override @SuppressWarnings("unchecked") public QueryRunner mergeResults(QueryRunner runner) diff --git a/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java b/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java index 0e0750a45dd9..3094d60ffc93 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java @@ -62,7 +62,7 @@ import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.query.topn.TopNQueryMetricsFactory; import org.apache.druid.query.topn.TopNQueryQueryToolChest; -import org.apache.druid.query.union.RealUnionQueryQueryToolChest; +import org.apache.druid.query.union.UnionQueryQueryToolChest; import org.apache.druid.query.union.UnionQuery; import java.util.Map; @@ -88,7 +88,7 @@ public class QueryToolChestModule implements Module .put(TimeseriesQuery.class, TimeseriesQueryQueryToolChest.class) .put(TopNQuery.class, TopNQueryQueryToolChest.class) .put(WindowOperatorQuery.class, WindowOperatorQueryQueryToolChest.class) - .put(UnionQuery.class, RealUnionQueryQueryToolChest.class) + .put(UnionQuery.class, UnionQueryQueryToolChest.class) .build(); @Override diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index ee7398bf2c23..0aca0d47b1ac 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -451,9 +451,19 @@ private DataSource inlineIfNecessary( .toString() ) ); - warehouse.getToolChest(subQuery).setWarehouse(warehouse); - queryResults = subQueryWithSerialization - .getRunner(this) + QueryToolChest subQueryToolChest = warehouse.getToolChest(subQuery); + subQueryToolChest.setWarehouse(warehouse); + +// final QueryRunner subQueryRunner ; + final QueryRunner subQueryRunner; + if (subQueryToolChest instanceof QueryExecSomething) { + subQueryRunner = ((QueryExecSomething) subQueryToolChest) + .executeQuery2(warehouse, subQueryWithSerialization, this); + }else { + subQueryRunner = subQueryWithSerialization.getRunner(this); + } + + queryResults = subQueryRunner .run(QueryPlus.wrap(subQueryWithSerialization), DirectDruidClient.makeResponseContextForQuery()); } From b3c8ceb95ffdd84e77d95cc6a175bf52cd76d142 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 15:28:39 +0000 Subject: [PATCH 047/149] rename/etc --- .../src/main/java/org/apache/druid/query/union/UnionQuery.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index 10705f1cd9ec..0fe5dc316c5f 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -110,7 +110,7 @@ public String getType() @Override public QueryRunner getRunner(QuerySegmentWalker walker) { - throw DruidException.defensive("This is not supported"); + throw DruidException.defensive("Use QueryToolChest to get a Runner"); } @Override From 02b126c390d5d917021d456eff0f633968e61b5a Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 15:39:59 +0000 Subject: [PATCH 048/149] cleanup --- .../java/org/apache/druid/query/union/UnionQuery.java | 10 ++++++---- 1 file changed, 6 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index 0fe5dc316c5f..f1f2bb03e3d2 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -25,6 +25,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; @@ -42,6 +43,7 @@ import org.joda.time.Interval; import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; @@ -56,9 +58,9 @@ public class UnionQuery implements Query @JsonProperty("queries") protected final List> queries; - public UnionQuery(List> queries2) + public UnionQuery(List> queries) { - this(queries2, queries2.get(0).getContext()); + this(queries, queries.get(0).getContext()); } @JsonCreator @@ -116,7 +118,7 @@ public QueryRunner getRunner(QuerySegmentWalker walker) @Override public List getIntervals() { - throw DruidException.defensive("This is not supported"); + return Collections.emptyList(); } @Override @@ -128,7 +130,7 @@ public Duration getDuration() @Override public Granularity getGranularity() { - throw DruidException.defensive("This is not supported"); + return Granularities.ALL; } @Override From 451d29de251ddf44f1c97b698a2539e106325a75 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 15:56:04 +0000 Subject: [PATCH 049/149] split basequery in two --- .../org/apache/druid/query/BaseQuery.java | 65 ++++------ .../druid/query/SingleDataSourceQuery.java | 116 ++++++++++++++++++ .../DataSourceMetadataQuery.java | 4 +- .../druid/query/groupby/GroupByQuery.java | 4 +- .../metadata/SegmentMetadataQuery.java | 13 +- .../query/operator/WindowOperatorQuery.java | 5 +- .../apache/druid/query/scan/ScanQuery.java | 5 +- .../druid/query/search/SearchQuery.java | 4 +- .../query/timeboundary/TimeBoundaryQuery.java | 4 +- .../query/timeseries/TimeseriesQuery.java | 5 +- .../apache/druid/query/topn/TopNQuery.java | 5 +- ...TimelineMissingSegmentQueryRunnerTest.java | 2 +- .../org/apache/druid/query/TestQuery.java | 3 +- .../CachingClusteredClientPerfTest.java | 3 +- .../coordination/ServerManagerTest.java | 4 +- .../server/log/LoggingRequestLoggerTest.java | 4 +- 16 files changed, 169 insertions(+), 77 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/SingleDataSourceQuery.java diff --git a/processing/src/main/java/org/apache/druid/query/BaseQuery.java b/processing/src/main/java/org/apache/druid/query/BaseQuery.java index bbfa570e3094..ce74d2918c38 100644 --- a/processing/src/main/java/org/apache/druid/query/BaseQuery.java +++ b/processing/src/main/java/org/apache/druid/query/BaseQuery.java @@ -56,47 +56,47 @@ public static void checkInterrupted() public static final String QUERY_ID = "queryId"; public static final String SUB_QUERY_ID = "subQueryId"; public static final String SQL_QUERY_ID = "sqlQueryId"; - private final DataSource dataSource; private final QueryContext context; private final QuerySegmentSpec querySegmentSpec; - private volatile Duration duration; + private final Duration duration; private final Granularity granularity; public BaseQuery( - DataSource dataSource, QuerySegmentSpec querySegmentSpec, Map context ) { - this(dataSource, querySegmentSpec, context, Granularities.ALL); + this(querySegmentSpec, context, Granularities.ALL); } public BaseQuery( - DataSource dataSource, QuerySegmentSpec querySegmentSpec, Map context, Granularity granularity ) { - Preconditions.checkNotNull(dataSource, "dataSource can't be null"); Preconditions.checkNotNull(querySegmentSpec, "querySegmentSpec can't be null"); Preconditions.checkNotNull(granularity, "Must specify a granularity"); - this.dataSource = dataSource; this.context = QueryContext.of(context); this.querySegmentSpec = querySegmentSpec; this.granularity = granularity; + this.duration = computeDuration(); } - @JsonProperty - @Override - public DataSource getDataSource() + private final Duration computeDuration() { - return dataSource; + Duration totalDuration = new Duration(0); + for (Interval interval : querySegmentSpec.getIntervals()) { + if (interval != null) { + totalDuration = totalDuration.plus(interval.toDuration()); + } + } + return totalDuration; } @JsonProperty("intervals") - public QuerySegmentSpec getQuerySegmentSpec() + public final QuerySegmentSpec getQuerySegmentSpec() { return querySegmentSpec; } @@ -117,36 +117,26 @@ public static QuerySegmentSpec getQuerySegmentSpecForLookUp(BaseQuery query) } @Override - public List getIntervals() + public final List getIntervals() { return querySegmentSpec.getIntervals(); } @Override - public Duration getDuration() + public final Duration getDuration() { - if (duration == null) { - Duration totalDuration = new Duration(0); - for (Interval interval : querySegmentSpec.getIntervals()) { - if (interval != null) { - totalDuration = totalDuration.plus(interval.toDuration()); - } - } - duration = totalDuration; - } - return duration; } @Override @JsonProperty - public Granularity getGranularity() + public final Granularity getGranularity() { return granularity; } @Override - public DateTimeZone getTimezone() + public final DateTimeZone getTimezone() { return granularity instanceof PeriodGranularity ? ((PeriodGranularity) granularity).getTimeZone() @@ -156,27 +146,17 @@ public DateTimeZone getTimezone() @Override @JsonProperty @JsonInclude(Include.NON_DEFAULT) - public Map getContext() + public final Map getContext() { return context.asMap(); } @Override - public QueryContext context() + public final QueryContext context() { return context; } - /** - * @deprecated use {@link #computeOverriddenContext(Map, Map) computeOverriddenContext(getContext(), overrides))} - * instead. This method may be removed in the next minor or major version of Druid. - */ - @Deprecated - protected Map computeOverridenContext(final Map overrides) - { - return computeOverriddenContext(getContext(), overrides); - } - public static Map computeOverriddenContext( final Map context, final Map overrides @@ -240,18 +220,15 @@ public boolean equals(Object o) } BaseQuery baseQuery = (BaseQuery) o; - // Must use getDuration() instead of "duration" because duration is lazily computed. - return Objects.equals(dataSource, baseQuery.dataSource) && - Objects.equals(context, baseQuery.context) && + return Objects.equals(context, baseQuery.context) && Objects.equals(querySegmentSpec, baseQuery.querySegmentSpec) && - Objects.equals(getDuration(), baseQuery.getDuration()) && + Objects.equals(duration, baseQuery.getDuration()) && Objects.equals(granularity, baseQuery.granularity); } @Override public int hashCode() { - // Must use getDuration() instead of "duration" because duration is lazily computed. - return Objects.hash(dataSource, context, querySegmentSpec, getDuration(), granularity); + return Objects.hash(context, querySegmentSpec, duration, granularity); } } diff --git a/processing/src/main/java/org/apache/druid/query/SingleDataSourceQuery.java b/processing/src/main/java/org/apache/druid/query/SingleDataSourceQuery.java new file mode 100644 index 000000000000..838759424419 --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/SingleDataSourceQuery.java @@ -0,0 +1,116 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query; + +import com.fasterxml.jackson.annotation.JsonProperty; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Ordering; +import org.apache.druid.guice.annotations.ExtensionPoint; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.granularity.Granularity; +import org.apache.druid.query.spec.QuerySegmentSpec; +import java.util.Map; +import java.util.Objects; + +/** + * + */ +@ExtensionPoint +public abstract class SingleDataSourceQuery extends BaseQuery implements Query +{ + private final DataSource dataSource; + + public SingleDataSourceQuery( + DataSource dataSource, + QuerySegmentSpec querySegmentSpec, + Map context + ) + { + this(dataSource, querySegmentSpec, context, Granularities.ALL); + } + + public SingleDataSourceQuery( + DataSource dataSource, + QuerySegmentSpec querySegmentSpec, + Map context, + Granularity granularity + ) + { + super(querySegmentSpec, context, granularity); + + this.dataSource = dataSource; + } + + @JsonProperty + @Override + public DataSource getDataSource() + { + return dataSource; + } + + @Override + public QueryRunner getRunner(QuerySegmentWalker walker) + { + return getQuerySegmentSpecForLookUp(this).lookup(this, walker); + } + + @VisibleForTesting + public static QuerySegmentSpec getQuerySegmentSpecForLookUp(SingleDataSourceQuery query) + { + DataSource queryDataSource = query.getDataSource(); + return queryDataSource.getAnalysis() + .getBaseQuerySegmentSpec() + .orElseGet(query::getQuerySegmentSpec); + } + + /** + * Default implementation of {@link Query#getResultOrdering()} that uses {@link Ordering#natural()}. + * + * If your query result type T is not Comparable, you must override this method. + */ + @Override + @SuppressWarnings("unchecked") // assumes T is Comparable; see method javadoc + public Ordering getResultOrdering() + { + return (Ordering) Ordering.natural(); + } + + @Override + public boolean equals(Object o) + { + if (this == o) { + return true; + } + if (o == null || getClass() != o.getClass()) { + return false; + } + if(!super.equals(o)) { + return false; + } + SingleDataSourceQuery baseQuery = (SingleDataSourceQuery) o; + return Objects.equals(dataSource, baseQuery.dataSource); + } + + @Override + public int hashCode() + { + return Objects.hash(super.hashCode(), dataSource); + } +} diff --git a/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQuery.java b/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQuery.java index 5c43c6445bc9..e335e91e5aa4 100644 --- a/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQuery.java +++ b/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQuery.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.Query; @@ -40,7 +40,7 @@ /** */ -public class DataSourceMetadataQuery extends BaseQuery> +public class DataSourceMetadataQuery extends SingleDataSourceQuery> { private static final QuerySegmentSpec DEFAULT_SEGMENT_SPEC = new MultipleIntervalSegmentSpec(Intervals.ONLY_ETERNITY); diff --git a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java index 68c071060633..84011580d8ad 100644 --- a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java @@ -42,6 +42,7 @@ import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.DimensionComparisonUtils; import org.apache.druid.query.Queries; @@ -72,6 +73,7 @@ import org.joda.time.Interval; import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -85,7 +87,7 @@ /** * */ -public class GroupByQuery extends BaseQuery implements SupportRowSignature +public class GroupByQuery extends SingleDataSourceQuery implements SupportRowSignature { public static final String CTX_KEY_SORT_BY_DIMS_FIRST = "sortByDimsFirst"; public static final String CTX_TIMESTAMP_RESULT_FIELD = "timestampResultField"; diff --git a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java index f2d434bab8a5..1ee11bbdfd4b 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java @@ -26,7 +26,7 @@ import org.apache.druid.java.util.common.Cacheable; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.Query; @@ -36,14 +36,11 @@ import org.apache.druid.query.metadata.SegmentMetadataQueryConfig; import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; import org.apache.druid.query.spec.QuerySegmentSpec; -import org.joda.time.Interval; - import java.util.EnumSet; -import java.util.List; import java.util.Map; import java.util.Objects; -public class SegmentMetadataQuery extends BaseQuery +public class SegmentMetadataQuery extends SingleDataSourceQuery { private static final QuerySegmentSpec DEFAULT_SEGMENT_SPEC = new MultipleIntervalSegmentSpec(Intervals.ONLY_ETERNITY); @@ -239,12 +236,6 @@ public SegmentMetadataQuery withFinalizedAnalysisTypes(SegmentMetadataQueryConfi .build(); } - @Override - public List getIntervals() - { - return this.getQuerySegmentSpec().getIntervals(); - } - @Override public String toString() { diff --git a/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java index aed132b745ee..b782190e3236 100644 --- a/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java +++ b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.java.util.common.IAE; -import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.InlineDataSource; import org.apache.druid.query.Order; @@ -36,6 +36,7 @@ import org.apache.druid.segment.column.RowSignature; import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -52,7 +53,7 @@ * The assumptions on the incoming data are defined by the operators. At initial time of writing, there is a baked * in assumption that data has been sorted "correctly" before this runs. */ -public class WindowOperatorQuery extends BaseQuery +public class WindowOperatorQuery extends SingleDataSourceQuery { private static DataSource validateMaybeRewriteDataSource(DataSource dataSource, boolean hasLeafs) { diff --git a/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java b/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java index 037148f50c3a..4c579c194a05 100644 --- a/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java +++ b/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java @@ -31,7 +31,7 @@ import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.UOE; -import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.InlineDataSource; @@ -50,6 +50,7 @@ import org.apache.druid.segment.column.RowSignature.Builder; import javax.annotation.Nullable; + import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -57,7 +58,7 @@ import java.util.Objects; import java.util.Set; -public class ScanQuery extends BaseQuery +public class ScanQuery extends SingleDataSourceQuery { public enum ResultFormat { diff --git a/processing/src/main/java/org/apache/druid/query/search/SearchQuery.java b/processing/src/main/java/org/apache/druid/query/search/SearchQuery.java index aa10b6ab5790..fdb9f9c1135f 100644 --- a/processing/src/main/java/org/apache/druid/query/search/SearchQuery.java +++ b/processing/src/main/java/org/apache/druid/query/search/SearchQuery.java @@ -25,7 +25,7 @@ import com.google.common.base.Preconditions; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.Query; @@ -42,7 +42,7 @@ /** */ -public class SearchQuery extends BaseQuery> +public class SearchQuery extends SingleDataSourceQuery> { private static final SearchSortSpec DEFAULT_SORT_SPEC = new SearchSortSpec(StringComparators.LEXICOGRAPHIC); diff --git a/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java b/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java index 47412f7d7d50..919fbe0e25c2 100644 --- a/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java +++ b/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java @@ -26,7 +26,7 @@ import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.Query; @@ -44,7 +44,7 @@ /** */ -public class TimeBoundaryQuery extends BaseQuery> +public class TimeBoundaryQuery extends SingleDataSourceQuery> { private static final QuerySegmentSpec DEFAULT_SEGMENT_SPEC = new MultipleIntervalSegmentSpec(Intervals.ONLY_ETERNITY); public static final String MAX_TIME = "maxTime"; diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java index 1a6f8e669fd3..7a1f48033e85 100644 --- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java +++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java @@ -28,7 +28,7 @@ import com.google.common.collect.Ordering; import org.apache.commons.lang3.StringUtils; import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.PerSegmentQueryOptimizationContext; @@ -47,6 +47,7 @@ import org.apache.druid.segment.column.RowSignature.Finalization; import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -57,7 +58,7 @@ /** */ @JsonTypeName("timeseries") -public class TimeseriesQuery extends BaseQuery> implements SupportRowSignature +public class TimeseriesQuery extends SingleDataSourceQuery> implements SupportRowSignature { public static final String CTX_GRAND_TOTAL = "grandTotal"; public static final String SKIP_EMPTY_BUCKETS = "skipEmptyBuckets"; diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java b/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java index 3178ac15a183..e13670f8f892 100644 --- a/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java +++ b/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java @@ -25,7 +25,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.PerSegmentQueryOptimizationContext; import org.apache.druid.query.Queries; @@ -40,6 +40,7 @@ import org.apache.druid.segment.column.RowSignature; import javax.annotation.Nullable; + import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -49,7 +50,7 @@ /** */ -public class TopNQuery extends BaseQuery> +public class TopNQuery extends SingleDataSourceQuery> { public static final String TOPN = "topN"; diff --git a/processing/src/test/java/org/apache/druid/query/ReportTimelineMissingSegmentQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/ReportTimelineMissingSegmentQueryRunnerTest.java index 34cb3dbcdea5..d5f7678121b6 100644 --- a/processing/src/test/java/org/apache/druid/query/ReportTimelineMissingSegmentQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/ReportTimelineMissingSegmentQueryRunnerTest.java @@ -66,7 +66,7 @@ public void testRunWithMultipleSegments() Assert.assertEquals(missingSegments, responseContext.getMissingSegments()); } - private static class TestQuery extends BaseQuery + private static class TestQuery extends SingleDataSourceQuery { private TestQuery() { diff --git a/processing/src/test/java/org/apache/druid/query/TestQuery.java b/processing/src/test/java/org/apache/druid/query/TestQuery.java index 186b02444b52..e8acda35f8e9 100644 --- a/processing/src/test/java/org/apache/druid/query/TestQuery.java +++ b/processing/src/test/java/org/apache/druid/query/TestQuery.java @@ -23,10 +23,11 @@ import org.apache.druid.query.spec.QuerySegmentSpec; import javax.annotation.Nullable; + import java.util.Map; import java.util.Set; -public class TestQuery extends BaseQuery +public class TestQuery extends SingleDataSourceQuery { @Nullable private Set requiredColumns; diff --git a/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java b/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java index b31799cf8b4c..7d01ba7f6607 100644 --- a/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java +++ b/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java @@ -34,6 +34,7 @@ import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.TestSequence; import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.BrokerParallelMergeConfig; import org.apache.druid.query.DataSource; import org.apache.druid.query.Query; @@ -194,7 +195,7 @@ public Sequence run( } } - private static class TestQuery extends BaseQuery + private static class TestQuery extends SingleDataSourceQuery { private QuerySegmentSpec spec; diff --git a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java index 69a7c08f2be9..8bf8ac95f630 100644 --- a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java @@ -44,7 +44,7 @@ import org.apache.druid.java.util.common.guava.YieldingAccumulator; import org.apache.druid.java.util.common.guava.YieldingSequenceBase; import org.apache.druid.java.util.emitter.EmittingLogger; -import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.ConcatQueryRunner; import org.apache.druid.query.DataSource; import org.apache.druid.query.DefaultQueryMetrics; @@ -594,7 +594,7 @@ public void testGetQueryRunnerForSegmentsForUnknownQueryThrowingException() expectedException.expect(QueryUnsupportedException.class); expectedException.expectMessage("Unknown query type"); serverManager.getQueryRunnerForSegments( - new BaseQuery( + new SingleDataSourceQuery( new TableDataSource("test"), new MultipleSpecificSegmentSpec(descriptors), new HashMap<>() diff --git a/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java b/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java index 3dc9696d4f4c..29f9fd15278e 100644 --- a/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java +++ b/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java @@ -27,7 +27,7 @@ import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.jackson.JacksonUtils; -import org.apache.druid.query.BaseQuery; +import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Query; import org.apache.druid.query.QueryDataSource; @@ -319,7 +319,7 @@ private static Map readContextMap(byte[] bytes) throws Exception } @JsonTypeName("fake") -class FakeQuery extends BaseQuery +class FakeQuery extends SingleDataSourceQuery { public FakeQuery( DataSource dataSource, From 523d82d8f77300e3d47499fde8bd16a273361a97 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 16:02:11 +0000 Subject: [PATCH 050/149] use abstract --- .../org/apache/druid/query/BaseQuery.java | 8 +-- .../spec/MultipleIntervalSegmentSpec.java | 6 +++ .../apache/druid/query/union/UnionQuery.java | 51 ++++--------------- 3 files changed, 19 insertions(+), 46 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/BaseQuery.java b/processing/src/main/java/org/apache/druid/query/BaseQuery.java index ce74d2918c38..11adcdf74979 100644 --- a/processing/src/main/java/org/apache/druid/query/BaseQuery.java +++ b/processing/src/main/java/org/apache/druid/query/BaseQuery.java @@ -56,10 +56,10 @@ public static void checkInterrupted() public static final String QUERY_ID = "queryId"; public static final String SUB_QUERY_ID = "subQueryId"; public static final String SQL_QUERY_ID = "sqlQueryId"; - private final QueryContext context; - private final QuerySegmentSpec querySegmentSpec; - private final Duration duration; - private final Granularity granularity; + protected final QueryContext context; + protected final QuerySegmentSpec querySegmentSpec; + protected final Duration duration; + protected final Granularity granularity; public BaseQuery( QuerySegmentSpec querySegmentSpec, diff --git a/processing/src/main/java/org/apache/druid/query/spec/MultipleIntervalSegmentSpec.java b/processing/src/main/java/org/apache/druid/query/spec/MultipleIntervalSegmentSpec.java index 7afcd2e13240..1aeb0188324d 100644 --- a/processing/src/main/java/org/apache/druid/query/spec/MultipleIntervalSegmentSpec.java +++ b/processing/src/main/java/org/apache/druid/query/spec/MultipleIntervalSegmentSpec.java @@ -21,12 +21,14 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; +import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.JodaUtils; import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; import org.joda.time.Interval; +import java.util.Collections; import java.util.List; import java.util.Objects; @@ -34,6 +36,10 @@ */ public class MultipleIntervalSegmentSpec implements QuerySegmentSpec { + public static final MultipleIntervalSegmentSpec ETERNITY = new MultipleIntervalSegmentSpec( + Collections.singletonList(Intervals.ETERNITY) + ); + private final List intervals; @JsonCreator diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index f1f2bb03e3d2..573ad2325f9f 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -25,8 +25,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; import org.apache.druid.error.DruidException; -import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Query; @@ -36,22 +34,19 @@ import org.apache.druid.query.UnionDataSource; import org.apache.druid.query.filter.DimFilter; import org.apache.druid.query.planning.DataSourceAnalysis; +import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.segment.SegmentReference; -import org.joda.time.DateTimeZone; -import org.joda.time.Duration; -import org.joda.time.Interval; - import java.util.ArrayList; -import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; -public class UnionQuery implements Query +public class UnionQuery extends BaseQuery { + @JsonProperty("context") protected final Map context; @@ -60,14 +55,16 @@ public class UnionQuery implements Query public UnionQuery(List> queries) { - this(queries, queries.get(0).getContext()); + this(MultipleIntervalSegmentSpec.ETERNITY, queries, queries.get(0).getContext()); } @JsonCreator public UnionQuery( + @JsonProperty("intervals") QuerySegmentSpec intervals, @JsonProperty("queries") List> queries, @JsonProperty("context") Map context) { + super(intervals, context); Preconditions.checkArgument(queries.size() > 1, "union with fewer than 2 queries makes no sense"); this.queries = queries; this.context = context; @@ -115,36 +112,6 @@ public QueryRunner getRunner(QuerySegmentWalker walker) throw DruidException.defensive("Use QueryToolChest to get a Runner"); } - @Override - public List getIntervals() - { - return Collections.emptyList(); - } - - @Override - public Duration getDuration() - { - throw DruidException.defensive("This is not supported"); - } - - @Override - public Granularity getGranularity() - { - return Granularities.ALL; - } - - @Override - public DateTimeZone getTimezone() - { - throw DruidException.defensive("This is not supported"); - } - - @Override - public Map getContext() - { - return context; - } - @Override public Ordering getResultOrdering() { @@ -155,13 +122,13 @@ public Ordering getResultOrdering() public Query withOverriddenContext(Map contextOverrides) { List> newQueries = mapQueries(q -> q.withOverriddenContext(contextOverrides)); - return new UnionQuery(newQueries, QueryContexts.override(getContext(), contextOverrides)); + return new UnionQuery(getQuerySegmentSpec(), newQueries, QueryContexts.override(getContext(), contextOverrides)); } @Override public Query withQuerySegmentSpec(QuerySegmentSpec spec) { - throw new RuntimeException("FIXME: Unimplemented!"); + return new UnionQuery(spec, queries, context); } @Override @@ -202,7 +169,7 @@ public Query withDataSources(List children) for (int i = 0; i < queries.size(); i++) { newQueries.add(queries.get(i).withDataSource(children.get(i))); } - return new UnionQuery(newQueries, context); + return new UnionQuery(querySegmentSpec, newQueries, context); } List> mapQueries(Function, Query> mapFn) From 13abfb270b9775883e2199d94708870a84dcf0a1 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 16:08:31 +0000 Subject: [PATCH 051/149] undo basequery split --- .../org/apache/druid/query/BaseQuery.java | 71 +++++++---- .../druid/query/SingleDataSourceQuery.java | 116 ------------------ .../DataSourceMetadataQuery.java | 4 +- .../druid/query/groupby/GroupByQuery.java | 4 +- .../metadata/SegmentMetadataQuery.java | 13 +- .../query/operator/WindowOperatorQuery.java | 5 +- .../apache/druid/query/scan/ScanQuery.java | 5 +- .../druid/query/search/SearchQuery.java | 4 +- .../spec/MultipleIntervalSegmentSpec.java | 6 - .../query/timeboundary/TimeBoundaryQuery.java | 4 +- .../query/timeseries/TimeseriesQuery.java | 5 +- .../apache/druid/query/topn/TopNQuery.java | 5 +- .../apache/druid/query/union/UnionQuery.java | 51 ++++++-- ...TimelineMissingSegmentQueryRunnerTest.java | 2 +- .../org/apache/druid/query/TestQuery.java | 3 +- .../CachingClusteredClientPerfTest.java | 3 +- .../coordination/ServerManagerTest.java | 4 +- .../server/log/LoggingRequestLoggerTest.java | 4 +- 18 files changed, 122 insertions(+), 187 deletions(-) delete mode 100644 processing/src/main/java/org/apache/druid/query/SingleDataSourceQuery.java diff --git a/processing/src/main/java/org/apache/druid/query/BaseQuery.java b/processing/src/main/java/org/apache/druid/query/BaseQuery.java index 11adcdf74979..bbfa570e3094 100644 --- a/processing/src/main/java/org/apache/druid/query/BaseQuery.java +++ b/processing/src/main/java/org/apache/druid/query/BaseQuery.java @@ -56,47 +56,47 @@ public static void checkInterrupted() public static final String QUERY_ID = "queryId"; public static final String SUB_QUERY_ID = "subQueryId"; public static final String SQL_QUERY_ID = "sqlQueryId"; - protected final QueryContext context; - protected final QuerySegmentSpec querySegmentSpec; - protected final Duration duration; - protected final Granularity granularity; + private final DataSource dataSource; + private final QueryContext context; + private final QuerySegmentSpec querySegmentSpec; + private volatile Duration duration; + private final Granularity granularity; public BaseQuery( + DataSource dataSource, QuerySegmentSpec querySegmentSpec, Map context ) { - this(querySegmentSpec, context, Granularities.ALL); + this(dataSource, querySegmentSpec, context, Granularities.ALL); } public BaseQuery( + DataSource dataSource, QuerySegmentSpec querySegmentSpec, Map context, Granularity granularity ) { + Preconditions.checkNotNull(dataSource, "dataSource can't be null"); Preconditions.checkNotNull(querySegmentSpec, "querySegmentSpec can't be null"); Preconditions.checkNotNull(granularity, "Must specify a granularity"); + this.dataSource = dataSource; this.context = QueryContext.of(context); this.querySegmentSpec = querySegmentSpec; this.granularity = granularity; - this.duration = computeDuration(); } - private final Duration computeDuration() + @JsonProperty + @Override + public DataSource getDataSource() { - Duration totalDuration = new Duration(0); - for (Interval interval : querySegmentSpec.getIntervals()) { - if (interval != null) { - totalDuration = totalDuration.plus(interval.toDuration()); - } - } - return totalDuration; + return dataSource; } @JsonProperty("intervals") - public final QuerySegmentSpec getQuerySegmentSpec() + public QuerySegmentSpec getQuerySegmentSpec() { return querySegmentSpec; } @@ -117,26 +117,36 @@ public static QuerySegmentSpec getQuerySegmentSpecForLookUp(BaseQuery query) } @Override - public final List getIntervals() + public List getIntervals() { return querySegmentSpec.getIntervals(); } @Override - public final Duration getDuration() + public Duration getDuration() { + if (duration == null) { + Duration totalDuration = new Duration(0); + for (Interval interval : querySegmentSpec.getIntervals()) { + if (interval != null) { + totalDuration = totalDuration.plus(interval.toDuration()); + } + } + duration = totalDuration; + } + return duration; } @Override @JsonProperty - public final Granularity getGranularity() + public Granularity getGranularity() { return granularity; } @Override - public final DateTimeZone getTimezone() + public DateTimeZone getTimezone() { return granularity instanceof PeriodGranularity ? ((PeriodGranularity) granularity).getTimeZone() @@ -146,17 +156,27 @@ public final DateTimeZone getTimezone() @Override @JsonProperty @JsonInclude(Include.NON_DEFAULT) - public final Map getContext() + public Map getContext() { return context.asMap(); } @Override - public final QueryContext context() + public QueryContext context() { return context; } + /** + * @deprecated use {@link #computeOverriddenContext(Map, Map) computeOverriddenContext(getContext(), overrides))} + * instead. This method may be removed in the next minor or major version of Druid. + */ + @Deprecated + protected Map computeOverridenContext(final Map overrides) + { + return computeOverriddenContext(getContext(), overrides); + } + public static Map computeOverriddenContext( final Map context, final Map overrides @@ -220,15 +240,18 @@ public boolean equals(Object o) } BaseQuery baseQuery = (BaseQuery) o; - return Objects.equals(context, baseQuery.context) && + // Must use getDuration() instead of "duration" because duration is lazily computed. + return Objects.equals(dataSource, baseQuery.dataSource) && + Objects.equals(context, baseQuery.context) && Objects.equals(querySegmentSpec, baseQuery.querySegmentSpec) && - Objects.equals(duration, baseQuery.getDuration()) && + Objects.equals(getDuration(), baseQuery.getDuration()) && Objects.equals(granularity, baseQuery.granularity); } @Override public int hashCode() { - return Objects.hash(context, querySegmentSpec, duration, granularity); + // Must use getDuration() instead of "duration" because duration is lazily computed. + return Objects.hash(dataSource, context, querySegmentSpec, getDuration(), granularity); } } diff --git a/processing/src/main/java/org/apache/druid/query/SingleDataSourceQuery.java b/processing/src/main/java/org/apache/druid/query/SingleDataSourceQuery.java deleted file mode 100644 index 838759424419..000000000000 --- a/processing/src/main/java/org/apache/druid/query/SingleDataSourceQuery.java +++ /dev/null @@ -1,116 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query; - -import com.fasterxml.jackson.annotation.JsonProperty; -import com.google.common.annotations.VisibleForTesting; -import com.google.common.collect.Ordering; -import org.apache.druid.guice.annotations.ExtensionPoint; -import org.apache.druid.java.util.common.granularity.Granularities; -import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.query.spec.QuerySegmentSpec; -import java.util.Map; -import java.util.Objects; - -/** - * - */ -@ExtensionPoint -public abstract class SingleDataSourceQuery extends BaseQuery implements Query -{ - private final DataSource dataSource; - - public SingleDataSourceQuery( - DataSource dataSource, - QuerySegmentSpec querySegmentSpec, - Map context - ) - { - this(dataSource, querySegmentSpec, context, Granularities.ALL); - } - - public SingleDataSourceQuery( - DataSource dataSource, - QuerySegmentSpec querySegmentSpec, - Map context, - Granularity granularity - ) - { - super(querySegmentSpec, context, granularity); - - this.dataSource = dataSource; - } - - @JsonProperty - @Override - public DataSource getDataSource() - { - return dataSource; - } - - @Override - public QueryRunner getRunner(QuerySegmentWalker walker) - { - return getQuerySegmentSpecForLookUp(this).lookup(this, walker); - } - - @VisibleForTesting - public static QuerySegmentSpec getQuerySegmentSpecForLookUp(SingleDataSourceQuery query) - { - DataSource queryDataSource = query.getDataSource(); - return queryDataSource.getAnalysis() - .getBaseQuerySegmentSpec() - .orElseGet(query::getQuerySegmentSpec); - } - - /** - * Default implementation of {@link Query#getResultOrdering()} that uses {@link Ordering#natural()}. - * - * If your query result type T is not Comparable, you must override this method. - */ - @Override - @SuppressWarnings("unchecked") // assumes T is Comparable; see method javadoc - public Ordering getResultOrdering() - { - return (Ordering) Ordering.natural(); - } - - @Override - public boolean equals(Object o) - { - if (this == o) { - return true; - } - if (o == null || getClass() != o.getClass()) { - return false; - } - if(!super.equals(o)) { - return false; - } - SingleDataSourceQuery baseQuery = (SingleDataSourceQuery) o; - return Objects.equals(dataSource, baseQuery.dataSource); - } - - @Override - public int hashCode() - { - return Objects.hash(super.hashCode(), dataSource); - } -} diff --git a/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQuery.java b/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQuery.java index e335e91e5aa4..5c43c6445bc9 100644 --- a/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQuery.java +++ b/processing/src/main/java/org/apache/druid/query/datasourcemetadata/DataSourceMetadataQuery.java @@ -23,7 +23,7 @@ import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; -import org.apache.druid.query.SingleDataSourceQuery; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.Query; @@ -40,7 +40,7 @@ /** */ -public class DataSourceMetadataQuery extends SingleDataSourceQuery> +public class DataSourceMetadataQuery extends BaseQuery> { private static final QuerySegmentSpec DEFAULT_SEGMENT_SPEC = new MultipleIntervalSegmentSpec(Intervals.ONLY_ETERNITY); diff --git a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java index 84011580d8ad..68c071060633 100644 --- a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java @@ -42,7 +42,6 @@ import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.query.BaseQuery; -import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.DimensionComparisonUtils; import org.apache.druid.query.Queries; @@ -73,7 +72,6 @@ import org.joda.time.Interval; import javax.annotation.Nullable; - import java.util.ArrayList; import java.util.Arrays; import java.util.Comparator; @@ -87,7 +85,7 @@ /** * */ -public class GroupByQuery extends SingleDataSourceQuery implements SupportRowSignature +public class GroupByQuery extends BaseQuery implements SupportRowSignature { public static final String CTX_KEY_SORT_BY_DIMS_FIRST = "sortByDimsFirst"; public static final String CTX_TIMESTAMP_RESULT_FIELD = "timestampResultField"; diff --git a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java index 1ee11bbdfd4b..f2d434bab8a5 100644 --- a/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java +++ b/processing/src/main/java/org/apache/druid/query/metadata/metadata/SegmentMetadataQuery.java @@ -26,7 +26,7 @@ import org.apache.druid.java.util.common.Cacheable; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.query.SingleDataSourceQuery; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.Query; @@ -36,11 +36,14 @@ import org.apache.druid.query.metadata.SegmentMetadataQueryConfig; import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; import org.apache.druid.query.spec.QuerySegmentSpec; +import org.joda.time.Interval; + import java.util.EnumSet; +import java.util.List; import java.util.Map; import java.util.Objects; -public class SegmentMetadataQuery extends SingleDataSourceQuery +public class SegmentMetadataQuery extends BaseQuery { private static final QuerySegmentSpec DEFAULT_SEGMENT_SPEC = new MultipleIntervalSegmentSpec(Intervals.ONLY_ETERNITY); @@ -236,6 +239,12 @@ public SegmentMetadataQuery withFinalizedAnalysisTypes(SegmentMetadataQueryConfi .build(); } + @Override + public List getIntervals() + { + return this.getQuerySegmentSpec().getIntervals(); + } + @Override public String toString() { diff --git a/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java index b782190e3236..aed132b745ee 100644 --- a/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java +++ b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java @@ -22,7 +22,7 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; import org.apache.druid.java.util.common.IAE; -import org.apache.druid.query.SingleDataSourceQuery; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.InlineDataSource; import org.apache.druid.query.Order; @@ -36,7 +36,6 @@ import org.apache.druid.segment.column.RowSignature; import javax.annotation.Nullable; - import java.util.ArrayList; import java.util.List; import java.util.Map; @@ -53,7 +52,7 @@ * The assumptions on the incoming data are defined by the operators. At initial time of writing, there is a baked * in assumption that data has been sorted "correctly" before this runs. */ -public class WindowOperatorQuery extends SingleDataSourceQuery +public class WindowOperatorQuery extends BaseQuery { private static DataSource validateMaybeRewriteDataSource(DataSource dataSource, boolean hasLeafs) { diff --git a/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java b/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java index 4c579c194a05..037148f50c3a 100644 --- a/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java +++ b/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java @@ -31,7 +31,7 @@ import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.Pair; import org.apache.druid.java.util.common.UOE; -import org.apache.druid.query.SingleDataSourceQuery; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.InlineDataSource; @@ -50,7 +50,6 @@ import org.apache.druid.segment.column.RowSignature.Builder; import javax.annotation.Nullable; - import java.util.Collections; import java.util.Comparator; import java.util.List; @@ -58,7 +57,7 @@ import java.util.Objects; import java.util.Set; -public class ScanQuery extends SingleDataSourceQuery +public class ScanQuery extends BaseQuery { public enum ResultFormat { diff --git a/processing/src/main/java/org/apache/druid/query/search/SearchQuery.java b/processing/src/main/java/org/apache/druid/query/search/SearchQuery.java index fdb9f9c1135f..aa10b6ab5790 100644 --- a/processing/src/main/java/org/apache/druid/query/search/SearchQuery.java +++ b/processing/src/main/java/org/apache/druid/query/search/SearchQuery.java @@ -25,7 +25,7 @@ import com.google.common.base.Preconditions; import org.apache.druid.java.util.common.granularity.Granularities; import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.query.SingleDataSourceQuery; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.Query; @@ -42,7 +42,7 @@ /** */ -public class SearchQuery extends SingleDataSourceQuery> +public class SearchQuery extends BaseQuery> { private static final SearchSortSpec DEFAULT_SORT_SPEC = new SearchSortSpec(StringComparators.LEXICOGRAPHIC); diff --git a/processing/src/main/java/org/apache/druid/query/spec/MultipleIntervalSegmentSpec.java b/processing/src/main/java/org/apache/druid/query/spec/MultipleIntervalSegmentSpec.java index 1aeb0188324d..7afcd2e13240 100644 --- a/processing/src/main/java/org/apache/druid/query/spec/MultipleIntervalSegmentSpec.java +++ b/processing/src/main/java/org/apache/druid/query/spec/MultipleIntervalSegmentSpec.java @@ -21,14 +21,12 @@ import com.fasterxml.jackson.annotation.JsonCreator; import com.fasterxml.jackson.annotation.JsonProperty; -import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.JodaUtils; import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; import org.joda.time.Interval; -import java.util.Collections; import java.util.List; import java.util.Objects; @@ -36,10 +34,6 @@ */ public class MultipleIntervalSegmentSpec implements QuerySegmentSpec { - public static final MultipleIntervalSegmentSpec ETERNITY = new MultipleIntervalSegmentSpec( - Collections.singletonList(Intervals.ETERNITY) - ); - private final List intervals; @JsonCreator diff --git a/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java b/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java index 919fbe0e25c2..47412f7d7d50 100644 --- a/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java +++ b/processing/src/main/java/org/apache/druid/query/timeboundary/TimeBoundaryQuery.java @@ -26,7 +26,7 @@ import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.StringUtils; -import org.apache.druid.query.SingleDataSourceQuery; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.Query; @@ -44,7 +44,7 @@ /** */ -public class TimeBoundaryQuery extends SingleDataSourceQuery> +public class TimeBoundaryQuery extends BaseQuery> { private static final QuerySegmentSpec DEFAULT_SEGMENT_SPEC = new MultipleIntervalSegmentSpec(Intervals.ONLY_ETERNITY); public static final String MAX_TIME = "maxTime"; diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java index 7a1f48033e85..1a6f8e669fd3 100644 --- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java +++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java @@ -28,7 +28,7 @@ import com.google.common.collect.Ordering; import org.apache.commons.lang3.StringUtils; import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.query.SingleDataSourceQuery; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Druids; import org.apache.druid.query.PerSegmentQueryOptimizationContext; @@ -47,7 +47,6 @@ import org.apache.druid.segment.column.RowSignature.Finalization; import javax.annotation.Nullable; - import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -58,7 +57,7 @@ /** */ @JsonTypeName("timeseries") -public class TimeseriesQuery extends SingleDataSourceQuery> implements SupportRowSignature +public class TimeseriesQuery extends BaseQuery> implements SupportRowSignature { public static final String CTX_GRAND_TOTAL = "grandTotal"; public static final String SKIP_EMPTY_BUCKETS = "skipEmptyBuckets"; diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java b/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java index e13670f8f892..3178ac15a183 100644 --- a/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java +++ b/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java @@ -25,7 +25,7 @@ import com.google.common.base.Preconditions; import com.google.common.collect.ImmutableList; import org.apache.druid.java.util.common.granularity.Granularity; -import org.apache.druid.query.SingleDataSourceQuery; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.PerSegmentQueryOptimizationContext; import org.apache.druid.query.Queries; @@ -40,7 +40,6 @@ import org.apache.druid.segment.column.RowSignature; import javax.annotation.Nullable; - import java.util.ArrayList; import java.util.Collections; import java.util.List; @@ -50,7 +49,7 @@ /** */ -public class TopNQuery extends SingleDataSourceQuery> +public class TopNQuery extends BaseQuery> { public static final String TOPN = "topN"; diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index 573ad2325f9f..f1f2bb03e3d2 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -25,6 +25,8 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.Ordering; import org.apache.druid.error.DruidException; +import org.apache.druid.java.util.common.granularity.Granularities; +import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Query; @@ -34,19 +36,22 @@ import org.apache.druid.query.UnionDataSource; import org.apache.druid.query.filter.DimFilter; import org.apache.druid.query.planning.DataSourceAnalysis; -import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.segment.SegmentReference; +import org.joda.time.DateTimeZone; +import org.joda.time.Duration; +import org.joda.time.Interval; + import java.util.ArrayList; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Set; import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; -public class UnionQuery extends BaseQuery +public class UnionQuery implements Query { - @JsonProperty("context") protected final Map context; @@ -55,16 +60,14 @@ public class UnionQuery extends BaseQuery public UnionQuery(List> queries) { - this(MultipleIntervalSegmentSpec.ETERNITY, queries, queries.get(0).getContext()); + this(queries, queries.get(0).getContext()); } @JsonCreator public UnionQuery( - @JsonProperty("intervals") QuerySegmentSpec intervals, @JsonProperty("queries") List> queries, @JsonProperty("context") Map context) { - super(intervals, context); Preconditions.checkArgument(queries.size() > 1, "union with fewer than 2 queries makes no sense"); this.queries = queries; this.context = context; @@ -112,6 +115,36 @@ public QueryRunner getRunner(QuerySegmentWalker walker) throw DruidException.defensive("Use QueryToolChest to get a Runner"); } + @Override + public List getIntervals() + { + return Collections.emptyList(); + } + + @Override + public Duration getDuration() + { + throw DruidException.defensive("This is not supported"); + } + + @Override + public Granularity getGranularity() + { + return Granularities.ALL; + } + + @Override + public DateTimeZone getTimezone() + { + throw DruidException.defensive("This is not supported"); + } + + @Override + public Map getContext() + { + return context; + } + @Override public Ordering getResultOrdering() { @@ -122,13 +155,13 @@ public Ordering getResultOrdering() public Query withOverriddenContext(Map contextOverrides) { List> newQueries = mapQueries(q -> q.withOverriddenContext(contextOverrides)); - return new UnionQuery(getQuerySegmentSpec(), newQueries, QueryContexts.override(getContext(), contextOverrides)); + return new UnionQuery(newQueries, QueryContexts.override(getContext(), contextOverrides)); } @Override public Query withQuerySegmentSpec(QuerySegmentSpec spec) { - return new UnionQuery(spec, queries, context); + throw new RuntimeException("FIXME: Unimplemented!"); } @Override @@ -169,7 +202,7 @@ public Query withDataSources(List children) for (int i = 0; i < queries.size(); i++) { newQueries.add(queries.get(i).withDataSource(children.get(i))); } - return new UnionQuery(querySegmentSpec, newQueries, context); + return new UnionQuery(newQueries, context); } List> mapQueries(Function, Query> mapFn) diff --git a/processing/src/test/java/org/apache/druid/query/ReportTimelineMissingSegmentQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/ReportTimelineMissingSegmentQueryRunnerTest.java index d5f7678121b6..34cb3dbcdea5 100644 --- a/processing/src/test/java/org/apache/druid/query/ReportTimelineMissingSegmentQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/ReportTimelineMissingSegmentQueryRunnerTest.java @@ -66,7 +66,7 @@ public void testRunWithMultipleSegments() Assert.assertEquals(missingSegments, responseContext.getMissingSegments()); } - private static class TestQuery extends SingleDataSourceQuery + private static class TestQuery extends BaseQuery { private TestQuery() { diff --git a/processing/src/test/java/org/apache/druid/query/TestQuery.java b/processing/src/test/java/org/apache/druid/query/TestQuery.java index e8acda35f8e9..186b02444b52 100644 --- a/processing/src/test/java/org/apache/druid/query/TestQuery.java +++ b/processing/src/test/java/org/apache/druid/query/TestQuery.java @@ -23,11 +23,10 @@ import org.apache.druid.query.spec.QuerySegmentSpec; import javax.annotation.Nullable; - import java.util.Map; import java.util.Set; -public class TestQuery extends SingleDataSourceQuery +public class TestQuery extends BaseQuery { @Nullable private Set requiredColumns; diff --git a/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java b/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java index 7d01ba7f6607..b31799cf8b4c 100644 --- a/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java +++ b/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java @@ -34,7 +34,6 @@ import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.TestSequence; import org.apache.druid.query.BaseQuery; -import org.apache.druid.query.SingleDataSourceQuery; import org.apache.druid.query.BrokerParallelMergeConfig; import org.apache.druid.query.DataSource; import org.apache.druid.query.Query; @@ -195,7 +194,7 @@ public Sequence run( } } - private static class TestQuery extends SingleDataSourceQuery + private static class TestQuery extends BaseQuery { private QuerySegmentSpec spec; diff --git a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java index 8bf8ac95f630..69a7c08f2be9 100644 --- a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java @@ -44,7 +44,7 @@ import org.apache.druid.java.util.common.guava.YieldingAccumulator; import org.apache.druid.java.util.common.guava.YieldingSequenceBase; import org.apache.druid.java.util.emitter.EmittingLogger; -import org.apache.druid.query.SingleDataSourceQuery; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.ConcatQueryRunner; import org.apache.druid.query.DataSource; import org.apache.druid.query.DefaultQueryMetrics; @@ -594,7 +594,7 @@ public void testGetQueryRunnerForSegmentsForUnknownQueryThrowingException() expectedException.expect(QueryUnsupportedException.class); expectedException.expectMessage("Unknown query type"); serverManager.getQueryRunnerForSegments( - new SingleDataSourceQuery( + new BaseQuery( new TableDataSource("test"), new MultipleSpecificSegmentSpec(descriptors), new HashMap<>() diff --git a/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java b/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java index 29f9fd15278e..3dc9696d4f4c 100644 --- a/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java +++ b/server/src/test/java/org/apache/druid/server/log/LoggingRequestLoggerTest.java @@ -27,7 +27,7 @@ import org.apache.druid.java.util.common.DateTimes; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.jackson.JacksonUtils; -import org.apache.druid.query.SingleDataSourceQuery; +import org.apache.druid.query.BaseQuery; import org.apache.druid.query.DataSource; import org.apache.druid.query.Query; import org.apache.druid.query.QueryDataSource; @@ -319,7 +319,7 @@ private static Map readContextMap(byte[] bytes) throws Exception } @JsonTypeName("fake") -class FakeQuery extends SingleDataSourceQuery +class FakeQuery extends BaseQuery { public FakeQuery( DataSource dataSource, From 25039743093422a3a1906028b43318ff1a5eca76 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 16:12:52 +0000 Subject: [PATCH 052/149] rename/etc --- .../java/org/apache/druid/query/QueryExecSomething.java | 7 +++++-- .../apache/druid/query/union/UnionQueryQueryToolChest.java | 4 ++-- .../{RealUnionQueryRunner2.java => UnionQueryRunner.java} | 4 ++-- .../org/apache/druid/server/ClientQuerySegmentWalker.java | 4 ++-- 4 files changed, 11 insertions(+), 8 deletions(-) rename processing/src/main/java/org/apache/druid/query/union/{RealUnionQueryRunner2.java => UnionQueryRunner.java} (94%) diff --git a/processing/src/main/java/org/apache/druid/query/QueryExecSomething.java b/processing/src/main/java/org/apache/druid/query/QueryExecSomething.java index 2f18c40d2bd9..0e6f7f43cb4c 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryExecSomething.java +++ b/processing/src/main/java/org/apache/druid/query/QueryExecSomething.java @@ -21,7 +21,10 @@ public interface QueryExecSomething { - public QueryRunner executeQuery2(QueryToolChestWarehouse warehouse, - Query query, QuerySegmentWalker clientQuerySegmentWalker); + QueryRunner makeQueryRunner( + QueryToolChestWarehouse warehouse, + Query query, + QuerySegmentWalker walker + ); } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index be3c76b11717..d5b73e3ed989 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -50,10 +50,10 @@ public class UnionQueryQueryToolChest extends QueryToolChest { - public QueryRunner executeQuery2(QueryToolChestWarehouse warehouse, + public QueryRunner makeQueryRunner(QueryToolChestWarehouse warehouse, Query query, QuerySegmentWalker clientQuerySegmentWalker) { - RealUnionQueryRunner2 runner = new RealUnionQueryRunner2(warehouse, (UnionQuery) query, clientQuerySegmentWalker); + UnionQueryRunner runner = new UnionQueryRunner(warehouse, (UnionQuery) query, clientQuerySegmentWalker); setWarehouse(warehouse); return runner; } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner2.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java similarity index 94% rename from processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner2.java rename to processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java index 1868c1d36d2d..f6c6f4da2ca8 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner2.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java @@ -31,7 +31,7 @@ import java.util.ArrayList; import java.util.List; -class RealUnionQueryRunner2 implements QueryRunner +class UnionQueryRunner implements QueryRunner { private QueryToolChestWarehouse warehouse; @@ -39,7 +39,7 @@ class RealUnionQueryRunner2 implements QueryRunner private UnionQuery query; private List runners; - public RealUnionQueryRunner2(QueryToolChestWarehouse warehouse, UnionQuery query, + public UnionQueryRunner(QueryToolChestWarehouse warehouse, UnionQuery query, QuerySegmentWalker walker) { this.warehouse = warehouse; diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 0aca0d47b1ac..4a2b5e689310 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -183,7 +183,7 @@ public QueryRunner getQueryRunnerForIntervals(final Query query, final // Optional> toolchestExecResult = toolChest.executeQuery21(warehouse, query, this); if (toolChest instanceof QueryExecSomething) { QueryExecSomething t = (QueryExecSomething) toolChest; - return t.executeQuery2(warehouse, query, this); + return t.makeQueryRunner(warehouse, query, this); } // transform TableDataSource to GlobalTableDataSource when eligible @@ -458,7 +458,7 @@ private DataSource inlineIfNecessary( final QueryRunner subQueryRunner; if (subQueryToolChest instanceof QueryExecSomething) { subQueryRunner = ((QueryExecSomething) subQueryToolChest) - .executeQuery2(warehouse, subQueryWithSerialization, this); + .makeQueryRunner(warehouse, subQueryWithSerialization, this); }else { subQueryRunner = subQueryWithSerialization.getRunner(this); } From ae2192163e1970ac5d9c06182251b22644dc924c Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 16:22:42 +0000 Subject: [PATCH 053/149] na jo --- .../apache/druid/query/QueryToolChest.java | 4 +- .../query/union/UnionQueryQueryToolChest.java | 3 +- .../druid/query/union/UnionQueryRunner.java | 17 +- .../server/ClientQuerySegmentWalker.java | 1 - ...estUnionAllTwoQueriesBothQueriesAreJoin.iq | 153 +++++++++++++++++- 5 files changed, 157 insertions(+), 21 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java index 3ea27149d53c..f8b029806acc 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.TypeFactory; import com.google.common.base.Function; +import com.google.inject.Inject; import org.apache.druid.error.DruidException; import org.apache.druid.frame.allocation.MemoryAllocatorFactory; import org.apache.druid.guice.annotations.ExtensionPoint; @@ -424,9 +425,10 @@ public Optional> resultsAsFrames( return Optional.empty(); } + @Inject public void setWarehouse(QueryToolChestWarehouse warehouse) { - this.warehouse=warehouse; + this.warehouse = warehouse; } public Optional> executeQuery( diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index d5b73e3ed989..ce452e9f23e4 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -53,8 +53,7 @@ public class UnionQueryQueryToolChest extends QueryToolChest makeQueryRunner(QueryToolChestWarehouse warehouse, Query query, QuerySegmentWalker clientQuerySegmentWalker) { - UnionQueryRunner runner = new UnionQueryRunner(warehouse, (UnionQuery) query, clientQuerySegmentWalker); - setWarehouse(warehouse); + UnionQueryRunner runner = new UnionQueryRunner((UnionQuery) query, clientQuerySegmentWalker); return runner; } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java index f6c6f4da2ca8..e4f3de27670a 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java @@ -25,7 +25,6 @@ import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; -import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.context.ResponseContext; import java.util.ArrayList; @@ -33,19 +32,17 @@ class UnionQueryRunner implements QueryRunner { + private final QuerySegmentWalker walker; + private final UnionQuery query; + private final List runners; - private QueryToolChestWarehouse warehouse; - private QuerySegmentWalker walker; - private UnionQuery query; - private List runners; - - public UnionQueryRunner(QueryToolChestWarehouse warehouse, UnionQuery query, - QuerySegmentWalker walker) + public UnionQueryRunner( + UnionQuery query, + QuerySegmentWalker walker + ) { - this.warehouse = warehouse; this.query = query; this.walker = walker; - this.runners = makeSubQueryRunners(query); } diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 4a2b5e689310..24255fd2d396 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -452,7 +452,6 @@ private DataSource inlineIfNecessary( ) ); QueryToolChest subQueryToolChest = warehouse.getToolChest(subQuery); - subQueryToolChest.setWarehouse(warehouse); // final QueryRunner subQueryRunner ; final QueryRunner subQueryRunner; diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq index 65b1fac2fad7..4c692c3a6bda 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq @@ -9,13 +9,152 @@ !set outputformat mysql !use druidtest:/// (SELECT COUNT(*) FROM foo LEFT JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; -+--------+ -| EXPR$0 | -+--------+ -| 1 | -| 6 | -+--------+ -(2 rows) +org.apache.calcite.avatica.AvaticaSqlException: Error -1 (00000) : Error while executing SQL "(SELECT COUNT(*) FROM foo LEFT JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ": Remote driver error: QueryInterruptedException: (null exception message) -> NullPointerException: (null exception message) + at org.apache.calcite.avatica.Helper.createException(Helper.java:54) + at org.apache.calcite.avatica.Helper.createException(Helper.java:41) + at org.apache.calcite.avatica.AvaticaStatement.executeInternal(AvaticaStatement.java:164) + at org.apache.calcite.avatica.AvaticaStatement.executeQuery(AvaticaStatement.java:228) + at net.hydromatic.quidem.Quidem.checkResult(Quidem.java:299) + at net.hydromatic.quidem.Quidem.access$2600(Quidem.java:57) + at net.hydromatic.quidem.Quidem$ContextImpl.checkResult(Quidem.java:1636) + at net.hydromatic.quidem.Quidem$CheckResultCommand.execute(Quidem.java:941) + at net.hydromatic.quidem.Quidem$CompositeCommand.execute(Quidem.java:1437) + at net.hydromatic.quidem.Quidem.execute(Quidem.java:190) + at org.apache.druid.quidem.DruidQuidemTestBase$DruidQuidemRunner.run(DruidQuidemTestBase.java:158) + at org.apache.druid.quidem.DruidQuidemTestBase$DruidQuidemRunner.run(DruidQuidemTestBase.java:138) + at org.apache.druid.sql.calcite.QTestCase$1.run(QTestCase.java:73) + at org.apache.druid.sql.calcite.QueryTestRunner.run(QueryTestRunner.java:766) + at org.apache.druid.sql.calcite.QueryTestBuilder.run(QueryTestBuilder.java:304) + at org.apache.druid.sql.calcite.BaseCalciteQueryTest.testQuery(BaseCalciteQueryTest.java:733) + at org.apache.druid.sql.calcite.CalciteJoinQueryTest.testUnionAllTwoQueriesBothQueriesAreJoin(CalciteJoinQueryTest.java:4427) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.base/java.lang.reflect.Method.invoke(Method.java:566) + at org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:728) + at org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131) + at org.apache.druid.sql.calcite.NotYetSupported$NotYetSupportedProcessor.interceptTestMethod(NotYetSupported.java:134) + at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(InterceptingExecutableInvoker.java:103) + at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.lambda$invoke$0(InterceptingExecutableInvoker.java:93) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106) + at org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:156) + at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:147) + at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:86) + at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(InterceptingExecutableInvoker.java:103) + at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.lambda$invoke$0(InterceptingExecutableInvoker.java:93) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37) + at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:92) + at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:86) + at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$7(TestMethodTestDescriptor.java:218) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:214) + at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:139) + at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:69) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:151) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) + at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1541) + at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:155) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) + at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1541) + at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:155) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) + at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) + at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:35) + at org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57) + at org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:54) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:147) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:127) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:90) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:55) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:102) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:54) + at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:114) + at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:95) + at org.junit.platform.launcher.core.DefaultLauncherSession$DelegatingLauncher.execute(DefaultLauncherSession.java:91) + at org.junit.platform.launcher.core.SessionPerRequestLauncher.execute(SessionPerRequestLauncher.java:60) + at org.eclipse.jdt.internal.junit5.runner.JUnit5TestReference.run(JUnit5TestReference.java:98) + at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:40) + at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:529) + at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:756) + at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:452) + at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:210) +QueryInterruptedException{msg=null, code=Unknown exception, class=java.lang.NullPointerException, host=null} + at org.apache.druid.query.QueryInterruptedException.wrapIfNeeded(QueryInterruptedException.java:113) + at org.apache.druid.sql.avatica.ErrorHandler.sanitize(ErrorHandler.java:82) + at org.apache.druid.sql.avatica.DruidMeta.mapException(DruidMeta.java:395) + at org.apache.druid.sql.avatica.DruidMeta.prepareAndExecute(DruidMeta.java:362) + at org.apache.calcite.avatica.remote.LocalService.apply(LocalService.java:212) + at org.apache.calcite.avatica.remote.Service$PrepareAndExecuteRequest.accept(Service.java:952) + at org.apache.calcite.avatica.remote.Service$PrepareAndExecuteRequest.accept(Service.java:904) + at org.apache.calcite.avatica.remote.AbstractHandler.apply(AbstractHandler.java:94) + at org.apache.calcite.avatica.remote.JsonHandler.apply(JsonHandler.java:52) + at org.apache.calcite.avatica.server.AvaticaJsonHandler.handle(AvaticaJsonHandler.java:133) + at org.apache.druid.sql.avatica.DruidAvaticaJsonHandler.handle(DruidAvaticaJsonHandler.java:61) + at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) + at org.eclipse.jetty.server.Server.handle(Server.java:516) + at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:487) + at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:732) + at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:479) + at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) + at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) + at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) + at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) + at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) + at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) + at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) + at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) + at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) + at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) + at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) + at java.base/java.lang.Thread.run(Thread.java:829) +Caused by: java.lang.NullPointerException + at org.apache.druid.query.union.UnionQueryQueryToolChest.resultsAsArrays(UnionQueryQueryToolChest.java:129) + at org.apache.druid.query.union.UnionQueryQueryToolChest.resultsAsArrays(UnionQueryQueryToolChest.java:120) + at org.apache.druid.query.union.UnionQueryQueryToolChest.resultsAsArrays(UnionQueryQueryToolChest.java:1) + at org.apache.druid.server.ClientQuerySegmentWalker.materializeResultsAsArray(ClientQuerySegmentWalker.java:893) + at org.apache.druid.server.ClientQuerySegmentWalker.toInlineDataSource(ClientQuerySegmentWalker.java:725) + at org.apache.druid.server.ClientQuerySegmentWalker.inlineIfNecessary(ClientQuerySegmentWalker.java:469) + at org.apache.druid.server.ClientQuerySegmentWalker.getQueryRunnerForIntervals(ClientQuerySegmentWalker.java:232) + at org.apache.druid.server.SpecificSegmentsQuerySegmentWalker.getQueryRunnerForIntervals(SpecificSegmentsQuerySegmentWalker.java:223) + at org.apache.druid.query.spec.MultipleIntervalSegmentSpec.lookup(MultipleIntervalSegmentSpec.java:57) + at org.apache.druid.query.BaseQuery.getRunner(BaseQuery.java:107) + at org.apache.druid.query.QueryPlus.run(QueryPlus.java:158) + at org.apache.druid.server.QueryLifecycle.execute(QueryLifecycle.java:315) + at org.apache.druid.server.QueryLifecycle.runSimple(QueryLifecycle.java:162) + at org.apache.druid.sql.calcite.run.NativeQueryMaker.execute(NativeQueryMaker.java:195) + at org.apache.druid.sql.calcite.run.NativeQueryMaker.runQuery(NativeQueryMaker.java:152) + at org.apache.druid.sql.calcite.planner.QueryHandler.lambda$7(QueryHandler.java:568) + at org.apache.druid.sql.calcite.planner.PlannerResult.run(PlannerResult.java:62) + at org.apache.druid.sql.DirectStatement$ResultSet.run(DirectStatement.java:109) + at org.apache.druid.sql.DirectStatement.execute(DirectStatement.java:184) + at org.apache.druid.sql.avatica.DruidJdbcResultSet.lambda$0(DruidJdbcResultSet.java:239) + at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) + at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) + ... 1 more !ok LogicalUnion(all=[true]) From 5d2f088e9d4631dba1ab620c0daad41fc0cf969d Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 16:35:19 +0000 Subject: [PATCH 054/149] cleamp --- ...DefaultQueryRunnerFactoryConglomerate.java | 16 +++- .../query/union/UnionQueryQueryToolChest.java | 82 +------------------ 2 files changed, 14 insertions(+), 84 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java index 7f004ffa6343..48d2ace99a8a 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java @@ -26,22 +26,30 @@ /** */ -public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactoryConglomerate +public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactoryConglomerate, QueryToolChestWarehouse { private final Map, QueryRunnerFactory> factories; @Inject public DefaultQueryRunnerFactoryConglomerate(Map, QueryRunnerFactory> factories) { - // Accesses to IdentityHashMap should be faster than to HashMap or ImmutableMap. - // Class doesn't override Object.equals(). this.factories = new IdentityHashMap<>(factories); + for (QueryRunnerFactory factory :factories.values()) { + factory.getToolchest().setWarehouse(this); + } } @Override @SuppressWarnings("unchecked") public > QueryRunnerFactory findFactory(QueryType query) { - return (QueryRunnerFactory) factories.get(query.getClass()); + return factories.get(query.getClass()); + } + + @SuppressWarnings("unchecked") + @Override + public > QueryToolChest getToolChest(QueryType query) + { + return factories.get(query.getClass()).getToolchest(); } } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index ce452e9f23e4..2b87f1bdb89c 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -30,15 +30,12 @@ import org.apache.druid.query.Query; import org.apache.druid.query.QueryExecSomething; import org.apache.druid.query.QueryMetrics; -import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.aggregation.MetricManipulationFn; -import org.apache.druid.query.context.ResponseContext; import org.apache.druid.query.groupby.SupportRowSignature; -import org.apache.druid.query.operator.WindowOperatorQuery; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature.Finalization; @@ -53,23 +50,14 @@ public class UnionQueryQueryToolChest extends QueryToolChest makeQueryRunner(QueryToolChestWarehouse warehouse, Query query, QuerySegmentWalker clientQuerySegmentWalker) { - UnionQueryRunner runner = new UnionQueryRunner((UnionQuery) query, clientQuerySegmentWalker); - return runner; + return new UnionQueryRunner((UnionQuery) query, clientQuerySegmentWalker); } @Override @SuppressWarnings("unchecked") public QueryRunner mergeResults(QueryRunner runner) { - if (true) { - throw new UnsupportedOperationException("Not supported"); - } - - return new RealUnionResultSerializingQueryRunner( - (queryPlus, responseContext) -> { - return runner.run(queryPlus, responseContext); - } - ); + throw new UnsupportedOperationException("Not supported"); } @Override @@ -111,7 +99,6 @@ public Sequence resultsAsArrays( Sequence resultSequence) { List results = resultSequence.toList(); - List> resultSeqs = new ArrayList>(); for (int i = 0; i < results.size(); i++) { @@ -119,7 +106,6 @@ public Sequence resultsAsArrays( RealUnionResult realUnionResult = results.get(i); resultSeqs.add(resultsAsArrays(q, realUnionResult)); } - return Sequences.concat(resultSeqs); } @@ -130,13 +116,6 @@ private > Sequence resultsAsArrays(Query return toolChest.resultsAsArrays(q, realUnionResult.getResults()); } - @SuppressWarnings("unused") - private Sequence resultsAsArrays1(Query q, RealUnionResult realUnionResult) - { - warehouse.getToolChest(q); - return null; - } - @Override @SuppressWarnings({"unchecked", "rawtypes"}) public Optional> resultsAsFrames( @@ -147,61 +126,4 @@ public Optional> resultsAsFrames( { throw new UnsupportedOperationException("Not supported"); } - - /** - * This class exists to serialize the RealUnionResult that are used in this - * query and make it the return Sequence actually be a Sequence of rows or - * frames, as the query requires. This is relatively broken in a number of - * regards, the most obvious of which is that it is going to run counter to - * the stated class on the Generic of the QueryToolChest. That is, the code - * makes it look like you are getting a Sequence of RealUnionResult, but, by - * using this, the query will actually ultimately produce a Sequence of - * Object[] or Frames. This works because of type Erasure in Java (it's all - * Object at the end of the day). - *

- * While it might seem like this will break all sorts of things, the Generic - * type is actually there more as a type "hint" to make the writing of the - * ToolChest and Factory and stuff a bit more simple. Any caller of this - * cannot truly depend on the type anyway other than to just throw it across - * the wire, so this should just magically work even though it looks like it - * shouldn't even compile. - *

- * Not our proudest moment, but we use the tools available to us. - */ - @SuppressWarnings({"unchecked", "rawtypes"}) - private static class RealUnionResultSerializingQueryRunner implements QueryRunner - { - - private final QueryRunner baseQueryRunner; - - private RealUnionResultSerializingQueryRunner( - QueryRunner baseQueryRunner) - { - this.baseQueryRunner = baseQueryRunner; - } - - @Override - public Sequence run( - QueryPlus queryPlus, - ResponseContext responseContext) - { - throw new UnsupportedOperationException("Not supported"); - } - - /** - * Translates Sequence of RACs to a Sequence of Object[] - */ - private static Sequence asRows(final Sequence baseSequence, final WindowOperatorQuery query) - { - throw new UnsupportedOperationException("Not supported"); - } - - /** - * Translates a sequence of RACs to a Sequence of Frames - */ - private static Sequence asFrames(final Sequence baseSequence) - { - throw new UnsupportedOperationException("Not supported"); - } - } } From ae40a8ba8a8cfe9759389332a973777af343f445 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 16:35:57 +0000 Subject: [PATCH 055/149] fix --- .../apache/druid/query/QueryToolChest.java | 2 - ...estUnionAllTwoQueriesBothQueriesAreJoin.iq | 153 +----------------- 2 files changed, 7 insertions(+), 148 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java index f8b029806acc..1cc48a7be743 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java @@ -24,7 +24,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.TypeFactory; import com.google.common.base.Function; -import com.google.inject.Inject; import org.apache.druid.error.DruidException; import org.apache.druid.frame.allocation.MemoryAllocatorFactory; import org.apache.druid.guice.annotations.ExtensionPoint; @@ -425,7 +424,6 @@ public Optional> resultsAsFrames( return Optional.empty(); } - @Inject public void setWarehouse(QueryToolChestWarehouse warehouse) { this.warehouse = warehouse; diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq index 4c692c3a6bda..65b1fac2fad7 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq @@ -9,152 +9,13 @@ !set outputformat mysql !use druidtest:/// (SELECT COUNT(*) FROM foo LEFT JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; -org.apache.calcite.avatica.AvaticaSqlException: Error -1 (00000) : Error while executing SQL "(SELECT COUNT(*) FROM foo LEFT JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ": Remote driver error: QueryInterruptedException: (null exception message) -> NullPointerException: (null exception message) - at org.apache.calcite.avatica.Helper.createException(Helper.java:54) - at org.apache.calcite.avatica.Helper.createException(Helper.java:41) - at org.apache.calcite.avatica.AvaticaStatement.executeInternal(AvaticaStatement.java:164) - at org.apache.calcite.avatica.AvaticaStatement.executeQuery(AvaticaStatement.java:228) - at net.hydromatic.quidem.Quidem.checkResult(Quidem.java:299) - at net.hydromatic.quidem.Quidem.access$2600(Quidem.java:57) - at net.hydromatic.quidem.Quidem$ContextImpl.checkResult(Quidem.java:1636) - at net.hydromatic.quidem.Quidem$CheckResultCommand.execute(Quidem.java:941) - at net.hydromatic.quidem.Quidem$CompositeCommand.execute(Quidem.java:1437) - at net.hydromatic.quidem.Quidem.execute(Quidem.java:190) - at org.apache.druid.quidem.DruidQuidemTestBase$DruidQuidemRunner.run(DruidQuidemTestBase.java:158) - at org.apache.druid.quidem.DruidQuidemTestBase$DruidQuidemRunner.run(DruidQuidemTestBase.java:138) - at org.apache.druid.sql.calcite.QTestCase$1.run(QTestCase.java:73) - at org.apache.druid.sql.calcite.QueryTestRunner.run(QueryTestRunner.java:766) - at org.apache.druid.sql.calcite.QueryTestBuilder.run(QueryTestBuilder.java:304) - at org.apache.druid.sql.calcite.BaseCalciteQueryTest.testQuery(BaseCalciteQueryTest.java:733) - at org.apache.druid.sql.calcite.CalciteJoinQueryTest.testUnionAllTwoQueriesBothQueriesAreJoin(CalciteJoinQueryTest.java:4427) - at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.base/java.lang.reflect.Method.invoke(Method.java:566) - at org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:728) - at org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60) - at org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131) - at org.apache.druid.sql.calcite.NotYetSupported$NotYetSupportedProcessor.interceptTestMethod(NotYetSupported.java:134) - at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(InterceptingExecutableInvoker.java:103) - at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.lambda$invoke$0(InterceptingExecutableInvoker.java:93) - at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106) - at org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:156) - at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:147) - at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:86) - at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(InterceptingExecutableInvoker.java:103) - at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.lambda$invoke$0(InterceptingExecutableInvoker.java:93) - at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106) - at org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64) - at org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45) - at org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37) - at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:92) - at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:86) - at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$7(TestMethodTestDescriptor.java:218) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:214) - at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:139) - at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:69) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:151) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) - at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) - at java.base/java.util.ArrayList.forEach(ArrayList.java:1541) - at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:155) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) - at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) - at java.base/java.util.ArrayList.forEach(ArrayList.java:1541) - at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:155) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) - at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) - at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:35) - at org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57) - at org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:54) - at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:147) - at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:127) - at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:90) - at org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:55) - at org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:102) - at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:54) - at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:114) - at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:95) - at org.junit.platform.launcher.core.DefaultLauncherSession$DelegatingLauncher.execute(DefaultLauncherSession.java:91) - at org.junit.platform.launcher.core.SessionPerRequestLauncher.execute(SessionPerRequestLauncher.java:60) - at org.eclipse.jdt.internal.junit5.runner.JUnit5TestReference.run(JUnit5TestReference.java:98) - at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:40) - at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:529) - at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:756) - at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:452) - at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:210) -QueryInterruptedException{msg=null, code=Unknown exception, class=java.lang.NullPointerException, host=null} - at org.apache.druid.query.QueryInterruptedException.wrapIfNeeded(QueryInterruptedException.java:113) - at org.apache.druid.sql.avatica.ErrorHandler.sanitize(ErrorHandler.java:82) - at org.apache.druid.sql.avatica.DruidMeta.mapException(DruidMeta.java:395) - at org.apache.druid.sql.avatica.DruidMeta.prepareAndExecute(DruidMeta.java:362) - at org.apache.calcite.avatica.remote.LocalService.apply(LocalService.java:212) - at org.apache.calcite.avatica.remote.Service$PrepareAndExecuteRequest.accept(Service.java:952) - at org.apache.calcite.avatica.remote.Service$PrepareAndExecuteRequest.accept(Service.java:904) - at org.apache.calcite.avatica.remote.AbstractHandler.apply(AbstractHandler.java:94) - at org.apache.calcite.avatica.remote.JsonHandler.apply(JsonHandler.java:52) - at org.apache.calcite.avatica.server.AvaticaJsonHandler.handle(AvaticaJsonHandler.java:133) - at org.apache.druid.sql.avatica.DruidAvaticaJsonHandler.handle(DruidAvaticaJsonHandler.java:61) - at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) - at org.eclipse.jetty.server.Server.handle(Server.java:516) - at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:487) - at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:732) - at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:479) - at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) - at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) - at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) - at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) - at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) - at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) - at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) - at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) - at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) - at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) - at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) - at java.base/java.lang.Thread.run(Thread.java:829) -Caused by: java.lang.NullPointerException - at org.apache.druid.query.union.UnionQueryQueryToolChest.resultsAsArrays(UnionQueryQueryToolChest.java:129) - at org.apache.druid.query.union.UnionQueryQueryToolChest.resultsAsArrays(UnionQueryQueryToolChest.java:120) - at org.apache.druid.query.union.UnionQueryQueryToolChest.resultsAsArrays(UnionQueryQueryToolChest.java:1) - at org.apache.druid.server.ClientQuerySegmentWalker.materializeResultsAsArray(ClientQuerySegmentWalker.java:893) - at org.apache.druid.server.ClientQuerySegmentWalker.toInlineDataSource(ClientQuerySegmentWalker.java:725) - at org.apache.druid.server.ClientQuerySegmentWalker.inlineIfNecessary(ClientQuerySegmentWalker.java:469) - at org.apache.druid.server.ClientQuerySegmentWalker.getQueryRunnerForIntervals(ClientQuerySegmentWalker.java:232) - at org.apache.druid.server.SpecificSegmentsQuerySegmentWalker.getQueryRunnerForIntervals(SpecificSegmentsQuerySegmentWalker.java:223) - at org.apache.druid.query.spec.MultipleIntervalSegmentSpec.lookup(MultipleIntervalSegmentSpec.java:57) - at org.apache.druid.query.BaseQuery.getRunner(BaseQuery.java:107) - at org.apache.druid.query.QueryPlus.run(QueryPlus.java:158) - at org.apache.druid.server.QueryLifecycle.execute(QueryLifecycle.java:315) - at org.apache.druid.server.QueryLifecycle.runSimple(QueryLifecycle.java:162) - at org.apache.druid.sql.calcite.run.NativeQueryMaker.execute(NativeQueryMaker.java:195) - at org.apache.druid.sql.calcite.run.NativeQueryMaker.runQuery(NativeQueryMaker.java:152) - at org.apache.druid.sql.calcite.planner.QueryHandler.lambda$7(QueryHandler.java:568) - at org.apache.druid.sql.calcite.planner.PlannerResult.run(PlannerResult.java:62) - at org.apache.druid.sql.DirectStatement$ResultSet.run(DirectStatement.java:109) - at org.apache.druid.sql.DirectStatement.execute(DirectStatement.java:184) - at org.apache.druid.sql.avatica.DruidJdbcResultSet.lambda$0(DruidJdbcResultSet.java:239) - at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) - at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) - at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) - ... 1 more ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) !ok LogicalUnion(all=[true]) From ab03132cc3afb6005966cbf56d14bc6c0485786b Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 16:55:32 +0000 Subject: [PATCH 056/149] add Named --- .../apache/druid/sql/calcite/util/SqlTestFramework.java | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 71e163ea8c0e..01a3a935606d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -567,6 +567,7 @@ public SqlStatementFactory statementFactory() */ private class TestSetupModule implements DruidModule { + private static final String SQL_TEST_FRAME_WORK = "sqlTestFrameWork"; private final Builder builder; private final List subModules = Arrays.asList(new BuiltInTypesModule(), new TestSqlModule()); @@ -596,6 +597,10 @@ public void configure(Binder binder) binder.bind(DefaultColumnFormatConfig.class).toInstance(new DefaultColumnFormatConfig(null, null)); } + /* + * Ideally this should not have a Named annotation, but it clashes with {@link DruidProcessingModule}. + */ + @Named(SQL_TEST_FRAME_WORK) @Provides @LazySingleton public DruidProcessingConfig makeProcessingConfig() @@ -612,7 +617,7 @@ public TestBufferPool makeTestBufferPool() @Provides @LazySingleton - public TestGroupByBuffers makeTestGroupByBuffers(DruidProcessingConfig processingConfig) + public TestGroupByBuffers makeTestGroupByBuffers(@Named(SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig) { return QueryStackTests.makeGroupByBuffers(resourceCloser, processingConfig); } @@ -620,7 +625,7 @@ public TestGroupByBuffers makeTestGroupByBuffers(DruidProcessingConfig processin @Provides @LazySingleton public QueryRunnerFactoryConglomerate conglomerate( - DruidProcessingConfig processingConfig, + @Named(SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig, TestBufferPool testBufferPool, TestGroupByBuffers groupByBuffers) { From bba4f5720159db14d091406b7aacabaad2c9924d Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 11 Oct 2024 16:57:56 +0000 Subject: [PATCH 057/149] move class --- .../druid/query/ConglomerateBackedQueryToolChestWarehouse.java | 0 1 file changed, 0 insertions(+), 0 deletions(-) rename {processing/src/main => server/src/test}/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java (100%) diff --git a/processing/src/main/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java b/server/src/test/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java similarity index 100% rename from processing/src/main/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java rename to server/src/test/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java From 95aa74685da4b487210fbad3df514d2c2d9a367a Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Mon, 14 Oct 2024 07:37:56 +0000 Subject: [PATCH 058/149] enabling access to override factories --- .../apache/druid/server/QueryStackTests.java | 120 ++++++++++-------- .../sql/calcite/util/SqlTestFramework.java | 15 ++- 2 files changed, 73 insertions(+), 62 deletions(-) diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index fb964af16f2d..9414f4a4af47 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -298,68 +298,76 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat final Integer minTopNThreshold, final ObjectMapper jsonMapper, final TestBufferPool testBufferPool, - final TestGroupByBuffers groupByBuffers - ) + final TestGroupByBuffers groupByBuffers) { - final GroupByQueryRunnerFactory groupByQueryRunnerFactory = - GroupByQueryRunnerTest.makeQueryRunnerFactory( - jsonMapper, - new GroupByQueryConfig() - { - }, - groupByBuffers, - processingConfig - ); - final QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate( - ImmutableMap., QueryRunnerFactory>builder() - .put( - SegmentMetadataQuery.class, - new SegmentMetadataQueryRunnerFactory( - new SegmentMetadataQueryQueryToolChest( - new SegmentMetadataQueryConfig("P1W") - ), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ) + makeDefaultQueryRunnerFactories(processingConfig, minTopNThreshold, jsonMapper, testBufferPool, groupByBuffers) + ); + return conglomerate; + } + + @SuppressWarnings("rawtypes") + public static ImmutableMap, QueryRunnerFactory> makeDefaultQueryRunnerFactories( + final DruidProcessingConfig processingConfig, + final Integer minTopNThreshold, + final ObjectMapper jsonMapper, + final TestBufferPool testBufferPool, + final TestGroupByBuffers groupByBuffers) + { + final GroupByQueryRunnerFactory groupByQueryRunnerFactory = GroupByQueryRunnerTest.makeQueryRunnerFactory( + jsonMapper, + new GroupByQueryConfig() + { + }, + groupByBuffers, + processingConfig + ); + + return ImmutableMap., QueryRunnerFactory>builder() + .put( + SegmentMetadataQuery.class, + new SegmentMetadataQueryRunnerFactory( + new SegmentMetadataQueryQueryToolChest( + new SegmentMetadataQueryConfig("P1W") + ), + QueryRunnerTestHelper.NOOP_QUERYWATCHER ) - .put( - ScanQuery.class, - new ScanQueryRunnerFactory( - new ScanQueryQueryToolChest(DefaultGenericQueryMetricsFactory.instance()), - new ScanQueryEngine(), - new ScanQueryConfig() - ) + ) + .put( + ScanQuery.class, + new ScanQueryRunnerFactory( + new ScanQueryQueryToolChest(DefaultGenericQueryMetricsFactory.instance()), + new ScanQueryEngine(), + new ScanQueryConfig() ) - .put( - TimeseriesQuery.class, - new TimeseriesQueryRunnerFactory( - new TimeseriesQueryQueryToolChest(), - new TimeseriesQueryEngine(), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ) + ) + .put( + TimeseriesQuery.class, + new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER ) - .put( - TopNQuery.class, - new TopNQueryRunnerFactory( - testBufferPool, - new TopNQueryQueryToolChest(new TopNQueryConfig() - { - @Override - public int getMinTopNThreshold() - { - return minTopNThreshold; - } - }), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ) + ) + .put( + TopNQuery.class, + new TopNQueryRunnerFactory( + testBufferPool, + new TopNQueryQueryToolChest(new TopNQueryConfig() + { + @Override + public int getMinTopNThreshold() + { + return minTopNThreshold; + } + }), + QueryRunnerTestHelper.NOOP_QUERYWATCHER ) - .put(GroupByQuery.class, groupByQueryRunnerFactory) - .put(TimeBoundaryQuery.class, new TimeBoundaryQueryRunnerFactory(QueryRunnerTestHelper.NOOP_QUERYWATCHER)) - .put(WindowOperatorQuery.class, new WindowOperatorQueryQueryRunnerFactory()) - .build() - ); - - return conglomerate; + ) + .put(GroupByQuery.class, groupByQueryRunnerFactory) + .put(TimeBoundaryQuery.class, new TimeBoundaryQueryRunnerFactory(QueryRunnerTestHelper.NOOP_QUERYWATCHER)) + .put(WindowOperatorQuery.class, new WindowOperatorQueryQueryRunnerFactory()) + .build(); } public static JoinableFactory makeJoinableFactoryForLookup( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 01a3a935606d..26bb026b4a5d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -39,6 +39,7 @@ import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate; import org.apache.druid.query.DruidProcessingConfig; import org.apache.druid.query.GlobalTableDataSource; import org.apache.druid.query.QueryRunnerFactoryConglomerate; @@ -275,12 +276,14 @@ public QueryRunnerFactoryConglomerate createCongolmerate( DruidProcessingConfig processingConfig ) { - return QueryStackTests.createQueryRunnerFactoryConglomerate( - processingConfig, - builder.minTopNThreshold, - jsonMapper, - testBufferPool, - groupByBuffers + return new DefaultQueryRunnerFactoryConglomerate( + QueryStackTests.makeDefaultQueryRunnerFactories( + processingConfig, + builder.minTopNThreshold, + jsonMapper, + testBufferPool, + groupByBuffers + ) ); } From d4055ac0acb7a962927b8102b33c670c47ecc17f Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Mon, 14 Oct 2024 13:50:45 +0000 Subject: [PATCH 059/149] dead end --- .../druid/java/util/common/io/Closer.java | 2 +- .../apache/druid/server/QueryStackTests.java | 64 +++++++++++++++ .../sql/calcite/util/SqlTestFramework.java | 79 +++++++++++++------ 3 files changed, 120 insertions(+), 25 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/java/util/common/io/Closer.java b/processing/src/main/java/org/apache/druid/java/util/common/io/Closer.java index a366f6c3fa26..d916aa12a7ce 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/io/Closer.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/io/Closer.java @@ -104,7 +104,7 @@ public static Closer create() private final Deque stack = new ArrayDeque<>(4); private Throwable thrown; - private Closer() + public Closer() { } diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index 9414f4a4af47..a1fe8f292f22 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -22,9 +22,14 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.inject.Binder; +import com.google.inject.Inject; import com.google.inject.Injector; +import com.google.inject.Module; +import com.google.inject.multibindings.MapBinder; import org.apache.druid.client.cache.Cache; import org.apache.druid.client.cache.CacheConfig; +import org.apache.druid.guice.DruidBinders; import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.guice.ExpressionModule; import org.apache.druid.guice.SegmentWranglerModule; @@ -306,6 +311,65 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat return conglomerate; } + public static class DefaultRunnersModule implements Module { + + @Inject + public + DruidProcessingConfig processingConfig; +//// @Inject +//// public Integer minTopNThreshold; + @Inject + public ObjectMapper jsonMapper; + @Inject + public TestBufferPool testBufferPool; +// @Inject +// public TestGroupByBuffers groupByBuffers; + +// @Inject +// private void asd(TestBufferPool aa) +// { +// if (true) { +// throw new RuntimeException("FIXME: Unimplemented!"); +// } +// +// } + + @Override + public void configure(Binder binder) + { + MapBinder, QueryRunnerFactory> runnerFactoryBinder = DruidBinders.queryRunnerFactoryBinder(binder); + runnerFactoryBinder.addBinding(ScanQuery.class).toProvider(this::asd); +// runnerFactoryBinder.addBinding(SegmentMetadataQuery.class).to(SegmentMetadataQueryRunnerFactory.class); + } + + + public ScanQueryRunnerFactory asd() + { + return new ScanQueryRunnerFactory( + new ScanQueryQueryToolChest(DefaultGenericQueryMetricsFactory.instance()), + new ScanQueryEngine(), + new ScanQueryConfig() + ); + } + +// @Provides +// public GroupByQueryRunnerFactory makeGroupByQueryRunnerFactory ( +// final DruidProcessingConfig processingConfig, +// final Integer minTopNThreshold, +// final ObjectMapper jsonMapper, +// final TestBufferPool testBufferPool, +// final TestGroupByBuffers groupByBuffers, +// final Binder binder) +// { +// ImmutableMap, QueryRunnerFactory> defaultFactories = makeDefaultQueryRunnerFactories(processingConfig, minTopNThreshold, jsonMapper, testBufferPool, groupByBuffers); +// +// return null; +// } +// + + + } + @SuppressWarnings("rawtypes") public static ImmutableMap, QueryRunnerFactory> makeDefaultQueryRunnerFactories( final DruidProcessingConfig processingConfig, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 26bb026b4a5d..3b564ce83278 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -24,6 +24,7 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.inject.Binder; +import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.Provides; @@ -264,6 +265,9 @@ public void gatherProperties(Properties properties) @Override public void configureGuice(DruidInjectorBuilder builder) { + builder.addClass(SubTestModule.class); + + builder.addClass(QueryStackTests.DefaultRunnersModule.class); } @Override @@ -555,6 +559,53 @@ public SqlStatementFactory statementFactory() } } + private static final String SQL_TEST_FRAME_WORK = "sqlTestFrameWork"; + + public static class SubTestModule implements Module + { + private final int mergeBufferCount; + private final Closer resourceCloser; + + @Inject + public SubTestModule( + + ) + { + this.resourceCloser = new Closer(); + this.mergeBufferCount = 11; + } + /* + * Ideally this should not have a Named annotation, but it clashes with + * {@link DruidProcessingModule}. + */ + @Named(SQL_TEST_FRAME_WORK) + @Provides + @LazySingleton + public DruidProcessingConfig makeProcessingConfig() + { + return QueryStackTests.getProcessingConfig(mergeBufferCount); + } + + @Provides + @LazySingleton + public TestBufferPool makeTestBufferPool() + { + return QueryStackTests.makeTestBufferPool(resourceCloser); + } + + @Provides + @LazySingleton + public TestGroupByBuffers makeTestGroupByBuffers( + @Named(SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig) + { + return QueryStackTests.makeGroupByBuffers(resourceCloser, processingConfig); + } + + @Override + public void configure(Binder binder) + { + } + } /** * Guice module to create the various query framework items. By creating items within * a module, later items can depend on those created earlier by grabbing them from the @@ -570,7 +621,6 @@ public SqlStatementFactory statementFactory() */ private class TestSetupModule implements DruidModule { - private static final String SQL_TEST_FRAME_WORK = "sqlTestFrameWork"; private final Builder builder; private final List subModules = Arrays.asList(new BuiltInTypesModule(), new TestSqlModule()); @@ -598,31 +648,12 @@ public void configure(Binder binder) binder.bind(DruidOperatorTable.class).in(LazySingleton.class); binder.bind(DataSegment.PruneSpecsHolder.class).toInstance(DataSegment.PruneSpecsHolder.DEFAULT); binder.bind(DefaultColumnFormatConfig.class).toInstance(new DefaultColumnFormatConfig(null, null)); - } - /* - * Ideally this should not have a Named annotation, but it clashes with {@link DruidProcessingModule}. - */ - @Named(SQL_TEST_FRAME_WORK) - @Provides - @LazySingleton - public DruidProcessingConfig makeProcessingConfig() - { - return QueryStackTests.getProcessingConfig(builder.mergeBufferCount); - } +// binder.install(new SubTestModule(resourceCloser, builder.mergeBufferCount)); +// QueryStackTests.DefaultRunnersModule module = new QueryStackTests.DefaultRunnersModule(); +// binder.requestInjection(module); +// binder.install(module); - @Provides - @LazySingleton - public TestBufferPool makeTestBufferPool() - { - return QueryStackTests.makeTestBufferPool(resourceCloser); - } - - @Provides - @LazySingleton - public TestGroupByBuffers makeTestGroupByBuffers(@Named(SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig) - { - return QueryStackTests.makeGroupByBuffers(resourceCloser, processingConfig); } @Provides From 87867073eb1ecd319cbab60f7b53319b9de0cd79 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Mon, 14 Oct 2024 13:50:47 +0000 Subject: [PATCH 060/149] Revert "dead end" This reverts commit d4055ac0acb7a962927b8102b33c670c47ecc17f. --- .../druid/java/util/common/io/Closer.java | 2 +- .../apache/druid/server/QueryStackTests.java | 64 --------------- .../sql/calcite/util/SqlTestFramework.java | 79 ++++++------------- 3 files changed, 25 insertions(+), 120 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/java/util/common/io/Closer.java b/processing/src/main/java/org/apache/druid/java/util/common/io/Closer.java index d916aa12a7ce..a366f6c3fa26 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/io/Closer.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/io/Closer.java @@ -104,7 +104,7 @@ public static Closer create() private final Deque stack = new ArrayDeque<>(4); private Throwable thrown; - public Closer() + private Closer() { } diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index a1fe8f292f22..9414f4a4af47 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -22,14 +22,9 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; -import com.google.inject.Binder; -import com.google.inject.Inject; import com.google.inject.Injector; -import com.google.inject.Module; -import com.google.inject.multibindings.MapBinder; import org.apache.druid.client.cache.Cache; import org.apache.druid.client.cache.CacheConfig; -import org.apache.druid.guice.DruidBinders; import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.guice.ExpressionModule; import org.apache.druid.guice.SegmentWranglerModule; @@ -311,65 +306,6 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat return conglomerate; } - public static class DefaultRunnersModule implements Module { - - @Inject - public - DruidProcessingConfig processingConfig; -//// @Inject -//// public Integer minTopNThreshold; - @Inject - public ObjectMapper jsonMapper; - @Inject - public TestBufferPool testBufferPool; -// @Inject -// public TestGroupByBuffers groupByBuffers; - -// @Inject -// private void asd(TestBufferPool aa) -// { -// if (true) { -// throw new RuntimeException("FIXME: Unimplemented!"); -// } -// -// } - - @Override - public void configure(Binder binder) - { - MapBinder, QueryRunnerFactory> runnerFactoryBinder = DruidBinders.queryRunnerFactoryBinder(binder); - runnerFactoryBinder.addBinding(ScanQuery.class).toProvider(this::asd); -// runnerFactoryBinder.addBinding(SegmentMetadataQuery.class).to(SegmentMetadataQueryRunnerFactory.class); - } - - - public ScanQueryRunnerFactory asd() - { - return new ScanQueryRunnerFactory( - new ScanQueryQueryToolChest(DefaultGenericQueryMetricsFactory.instance()), - new ScanQueryEngine(), - new ScanQueryConfig() - ); - } - -// @Provides -// public GroupByQueryRunnerFactory makeGroupByQueryRunnerFactory ( -// final DruidProcessingConfig processingConfig, -// final Integer minTopNThreshold, -// final ObjectMapper jsonMapper, -// final TestBufferPool testBufferPool, -// final TestGroupByBuffers groupByBuffers, -// final Binder binder) -// { -// ImmutableMap, QueryRunnerFactory> defaultFactories = makeDefaultQueryRunnerFactories(processingConfig, minTopNThreshold, jsonMapper, testBufferPool, groupByBuffers); -// -// return null; -// } -// - - - } - @SuppressWarnings("rawtypes") public static ImmutableMap, QueryRunnerFactory> makeDefaultQueryRunnerFactories( final DruidProcessingConfig processingConfig, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 3b564ce83278..26bb026b4a5d 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -24,7 +24,6 @@ import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; import com.google.inject.Binder; -import com.google.inject.Inject; import com.google.inject.Injector; import com.google.inject.Module; import com.google.inject.Provides; @@ -265,9 +264,6 @@ public void gatherProperties(Properties properties) @Override public void configureGuice(DruidInjectorBuilder builder) { - builder.addClass(SubTestModule.class); - - builder.addClass(QueryStackTests.DefaultRunnersModule.class); } @Override @@ -559,53 +555,6 @@ public SqlStatementFactory statementFactory() } } - private static final String SQL_TEST_FRAME_WORK = "sqlTestFrameWork"; - - public static class SubTestModule implements Module - { - private final int mergeBufferCount; - private final Closer resourceCloser; - - @Inject - public SubTestModule( - - ) - { - this.resourceCloser = new Closer(); - this.mergeBufferCount = 11; - } - /* - * Ideally this should not have a Named annotation, but it clashes with - * {@link DruidProcessingModule}. - */ - @Named(SQL_TEST_FRAME_WORK) - @Provides - @LazySingleton - public DruidProcessingConfig makeProcessingConfig() - { - return QueryStackTests.getProcessingConfig(mergeBufferCount); - } - - @Provides - @LazySingleton - public TestBufferPool makeTestBufferPool() - { - return QueryStackTests.makeTestBufferPool(resourceCloser); - } - - @Provides - @LazySingleton - public TestGroupByBuffers makeTestGroupByBuffers( - @Named(SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig) - { - return QueryStackTests.makeGroupByBuffers(resourceCloser, processingConfig); - } - - @Override - public void configure(Binder binder) - { - } - } /** * Guice module to create the various query framework items. By creating items within * a module, later items can depend on those created earlier by grabbing them from the @@ -621,6 +570,7 @@ public void configure(Binder binder) */ private class TestSetupModule implements DruidModule { + private static final String SQL_TEST_FRAME_WORK = "sqlTestFrameWork"; private final Builder builder; private final List subModules = Arrays.asList(new BuiltInTypesModule(), new TestSqlModule()); @@ -648,12 +598,31 @@ public void configure(Binder binder) binder.bind(DruidOperatorTable.class).in(LazySingleton.class); binder.bind(DataSegment.PruneSpecsHolder.class).toInstance(DataSegment.PruneSpecsHolder.DEFAULT); binder.bind(DefaultColumnFormatConfig.class).toInstance(new DefaultColumnFormatConfig(null, null)); + } -// binder.install(new SubTestModule(resourceCloser, builder.mergeBufferCount)); -// QueryStackTests.DefaultRunnersModule module = new QueryStackTests.DefaultRunnersModule(); -// binder.requestInjection(module); -// binder.install(module); + /* + * Ideally this should not have a Named annotation, but it clashes with {@link DruidProcessingModule}. + */ + @Named(SQL_TEST_FRAME_WORK) + @Provides + @LazySingleton + public DruidProcessingConfig makeProcessingConfig() + { + return QueryStackTests.getProcessingConfig(builder.mergeBufferCount); + } + @Provides + @LazySingleton + public TestBufferPool makeTestBufferPool() + { + return QueryStackTests.makeTestBufferPool(resourceCloser); + } + + @Provides + @LazySingleton + public TestGroupByBuffers makeTestGroupByBuffers(@Named(SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig) + { + return QueryStackTests.makeGroupByBuffers(resourceCloser, processingConfig); } @Provides From 00129b25e2d1ea0eff07893938de20c9dd774b5a Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Mon, 14 Oct 2024 14:00:35 +0000 Subject: [PATCH 061/149] this may actually work --- .../sql/calcite/util/SqlTestFramework.java | 61 +++++++++++++++---- 1 file changed, 50 insertions(+), 11 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 26bb026b4a5d..b33f6a50b27e 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -42,6 +42,8 @@ import org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate; import org.apache.druid.query.DruidProcessingConfig; import org.apache.druid.query.GlobalTableDataSource; +import org.apache.druid.query.Query; +import org.apache.druid.query.QueryRunnerFactory; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.TestBufferPool; @@ -86,6 +88,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; @@ -264,8 +267,37 @@ public void gatherProperties(Properties properties) @Override public void configureGuice(DruidInjectorBuilder builder) { + builder.addModule(new x()); } + class x implements Module + { + + + + @Provides + public @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> createCongolmerate( + ObjectMapper jsonMapper, + final TestBufferPool testBufferPool, + final TestGroupByBuffers groupByBuffers, + @Named(SqlTestFramework.SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig) + { + return QueryStackTests.makeDefaultQueryRunnerFactories( + processingConfig, + 111, + jsonMapper, + testBufferPool, + groupByBuffers + ); + } + + @Override + public void configure(Binder binder) + { + } + + } + @Override public QueryRunnerFactoryConglomerate createCongolmerate( Builder builder, @@ -555,6 +587,7 @@ public SqlStatementFactory statementFactory() } } + public static final String SQL_TEST_FRAME_WORK = "sqlTestFrameWork"; /** * Guice module to create the various query framework items. By creating items within * a module, later items can depend on those created earlier by grabbing them from the @@ -570,7 +603,6 @@ public SqlStatementFactory statementFactory() */ private class TestSetupModule implements DruidModule { - private static final String SQL_TEST_FRAME_WORK = "sqlTestFrameWork"; private final Builder builder; private final List subModules = Arrays.asList(new BuiltInTypesModule(), new TestSqlModule()); @@ -630,16 +662,23 @@ public TestGroupByBuffers makeTestGroupByBuffers(@Named(SQL_TEST_FRAME_WORK) Dru public QueryRunnerFactoryConglomerate conglomerate( @Named(SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig, TestBufferPool testBufferPool, - TestGroupByBuffers groupByBuffers) - { - return componentSupplier.createCongolmerate( - builder, - resourceCloser, - queryJsonMapper(), - testBufferPool, - groupByBuffers, - processingConfig - ); + TestGroupByBuffers groupByBuffers, + @Named(SQL_TEST_FRAME_WORK) + Map, QueryRunnerFactory> fact + ) + { + return new DefaultQueryRunnerFactoryConglomerate( + fact + ); + +// return componentSupplier.createCongolmerate( +// builder, +// resourceCloser, +// queryJsonMapper(), +// testBufferPool, +// groupByBuffers, +// processingConfig +// ); } @Provides From b5063db6056834d3117e76a327348f71cdcac722 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Mon, 14 Oct 2024 18:22:14 +0000 Subject: [PATCH 062/149] simpler --- ...AsBrokerQueryComponentSupplierWrapper.java | 4 ++-- .../druid/quidem/DruidAvaticaTestDriver.java | 4 ++-- .../sql/calcite/util/SqlTestFramework.java | 24 +++++++++++++++---- 3 files changed, 23 insertions(+), 9 deletions(-) diff --git a/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java b/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java index 62f9e6109fe4..d6fde885944e 100644 --- a/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java +++ b/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java @@ -158,12 +158,12 @@ public void configureGuice(CoreInjectorBuilder builder, List overrideMod } @Override - public QueryRunnerFactoryConglomerate createCongolmerate(Builder builder, Closer resourceCloser, + public QueryRunnerFactoryConglomerate createCongolmerate1(Builder builder, Closer resourceCloser, ObjectMapper jsonMapper, TestBufferPool testBufferPool, TestGroupByBuffers groupByBuffers, DruidProcessingConfig processingConfig) { return delegate - .createCongolmerate(builder, resourceCloser, jsonMapper, testBufferPool, groupByBuffers, processingConfig); + .createCongolmerate1(builder, resourceCloser, jsonMapper, testBufferPool, groupByBuffers, processingConfig); } @Override public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(QueryRunnerFactoryConglomerate conglomerate, diff --git a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java index 149ab47ff22e..5244b21b0d88 100644 --- a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java +++ b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java @@ -241,12 +241,12 @@ public void configureGuice(DruidInjectorBuilder builder) } @Override - public QueryRunnerFactoryConglomerate createCongolmerate(Builder builder, Closer resourceCloser, + public QueryRunnerFactoryConglomerate createCongolmerate1(Builder builder, Closer resourceCloser, ObjectMapper jsonMapper, TestBufferPool testBufferPool, TestGroupByBuffers groupByBuffers, DruidProcessingConfig processingConfig) { return delegate - .createCongolmerate(builder, resourceCloser, jsonMapper, testBufferPool, groupByBuffers, processingConfig); + .createCongolmerate1(builder, resourceCloser, jsonMapper, testBufferPool, groupByBuffers, processingConfig); } @Override diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index b33f6a50b27e..2aaa7cab8ecb 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -161,7 +161,7 @@ public interface QueryComponentSupplier extends Closeable */ void configureGuice(DruidInjectorBuilder builder); - QueryRunnerFactoryConglomerate createCongolmerate( + QueryRunnerFactoryConglomerate createCongolmerate1( Builder builder, Closer resourceCloser, ObjectMapper jsonMapper, @@ -267,14 +267,11 @@ public void gatherProperties(Properties properties) @Override public void configureGuice(DruidInjectorBuilder builder) { - builder.addModule(new x()); } class x implements Module { - - @Provides public @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> createCongolmerate( ObjectMapper jsonMapper, @@ -299,7 +296,7 @@ public void configure(Binder binder) } @Override - public QueryRunnerFactoryConglomerate createCongolmerate( + public QueryRunnerFactoryConglomerate createCongolmerate1( Builder builder, Closer resourceCloser, ObjectMapper jsonMapper, @@ -632,6 +629,23 @@ public void configure(Binder binder) binder.bind(DefaultColumnFormatConfig.class).toInstance(new DefaultColumnFormatConfig(null, null)); } + + @Provides + public @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> make1( + ObjectMapper jsonMapper, + final TestBufferPool testBufferPool, + final TestGroupByBuffers groupByBuffers, + @Named(SqlTestFramework.SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig) + { + return QueryStackTests.makeDefaultQueryRunnerFactories( + processingConfig, + builder.mergeBufferCount, + jsonMapper, + testBufferPool, + groupByBuffers + ); + } + /* * Ideally this should not have a Named annotation, but it clashes with {@link DruidProcessingModule}. */ From 84545f591de295e7711c45ec022f606e92237d37 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 10:59:17 +0000 Subject: [PATCH 063/149] cleanup via abstract delegate --- ...AsBrokerQueryComponentSupplierWrapper.java | 94 +------------------ .../druid/quidem/DruidAvaticaTestDriver.java | 79 +--------------- .../sql/calcite/util/SqlTestFramework.java | 89 ++++++++++++++++++ 3 files changed, 98 insertions(+), 164 deletions(-) diff --git a/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java b/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java index d6fde885944e..d1fa6a349ba4 100644 --- a/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java +++ b/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java @@ -19,11 +19,9 @@ package org.apache.druid.quidem; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.inject.AbstractModule; -import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.Provides; @@ -44,7 +42,6 @@ import org.apache.druid.guice.BrokerProcessingModule; import org.apache.druid.guice.BrokerServiceModule; import org.apache.druid.guice.CoordinatorDiscoveryModule; -import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.guice.ExpressionModule; import org.apache.druid.guice.ExtensionsModule; import org.apache.druid.guice.JacksonConfigManagerModule; @@ -73,23 +70,14 @@ import org.apache.druid.initialization.Log4jShutterDownerModule; import org.apache.druid.initialization.ServerInjectorBuilder; import org.apache.druid.initialization.TombstoneDataStorageModule; -import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.metadata.storage.derby.DerbyMetadataStorageDruidModule; -import org.apache.druid.query.DruidProcessingConfig; -import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.RetryQueryRunnerConfig; -import org.apache.druid.query.TestBufferPool; -import org.apache.druid.query.groupby.TestGroupByBuffers; -import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.rpc.guice.ServiceClientModule; -import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.writeout.SegmentWriteOutMediumModule; import org.apache.druid.server.BrokerQueryResource; import org.apache.druid.server.ClientInfoResource; import org.apache.druid.server.DruidNode; -import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.ResponseContextConfig; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.SubqueryGuardrailHelper; import org.apache.druid.server.SubqueryGuardrailHelperProvider; import org.apache.druid.server.coordination.ServerType; @@ -104,50 +92,33 @@ import org.apache.druid.server.metrics.SubqueryCountStatsProvider; import org.apache.druid.server.router.TieredBrokerConfig; import org.apache.druid.server.security.TLSCertificateCheckerModule; -import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCache; import org.apache.druid.sql.calcite.schema.DruidSchemaName; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SqlTestFramework; -import org.apache.druid.sql.calcite.util.SqlTestFramework.Builder; -import org.apache.druid.sql.calcite.util.SqlTestFramework.PlannerComponentSupplier; import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplier; +import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplierDelegate; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.storage.StorageConnectorModule; import org.apache.druid.timeline.PruneLoadSpec; import org.eclipse.jetty.server.Server; -import java.io.IOException; import java.util.List; import java.util.Properties; /** * A wrapper class to expose a {@link QueryComponentSupplier} as a Broker service. */ -public class ExposedAsBrokerQueryComponentSupplierWrapper implements QueryComponentSupplier +public class ExposedAsBrokerQueryComponentSupplierWrapper extends QueryComponentSupplierDelegate { - private QueryComponentSupplier delegate; - public ExposedAsBrokerQueryComponentSupplierWrapper(QueryComponentSupplier delegate) { - this.delegate = delegate; - } - - @Override - public void gatherProperties(Properties properties) - { - delegate.gatherProperties(properties); - } - - @Override - public void configureGuice(DruidInjectorBuilder builder) - { + super(delegate); } @Override public void configureGuice(CoreInjectorBuilder builder, List overrideModules) { - delegate.configureGuice(builder); + super.configureGuice(builder); installForServerModules(builder); builder.add(new QueryRunnerFactoryModule()); @@ -157,57 +128,6 @@ public void configureGuice(CoreInjectorBuilder builder, List overrideMod builder.add(QuidemCaptureModule.class); } - @Override - public QueryRunnerFactoryConglomerate createCongolmerate1(Builder builder, Closer resourceCloser, - ObjectMapper jsonMapper, TestBufferPool testBufferPool, TestGroupByBuffers groupByBuffers, - DruidProcessingConfig processingConfig) - { - return delegate - .createCongolmerate1(builder, resourceCloser, jsonMapper, testBufferPool, groupByBuffers, processingConfig); - } - @Override - public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(QueryRunnerFactoryConglomerate conglomerate, - JoinableFactoryWrapper joinableFactory, Injector injector) - { - return delegate.createQuerySegmentWalker(conglomerate, joinableFactory, injector); - } - - @Override - public SqlEngine createEngine(QueryLifecycleFactory qlf, ObjectMapper objectMapper, Injector injector) - { - return delegate.createEngine(qlf, objectMapper, injector); - } - - @Override - public void configureJsonMapper(ObjectMapper mapper) - { - delegate.configureJsonMapper(mapper); - } - - @Override - public JoinableFactoryWrapper createJoinableFactoryWrapper(LookupExtractorFactoryContainerProvider lookupProvider) - { - return delegate.createJoinableFactoryWrapper(lookupProvider); - } - - @Override - public void finalizeTestFramework(SqlTestFramework sqlTestFramework) - { - delegate.finalizeTestFramework(sqlTestFramework); - } - - @Override - public void close() throws IOException - { - delegate.close(); - } - - @Override - public PlannerComponentSupplier getPlannerComponentSupplier() - { - return delegate.getPlannerComponentSupplier(); - } - public static class BrokerTestModule extends AbstractModule { @Override @@ -341,10 +261,4 @@ static List brokerModules() } ); } - - @Override - public Boolean isExplainSupported() - { - return delegate.isExplainSupported(); - } } diff --git a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java index 5244b21b0d88..66074055abc5 100644 --- a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java +++ b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Binder; -import com.google.inject.Injector; import com.google.inject.Provides; import com.google.inject.name.Named; import com.google.inject.name.Names; @@ -32,14 +31,8 @@ import org.apache.druid.java.util.common.FileUtils; import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.io.Closer; -import org.apache.druid.query.DruidProcessingConfig; import org.apache.druid.query.QueryRunnerFactoryConglomerate; -import org.apache.druid.query.TestBufferPool; -import org.apache.druid.query.groupby.TestGroupByBuffers; -import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; -import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.server.DruidNode; -import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.avatica.AvaticaMonitor; import org.apache.druid.sql.avatica.DruidAvaticaJsonHandler; @@ -48,13 +41,10 @@ import org.apache.druid.sql.calcite.SqlTestFrameworkConfig.ConfigurationInstance; import org.apache.druid.sql.calcite.SqlTestFrameworkConfig.SqlTestFrameworkConfigStore; import org.apache.druid.sql.calcite.planner.PlannerConfig; -import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SqlTestFramework; -import org.apache.druid.sql.calcite.util.SqlTestFramework.Builder; -import org.apache.druid.sql.calcite.util.SqlTestFramework.PlannerComponentSupplier; import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplier; +import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplierDelegate; import org.apache.druid.sql.hook.DruidHookDispatcher; import org.apache.http.client.utils.URIBuilder; import org.eclipse.jetty.server.Server; @@ -209,27 +199,20 @@ protected AbstractAvaticaHandler getAvaticaHandler(final DruidMeta druidMeta) } } - static class AvaticaBasedTestConnectionSupplier implements QueryComponentSupplier + static class AvaticaBasedTestConnectionSupplier extends QueryComponentSupplierDelegate { - private QueryComponentSupplier delegate; private AvaticaBasedConnectionModule connectionModule; public AvaticaBasedTestConnectionSupplier(QueryComponentSupplier delegate) { - this.delegate = delegate; + super(delegate); this.connectionModule = new AvaticaBasedConnectionModule(); } - @Override - public void gatherProperties(Properties properties) - { - delegate.gatherProperties(properties); - } - @Override public void configureGuice(DruidInjectorBuilder builder) { - delegate.configureGuice(builder); + super.configureGuice(builder); builder.addModule(connectionModule); builder.addModule( binder -> { @@ -240,63 +223,11 @@ public void configureGuice(DruidInjectorBuilder builder) ); } - @Override - public QueryRunnerFactoryConglomerate createCongolmerate1(Builder builder, Closer resourceCloser, - ObjectMapper jsonMapper, TestBufferPool testBufferPool, TestGroupByBuffers groupByBuffers, - DruidProcessingConfig processingConfig) - { - return delegate - .createCongolmerate1(builder, resourceCloser, jsonMapper, testBufferPool, groupByBuffers, processingConfig); - } - - @Override - public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(QueryRunnerFactoryConglomerate conglomerate, - JoinableFactoryWrapper joinableFactory, Injector injector) - { - return delegate.createQuerySegmentWalker(conglomerate, joinableFactory, injector); - } - - @Override - public SqlEngine createEngine(QueryLifecycleFactory qlf, ObjectMapper objectMapper, Injector injector) - { - return delegate.createEngine(qlf, objectMapper, injector); - } - - @Override - public void configureJsonMapper(ObjectMapper mapper) - { - delegate.configureJsonMapper(mapper); - } - - @Override - public JoinableFactoryWrapper createJoinableFactoryWrapper(LookupExtractorFactoryContainerProvider lookupProvider) - { - return delegate.createJoinableFactoryWrapper(lookupProvider); - } - - @Override - public void finalizeTestFramework(SqlTestFramework sqlTestFramework) - { - delegate.finalizeTestFramework(sqlTestFramework); - } - @Override public void close() throws IOException { connectionModule.close(); - delegate.close(); - } - - @Override - public PlannerComponentSupplier getPlannerComponentSupplier() - { - return delegate.getPlannerComponentSupplier(); - } - - @Override - public Boolean isExplainSupported() - { - return delegate.isExplainSupported(); + super.close(); } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 2aaa7cab8ecb..a33175e34361 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -217,6 +217,95 @@ default void configureGuice(CoreInjectorBuilder injectorBuilder, List ov Boolean isExplainSupported(); } + public static class QueryComponentSupplierDelegate implements QueryComponentSupplier { + + private final QueryComponentSupplier delegate; + + public QueryComponentSupplierDelegate(QueryComponentSupplier delegate) + { + this.delegate = delegate; + } + + @Override + public void gatherProperties(Properties properties) + { + delegate.gatherProperties(properties); + } + + @Override + public void configureGuice(DruidInjectorBuilder builder) + { + delegate.configureGuice(builder); + } + + @Override + public QueryRunnerFactoryConglomerate createCongolmerate1( + Builder builder, + Closer resourceCloser, + ObjectMapper jsonMapper, + TestBufferPool testBufferPool, + TestGroupByBuffers groupByBuffers, + DruidProcessingConfig processingConfig) + { + return delegate + .createCongolmerate1(builder, resourceCloser, jsonMapper, testBufferPool, groupByBuffers, processingConfig); + } + + @Override + public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker( + QueryRunnerFactoryConglomerate conglomerate, + JoinableFactoryWrapper joinableFactory, + Injector injector) + { + return delegate.createQuerySegmentWalker(conglomerate, joinableFactory, injector); + } + + @Override + public SqlEngine createEngine( + QueryLifecycleFactory qlf, + ObjectMapper objectMapper, + Injector injector) + { + return delegate.createEngine(qlf, objectMapper, injector); + } + + @Override + public void configureJsonMapper(ObjectMapper mapper) + { + delegate.configureJsonMapper(mapper); + } + + @Override + public JoinableFactoryWrapper createJoinableFactoryWrapper(LookupExtractorFactoryContainerProvider lookupProvider) + { + return delegate.createJoinableFactoryWrapper(lookupProvider); + } + + @Override + public void finalizeTestFramework(SqlTestFramework sqlTestFramework) + { + delegate.finalizeTestFramework(sqlTestFramework); + } + + @Override + public PlannerComponentSupplier getPlannerComponentSupplier() + { + return delegate.getPlannerComponentSupplier(); + } + + @Override + public void close() throws IOException + { + delegate.close(); + } + + @Override + public Boolean isExplainSupported() + { + return delegate.isExplainSupported(); + } + } + public interface PlannerComponentSupplier { Set extensionCalciteRules(); From 25fa7785eaf9c1b3e8c6d09f3dd779774f54a137 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 11:00:55 +0000 Subject: [PATCH 064/149] make abstract --- .../org/apache/druid/sql/calcite/util/SqlTestFramework.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index a33175e34361..cdf994899c06 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -217,7 +217,7 @@ default void configureGuice(CoreInjectorBuilder injectorBuilder, List ov Boolean isExplainSupported(); } - public static class QueryComponentSupplierDelegate implements QueryComponentSupplier { + public static abstract class QueryComponentSupplierDelegate implements QueryComponentSupplier { private final QueryComponentSupplier delegate; From 949118f0ed41b87b0d5df38dbb5211668b486f0c Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 11:02:26 +0000 Subject: [PATCH 065/149] cleanup --- .../sql/calcite/util/SqlTestFramework.java | 26 ------------------- 1 file changed, 26 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index cdf994899c06..10ff0d099372 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -358,32 +358,6 @@ public void configureGuice(DruidInjectorBuilder builder) { } - class x implements Module - { - - @Provides - public @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> createCongolmerate( - ObjectMapper jsonMapper, - final TestBufferPool testBufferPool, - final TestGroupByBuffers groupByBuffers, - @Named(SqlTestFramework.SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig) - { - return QueryStackTests.makeDefaultQueryRunnerFactories( - processingConfig, - 111, - jsonMapper, - testBufferPool, - groupByBuffers - ); - } - - @Override - public void configure(Binder binder) - { - } - - } - @Override public QueryRunnerFactoryConglomerate createCongolmerate1( Builder builder, From b59771ca0ebd0d7c428b53e73dc06ddc103ea50a Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 11:08:16 +0000 Subject: [PATCH 066/149] cleanup/etc --- .../sql/calcite/util/SqlTestFramework.java | 42 +++++++------------ 1 file changed, 16 insertions(+), 26 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 10ff0d099372..a40868c857b5 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -694,19 +694,25 @@ public void configure(Binder binder) @Provides - public @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> make1( + public @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> makeRunnerFactories( ObjectMapper jsonMapper, final TestBufferPool testBufferPool, final TestGroupByBuffers groupByBuffers, @Named(SqlTestFramework.SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig) { - return QueryStackTests.makeDefaultQueryRunnerFactories( - processingConfig, - builder.mergeBufferCount, - jsonMapper, - testBufferPool, - groupByBuffers - ); + return ImmutableMap., QueryRunnerFactory>builder() + .putAll( + QueryStackTests + .makeDefaultQueryRunnerFactories( + processingConfig, + builder.mergeBufferCount, + jsonMapper, + testBufferPool, + groupByBuffers + ) + ) + .build(); + } /* @@ -737,25 +743,9 @@ public TestGroupByBuffers makeTestGroupByBuffers(@Named(SQL_TEST_FRAME_WORK) Dru @Provides @LazySingleton public QueryRunnerFactoryConglomerate conglomerate( - @Named(SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig, - TestBufferPool testBufferPool, - TestGroupByBuffers groupByBuffers, - @Named(SQL_TEST_FRAME_WORK) - Map, QueryRunnerFactory> fact - ) + @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> factories) { - return new DefaultQueryRunnerFactoryConglomerate( - fact - ); - -// return componentSupplier.createCongolmerate( -// builder, -// resourceCloser, -// queryJsonMapper(), -// testBufferPool, -// groupByBuffers, -// processingConfig -// ); + return new DefaultQueryRunnerFactoryConglomerate(factories); } @Provides From 01cc4f59a81302a7f3b6aafcbc4034e7284c623d Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 11:10:56 +0000 Subject: [PATCH 067/149] cleanup/etc --- .../apache/druid/sql/calcite/util/SqlTestFramework.java | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index a40868c857b5..e83194d9f436 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -161,7 +161,7 @@ public interface QueryComponentSupplier extends Closeable */ void configureGuice(DruidInjectorBuilder builder); - QueryRunnerFactoryConglomerate createCongolmerate1( + QueryRunnerFactoryConglomerate createCongolmerate( Builder builder, Closer resourceCloser, ObjectMapper jsonMapper, @@ -239,7 +239,7 @@ public void configureGuice(DruidInjectorBuilder builder) } @Override - public QueryRunnerFactoryConglomerate createCongolmerate1( + public QueryRunnerFactoryConglomerate createCongolmerate( Builder builder, Closer resourceCloser, ObjectMapper jsonMapper, @@ -248,7 +248,7 @@ public QueryRunnerFactoryConglomerate createCongolmerate1( DruidProcessingConfig processingConfig) { return delegate - .createCongolmerate1(builder, resourceCloser, jsonMapper, testBufferPool, groupByBuffers, processingConfig); + .createCongolmerate(builder, resourceCloser, jsonMapper, testBufferPool, groupByBuffers, processingConfig); } @Override @@ -359,7 +359,7 @@ public void configureGuice(DruidInjectorBuilder builder) } @Override - public QueryRunnerFactoryConglomerate createCongolmerate1( + public QueryRunnerFactoryConglomerate createCongolmerate( Builder builder, Closer resourceCloser, ObjectMapper jsonMapper, From 362bd1849a9f9a05488cda5d577bd4844496cb7d Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 11:12:16 +0000 Subject: [PATCH 068/149] remove createConglomerate --- .../sql/calcite/util/SqlTestFramework.java | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index e83194d9f436..ea4a37e49e11 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -161,15 +161,6 @@ public interface QueryComponentSupplier extends Closeable */ void configureGuice(DruidInjectorBuilder builder); - QueryRunnerFactoryConglomerate createCongolmerate( - Builder builder, - Closer resourceCloser, - ObjectMapper jsonMapper, - TestBufferPool testBufferPool, - TestGroupByBuffers groupByBuffers, - DruidProcessingConfig processingConfig - ); - SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker( QueryRunnerFactoryConglomerate conglomerate, JoinableFactoryWrapper joinableFactory, @@ -238,19 +229,6 @@ public void configureGuice(DruidInjectorBuilder builder) delegate.configureGuice(builder); } - @Override - public QueryRunnerFactoryConglomerate createCongolmerate( - Builder builder, - Closer resourceCloser, - ObjectMapper jsonMapper, - TestBufferPool testBufferPool, - TestGroupByBuffers groupByBuffers, - DruidProcessingConfig processingConfig) - { - return delegate - .createCongolmerate(builder, resourceCloser, jsonMapper, testBufferPool, groupByBuffers, processingConfig); - } - @Override public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker( QueryRunnerFactoryConglomerate conglomerate, @@ -358,27 +336,6 @@ public void configureGuice(DruidInjectorBuilder builder) { } - @Override - public QueryRunnerFactoryConglomerate createCongolmerate( - Builder builder, - Closer resourceCloser, - ObjectMapper jsonMapper, - final TestBufferPool testBufferPool, - final TestGroupByBuffers groupByBuffers, - DruidProcessingConfig processingConfig - ) - { - return new DefaultQueryRunnerFactoryConglomerate( - QueryStackTests.makeDefaultQueryRunnerFactories( - processingConfig, - builder.minTopNThreshold, - jsonMapper, - testBufferPool, - groupByBuffers - ) - ); - } - @Override public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker( final QueryRunnerFactoryConglomerate conglomerate, From 94ece7a28b90c9304488b475c31fe4ddc657183e Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 11:18:51 +0000 Subject: [PATCH 069/149] fix style --- .../org/apache/druid/sql/calcite/util/SqlTestFramework.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index ea4a37e49e11..025e425bc540 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -208,7 +208,8 @@ default void configureGuice(CoreInjectorBuilder injectorBuilder, List ov Boolean isExplainSupported(); } - public static abstract class QueryComponentSupplierDelegate implements QueryComponentSupplier { + public abstract static class QueryComponentSupplierDelegate implements QueryComponentSupplier + { private final QueryComponentSupplier delegate; @@ -605,6 +606,7 @@ public SqlStatementFactory statementFactory() } public static final String SQL_TEST_FRAME_WORK = "sqlTestFrameWork"; + /** * Guice module to create the various query framework items. By creating items within * a module, later items can depend on those created earlier by grabbing them from the From 078f7c9779a5b3644150cfd007d84b5fbc712191 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 11:34:50 +0000 Subject: [PATCH 070/149] add wrapConglomerate --- .../sql/calcite/util/SqlTestFramework.java | 18 ++++++++++++++++-- 1 file changed, 16 insertions(+), 2 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 025e425bc540..c333d0ec6d52 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -206,11 +206,12 @@ default void configureGuice(CoreInjectorBuilder injectorBuilder, List ov * MSQ right now needs a full query run. */ Boolean isExplainSupported(); + + QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate); } public abstract static class QueryComponentSupplierDelegate implements QueryComponentSupplier { - private final QueryComponentSupplier delegate; public QueryComponentSupplierDelegate(QueryComponentSupplier delegate) @@ -283,6 +284,12 @@ public Boolean isExplainSupported() { return delegate.isExplainSupported(); } + + @Override + public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate) + { + return delegate.wrapConglomerate(conglomerate); + } } public interface PlannerComponentSupplier @@ -405,6 +412,12 @@ public Boolean isExplainSupported() { return true; } + + @Override + public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate) + { + return conglomerate; + } } public static class StandardPlannerComponentSupplier implements PlannerComponentSupplier @@ -704,7 +717,8 @@ public TestGroupByBuffers makeTestGroupByBuffers(@Named(SQL_TEST_FRAME_WORK) Dru public QueryRunnerFactoryConglomerate conglomerate( @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> factories) { - return new DefaultQueryRunnerFactoryConglomerate(factories); + QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate(factories); + return componentSupplier.wrapConglomerate(conglomerate); } @Provides From d0cbaf57b76272fcc2cfcc7ed4c600604d0f66ea Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 11:51:16 +0000 Subject: [PATCH 071/149] add closer --- .../druid/sql/calcite/util/SqlTestFramework.java | 12 +++++++----- 1 file changed, 7 insertions(+), 5 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index c333d0ec6d52..228b83fd203b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -207,7 +207,7 @@ default void configureGuice(CoreInjectorBuilder injectorBuilder, List ov */ Boolean isExplainSupported(); - QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate); + QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate, Closer resourceCloser); } public abstract static class QueryComponentSupplierDelegate implements QueryComponentSupplier @@ -286,9 +286,10 @@ public Boolean isExplainSupported() } @Override - public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate) + public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate, + Closer resourceCloser) { - return delegate.wrapConglomerate(conglomerate); + return delegate.wrapConglomerate(conglomerate, resourceCloser); } } @@ -414,7 +415,8 @@ public Boolean isExplainSupported() } @Override - public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate) + public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate, + Closer resourceCloser) { return conglomerate; } @@ -718,7 +720,7 @@ public QueryRunnerFactoryConglomerate conglomerate( @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> factories) { QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate(factories); - return componentSupplier.wrapConglomerate(conglomerate); + return componentSupplier.wrapConglomerate(conglomerate, resourceCloser); } @Provides From d38aa5ea36edc87d96a9e2d74a9a85958a92d0d0 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 12:16:21 +0000 Subject: [PATCH 072/149] normalize --- .../apache/druid/messages/client/MessageRelayClientImpl.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java b/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java index b2de6581a720..b238cdee0aa7 100644 --- a/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java +++ b/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java @@ -69,7 +69,7 @@ public ListenableFuture> getMessages( startWatermark ); - ListenableFuture asyncRequest = (ListenableFuture) serviceClient.asyncRequest( + ListenableFuture asyncRequest = serviceClient.asyncRequest( new RequestBuilder(HttpMethod.GET, path), new BytesFullResponseHandler() ); From 55d87d8869815613486d44c289861ad2c8203020 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 12:21:54 +0000 Subject: [PATCH 073/149] remove class --- .../query/DefaultQueryRunnerFactoryConglomerate.java | 8 +------- .../druid/query/QueryRunnerFactoryConglomerate.java | 11 ++++++++++- .../QueryRunnerBasedOnClusteredClientTestBase.java | 2 +- .../metadata/SegmentMetadataCacheTestBase.java | 3 +-- .../java/org/apache/druid/server/QueryStackTests.java | 3 +-- .../druid/sql/calcite/util/QueryFrameworkUtils.java | 3 +-- 6 files changed, 15 insertions(+), 15 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java index 48d2ace99a8a..04298a7fa003 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java @@ -26,7 +26,7 @@ /** */ -public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactoryConglomerate, QueryToolChestWarehouse +public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactoryConglomerate { private final Map, QueryRunnerFactory> factories; @@ -46,10 +46,4 @@ public > QueryRunnerFactory findFact return factories.get(query.getClass()); } - @SuppressWarnings("unchecked") - @Override - public > QueryToolChest getToolChest(QueryType query) - { - return factories.get(query.getClass()).getToolchest(); - } } diff --git a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java index da89d81c8dda..6a753656e574 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java @@ -21,7 +21,16 @@ /** */ -public interface QueryRunnerFactoryConglomerate +public interface QueryRunnerFactoryConglomerate extends QueryToolChestWarehouse { > QueryRunnerFactory findFactory(QueryType query); + + default > QueryToolChest getToolChest(QueryType query) + { + QueryRunnerFactory factory = findFactory(query); + if (factory == null) { + return null; + } + return factory.getToolchest(); + } } diff --git a/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java b/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java index 68da75759176..4cb081c8e11c 100644 --- a/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java +++ b/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java @@ -109,7 +109,7 @@ protected QueryRunnerBasedOnClusteredClientTestBase() CLOSER, TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD ); - toolChestWarehouse = new ConglomerateBackedQueryToolChestWarehouse(conglomerate); + toolChestWarehouse = conglomerate; } @AfterClass diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java index f08d5942f146..1fcd27be29a5 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java @@ -29,7 +29,6 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.io.Closer; -import org.apache.druid.query.ConglomerateBackedQueryToolChestWarehouse; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; import org.apache.druid.query.QueryRunnerFactoryConglomerate; @@ -111,7 +110,7 @@ public void setUpCommon() { resourceCloser = Closer.create(); conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser); - queryToolChestWarehouse = new ConglomerateBackedQueryToolChestWarehouse(conglomerate); + queryToolChestWarehouse = conglomerate; } public void setUpData() throws Exception diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index 9414f4a4af47..98c2ed652afe 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -33,7 +33,6 @@ import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.query.BrokerParallelMergeConfig; -import org.apache.druid.query.ConglomerateBackedQueryToolChestWarehouse; import org.apache.druid.query.DataSource; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate; @@ -135,7 +134,7 @@ public static ClientQuerySegmentWalker createClientQuerySegmentWalker( emitter, clusterWalker, localWalker, - new ConglomerateBackedQueryToolChestWarehouse(conglomerate), + conglomerate, joinableFactory, new RetryQueryRunnerConfig(), injector.getInstance(ObjectMapper.class), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index d46067639479..6af708f2a72a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -28,7 +28,6 @@ import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.java.util.emitter.core.NoopEmitter; import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.query.ConglomerateBackedQueryToolChestWarehouse; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; import org.apache.druid.query.GlobalTableDataSource; @@ -88,7 +87,7 @@ public static QueryLifecycleFactory createMockQueryLifecycleFactory( ) { return new QueryLifecycleFactory( - new ConglomerateBackedQueryToolChestWarehouse(conglomerate), + conglomerate, walker, new DefaultGenericQueryMetricsFactory(), new ServiceEmitter("dummy", "dummy", new NoopEmitter()), From 2fb67945c3dae3ca195603b38834dce96a3a5714 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 12:53:29 +0000 Subject: [PATCH 074/149] updates --- ...yExecSomething.java => QueryExecutor.java} | 8 +++-- .../query/QueryRunnerFactoryConglomerate.java | 4 ++- .../apache/druid/query/QueryToolChest.java | 10 +++--- .../query/union/UnionQueryQueryToolChest.java | 4 +-- ...tory.java => UnionQueryRunnerFactory.java} | 32 ++++++----------- .../druid/guice/QueryRunnerFactoryModule.java | 4 +-- .../server/ClientQuerySegmentWalker.java | 12 +++---- ...lomerateBackedQueryToolChestWarehouse.java | 36 ------------------- .../sql/calcite/util/SqlTestFramework.java | 21 +++++++++++ 9 files changed, 55 insertions(+), 76 deletions(-) rename processing/src/main/java/org/apache/druid/query/{QueryExecSomething.java => QueryExecutor.java} (82%) rename processing/src/main/java/org/apache/druid/query/union/{RealUnionQueryRunnerFactory.java => UnionQueryRunnerFactory.java} (74%) delete mode 100644 server/src/test/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java diff --git a/processing/src/main/java/org/apache/druid/query/QueryExecSomething.java b/processing/src/main/java/org/apache/druid/query/QueryExecutor.java similarity index 82% rename from processing/src/main/java/org/apache/druid/query/QueryExecSomething.java rename to processing/src/main/java/org/apache/druid/query/QueryExecutor.java index 0e6f7f43cb4c..a94dbd11c9e9 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryExecSomething.java +++ b/processing/src/main/java/org/apache/druid/query/QueryExecutor.java @@ -19,12 +19,16 @@ package org.apache.druid.query; -public interface QueryExecSomething +/** + * Executes the query by utilizing the given walker. + * + * {@link QueryToolChest}-s implementing this interface will get control over the execution of the query. + */ +public interface QueryExecutor { QueryRunner makeQueryRunner( QueryToolChestWarehouse warehouse, Query query, QuerySegmentWalker walker ); - } diff --git a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java index 6a753656e574..81cceee4afeb 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java @@ -19,6 +19,8 @@ package org.apache.druid.query; +import org.apache.druid.error.DruidException; + /** */ public interface QueryRunnerFactoryConglomerate extends QueryToolChestWarehouse @@ -29,7 +31,7 @@ default > QueryToolChest getToolChes { QueryRunnerFactory factory = findFactory(query); if (factory == null) { - return null; + throw DruidException.defensive(query.getClass().getName() + " is not registered"); } return factory.getToolchest(); } diff --git a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java index 1cc48a7be743..e158da3edbab 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java @@ -72,6 +72,11 @@ protected QueryToolChest() } } + public void setWarehouse(QueryToolChestWarehouse warehouse) + { + this.warehouse = warehouse; + } + public final JavaType getBaseResultType() { return baseResultType; @@ -424,11 +429,6 @@ public Optional> resultsAsFrames( return Optional.empty(); } - public void setWarehouse(QueryToolChestWarehouse warehouse) - { - this.warehouse = warehouse; - } - public Optional> executeQuery( // ideally; it should know about the warehouse QueryToolChestWarehouse warehouse, diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 2b87f1bdb89c..c1fe4f0dc9c4 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -28,7 +28,7 @@ import org.apache.druid.query.DefaultQueryMetrics; import org.apache.druid.query.FrameSignaturePair; import org.apache.druid.query.Query; -import org.apache.druid.query.QueryExecSomething; +import org.apache.druid.query.QueryExecutor; import org.apache.druid.query.QueryMetrics; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; @@ -44,7 +44,7 @@ import java.util.Optional; public class UnionQueryQueryToolChest extends QueryToolChest - implements QueryExecSomething + implements QueryExecutor { public QueryRunner makeQueryRunner(QueryToolChestWarehouse warehouse, diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunnerFactory.java similarity index 74% rename from processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java rename to processing/src/main/java/org/apache/druid/query/union/UnionQueryRunnerFactory.java index 281c29bf9eb0..d493e52b1144 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunnerFactory.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunnerFactory.java @@ -19,52 +19,40 @@ package org.apache.druid.query.union; +import com.google.inject.Inject; +import org.apache.druid.error.DruidException; import org.apache.druid.query.QueryProcessingPool; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QueryRunnerFactory; import org.apache.druid.query.QueryToolChest; import org.apache.druid.segment.Segment; -public class RealUnionQueryRunnerFactory implements QueryRunnerFactory +public class UnionQueryRunnerFactory implements QueryRunnerFactory { + private UnionQueryQueryToolChest toolChest; - private static UnionQueryQueryToolChest tt; - - public RealUnionQueryRunnerFactory(String string) + @Inject + public UnionQueryRunnerFactory(UnionQueryQueryToolChest toolChest) { - int asd=1; + this.toolChest = toolChest; } @Override public QueryRunner createRunner(Segment segment) { - if(true) - { - throw new RuntimeException("FIXME: Unimplemented!"); - } - return null; - + throw DruidException.defensive("not expected to be used"); } @Override public QueryRunner mergeRunners(QueryProcessingPool queryProcessingPool, Iterable> queryRunners) { - if(true) - { - throw new RuntimeException("FIXME: Unimplemented!"); - } - return null; - + throw DruidException.defensive("not expected to be used"); } @Override public QueryToolChest getToolchest() { - if (tt == null) { - tt = new UnionQueryQueryToolChest(); - } - return tt; + return toolChest; } - } diff --git a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java index f71228a3fbee..e3e685a83a44 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java @@ -46,7 +46,7 @@ import org.apache.druid.query.timeseries.TimeseriesQueryRunnerFactory; import org.apache.druid.query.topn.TopNQuery; import org.apache.druid.query.topn.TopNQueryRunnerFactory; -import org.apache.druid.query.union.RealUnionQueryRunnerFactory; +import org.apache.druid.query.union.UnionQueryRunnerFactory; import org.apache.druid.query.union.UnionQuery; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QuerySchedulerProvider; @@ -68,7 +68,7 @@ public class QueryRunnerFactoryModule extends QueryToolChestModule .put(TimeseriesQuery.class, TimeseriesQueryRunnerFactory.class) .put(TopNQuery.class, TopNQueryRunnerFactory.class) .put(WindowOperatorQuery.class, WindowOperatorQueryQueryRunnerFactory.class) - .put(UnionQuery.class, RealUnionQueryRunnerFactory.class) + .put(UnionQuery.class, UnionQueryRunnerFactory.class) .build(); @Override diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 24255fd2d396..52c7a37e63ab 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -49,7 +49,7 @@ import org.apache.druid.query.Query; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.QueryDataSource; -import org.apache.druid.query.QueryExecSomething; +import org.apache.druid.query.QueryExecutor; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; @@ -181,8 +181,8 @@ public QueryRunner getQueryRunnerForIntervals(final Query query, final final QueryToolChest> toolChest = warehouse.getToolChest(query); // Optional> toolchestExecResult = toolChest.executeQuery21(warehouse, query, this); - if (toolChest instanceof QueryExecSomething) { - QueryExecSomething t = (QueryExecSomething) toolChest; + if (toolChest instanceof QueryExecutor) { + QueryExecutor t = (QueryExecutor) toolChest; return t.makeQueryRunner(warehouse, query, this); } @@ -317,7 +317,7 @@ private boolean canRunQueryUsingClusterWalker(Query query) final DataSourceAnalysis analysis = query.getDataSourceAnalysis(); final QueryToolChest> toolChest = warehouse.getToolChest(query); - if (toolChest instanceof QueryExecSomething) { + if (toolChest instanceof QueryExecutor) { return true; } @@ -455,8 +455,8 @@ private DataSource inlineIfNecessary( // final QueryRunner subQueryRunner ; final QueryRunner subQueryRunner; - if (subQueryToolChest instanceof QueryExecSomething) { - subQueryRunner = ((QueryExecSomething) subQueryToolChest) + if (subQueryToolChest instanceof QueryExecutor) { + subQueryRunner = ((QueryExecutor) subQueryToolChest) .makeQueryRunner(warehouse, subQueryWithSerialization, this); }else { subQueryRunner = subQueryWithSerialization.getRunner(this); diff --git a/server/src/test/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java b/server/src/test/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java deleted file mode 100644 index f80ac15987fd..000000000000 --- a/server/src/test/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query; - -public class ConglomerateBackedQueryToolChestWarehouse implements QueryToolChestWarehouse -{ - private QueryRunnerFactoryConglomerate conglomerate; - - public ConglomerateBackedQueryToolChestWarehouse(QueryRunnerFactoryConglomerate conglomerate) - { - this.conglomerate = conglomerate; - } - - @Override - public > QueryToolChest getToolChest(final QueryType query) - { - return conglomerate.findFactory(query).getToolchest(); - } -} diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 228b83fd203b..ceebefb5094e 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -50,6 +50,8 @@ import org.apache.druid.query.groupby.TestGroupByBuffers; import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.query.topn.TopNQueryConfig; +import org.apache.druid.query.union.UnionQueryRunnerFactory; +import org.apache.druid.query.union.UnionQuery; import org.apache.druid.quidem.TestSqlModule; import org.apache.druid.segment.DefaultColumnFormatConfig; import org.apache.druid.segment.join.JoinableFactoryWrapper; @@ -87,6 +89,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.Arrays; +import java.util.HashMap; import java.util.List; import java.util.Map; import java.util.Properties; @@ -208,6 +211,8 @@ default void configureGuice(CoreInjectorBuilder injectorBuilder, List ov Boolean isExplainSupported(); QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate, Closer resourceCloser); + + Map, ? extends QueryRunnerFactory> makeRunnerFactories(Injector injector); } public abstract static class QueryComponentSupplierDelegate implements QueryComponentSupplier @@ -291,6 +296,12 @@ public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglom { return delegate.wrapConglomerate(conglomerate, resourceCloser); } + + @Override + public Map, ? extends QueryRunnerFactory> makeRunnerFactories(Injector injector) + { + return delegate.makeRunnerFactories(injector); + } } public interface PlannerComponentSupplier @@ -420,6 +431,15 @@ public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglom { return conglomerate; } + + @Override + public Map, ? extends QueryRunnerFactory> makeRunnerFactories(Injector injector) + { + Map map =new HashMap<>(); + UnionQueryRunnerFactory factory = injector.getInstance(UnionQueryRunnerFactory.class); + map.put(UnionQuery.class, factory); + return map; + } } public static class StandardPlannerComponentSupplier implements PlannerComponentSupplier @@ -685,6 +705,7 @@ public void configure(Binder binder) groupByBuffers ) ) + .putAll(componentSupplier.makeRunnerFactories(injector)) .build(); } From 6533e61a084664467b4e44c3add4b52f3459b431 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 12:53:48 +0000 Subject: [PATCH 075/149] change signature --- .../src/main/java/org/apache/druid/query/QueryExecutor.java | 2 +- .../apache/druid/query/union/UnionQueryQueryToolChest.java | 4 ++-- .../org/apache/druid/server/ClientQuerySegmentWalker.java | 4 ++-- 3 files changed, 5 insertions(+), 5 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryExecutor.java b/processing/src/main/java/org/apache/druid/query/QueryExecutor.java index a94dbd11c9e9..8b11fe6da431 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryExecutor.java +++ b/processing/src/main/java/org/apache/druid/query/QueryExecutor.java @@ -27,8 +27,8 @@ public interface QueryExecutor { QueryRunner makeQueryRunner( - QueryToolChestWarehouse warehouse, Query query, + QueryToolChestWarehouse warehouse, QuerySegmentWalker walker ); } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index c1fe4f0dc9c4..7eaf685b3560 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -47,8 +47,8 @@ public class UnionQueryQueryToolChest extends QueryToolChest { - public QueryRunner makeQueryRunner(QueryToolChestWarehouse warehouse, - Query query, QuerySegmentWalker clientQuerySegmentWalker) + public QueryRunner makeQueryRunner(Query query, + QueryToolChestWarehouse warehouse, QuerySegmentWalker clientQuerySegmentWalker) { return new UnionQueryRunner((UnionQuery) query, clientQuerySegmentWalker); } diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 52c7a37e63ab..3a78a50ab14c 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -183,7 +183,7 @@ public QueryRunner getQueryRunnerForIntervals(final Query query, final // Optional> toolchestExecResult = toolChest.executeQuery21(warehouse, query, this); if (toolChest instanceof QueryExecutor) { QueryExecutor t = (QueryExecutor) toolChest; - return t.makeQueryRunner(warehouse, query, this); + return t.makeQueryRunner(query, warehouse, this); } // transform TableDataSource to GlobalTableDataSource when eligible @@ -457,7 +457,7 @@ private DataSource inlineIfNecessary( final QueryRunner subQueryRunner; if (subQueryToolChest instanceof QueryExecutor) { subQueryRunner = ((QueryExecutor) subQueryToolChest) - .makeQueryRunner(warehouse, subQueryWithSerialization, this); + .makeQueryRunner(subQueryWithSerialization, warehouse, this); }else { subQueryRunner = subQueryWithSerialization.getRunner(this); } From a1d8748700f90f705f090235c4de70d19c99f9df Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 12:57:42 +0000 Subject: [PATCH 076/149] update --- .../main/java/org/apache/druid/query/QueryToolChest.java | 9 --------- .../apache/druid/server/ClientQuerySegmentWalker.java | 4 ++-- 2 files changed, 2 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java index e158da3edbab..2c814553bfc1 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java @@ -429,15 +429,6 @@ public Optional> resultsAsFrames( return Optional.empty(); } - public Optional> executeQuery( - // ideally; it should know about the warehouse - QueryToolChestWarehouse warehouse, - Query query, - QuerySegmentWalker clientQuerySegmentWalker) - { - return Optional.empty(); - } - public boolean canExecuteFully(Query query) { DataSource dataSourceFromQuery = query.getDataSource(); diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 3a78a50ab14c..aa77fba2a96b 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -314,12 +314,12 @@ private boolean canRunQueryUsingLocalWalker(Query query) */ private boolean canRunQueryUsingClusterWalker(Query query) { - final DataSourceAnalysis analysis = query.getDataSourceAnalysis(); final QueryToolChest> toolChest = warehouse.getToolChest(query); - if (toolChest instanceof QueryExecutor) { + // these type of queries should be able to run return true; } + final DataSourceAnalysis analysis = query.getDataSourceAnalysis(); // 1) Must be based on a concrete table (the only shape the Druid cluster can handle). // 2) If there is an outer query, it must be handleable by the query toolchest (the cluster walker does not handle From 787a332643da3a1df1e98246309bba1d1b8ee1f0 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 13:09:19 +0000 Subject: [PATCH 077/149] cleanup/move/etc --- .../main/java/org/apache/druid/query/Query.java | 2 +- .../org/apache/druid/query/QueryDataSource.java | 15 +-------------- .../druid/query/planning/DataSourceAnalysis.java | 2 +- 3 files changed, 3 insertions(+), 16 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/Query.java b/processing/src/main/java/org/apache/druid/query/Query.java index 55fc0dce5ad4..50bcd7db4016 100644 --- a/processing/src/main/java/org/apache/druid/query/Query.java +++ b/processing/src/main/java/org/apache/druid/query/Query.java @@ -308,6 +308,6 @@ default Query withDataSources(List children) default DataSourceAnalysis getDataSourceAnalysis() { - return getDataSource().getAnalysis(); + return getDataSource().getAnalysis().maybeWithBaseQuery(this); } } diff --git a/processing/src/main/java/org/apache/druid/query/QueryDataSource.java b/processing/src/main/java/org/apache/druid/query/QueryDataSource.java index 75170ff09e56..0be20456e85e 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryDataSource.java +++ b/processing/src/main/java/org/apache/druid/query/QueryDataSource.java @@ -23,12 +23,9 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; -import org.apache.druid.java.util.common.IAE; import org.apache.druid.query.planning.DataSourceAnalysis; -import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.SegmentReference; -import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -119,17 +116,7 @@ public byte[] getCacheKey() public DataSourceAnalysis getAnalysis() { final Query subQuery = this.getQuery(); - if (subQuery instanceof BaseQuery) { - final DataSource current = subQuery.getDataSource(); - return current.getAnalysis().maybeWithBaseQuery(subQuery); - } - if(subQuery instanceof UnionQuery) { - return new DataSourceAnalysis(this, null, null, Collections.emptyList()); - } - // We must verify that the subQuery is a BaseQuery, because it is required to make - // "DataSourceAnalysis.getBaseQuerySegmentSpec" work properly. - // All built-in query types are BaseQuery, so we only expect this with funky extension queries. - throw new IAE("Cannot analyze subquery of class[%s]", subQuery.getClass().getName()); + return subQuery.getDataSourceAnalysis(); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/planning/DataSourceAnalysis.java b/processing/src/main/java/org/apache/druid/query/planning/DataSourceAnalysis.java index 14a5192d0f26..b7d7db7c8e3a 100644 --- a/processing/src/main/java/org/apache/druid/query/planning/DataSourceAnalysis.java +++ b/processing/src/main/java/org/apache/druid/query/planning/DataSourceAnalysis.java @@ -182,7 +182,7 @@ public Optional getBaseQuerySegmentSpec() */ public DataSourceAnalysis maybeWithBaseQuery(Query query) { - if (!getBaseQuery().isPresent()) { + if (!getBaseQuery().isPresent() && query instanceof BaseQuery) { return new DataSourceAnalysis(baseDataSource, query, joinBaseTableFilter, preJoinableClauses); } return this; From c78c93551f647f2c38893bbf06939fc33bf27bb3 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 13:12:17 +0000 Subject: [PATCH 078/149] cleanup --- processing/src/main/java/org/apache/druid/query/QueryPlus.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryPlus.java b/processing/src/main/java/org/apache/druid/query/QueryPlus.java index f862fd7b825a..53697834a77f 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryPlus.java +++ b/processing/src/main/java/org/apache/druid/query/QueryPlus.java @@ -23,9 +23,9 @@ import com.google.common.collect.ImmutableMap; import org.apache.druid.error.DruidException; import org.apache.druid.guice.annotations.PublicApi; -import org.apache.druid.java.util.common.ISE; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.query.context.ResponseContext; + import javax.annotation.Nullable; /** From 0d99b619a0624d16a6632cc3e2caf3426e87f755 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 13:15:19 +0000 Subject: [PATCH 079/149] cleanup --- processing/src/main/java/org/apache/druid/query/QueryPlus.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryPlus.java b/processing/src/main/java/org/apache/druid/query/QueryPlus.java index 53697834a77f..ae71895b8e41 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryPlus.java +++ b/processing/src/main/java/org/apache/druid/query/QueryPlus.java @@ -62,7 +62,7 @@ public Query getQuery() public > C unwrapQuery(Class clazz) { - if(clazz.isInstance(query)) { + if (clazz.isInstance(query)) { return (C) query; } throw DruidException.defensive("Encountered unexpected query type [%s] instead of [%s]", query.getClass(), clazz); From 874438f6d8767ba1428bcd05da98da83f5cda711 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 13:20:24 +0000 Subject: [PATCH 080/149] rename/etc --- .../java/org/apache/druid/query/FluentQueryRunner.java | 2 +- ...ueryRunner.java => UnionDataSourceQueryRunner.java} | 4 ++-- .../druid/query/union/UnionQueryQueryToolChest.java | 10 ++++++---- .../org/apache/druid/query/union/UnionQueryRunner.java | 2 +- .../org/apache/druid/query/UnionQueryRunnerTest.java | 2 +- .../timeseries/TimeSeriesUnionQueryRunnerTest.java | 4 ++-- 6 files changed, 13 insertions(+), 11 deletions(-) rename processing/src/main/java/org/apache/druid/query/{UnionQueryRunner.java => UnionDataSourceQueryRunner.java} (97%) diff --git a/processing/src/main/java/org/apache/druid/query/FluentQueryRunner.java b/processing/src/main/java/org/apache/druid/query/FluentQueryRunner.java index dcef6fa06514..e6f52d47b541 100644 --- a/processing/src/main/java/org/apache/druid/query/FluentQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/FluentQueryRunner.java @@ -64,7 +64,7 @@ public FluentQueryRunner applyPostMergeDecoration() public FluentQueryRunner applyPreMergeDecoration() { - return from(new UnionQueryRunner<>(toolChest.preMergeQueryDecoration(baseRunner))); + return from(new UnionDataSourceQueryRunner<>(toolChest.preMergeQueryDecoration(baseRunner))); } public FluentQueryRunner emitCPUTimeMetric(ServiceEmitter emitter) diff --git a/processing/src/main/java/org/apache/druid/query/UnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/UnionDataSourceQueryRunner.java similarity index 97% rename from processing/src/main/java/org/apache/druid/query/UnionQueryRunner.java rename to processing/src/main/java/org/apache/druid/query/UnionDataSourceQueryRunner.java index 07b761138f4d..5eaa84a5e491 100644 --- a/processing/src/main/java/org/apache/druid/query/UnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/UnionDataSourceQueryRunner.java @@ -34,11 +34,11 @@ import java.util.stream.Collectors; import java.util.stream.IntStream; -public class UnionQueryRunner implements QueryRunner +public class UnionDataSourceQueryRunner implements QueryRunner { private final QueryRunner baseRunner; - public UnionQueryRunner( + public UnionDataSourceQueryRunner( QueryRunner baseRunner ) { diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 7eaf685b3560..75361186aec6 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -22,6 +22,7 @@ import com.fasterxml.jackson.core.type.TypeReference; import com.google.common.base.Function; import com.google.common.base.Functions; +import org.apache.druid.error.DruidException; import org.apache.druid.frame.allocation.MemoryAllocatorFactory; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; @@ -85,11 +86,12 @@ public TypeReference getResultTypeReference() @Override public RowSignature resultArraySignature(UnionQuery query) { - Query q0 = query.queries.get(0); - if (q0 instanceof SupportRowSignature) { - return ((SupportRowSignature) q0).getResultRowSignature(Finalization.UNKNOWN); + for (Query q: query.queries) { + if (q instanceof SupportRowSignature) { + return ((SupportRowSignature) q).getResultRowSignature(Finalization.UNKNOWN); + } } - throw new UnsupportedOperationException("Not supported"); + throw DruidException.defensive("None of the subqueries support row signature"); } @Override diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java index e4f3de27670a..4f15e1907f6c 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java @@ -76,4 +76,4 @@ private RealUnionResult makeUnionResult(QueryRunner runner, QueryPlus wit Sequence seq = runner.run(withQuery, responseContext); return new RealUnionResult(seq); } -} \ No newline at end of file +} diff --git a/processing/src/test/java/org/apache/druid/query/UnionQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/UnionQueryRunnerTest.java index a9ce7a9471b2..332f74022e3f 100644 --- a/processing/src/test/java/org/apache/druid/query/UnionQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/UnionQueryRunnerTest.java @@ -56,7 +56,7 @@ public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) } } }; - UnionQueryRunner runner = new UnionQueryRunner(baseRunner); + UnionDataSourceQueryRunner runner = new UnionDataSourceQueryRunner(baseRunner); // Make a dummy query with Union datasource Query q = Druids.newTimeseriesQueryBuilder() .dataSource( diff --git a/processing/src/test/java/org/apache/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java index 4cca90c083d5..a8644e558e8c 100644 --- a/processing/src/test/java/org/apache/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/timeseries/TimeSeriesUnionQueryRunnerTest.java @@ -32,7 +32,7 @@ import org.apache.druid.query.Result; import org.apache.druid.query.TableDataSource; import org.apache.druid.query.UnionDataSource; -import org.apache.druid.query.UnionQueryRunner; +import org.apache.druid.query.UnionDataSourceQueryRunner; import org.apache.druid.query.aggregation.LongSumAggregatorFactory; import org.apache.druid.query.context.ResponseContext; import org.apache.druid.segment.TestHelper; @@ -172,7 +172,7 @@ public void testUnionResultMerging() ); QueryRunner mergingrunner = toolChest.mergeResults( - new UnionQueryRunner<>( + new UnionDataSourceQueryRunner<>( new QueryRunner>() { @Override From 29756bcc61f9a640624e767cb110a85081577727 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 13:28:05 +0000 Subject: [PATCH 081/149] rename/etc --- .../druid/query/union/UnionQueryQueryToolChest.java | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 75361186aec6..58066ed8bc31 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -106,7 +106,8 @@ public Sequence resultsAsArrays( for (int i = 0; i < results.size(); i++) { Query q = query.queries.get(i); RealUnionResult realUnionResult = results.get(i); - resultSeqs.add(resultsAsArrays(q, realUnionResult)); + Sequence resultsAsArrays2 = resultsAsArrays2(q, realUnionResult); + resultSeqs.add(resultsAsArrays2); } return Sequences.concat(resultSeqs); } @@ -118,6 +119,14 @@ private > Sequence resultsAsArrays(Query return toolChest.resultsAsArrays(q, realUnionResult.getResults()); } + + private Sequence resultsAsArrays2(Query q, + RealUnionResult realUnionResult) + { + QueryToolChest toolChest = warehouse.getToolChest(q); + return toolChest.resultsAsArrays(q, realUnionResult.getResults()); + } + @Override @SuppressWarnings({"unchecked", "rawtypes"}) public Optional> resultsAsFrames( From 97c3749d702c2009f5a8e6dc89acb76a0a0d1c36 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 13:30:42 +0000 Subject: [PATCH 082/149] x --- .../query/union/UnionQueryQueryToolChest.java | 38 +++++++++---------- 1 file changed, 19 insertions(+), 19 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 58066ed8bc31..59eefffa648c 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -86,7 +86,7 @@ public TypeReference getResultTypeReference() @Override public RowSignature resultArraySignature(UnionQuery query) { - for (Query q: query.queries) { + for (Query q : query.queries) { if (q instanceof SupportRowSignature) { return ((SupportRowSignature) q).getResultRowSignature(Finalization.UNKNOWN); } @@ -106,27 +106,13 @@ public Sequence resultsAsArrays( for (int i = 0; i < results.size(); i++) { Query q = query.queries.get(i); RealUnionResult realUnionResult = results.get(i); - Sequence resultsAsArrays2 = resultsAsArrays2(q, realUnionResult); - resultSeqs.add(resultsAsArrays2); + QueryToolChest toolChest = warehouse.getToolChest(q); + Sequence queryResults = toolChest.resultsAsArrays(q, realUnionResult.getResults()); + resultSeqs.add(queryResults); } return Sequences.concat(resultSeqs); } - private > Sequence resultsAsArrays(QueryType q, - RealUnionResult realUnionResult) - { - QueryToolChest toolChest = warehouse.getToolChest(q); - return toolChest.resultsAsArrays(q, realUnionResult.getResults()); - } - - - private Sequence resultsAsArrays2(Query q, - RealUnionResult realUnionResult) - { - QueryToolChest toolChest = warehouse.getToolChest(q); - return toolChest.resultsAsArrays(q, realUnionResult.getResults()); - } - @Override @SuppressWarnings({"unchecked", "rawtypes"}) public Optional> resultsAsFrames( @@ -135,6 +121,20 @@ public Optional> resultsAsFrames( MemoryAllocatorFactory memoryAllocatorFactory, boolean useNestedForUnknownTypes) { - throw new UnsupportedOperationException("Not supported"); + List results = resultSequence.toList(); + List> resultSeqs = new ArrayList<>(); + + for (int i = 0; i < results.size(); i++) { + Query q = query.queries.get(i); + RealUnionResult realUnionResult = results.get(i); + QueryToolChest toolChest = warehouse.getToolChest(q); + Optional> queryResults = toolChest + .resultsAsFrames(query, resultSequence, memoryAllocatorFactory, useNestedForUnknownTypes); + if (!queryResults.isPresent()) { + return Optional.empty(); + } + resultSeqs.add(queryResults.get()); + } + return Optional.of(Sequences.concat(resultSeqs)); } } From f9ff96c626f168ebee10186ac6ed83c68c9e0a93 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 13:32:25 +0000 Subject: [PATCH 083/149] x --- .../druid/query/DefaultQueryRunnerFactoryConglomerate.java | 2 +- .../org/apache/druid/query/groupby/SupportRowSignature.java | 3 +-- .../org/apache/druid/query/timeseries/TimeseriesQuery.java | 3 ++- 3 files changed, 4 insertions(+), 4 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java index 04298a7fa003..33287b9eebde 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java @@ -34,7 +34,7 @@ public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactory public DefaultQueryRunnerFactoryConglomerate(Map, QueryRunnerFactory> factories) { this.factories = new IdentityHashMap<>(factories); - for (QueryRunnerFactory factory :factories.values()) { + for (QueryRunnerFactory factory : factories.values()) { factory.getToolchest().setWarehouse(this); } } diff --git a/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java b/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java index 9698c3b9bf26..5f0c56c0ae3f 100644 --- a/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java +++ b/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java @@ -23,6 +23,5 @@ public interface SupportRowSignature { - public RowSignature getResultRowSignature(final RowSignature.Finalization finalization); - + RowSignature getResultRowSignature(RowSignature.Finalization finalization); } diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java index 1a6f8e669fd3..c3e805a93150 100644 --- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java +++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java @@ -184,7 +184,8 @@ public boolean isSkipEmptyBuckets() } @Override - public RowSignature getResultRowSignature(Finalization finalization) { + public RowSignature getResultRowSignature(Finalization finalization) + { return getResultSignature(finalization); } From 5e6eda12fe018a9cafa82b5a57597022cfd83648 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 13:33:29 +0000 Subject: [PATCH 084/149] x --- .../org/apache/druid/guice/QueryRunnerFactoryModule.java | 2 +- .../java/org/apache/druid/guice/QueryToolChestModule.java | 2 +- .../org/apache/druid/server/ClientQuerySegmentWalker.java | 7 ++----- 3 files changed, 4 insertions(+), 7 deletions(-) diff --git a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java index e3e685a83a44..6d7008944134 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java @@ -46,8 +46,8 @@ import org.apache.druid.query.timeseries.TimeseriesQueryRunnerFactory; import org.apache.druid.query.topn.TopNQuery; import org.apache.druid.query.topn.TopNQueryRunnerFactory; -import org.apache.druid.query.union.UnionQueryRunnerFactory; import org.apache.druid.query.union.UnionQuery; +import org.apache.druid.query.union.UnionQueryRunnerFactory; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QuerySchedulerProvider; diff --git a/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java b/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java index 3094d60ffc93..3ef5b7409539 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java @@ -62,8 +62,8 @@ import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.query.topn.TopNQueryMetricsFactory; import org.apache.druid.query.topn.TopNQueryQueryToolChest; -import org.apache.druid.query.union.UnionQueryQueryToolChest; import org.apache.druid.query.union.UnionQuery; +import org.apache.druid.query.union.UnionQueryQueryToolChest; import java.util.Map; diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index aa77fba2a96b..5f5c10afc9e6 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -180,10 +180,8 @@ public QueryRunner getQueryRunnerForIntervals(final Query query, final { final QueryToolChest> toolChest = warehouse.getToolChest(query); -// Optional> toolchestExecResult = toolChest.executeQuery21(warehouse, query, this); if (toolChest instanceof QueryExecutor) { - QueryExecutor t = (QueryExecutor) toolChest; - return t.makeQueryRunner(query, warehouse, this); + return ((QueryExecutor) toolChest).makeQueryRunner(query, warehouse, this); } // transform TableDataSource to GlobalTableDataSource when eligible @@ -453,12 +451,11 @@ private DataSource inlineIfNecessary( ); QueryToolChest subQueryToolChest = warehouse.getToolChest(subQuery); -// final QueryRunner subQueryRunner ; final QueryRunner subQueryRunner; if (subQueryToolChest instanceof QueryExecutor) { subQueryRunner = ((QueryExecutor) subQueryToolChest) .makeQueryRunner(subQueryWithSerialization, warehouse, this); - }else { + } else { subQueryRunner = subQueryWithSerialization.getRunner(this); } From 22bdd3ca0dddf138f40f1a77132ba42353ffcc94 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 13:35:02 +0000 Subject: [PATCH 085/149] x --- .../druid/sql/calcite/BaseCalciteQueryTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index fa5ad7380ec1..1963a755e700 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -1338,7 +1338,7 @@ protected static boolean isRewriteJoinToFilter(final Map queryCo public static Query recursivelyClearContext(final Query query, ObjectMapper queryJsonMapper) { try { - Query newQuery = query.withDataSources(recursivelyClearContext2(query.getDataSources(), queryJsonMapper)); + Query newQuery = query.withDataSources(recursivelyClearDatasource(query.getDataSources(), queryJsonMapper)); final JsonNode newQueryNode = queryJsonMapper.valueToTree(newQuery); ((ObjectNode) newQueryNode).remove("context"); return queryJsonMapper.treeToValue(newQueryNode, Query.class); @@ -1348,12 +1348,12 @@ public static Query recursivelyClearContext(final Query query, ObjectM } } - private static List recursivelyClearContext2(final List dataSource, ObjectMapper queryJsonMapper) + private static List recursivelyClearDatasource(final List dataSources, + ObjectMapper queryJsonMapper) { - - List ret=new ArrayList(); - for (DataSource dataSource2 : dataSource) { - ret.add(recursivelyClearContext(dataSource2, queryJsonMapper)); + List ret = new ArrayList(); + for (DataSource dataSource : dataSources) { + ret.add(recursivelyClearContext(dataSource, queryJsonMapper)); } return ret; } From 0f36ec6ca8bade7be47fe188901682c0feb533b8 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 13:35:18 +0000 Subject: [PATCH 086/149] x --- .../org/apache/druid/sql/calcite/util/SqlTestFramework.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index ceebefb5094e..f193c85ac1dc 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -50,8 +50,8 @@ import org.apache.druid.query.groupby.TestGroupByBuffers; import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.query.topn.TopNQueryConfig; -import org.apache.druid.query.union.UnionQueryRunnerFactory; import org.apache.druid.query.union.UnionQuery; +import org.apache.druid.query.union.UnionQueryRunnerFactory; import org.apache.druid.quidem.TestSqlModule; import org.apache.druid.segment.DefaultColumnFormatConfig; import org.apache.druid.segment.join.JoinableFactoryWrapper; @@ -435,7 +435,7 @@ public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglom @Override public Map, ? extends QueryRunnerFactory> makeRunnerFactories(Injector injector) { - Map map =new HashMap<>(); + Map map = new HashMap<>(); UnionQueryRunnerFactory factory = injector.getInstance(UnionQueryRunnerFactory.class); map.put(UnionQuery.class, factory); return map; From 4eae97892edd701de97844b70fa74e4cc6cb79bb Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 13:40:04 +0000 Subject: [PATCH 087/149] x --- .../druid/query/QuerySegmentWalker.java | 5 -- .../query/planning/DataSourceAnalysis.java | 5 ++ .../query/union/RealUnionQueryRunner.java | 59 ------------------- 3 files changed, 5 insertions(+), 64 deletions(-) delete mode 100644 processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java diff --git a/processing/src/main/java/org/apache/druid/query/QuerySegmentWalker.java b/processing/src/main/java/org/apache/druid/query/QuerySegmentWalker.java index 8e57a1086d4f..7084a80935d1 100644 --- a/processing/src/main/java/org/apache/druid/query/QuerySegmentWalker.java +++ b/processing/src/main/java/org/apache/druid/query/QuerySegmentWalker.java @@ -54,9 +54,4 @@ public interface QuerySegmentWalker * @return the Queryable object with the given SegmentSpecs */ QueryRunner getQueryRunnerForSegments(Query query, Iterable specs); - - default QueryRunner executeQuery(Query query) - { - throw new UnsupportedOperationException("executeQuery is not implemented"); - } } diff --git a/processing/src/main/java/org/apache/druid/query/planning/DataSourceAnalysis.java b/processing/src/main/java/org/apache/druid/query/planning/DataSourceAnalysis.java index b7d7db7c8e3a..23f372c43900 100644 --- a/processing/src/main/java/org/apache/druid/query/planning/DataSourceAnalysis.java +++ b/processing/src/main/java/org/apache/druid/query/planning/DataSourceAnalysis.java @@ -301,6 +301,11 @@ public String toString() */ public boolean isGlobal() { + for (PreJoinableClause preJoinableClause : preJoinableClauses) { + if (!preJoinableClause.getDataSource().isGlobal()) { + return false; + } + } return baseDataSource.isGlobal(); } } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java deleted file mode 100644 index 39c00250730e..000000000000 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionQueryRunner.java +++ /dev/null @@ -1,59 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.union; - -import org.apache.druid.java.util.common.guava.Sequence; -import org.apache.druid.java.util.common.guava.Sequences; -import org.apache.druid.query.Query; -import org.apache.druid.query.QueryPlus; -import org.apache.druid.query.QueryRunner; -import org.apache.druid.query.QuerySegmentWalker; -import org.apache.druid.query.context.ResponseContext; -import java.util.ArrayList; -import java.util.List; - -public class RealUnionQueryRunner implements QueryRunner -{ - private QuerySegmentWalker walker; - - public RealUnionQueryRunner(QuerySegmentWalker walker) - { - this.walker = walker; - } - - @Override - public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) - { - UnionQuery unionQuery = queryPlus.unwrapQuery(UnionQuery.class); - - List seqs = new ArrayList(); - for (Query query : unionQuery.queries) { - seqs.add(makeUnionResult(queryPlus.withQuery(query), responseContext)); - } - return Sequences.simple(seqs); - } - - private RealUnionResult makeUnionResult(QueryPlus withQuery, ResponseContext responseContext) - { - QueryRunner runner = withQuery.getQuery().getRunner(walker); - Sequence seq = runner.run(withQuery, responseContext); - return new RealUnionResult(seq); - } -} From 52d5fdac0ed83fec6348ee5e084d7735876a454d Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 13:55:59 +0000 Subject: [PATCH 088/149] warehouseprovider-try --- ...DefaultQueryRunnerFactoryConglomerate.java | 3 - .../apache/druid/query/QueryToolChest.java | 19 ++- .../query/union/UnionQueryQueryToolChest.java | 4 +- ...estUnionAllTwoQueriesBothQueriesAreJoin.iq | 152 +++++++++++++++++- 4 files changed, 163 insertions(+), 15 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java index 33287b9eebde..0caf02d619a4 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java @@ -34,9 +34,6 @@ public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactory public DefaultQueryRunnerFactoryConglomerate(Map, QueryRunnerFactory> factories) { this.factories = new IdentityHashMap<>(factories); - for (QueryRunnerFactory factory : factories.values()) { - factory.getToolchest().setWarehouse(this); - } } @Override diff --git a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java index 2c814553bfc1..253de32299bb 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java @@ -24,6 +24,8 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.TypeFactory; import com.google.common.base.Function; +import com.google.inject.Inject; +import com.google.inject.Provider; import org.apache.druid.error.DruidException; import org.apache.druid.frame.allocation.MemoryAllocatorFactory; import org.apache.druid.guice.annotations.ExtensionPoint; @@ -47,7 +49,7 @@ public abstract class QueryToolChest warehouseProvider; protected QueryToolChest() { @@ -72,9 +74,10 @@ protected QueryToolChest() } } - public void setWarehouse(QueryToolChestWarehouse warehouse) + @Inject + public void setWarehouse(Provider warehouseProvider) { - this.warehouse = warehouse; + this.warehouseProvider = warehouseProvider; } public final JavaType getBaseResultType() @@ -435,4 +438,14 @@ public boolean canExecuteFully(Query query) return (!(dataSourceFromQuery instanceof QueryDataSource) || canPerformSubquery(((QueryDataSource) dataSourceFromQuery).getQuery())); } + + public final QueryToolChestWarehouse getWarehouse() + { + return warehouseProvider.get(); + } + + public final QueryToolChest> getToolchest(Query query) + { + return getWarehouse().getToolChest(query); + } } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 59eefffa648c..692d0ea15e71 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -106,7 +106,7 @@ public Sequence resultsAsArrays( for (int i = 0; i < results.size(); i++) { Query q = query.queries.get(i); RealUnionResult realUnionResult = results.get(i); - QueryToolChest toolChest = warehouse.getToolChest(q); + QueryToolChest toolChest = getToolchest(q); Sequence queryResults = toolChest.resultsAsArrays(q, realUnionResult.getResults()); resultSeqs.add(queryResults); } @@ -127,7 +127,7 @@ public Optional> resultsAsFrames( for (int i = 0; i < results.size(); i++) { Query q = query.queries.get(i); RealUnionResult realUnionResult = results.get(i); - QueryToolChest toolChest = warehouse.getToolChest(q); + QueryToolChest toolChest = getToolchest(q); Optional> queryResults = toolChest .resultsAsFrames(query, resultSequence, memoryAllocatorFactory, useNestedForUnknownTypes); if (!queryResults.isPresent()) { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq index 65b1fac2fad7..ef4fc61a0fc8 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq @@ -9,13 +9,151 @@ !set outputformat mysql !use druidtest:/// (SELECT COUNT(*) FROM foo LEFT JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; -+--------+ -| EXPR$0 | -+--------+ -| 1 | -| 6 | -+--------+ -(2 rows) +org.apache.calcite.avatica.AvaticaSqlException: Error -1 (00000) : Error while executing SQL "(SELECT COUNT(*) FROM foo LEFT JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ": Remote driver error: QueryInterruptedException: (null exception message) -> NullPointerException: (null exception message) + at org.apache.calcite.avatica.Helper.createException(Helper.java:54) + at org.apache.calcite.avatica.Helper.createException(Helper.java:41) + at org.apache.calcite.avatica.AvaticaStatement.executeInternal(AvaticaStatement.java:164) + at org.apache.calcite.avatica.AvaticaStatement.executeQuery(AvaticaStatement.java:228) + at net.hydromatic.quidem.Quidem.checkResult(Quidem.java:299) + at net.hydromatic.quidem.Quidem.access$2600(Quidem.java:57) + at net.hydromatic.quidem.Quidem$ContextImpl.checkResult(Quidem.java:1636) + at net.hydromatic.quidem.Quidem$CheckResultCommand.execute(Quidem.java:941) + at net.hydromatic.quidem.Quidem$CompositeCommand.execute(Quidem.java:1437) + at net.hydromatic.quidem.Quidem.execute(Quidem.java:190) + at org.apache.druid.quidem.DruidQuidemTestBase$DruidQuidemRunner.run(DruidQuidemTestBase.java:158) + at org.apache.druid.quidem.DruidQuidemTestBase$DruidQuidemRunner.run(DruidQuidemTestBase.java:138) + at org.apache.druid.sql.calcite.QTestCase$1.run(QTestCase.java:73) + at org.apache.druid.sql.calcite.QueryTestRunner.run(QueryTestRunner.java:766) + at org.apache.druid.sql.calcite.QueryTestBuilder.run(QueryTestBuilder.java:304) + at org.apache.druid.sql.calcite.BaseCalciteQueryTest.testQuery(BaseCalciteQueryTest.java:733) + at org.apache.druid.sql.calcite.CalciteJoinQueryTest.testUnionAllTwoQueriesBothQueriesAreJoin(CalciteJoinQueryTest.java:4427) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) + at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) + at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) + at java.base/java.lang.reflect.Method.invoke(Method.java:566) + at org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:728) + at org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131) + at org.apache.druid.sql.calcite.NotYetSupported$NotYetSupportedProcessor.interceptTestMethod(NotYetSupported.java:134) + at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(InterceptingExecutableInvoker.java:103) + at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.lambda$invoke$0(InterceptingExecutableInvoker.java:93) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106) + at org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:156) + at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:147) + at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:86) + at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(InterceptingExecutableInvoker.java:103) + at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.lambda$invoke$0(InterceptingExecutableInvoker.java:93) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45) + at org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37) + at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:92) + at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:86) + at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$7(TestMethodTestDescriptor.java:218) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:214) + at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:139) + at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:69) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:151) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) + at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1541) + at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:155) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) + at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) + at java.base/java.util.ArrayList.forEach(ArrayList.java:1541) + at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:155) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) + at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) + at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) + at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) + at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:35) + at org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57) + at org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:54) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:147) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:127) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:90) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:55) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:102) + at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:54) + at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:114) + at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:95) + at org.junit.platform.launcher.core.DefaultLauncherSession$DelegatingLauncher.execute(DefaultLauncherSession.java:91) + at org.junit.platform.launcher.core.SessionPerRequestLauncher.execute(SessionPerRequestLauncher.java:60) + at org.eclipse.jdt.internal.junit5.runner.JUnit5TestReference.run(JUnit5TestReference.java:98) + at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:40) + at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:529) + at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:756) + at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:452) + at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:210) +QueryInterruptedException{msg=null, code=Unknown exception, class=java.lang.NullPointerException, host=null} + at org.apache.druid.query.QueryInterruptedException.wrapIfNeeded(QueryInterruptedException.java:113) + at org.apache.druid.sql.avatica.ErrorHandler.sanitize(ErrorHandler.java:82) + at org.apache.druid.sql.avatica.DruidMeta.mapException(DruidMeta.java:395) + at org.apache.druid.sql.avatica.DruidMeta.prepareAndExecute(DruidMeta.java:362) + at org.apache.calcite.avatica.remote.LocalService.apply(LocalService.java:212) + at org.apache.calcite.avatica.remote.Service$PrepareAndExecuteRequest.accept(Service.java:952) + at org.apache.calcite.avatica.remote.Service$PrepareAndExecuteRequest.accept(Service.java:904) + at org.apache.calcite.avatica.remote.AbstractHandler.apply(AbstractHandler.java:94) + at org.apache.calcite.avatica.remote.JsonHandler.apply(JsonHandler.java:52) + at org.apache.calcite.avatica.server.AvaticaJsonHandler.handle(AvaticaJsonHandler.java:133) + at org.apache.druid.sql.avatica.DruidAvaticaJsonHandler.handle(DruidAvaticaJsonHandler.java:61) + at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) + at org.eclipse.jetty.server.Server.handle(Server.java:516) + at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:487) + at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:732) + at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:479) + at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) + at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) + at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) + at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) + at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) + at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) + at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) + at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) + at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) + at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) + at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) + at java.base/java.lang.Thread.run(Thread.java:829) +Caused by: java.lang.NullPointerException + at org.apache.druid.query.union.UnionQueryQueryToolChest.resultsAsArrays(UnionQueryQueryToolChest.java:109) + at org.apache.druid.query.union.UnionQueryQueryToolChest.resultsAsArrays(UnionQueryQueryToolChest.java:1) + at org.apache.druid.server.ClientQuerySegmentWalker.materializeResultsAsArray(ClientQuerySegmentWalker.java:890) + at org.apache.druid.server.ClientQuerySegmentWalker.toInlineDataSource(ClientQuerySegmentWalker.java:722) + at org.apache.druid.server.ClientQuerySegmentWalker.inlineIfNecessary(ClientQuerySegmentWalker.java:466) + at org.apache.druid.server.ClientQuerySegmentWalker.getQueryRunnerForIntervals(ClientQuerySegmentWalker.java:230) + at org.apache.druid.server.SpecificSegmentsQuerySegmentWalker.getQueryRunnerForIntervals(SpecificSegmentsQuerySegmentWalker.java:223) + at org.apache.druid.query.spec.MultipleIntervalSegmentSpec.lookup(MultipleIntervalSegmentSpec.java:57) + at org.apache.druid.query.BaseQuery.getRunner(BaseQuery.java:107) + at org.apache.druid.query.QueryPlus.run(QueryPlus.java:158) + at org.apache.druid.server.QueryLifecycle.execute(QueryLifecycle.java:315) + at org.apache.druid.server.QueryLifecycle.runSimple(QueryLifecycle.java:162) + at org.apache.druid.sql.calcite.run.NativeQueryMaker.execute(NativeQueryMaker.java:195) + at org.apache.druid.sql.calcite.run.NativeQueryMaker.runQuery(NativeQueryMaker.java:152) + at org.apache.druid.sql.calcite.planner.QueryHandler.lambda$7(QueryHandler.java:568) + at org.apache.druid.sql.calcite.planner.PlannerResult.run(PlannerResult.java:62) + at org.apache.druid.sql.DirectStatement$ResultSet.run(DirectStatement.java:109) + at org.apache.druid.sql.DirectStatement.execute(DirectStatement.java:184) + at org.apache.druid.sql.avatica.DruidJdbcResultSet.lambda$0(DruidJdbcResultSet.java:239) + at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) + at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) + at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) + ... 1 more !ok LogicalUnion(all=[true]) From 4a2c095d7c686731305422aee79927ae0c31d1cd Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 13:56:01 +0000 Subject: [PATCH 089/149] Revert "warehouseprovider-try" This reverts commit 52d5fdac0ed83fec6348ee5e084d7735876a454d. --- ...DefaultQueryRunnerFactoryConglomerate.java | 3 + .../apache/druid/query/QueryToolChest.java | 19 +-- .../query/union/UnionQueryQueryToolChest.java | 4 +- ...estUnionAllTwoQueriesBothQueriesAreJoin.iq | 152 +----------------- 4 files changed, 15 insertions(+), 163 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java index 0caf02d619a4..33287b9eebde 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java @@ -34,6 +34,9 @@ public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactory public DefaultQueryRunnerFactoryConglomerate(Map, QueryRunnerFactory> factories) { this.factories = new IdentityHashMap<>(factories); + for (QueryRunnerFactory factory : factories.values()) { + factory.getToolchest().setWarehouse(this); + } } @Override diff --git a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java index 253de32299bb..2c814553bfc1 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java @@ -24,8 +24,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.TypeFactory; import com.google.common.base.Function; -import com.google.inject.Inject; -import com.google.inject.Provider; import org.apache.druid.error.DruidException; import org.apache.druid.frame.allocation.MemoryAllocatorFactory; import org.apache.druid.guice.annotations.ExtensionPoint; @@ -49,7 +47,7 @@ public abstract class QueryToolChest warehouseProvider; + protected QueryToolChestWarehouse warehouse; protected QueryToolChest() { @@ -74,10 +72,9 @@ protected QueryToolChest() } } - @Inject - public void setWarehouse(Provider warehouseProvider) + public void setWarehouse(QueryToolChestWarehouse warehouse) { - this.warehouseProvider = warehouseProvider; + this.warehouse = warehouse; } public final JavaType getBaseResultType() @@ -438,14 +435,4 @@ public boolean canExecuteFully(Query query) return (!(dataSourceFromQuery instanceof QueryDataSource) || canPerformSubquery(((QueryDataSource) dataSourceFromQuery).getQuery())); } - - public final QueryToolChestWarehouse getWarehouse() - { - return warehouseProvider.get(); - } - - public final QueryToolChest> getToolchest(Query query) - { - return getWarehouse().getToolChest(query); - } } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 692d0ea15e71..59eefffa648c 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -106,7 +106,7 @@ public Sequence resultsAsArrays( for (int i = 0; i < results.size(); i++) { Query q = query.queries.get(i); RealUnionResult realUnionResult = results.get(i); - QueryToolChest toolChest = getToolchest(q); + QueryToolChest toolChest = warehouse.getToolChest(q); Sequence queryResults = toolChest.resultsAsArrays(q, realUnionResult.getResults()); resultSeqs.add(queryResults); } @@ -127,7 +127,7 @@ public Optional> resultsAsFrames( for (int i = 0; i < results.size(); i++) { Query q = query.queries.get(i); RealUnionResult realUnionResult = results.get(i); - QueryToolChest toolChest = getToolchest(q); + QueryToolChest toolChest = warehouse.getToolChest(q); Optional> queryResults = toolChest .resultsAsFrames(query, resultSequence, memoryAllocatorFactory, useNestedForUnknownTypes); if (!queryResults.isPresent()) { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq index ef4fc61a0fc8..65b1fac2fad7 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteJoinQueryTest/testUnionAllTwoQueriesBothQueriesAreJoin.iq @@ -9,151 +9,13 @@ !set outputformat mysql !use druidtest:/// (SELECT COUNT(*) FROM foo LEFT JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ; -org.apache.calcite.avatica.AvaticaSqlException: Error -1 (00000) : Error while executing SQL "(SELECT COUNT(*) FROM foo LEFT JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k UNION ALL SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) ": Remote driver error: QueryInterruptedException: (null exception message) -> NullPointerException: (null exception message) - at org.apache.calcite.avatica.Helper.createException(Helper.java:54) - at org.apache.calcite.avatica.Helper.createException(Helper.java:41) - at org.apache.calcite.avatica.AvaticaStatement.executeInternal(AvaticaStatement.java:164) - at org.apache.calcite.avatica.AvaticaStatement.executeQuery(AvaticaStatement.java:228) - at net.hydromatic.quidem.Quidem.checkResult(Quidem.java:299) - at net.hydromatic.quidem.Quidem.access$2600(Quidem.java:57) - at net.hydromatic.quidem.Quidem$ContextImpl.checkResult(Quidem.java:1636) - at net.hydromatic.quidem.Quidem$CheckResultCommand.execute(Quidem.java:941) - at net.hydromatic.quidem.Quidem$CompositeCommand.execute(Quidem.java:1437) - at net.hydromatic.quidem.Quidem.execute(Quidem.java:190) - at org.apache.druid.quidem.DruidQuidemTestBase$DruidQuidemRunner.run(DruidQuidemTestBase.java:158) - at org.apache.druid.quidem.DruidQuidemTestBase$DruidQuidemRunner.run(DruidQuidemTestBase.java:138) - at org.apache.druid.sql.calcite.QTestCase$1.run(QTestCase.java:73) - at org.apache.druid.sql.calcite.QueryTestRunner.run(QueryTestRunner.java:766) - at org.apache.druid.sql.calcite.QueryTestBuilder.run(QueryTestBuilder.java:304) - at org.apache.druid.sql.calcite.BaseCalciteQueryTest.testQuery(BaseCalciteQueryTest.java:733) - at org.apache.druid.sql.calcite.CalciteJoinQueryTest.testUnionAllTwoQueriesBothQueriesAreJoin(CalciteJoinQueryTest.java:4427) - at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke0(Native Method) - at java.base/jdk.internal.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) - at java.base/jdk.internal.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) - at java.base/java.lang.reflect.Method.invoke(Method.java:566) - at org.junit.platform.commons.util.ReflectionUtils.invokeMethod(ReflectionUtils.java:728) - at org.junit.jupiter.engine.execution.MethodInvocation.proceed(MethodInvocation.java:60) - at org.junit.jupiter.engine.execution.InvocationInterceptorChain$ValidatingInvocation.proceed(InvocationInterceptorChain.java:131) - at org.apache.druid.sql.calcite.NotYetSupported$NotYetSupportedProcessor.interceptTestMethod(NotYetSupported.java:134) - at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(InterceptingExecutableInvoker.java:103) - at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.lambda$invoke$0(InterceptingExecutableInvoker.java:93) - at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106) - at org.junit.jupiter.engine.extension.TimeoutExtension.intercept(TimeoutExtension.java:156) - at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestableMethod(TimeoutExtension.java:147) - at org.junit.jupiter.engine.extension.TimeoutExtension.interceptTestMethod(TimeoutExtension.java:86) - at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker$ReflectiveInterceptorCall.lambda$ofVoidMethod$0(InterceptingExecutableInvoker.java:103) - at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.lambda$invoke$0(InterceptingExecutableInvoker.java:93) - at org.junit.jupiter.engine.execution.InvocationInterceptorChain$InterceptedInvocation.proceed(InvocationInterceptorChain.java:106) - at org.junit.jupiter.engine.execution.InvocationInterceptorChain.proceed(InvocationInterceptorChain.java:64) - at org.junit.jupiter.engine.execution.InvocationInterceptorChain.chainAndInvoke(InvocationInterceptorChain.java:45) - at org.junit.jupiter.engine.execution.InvocationInterceptorChain.invoke(InvocationInterceptorChain.java:37) - at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:92) - at org.junit.jupiter.engine.execution.InterceptingExecutableInvoker.invoke(InterceptingExecutableInvoker.java:86) - at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.lambda$invokeTestMethod$7(TestMethodTestDescriptor.java:218) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.invokeTestMethod(TestMethodTestDescriptor.java:214) - at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:139) - at org.junit.jupiter.engine.descriptor.TestMethodTestDescriptor.execute(TestMethodTestDescriptor.java:69) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:151) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) - at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) - at java.base/java.util.ArrayList.forEach(ArrayList.java:1541) - at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:155) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) - at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) - at java.base/java.util.ArrayList.forEach(ArrayList.java:1541) - at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.invokeAll(SameThreadHierarchicalTestExecutorService.java:41) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$6(NodeTestTask.java:155) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$8(NodeTestTask.java:141) - at org.junit.platform.engine.support.hierarchical.Node.around(Node.java:137) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.lambda$executeRecursively$9(NodeTestTask.java:139) - at org.junit.platform.engine.support.hierarchical.ThrowableCollector.execute(ThrowableCollector.java:73) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.executeRecursively(NodeTestTask.java:138) - at org.junit.platform.engine.support.hierarchical.NodeTestTask.execute(NodeTestTask.java:95) - at org.junit.platform.engine.support.hierarchical.SameThreadHierarchicalTestExecutorService.submit(SameThreadHierarchicalTestExecutorService.java:35) - at org.junit.platform.engine.support.hierarchical.HierarchicalTestExecutor.execute(HierarchicalTestExecutor.java:57) - at org.junit.platform.engine.support.hierarchical.HierarchicalTestEngine.execute(HierarchicalTestEngine.java:54) - at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:147) - at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:127) - at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:90) - at org.junit.platform.launcher.core.EngineExecutionOrchestrator.lambda$execute$0(EngineExecutionOrchestrator.java:55) - at org.junit.platform.launcher.core.EngineExecutionOrchestrator.withInterceptedStreams(EngineExecutionOrchestrator.java:102) - at org.junit.platform.launcher.core.EngineExecutionOrchestrator.execute(EngineExecutionOrchestrator.java:54) - at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:114) - at org.junit.platform.launcher.core.DefaultLauncher.execute(DefaultLauncher.java:95) - at org.junit.platform.launcher.core.DefaultLauncherSession$DelegatingLauncher.execute(DefaultLauncherSession.java:91) - at org.junit.platform.launcher.core.SessionPerRequestLauncher.execute(SessionPerRequestLauncher.java:60) - at org.eclipse.jdt.internal.junit5.runner.JUnit5TestReference.run(JUnit5TestReference.java:98) - at org.eclipse.jdt.internal.junit.runner.TestExecution.run(TestExecution.java:40) - at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:529) - at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.runTests(RemoteTestRunner.java:756) - at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.run(RemoteTestRunner.java:452) - at org.eclipse.jdt.internal.junit.runner.RemoteTestRunner.main(RemoteTestRunner.java:210) -QueryInterruptedException{msg=null, code=Unknown exception, class=java.lang.NullPointerException, host=null} - at org.apache.druid.query.QueryInterruptedException.wrapIfNeeded(QueryInterruptedException.java:113) - at org.apache.druid.sql.avatica.ErrorHandler.sanitize(ErrorHandler.java:82) - at org.apache.druid.sql.avatica.DruidMeta.mapException(DruidMeta.java:395) - at org.apache.druid.sql.avatica.DruidMeta.prepareAndExecute(DruidMeta.java:362) - at org.apache.calcite.avatica.remote.LocalService.apply(LocalService.java:212) - at org.apache.calcite.avatica.remote.Service$PrepareAndExecuteRequest.accept(Service.java:952) - at org.apache.calcite.avatica.remote.Service$PrepareAndExecuteRequest.accept(Service.java:904) - at org.apache.calcite.avatica.remote.AbstractHandler.apply(AbstractHandler.java:94) - at org.apache.calcite.avatica.remote.JsonHandler.apply(JsonHandler.java:52) - at org.apache.calcite.avatica.server.AvaticaJsonHandler.handle(AvaticaJsonHandler.java:133) - at org.apache.druid.sql.avatica.DruidAvaticaJsonHandler.handle(DruidAvaticaJsonHandler.java:61) - at org.eclipse.jetty.server.handler.HandlerWrapper.handle(HandlerWrapper.java:127) - at org.eclipse.jetty.server.Server.handle(Server.java:516) - at org.eclipse.jetty.server.HttpChannel.lambda$handle$1(HttpChannel.java:487) - at org.eclipse.jetty.server.HttpChannel.dispatch(HttpChannel.java:732) - at org.eclipse.jetty.server.HttpChannel.handle(HttpChannel.java:479) - at org.eclipse.jetty.server.HttpConnection.onFillable(HttpConnection.java:277) - at org.eclipse.jetty.io.AbstractConnection$ReadCallback.succeeded(AbstractConnection.java:311) - at org.eclipse.jetty.io.FillInterest.fillable(FillInterest.java:105) - at org.eclipse.jetty.io.ChannelEndPoint$1.run(ChannelEndPoint.java:104) - at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.runTask(EatWhatYouKill.java:338) - at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.doProduce(EatWhatYouKill.java:315) - at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.tryProduce(EatWhatYouKill.java:173) - at org.eclipse.jetty.util.thread.strategy.EatWhatYouKill.run(EatWhatYouKill.java:131) - at org.eclipse.jetty.util.thread.ReservedThreadExecutor$ReservedThread.run(ReservedThreadExecutor.java:409) - at org.eclipse.jetty.util.thread.QueuedThreadPool.runJob(QueuedThreadPool.java:883) - at org.eclipse.jetty.util.thread.QueuedThreadPool$Runner.run(QueuedThreadPool.java:1034) - at java.base/java.lang.Thread.run(Thread.java:829) -Caused by: java.lang.NullPointerException - at org.apache.druid.query.union.UnionQueryQueryToolChest.resultsAsArrays(UnionQueryQueryToolChest.java:109) - at org.apache.druid.query.union.UnionQueryQueryToolChest.resultsAsArrays(UnionQueryQueryToolChest.java:1) - at org.apache.druid.server.ClientQuerySegmentWalker.materializeResultsAsArray(ClientQuerySegmentWalker.java:890) - at org.apache.druid.server.ClientQuerySegmentWalker.toInlineDataSource(ClientQuerySegmentWalker.java:722) - at org.apache.druid.server.ClientQuerySegmentWalker.inlineIfNecessary(ClientQuerySegmentWalker.java:466) - at org.apache.druid.server.ClientQuerySegmentWalker.getQueryRunnerForIntervals(ClientQuerySegmentWalker.java:230) - at org.apache.druid.server.SpecificSegmentsQuerySegmentWalker.getQueryRunnerForIntervals(SpecificSegmentsQuerySegmentWalker.java:223) - at org.apache.druid.query.spec.MultipleIntervalSegmentSpec.lookup(MultipleIntervalSegmentSpec.java:57) - at org.apache.druid.query.BaseQuery.getRunner(BaseQuery.java:107) - at org.apache.druid.query.QueryPlus.run(QueryPlus.java:158) - at org.apache.druid.server.QueryLifecycle.execute(QueryLifecycle.java:315) - at org.apache.druid.server.QueryLifecycle.runSimple(QueryLifecycle.java:162) - at org.apache.druid.sql.calcite.run.NativeQueryMaker.execute(NativeQueryMaker.java:195) - at org.apache.druid.sql.calcite.run.NativeQueryMaker.runQuery(NativeQueryMaker.java:152) - at org.apache.druid.sql.calcite.planner.QueryHandler.lambda$7(QueryHandler.java:568) - at org.apache.druid.sql.calcite.planner.PlannerResult.run(PlannerResult.java:62) - at org.apache.druid.sql.DirectStatement$ResultSet.run(DirectStatement.java:109) - at org.apache.druid.sql.DirectStatement.execute(DirectStatement.java:184) - at org.apache.druid.sql.avatica.DruidJdbcResultSet.lambda$0(DruidJdbcResultSet.java:239) - at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264) - at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128) - at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628) - ... 1 more ++--------+ +| EXPR$0 | ++--------+ +| 1 | +| 6 | ++--------+ +(2 rows) !ok LogicalUnion(all=[true]) From e6ae62b20c7d538dbec70529debf99cbf2cae186 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 14:04:51 +0000 Subject: [PATCH 090/149] updates --- .../sql/calcite/CalciteArraysQueryTest.java | 2 +- .../druid/sql/calcite/CalciteQueryTest.java | 36 +++++++++---------- .../sql/calcite/DecoupledTestConfig.java | 6 +++- ...UnnestExtractionFn@NullHandling=default.iq | 4 +-- 4 files changed, 26 insertions(+), 22 deletions(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index 1cd61275421a..b31c96286c9a 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -7193,7 +7193,7 @@ public void testArrayToMvPostaggInline() ); } - // this is due to substring('',1') is null + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNNEST_SUBSTRING_EMPTY, separateDefaultModeTest = true) @Test public void testUnnestExtractionFn() { diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 28059f7f6bdc..24ba2a1b8d26 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -3117,26 +3117,26 @@ public void testUnionAllQueries() skipVectorize(); ImmutableList> queries = ImmutableList.of( Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .aggregators(aggregators(new CountAggregatorFactory("a0"))) - .context(QUERY_CONTEXT_DEFAULT) - .build(), + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators(aggregators(new CountAggregatorFactory("a0"))) + .context(QUERY_CONTEXT_DEFAULT) + .build(), Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .aggregators(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .context(QUERY_CONTEXT_DEFAULT) - .build(), + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .context(QUERY_CONTEXT_DEFAULT) + .build(), Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .aggregators(aggregators(new CountAggregatorFactory("a0"))) - .context(QUERY_CONTEXT_DEFAULT) - .build() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators(aggregators(new CountAggregatorFactory("a0"))) + .context(QUERY_CONTEXT_DEFAULT) + .build() ); if (testBuilder().isDecoupledMode()) { queries = ImmutableList.of( diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledTestConfig.java b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledTestConfig.java index 60d261c33e0a..ba46dea8008b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledTestConfig.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/DecoupledTestConfig.java @@ -161,7 +161,11 @@ enum QuidemTestCaseReason /** * Uses a UNION ALL query. */ - UNION_ALL_QUERY; + UNION_ALL_QUERY, + /** + * This is due to substring('',1') is null. + */ + UNNEST_SUBSTRING_EMPTY; public boolean isPresent() { diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=default.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=default.iq index 2acefa4270fd..bf46154190aa 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=default.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=default.iq @@ -1,5 +1,5 @@ -# testUnnestExtractionFn@NullHandling=default case-crc:404dc668 -# quidem testcase reason: UNNEST_DIFFERENT_RESULTSET +# testUnnestExtractionFn@NullHandling=default case-crc:33ee4d88 +# quidem testcase reason: UNNEST_SUBSTRING_EMPTY !set debug true !set defaultTimeout 300000 !set maxScatterGatherBytes 9223372036854775807 From ede7c6860d204b06d82e3b6680513e30df420fbe Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 14:28:49 +0000 Subject: [PATCH 091/149] undo indent change --- .../apache/druid/query/union/UnionQuery.java | 2 +- .../sql/calcite/CalciteArraysQueryTest.java | 3 +- .../sql/calcite/CalciteJoinQueryTest.java | 2 + .../druid/sql/calcite/CalciteQueryTest.java | 55 +++---- .../testUnionAllQueries.iq | 144 ++++++++++++++++++ 5 files changed, 173 insertions(+), 33 deletions(-) create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testUnionAllQueries.iq diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index f1f2bb03e3d2..aeeb1fdb7623 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -223,7 +223,7 @@ public String toString() public DataSourceAnalysis getDataSourceAnalysis() { OpagueDataSourceCover ds = new OpagueDataSourceCover(new UnionDataSource(getDataSources())); - return new DataSourceAnalysis(ds, null, null, null); + return new DataSourceAnalysis(ds, null, null, Collections.emptyList()); } private static class OpagueDataSourceCover implements DataSource diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index b31c96286c9a..024a6c3db485 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -7228,8 +7228,7 @@ public void testUnnestExtractionFn() ImmutableList.of( new Object[]{"a"}, new Object[]{"c"}, - new Object[]{"d"}, - new Object[]{""} + new Object[]{"d"} ) ); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java index 7c551954cb26..016e2a41e641 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteJoinQueryTest.java @@ -4338,6 +4338,8 @@ public void testUnionAllTwoQueriesLeftQueryIsJoin(Map queryConte cannotVectorize(); } + + testQuery( "(SELECT COUNT(*) FROM foo INNER JOIN lookup.lookyloo ON foo.dim1 = lookyloo.k) UNION ALL SELECT SUM(cnt) FROM foo", queryContext, diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 24ba2a1b8d26..33eb53ba5225 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -114,7 +114,6 @@ import org.apache.druid.query.topn.InvertedTopNMetricSpec; import org.apache.druid.query.topn.NumericTopNMetricSpec; import org.apache.druid.query.topn.TopNQueryBuilder; -import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.VirtualColumn; import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.ColumnHolder; @@ -3110,43 +3109,39 @@ public void testTopNWithSelectAndOrderByProjections() ); } + + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNION_ALL_QUERY) @Test public void testUnionAllQueries() { msqIncompatible(); skipVectorize(); - ImmutableList> queries = ImmutableList.of( - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .aggregators(aggregators(new CountAggregatorFactory("a0"))) - .context(QUERY_CONTEXT_DEFAULT) - .build(), - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .aggregators(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) - .context(QUERY_CONTEXT_DEFAULT) - .build(), - Druids.newTimeseriesQueryBuilder() - .dataSource(CalciteTests.DATASOURCE1) - .intervals(querySegmentSpec(Filtration.eternity())) - .granularity(Granularities.ALL) - .aggregators(aggregators(new CountAggregatorFactory("a0"))) - .context(QUERY_CONTEXT_DEFAULT) - .build() - ); - if (testBuilder().isDecoupledMode()) { - queries = ImmutableList.of( - new UnionQuery(queries) - ); - } testQuery( "SELECT COUNT(*) FROM foo UNION ALL SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo", - queries, + ImmutableList.of( + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators(aggregators(new CountAggregatorFactory("a0"))) + .context(QUERY_CONTEXT_DEFAULT) + .build(), + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators(aggregators(new LongSumAggregatorFactory("a0", "cnt"))) + .context(QUERY_CONTEXT_DEFAULT) + .build(), + Druids.newTimeseriesQueryBuilder() + .dataSource(CalciteTests.DATASOURCE1) + .intervals(querySegmentSpec(Filtration.eternity())) + .granularity(Granularities.ALL) + .aggregators(aggregators(new CountAggregatorFactory("a0"))) + .context(QUERY_CONTEXT_DEFAULT) + .build() + ), ImmutableList.of(new Object[]{6L}, new Object[]{6L}, new Object[]{6L}) ); } diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testUnionAllQueries.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testUnionAllQueries.iq new file mode 100644 index 000000000000..e4c98567392d --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testUnionAllQueries.iq @@ -0,0 +1,144 @@ +# testUnionAllQueries case-crc:c3323b00 +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +SELECT COUNT(*) FROM foo UNION ALL SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo; ++--------+ +| EXPR$0 | ++--------+ +| 6 | +| 6 | +| 6 | ++--------+ +(3 rows) + +!ok +LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalTableScan(table=[[druid, foo]]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalTableScan(table=[[druid, foo]]) + +!logicalPlan +DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan From b9e4b938a1dcaa808acbcc0b74c6a77a98c35a4e Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 14:29:28 +0000 Subject: [PATCH 092/149] add annotation --- .../test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java | 1 + 1 file changed, 1 insertion(+) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 33eb53ba5225..7a68d4e4315b 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -3146,6 +3146,7 @@ public void testUnionAllQueries() ); } + @DecoupledTestConfig(quidemReason = QuidemTestCaseReason.UNION_ALL_QUERY) @Test public void testUnionAllQueriesWithLimit() { From bd7eacc2f2d98927c31b8ae3457588fa75d448eb Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 14:29:58 +0000 Subject: [PATCH 093/149] add --- .../testUnionAllQueriesWithLimit.iq | 146 ++++++++++++++++++ 1 file changed, 146 insertions(+) create mode 100644 sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testUnionAllQueriesWithLimit.iq diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testUnionAllQueriesWithLimit.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testUnionAllQueriesWithLimit.iq new file mode 100644 index 000000000000..49fa808fe731 --- /dev/null +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testUnionAllQueriesWithLimit.iq @@ -0,0 +1,146 @@ +# testUnionAllQueriesWithLimit case-crc:30654774 +# quidem testcase reason: UNION_ALL_QUERY +!set debug true +!set defaultTimeout 300000 +!set maxScatterGatherBytes 9223372036854775807 +!set plannerStrategy DECOUPLED +!set sqlCurrentTimestamp 2000-01-01T00:00:00Z +!set sqlQueryId dummy +!set outputformat mysql +!use druidtest:/// +SELECT * FROM (SELECT COUNT(*) FROM foo UNION ALL SELECT SUM(cnt) FROM foo UNION ALL SELECT COUNT(*) FROM foo) LIMIT 2; ++--------+ +| EXPR$0 | ++--------+ +| 6 | +| 6 | ++--------+ +(2 rows) + +!ok +LogicalSort(fetch=[2]) + LogicalUnion(all=[true]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalTableScan(table=[[druid, foo]]) + LogicalAggregate(group=[{}], EXPR$0=[SUM($4)]) + LogicalTableScan(table=[[druid, foo]]) + LogicalAggregate(group=[{}], EXPR$0=[COUNT()]) + LogicalTableScan(table=[[druid, foo]]) + +!logicalPlan +DruidSort(fetch=[2], druid=[logical]) + DruidUnion(all=[true]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[SUM($4)], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + DruidAggregate(group=[{}], EXPR$0=[COUNT()], druid=[logical]) + DruidTableScan(table=[[druid, foo]], druid=[logical]) + +!druidPlan +{ + "queryType" : "scan", + "dataSource" : { + "type" : "query", + "query" : { + "queryType" : "union", + "queries" : [ { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "longSum", + "name" : "a0", + "fieldName" : "cnt" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + }, { + "queryType" : "timeseries", + "dataSource" : { + "type" : "table", + "name" : "foo" + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "granularity" : { + "type" : "all" + }, + "aggregations" : [ { + "type" : "count", + "name" : "a0" + } ], + "context" : { + "debug" : "true", + "defaultTimeout" : "300000", + "maxScatterGatherBytes" : "9223372036854775807", + "outputformat" : "MYSQL", + "plannerStrategy" : "DECOUPLED", + "sqlCurrentTimestamp" : "2000-01-01T00:00:00Z", + "sqlQueryId" : "dummy", + "sqlStringifyArrays" : false + } + } ], + "context" : null + } + }, + "intervals" : { + "type" : "intervals", + "intervals" : [ "-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z" ] + }, + "resultFormat" : "compactedList", + "limit" : 2, + "columns" : [ "a0" ], + "columnTypes" : [ "LONG" ], + "granularity" : { + "type" : "all" + }, + "legacy" : false +} +!nativePlan From 0bc26351416887a463cdb411f6067ef10900781b Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 14:53:30 +0000 Subject: [PATCH 094/149] better errors --- .../org/apache/druid/error/DruidException.java | 5 +++++ .../apache/druid/query/union/UnionQuery.java | 18 +++++++++--------- 2 files changed, 14 insertions(+), 9 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/error/DruidException.java b/processing/src/main/java/org/apache/druid/error/DruidException.java index 66190d13a91f..7ea155e373a6 100644 --- a/processing/src/main/java/org/apache/druid/error/DruidException.java +++ b/processing/src/main/java/org/apache/druid/error/DruidException.java @@ -524,4 +524,9 @@ public String getErrorCode() protected abstract DruidException makeException(DruidExceptionBuilder bob); } + public static String getCurrentMethodName() + { + StackTraceElement stackTraceElement = Thread.currentThread().getStackTrace()[2]; + return stackTraceElement.getClassName() + "." + stackTraceElement.getMethodName(); + } } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index aeeb1fdb7623..54316b10ceab 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -124,7 +124,7 @@ public List getIntervals() @Override public Duration getDuration() { - throw DruidException.defensive("This is not supported"); + throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); } @Override @@ -136,7 +136,7 @@ public Granularity getGranularity() @Override public DateTimeZone getTimezone() { - throw DruidException.defensive("This is not supported"); + throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); } @Override @@ -148,7 +148,7 @@ public Map getContext() @Override public Ordering getResultOrdering() { - throw new RuntimeException("FIXME: Unimplemented!"); + throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); } @Override @@ -161,7 +161,7 @@ public Query withOverriddenContext(Map contextO @Override public Query withQuerySegmentSpec(QuerySegmentSpec spec) { - throw new RuntimeException("FIXME: Unimplemented!"); + throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); } @Override @@ -191,7 +191,7 @@ public String getSubQueryId() @Override public Query withDataSource(DataSource dataSource) { - throw new RuntimeException("FIXME: Unimplemented!"); + throw new RuntimeException("This method is not supported. Use withDataSources instead!"); } @Override @@ -250,7 +250,7 @@ public List getChildren() @Override public DataSource withChildren(List children) { - throw DruidException.defensive().build("Not implemented"); + throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); } @Override @@ -274,13 +274,13 @@ public boolean isConcrete() @Override public Function createSegmentMapFunction(Query query, AtomicLong cpuTimeAcc) { - throw DruidException.defensive().build("Not implemented"); + throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); } @Override public DataSource withUpdatedDataSource(DataSource newSource) { - throw DruidException.defensive().build("Not implemented"); + throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); } @Override @@ -292,7 +292,7 @@ public byte[] getCacheKey() @Override public DataSourceAnalysis getAnalysis() { - throw DruidException.defensive().build("Not implemented"); + throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); } } } From cedc6de9dfafa65248ad56d4e447e9a9a2bb10f9 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 14:57:27 +0000 Subject: [PATCH 095/149] cosmetic --- .../src/main/java/org/apache/druid/error/DruidException.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/error/DruidException.java b/processing/src/main/java/org/apache/druid/error/DruidException.java index 7ea155e373a6..61ff9ccfe733 100644 --- a/processing/src/main/java/org/apache/druid/error/DruidException.java +++ b/processing/src/main/java/org/apache/druid/error/DruidException.java @@ -527,6 +527,6 @@ public String getErrorCode() public static String getCurrentMethodName() { StackTraceElement stackTraceElement = Thread.currentThread().getStackTrace()[2]; - return stackTraceElement.getClassName() + "." + stackTraceElement.getMethodName(); + return stackTraceElement.getClassName() + "#" + stackTraceElement.getMethodName(); } } From a57fae8dcb0ecaea6a6c77e908dff9961aa2b5b5 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 15:02:44 +0000 Subject: [PATCH 096/149] cleanup --- .../org/apache/druid/query/QueryRunnerFactoryConglomerate.java | 1 + .../main/java/org/apache/druid/query/groupby/GroupByQuery.java | 1 + .../src/main/java/org/apache/druid/query/union/UnionQuery.java | 3 +-- 3 files changed, 3 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java index 81cceee4afeb..ef3921475b57 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java @@ -27,6 +27,7 @@ public interface QueryRunnerFactoryConglomerate extends QueryToolChestWarehouse { > QueryRunnerFactory findFactory(QueryType query); + @Override default > QueryToolChest getToolChest(QueryType query) { QueryRunnerFactory factory = findFactory(query); diff --git a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java index 68c071060633..2a84331fa51f 100644 --- a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java @@ -350,6 +350,7 @@ public RowSignature getResultRowSignature() * * @see ResultRow for documentation about the order that fields will be in */ + @Override public RowSignature getResultRowSignature(final RowSignature.Finalization finalization) { if (finalization == RowSignature.Finalization.UNKNOWN) { diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index 54316b10ceab..d1b1091125d8 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -82,7 +82,6 @@ public DataSource getDataSource() @Override public List getDataSources() { - List dataSources = new ArrayList<>(); for (Query query : queries) { dataSources.add(query.getDataSource()); @@ -94,7 +93,6 @@ public List getDataSources() public boolean hasFilters() { return false; - } @Override @@ -220,6 +218,7 @@ public String toString() return "UnionQuery [context=" + context + ", queries=" + queries + "]"; } + @Override public DataSourceAnalysis getDataSourceAnalysis() { OpagueDataSourceCover ds = new OpagueDataSourceCover(new UnionDataSource(getDataSources())); From a05e363329051eab16fe903661965155befcf12a Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 15:03:06 +0000 Subject: [PATCH 097/149] executioncontext --- .../apache/druid/query/ExecutionContext.java | 33 +++++++++++++++++++ .../org/apache/druid/query/QueryExecutor.java | 3 +- 2 files changed, 34 insertions(+), 2 deletions(-) create mode 100644 processing/src/main/java/org/apache/druid/query/ExecutionContext.java diff --git a/processing/src/main/java/org/apache/druid/query/ExecutionContext.java b/processing/src/main/java/org/apache/druid/query/ExecutionContext.java new file mode 100644 index 000000000000..4b625807188f --- /dev/null +++ b/processing/src/main/java/org/apache/druid/query/ExecutionContext.java @@ -0,0 +1,33 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query; + +public class ExecutionContext +{ + public final QueryToolChestWarehouse warehouse; + public final QuerySegmentWalker walker; + + public ExecutionContext(QuerySegmentWalker walker, QueryToolChestWarehouse warehouse) + { + this.warehouse = warehouse; + this.walker = walker; + } + +} diff --git a/processing/src/main/java/org/apache/druid/query/QueryExecutor.java b/processing/src/main/java/org/apache/druid/query/QueryExecutor.java index 8b11fe6da431..ccb89a87aae2 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryExecutor.java +++ b/processing/src/main/java/org/apache/druid/query/QueryExecutor.java @@ -28,7 +28,6 @@ public interface QueryExecutor { QueryRunner makeQueryRunner( Query query, - QueryToolChestWarehouse warehouse, - QuerySegmentWalker walker + ExecutionContext executionContext ); } From 89e06b2a316af42a534cc8920dbdc5cfc03fa14b Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 15:03:08 +0000 Subject: [PATCH 098/149] Revert "executioncontext" This reverts commit a05e363329051eab16fe903661965155befcf12a. --- .../apache/druid/query/ExecutionContext.java | 33 ------------------- .../org/apache/druid/query/QueryExecutor.java | 3 +- 2 files changed, 2 insertions(+), 34 deletions(-) delete mode 100644 processing/src/main/java/org/apache/druid/query/ExecutionContext.java diff --git a/processing/src/main/java/org/apache/druid/query/ExecutionContext.java b/processing/src/main/java/org/apache/druid/query/ExecutionContext.java deleted file mode 100644 index 4b625807188f..000000000000 --- a/processing/src/main/java/org/apache/druid/query/ExecutionContext.java +++ /dev/null @@ -1,33 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query; - -public class ExecutionContext -{ - public final QueryToolChestWarehouse warehouse; - public final QuerySegmentWalker walker; - - public ExecutionContext(QuerySegmentWalker walker, QueryToolChestWarehouse warehouse) - { - this.warehouse = warehouse; - this.walker = walker; - } - -} diff --git a/processing/src/main/java/org/apache/druid/query/QueryExecutor.java b/processing/src/main/java/org/apache/druid/query/QueryExecutor.java index ccb89a87aae2..8b11fe6da431 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryExecutor.java +++ b/processing/src/main/java/org/apache/druid/query/QueryExecutor.java @@ -28,6 +28,7 @@ public interface QueryExecutor { QueryRunner makeQueryRunner( Query query, - ExecutionContext executionContext + QueryToolChestWarehouse warehouse, + QuerySegmentWalker walker ); } From e7debd2f905addb66bd82bc7f2700c842a3ba849 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 15:05:24 +0000 Subject: [PATCH 099/149] fix --- .../org/apache/druid/query/union/UnionQueryQueryToolChest.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 59eefffa648c..1a3102a4b2d4 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -48,6 +48,7 @@ public class UnionQueryQueryToolChest extends QueryToolChest { + @Override public QueryRunner makeQueryRunner(Query query, QueryToolChestWarehouse warehouse, QuerySegmentWalker clientQuerySegmentWalker) { @@ -129,7 +130,7 @@ public Optional> resultsAsFrames( RealUnionResult realUnionResult = results.get(i); QueryToolChest toolChest = warehouse.getToolChest(q); Optional> queryResults = toolChest - .resultsAsFrames(query, resultSequence, memoryAllocatorFactory, useNestedForUnknownTypes); + .resultsAsFrames(query, realUnionResult.getResults(), memoryAllocatorFactory, useNestedForUnknownTypes); if (!queryResults.isPresent()) { return Optional.empty(); } From b47d92b958497071aff148b6bd9afb3cd5f15d5a Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 15:20:33 +0000 Subject: [PATCH 100/149] cleanup/etc --- .../query/union/UnionQueryQueryToolChest.java | 66 ++++++++++++------- 1 file changed, 41 insertions(+), 25 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 1a3102a4b2d4..754d6123ebf6 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -95,23 +95,45 @@ public RowSignature resultArraySignature(UnionQuery query) throw DruidException.defensive("None of the subqueries support row signature"); } + abstract static class UnionSequenceMaker + { + public Optional> transform( + UnionQuery query, + Sequence resultSequence) + { + List results = resultSequence.toList(); + List> resultSeqs = new ArrayList<>(); + + for (int i = 0; i < results.size(); i++) { + Query q = query.queries.get(i); + RealUnionResult realUnionResult = results.get(i); + Optional> queryResults = transformResults(q, realUnionResult.getResults()); + if (!queryResults.isPresent()) { + return Optional.empty(); + } + resultSeqs.add(queryResults.get()); + } + return Optional.of(Sequences.concat(resultSeqs)); + } + + public abstract Optional> transformResults(Query q, Sequence results); + } + @Override @SuppressWarnings({"unchecked", "rawtypes"}) public Sequence resultsAsArrays( UnionQuery query, Sequence resultSequence) { - List results = resultSequence.toList(); - List> resultSeqs = new ArrayList>(); - - for (int i = 0; i < results.size(); i++) { - Query q = query.queries.get(i); - RealUnionResult realUnionResult = results.get(i); - QueryToolChest toolChest = warehouse.getToolChest(q); - Sequence queryResults = toolChest.resultsAsArrays(q, realUnionResult.getResults()); - resultSeqs.add(queryResults); - } - return Sequences.concat(resultSeqs); + return new UnionSequenceMaker() + { + @Override + public Optional> transformResults(Query query, Sequence results) + { + QueryToolChest toolChest = warehouse.getToolChest(query); + return Optional.of(toolChest.resultsAsArrays(query, results)); + } + }.transform(query, resultSequence).get(); } @Override @@ -122,20 +144,14 @@ public Optional> resultsAsFrames( MemoryAllocatorFactory memoryAllocatorFactory, boolean useNestedForUnknownTypes) { - List results = resultSequence.toList(); - List> resultSeqs = new ArrayList<>(); - - for (int i = 0; i < results.size(); i++) { - Query q = query.queries.get(i); - RealUnionResult realUnionResult = results.get(i); - QueryToolChest toolChest = warehouse.getToolChest(q); - Optional> queryResults = toolChest - .resultsAsFrames(query, realUnionResult.getResults(), memoryAllocatorFactory, useNestedForUnknownTypes); - if (!queryResults.isPresent()) { - return Optional.empty(); + return new UnionSequenceMaker() + { + @Override + public Optional> transformResults(Query query, Sequence results) + { + QueryToolChest toolChest = warehouse.getToolChest(query); + return toolChest.resultsAsFrames(query, results, memoryAllocatorFactory, useNestedForUnknownTypes); } - resultSeqs.add(queryResults.get()); - } - return Optional.of(Sequences.concat(resultSeqs)); + }.transform(query, resultSequence); } } From eebb276b7b45bd5f67668e1ea0c5264b7a03d22e Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 15:27:14 +0000 Subject: [PATCH 101/149] update message --- .../org/apache/druid/query/QueryRunnerFactoryConglomerate.java | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java index ef3921475b57..842d2f3d2793 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java @@ -32,7 +32,8 @@ default > QueryToolChest getToolChes { QueryRunnerFactory factory = findFactory(query); if (factory == null) { - throw DruidException.defensive(query.getClass().getName() + " is not registered"); + throw DruidException + .defensive("QueryRunnerFactory for QueryType [%s] is not registered!", query.getClass().getName()); } return factory.getToolchest(); } From 6e474634ae7b6612c86045f593bcb7b0e6f3775a Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Tue, 15 Oct 2024 15:29:57 +0000 Subject: [PATCH 102/149] remove NYS enum value --- .../test/java/org/apache/druid/sql/calcite/NotYetSupported.java | 1 - 1 file changed, 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java index 8dad971439c4..f456bae91f78 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/NotYetSupported.java @@ -89,7 +89,6 @@ enum Modes RESULT_MISMATCH(AssertionError.class, "(assertResulEquals|AssertionError: column content mismatch)"), LONG_CASTING(AssertionError.class, "expected: java.lang.Long"), UNSUPPORTED_NULL_ORDERING(DruidException.class, "(A|DE)SCENDING ordering with NULLS (LAST|FIRST)"), - UNION_WITH_COMPLEX_OPERAND(DruidException.class, "Only Table and Values are supported as inputs for Union"), UNION_MORE_STRICT_ROWTYPE_CHECK(DruidException.class, "Row signature mismatch in Union inputs"), SORT_REMOVE_TROUBLE(DruidException.class, "Calcite assertion violated.*Sort\\."), SORT_REMOVE_CONSTANT_KEYS_CONFLICT(DruidException.class, "not enough rules"), From 68a7a7065be390337150a18d3ee74b641b9b6a1f Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 16 Oct 2024 10:34:39 +0000 Subject: [PATCH 103/149] minify failure --- .../kttm_sunburst.iq | 22531 ---------------- 1 file changed, 22531 deletions(-) diff --git a/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/kttm_sunburst.iq b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/kttm_sunburst.iq index 7140ae5951b9..03229f55888d 100644 --- a/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/kttm_sunburst.iq +++ b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/kttm_sunburst.iq @@ -2,627 +2,7 @@ #started Wed Jul 10 19:19:15 IST 2024 !use druidtest:///?componentSupplier=KttmNestedComponentSupplier !set outputformat mysql -# Wed Jul 10 19:20:02 IST 2024 -SELECT -FLOOR(CAST(t."session_length" AS DOUBLE) / 10) * 10 AS "session_length" -FROM "kttm_nested" AS t -WHERE (TIMESTAMP '2019-08-25 00:00:00'<=CAST(t."__time" AS TIMESTAMP) AND CAST(t."__time" AS TIMESTAMP) Date: Wed, 16 Oct 2024 10:43:54 +0000 Subject: [PATCH 104/149] fix typo --- .../org/apache/druid/sql/calcite/util/SqlTestFramework.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 228b83fd203b..de09e97923cd 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -679,7 +679,7 @@ public void configure(Binder binder) QueryStackTests .makeDefaultQueryRunnerFactories( processingConfig, - builder.mergeBufferCount, + builder.minTopNThreshold, jsonMapper, testBufferPool, groupByBuffers From 150477edaa7ec8038e71ad2bfc7f99b3e84782f6 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 16 Oct 2024 10:44:08 +0000 Subject: [PATCH 105/149] undo temp changes --- .../kttm_sunburst.iq | 22531 ++++++++++++++++ 1 file changed, 22531 insertions(+) diff --git a/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/kttm_sunburst.iq b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/kttm_sunburst.iq index 03229f55888d..7140ae5951b9 100644 --- a/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/kttm_sunburst.iq +++ b/quidem-ut/src/test/quidem/org.apache.druid.quidem.QTest/kttm_sunburst.iq @@ -2,7 +2,627 @@ #started Wed Jul 10 19:19:15 IST 2024 !use druidtest:///?componentSupplier=KttmNestedComponentSupplier !set outputformat mysql +# Wed Jul 10 19:20:02 IST 2024 +SELECT +FLOOR(CAST(t."session_length" AS DOUBLE) / 10) * 10 AS "session_length" +FROM "kttm_nested" AS t +WHERE (TIMESTAMP '2019-08-25 00:00:00'<=CAST(t."__time" AS TIMESTAMP) AND CAST(t."__time" AS TIMESTAMP) Date: Thu, 17 Oct 2024 06:40:02 +0000 Subject: [PATCH 106/149] update --- .../testUnnestExtractionFn@NullHandling=sql.iq | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=sql.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=sql.iq index ca41ee34e6a6..57ce98e52b5e 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=sql.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteArraysQueryTest/testUnnestExtractionFn@NullHandling=sql.iq @@ -1,5 +1,5 @@ -# testUnnestExtractionFn@NullHandling=sql case-crc:404dc668 -# quidem testcase reason: UNNEST_DIFFERENT_RESULTSET +# testUnnestExtractionFn@NullHandling=sql case-crc:33ee4d88 +# quidem testcase reason: UNNEST_SUBSTRING_EMPTY !set debug true !set defaultTimeout 300000 !set maxScatterGatherBytes 9223372036854775807 From 8e8a2e3f1251b1a487d6b71d503ac9d8d075a618 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 06:44:48 +0000 Subject: [PATCH 107/149] remove method --- .../apache/druid/query/timeseries/TimeseriesQuery.java | 9 ++------- .../query/timeseries/TimeseriesQueryQueryToolChest.java | 6 ++---- 2 files changed, 4 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java index c3e805a93150..0ae7f4f13deb 100644 --- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java +++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java @@ -186,19 +186,14 @@ public boolean isSkipEmptyBuckets() @Override public RowSignature getResultRowSignature(Finalization finalization) { - return getResultSignature(finalization); - } - - @Deprecated - public RowSignature getResultSignature(final RowSignature.Finalization finalization) - { + final Finalization finalization1 = finalization; final RowSignature.Builder builder = RowSignature.builder(); builder.addTimeColumn(); String timestampResultField = getTimestampResultField(); if (StringUtils.isNotEmpty(timestampResultField)) { builder.add(timestampResultField, ColumnType.LONG); } - builder.addAggregators(aggregatorSpecs, finalization); + builder.addAggregators(aggregatorSpecs, finalization1); builder.addPostAggregators(postAggregatorSpecs); return builder.build(); } diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChest.java index 4e06da817ce7..acaf1427ce32 100644 --- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQueryQueryToolChest.java @@ -445,7 +445,7 @@ public Function, Result> ma @Override public RowSignature resultArraySignature(TimeseriesQuery query) { - return query.getResultSignature(RowSignature.Finalization.UNKNOWN); + return query.getResultRowSignature(RowSignature.Finalization.UNKNOWN); } @Override @@ -486,9 +486,7 @@ public Optional> resultsAsFrames( ) { final RowSignature rowSignature = - query.getResultSignature( - query.context().isFinalize(true) ? RowSignature.Finalization.YES : RowSignature.Finalization.NO - ); + query.getResultRowSignature(query.context().isFinalize(true) ? RowSignature.Finalization.YES : RowSignature.Finalization.NO); final Pair cursorAndCloseable = IterableRowsCursorHelper.getCursorFromSequence( resultsAsArrays(query, resultSequence), rowSignature From 371f8633fc094bf33b0441740a05efee4fa23bbe Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 06:50:04 +0000 Subject: [PATCH 108/149] cleanup --- .../query/union/UnionQueryQueryToolChest.java | 53 +++++++++---------- 1 file changed, 25 insertions(+), 28 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 754d6123ebf6..9ae0ae5923b1 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -47,7 +47,6 @@ public class UnionQueryQueryToolChest extends QueryToolChest implements QueryExecutor { - @Override public QueryRunner makeQueryRunner(Query query, QueryToolChestWarehouse warehouse, QuerySegmentWalker clientQuerySegmentWalker) @@ -79,9 +78,7 @@ public Function makePreComputeManipulatorFn( @Override public TypeReference getResultTypeReference() { - return new TypeReference() - { - }; + return null; } @Override @@ -95,30 +92,6 @@ public RowSignature resultArraySignature(UnionQuery query) throw DruidException.defensive("None of the subqueries support row signature"); } - abstract static class UnionSequenceMaker - { - public Optional> transform( - UnionQuery query, - Sequence resultSequence) - { - List results = resultSequence.toList(); - List> resultSeqs = new ArrayList<>(); - - for (int i = 0; i < results.size(); i++) { - Query q = query.queries.get(i); - RealUnionResult realUnionResult = results.get(i); - Optional> queryResults = transformResults(q, realUnionResult.getResults()); - if (!queryResults.isPresent()) { - return Optional.empty(); - } - resultSeqs.add(queryResults.get()); - } - return Optional.of(Sequences.concat(resultSeqs)); - } - - public abstract Optional> transformResults(Query q, Sequence results); - } - @Override @SuppressWarnings({"unchecked", "rawtypes"}) public Sequence resultsAsArrays( @@ -154,4 +127,28 @@ public Optional> transformResults(Query query, S } }.transform(query, resultSequence); } + + abstract static class UnionSequenceMaker + { + public Optional> transform( + UnionQuery query, + Sequence resultSequence) + { + List results = resultSequence.toList(); + List> resultSeqs = new ArrayList<>(); + + for (int i = 0; i < results.size(); i++) { + Query q = query.queries.get(i); + RealUnionResult realUnionResult = results.get(i); + Optional> queryResults = transformResults(q, realUnionResult.getResults()); + if (!queryResults.isPresent()) { + return Optional.empty(); + } + resultSeqs.add(queryResults.get()); + } + return Optional.of(Sequences.concat(resultSeqs)); + } + + public abstract Optional> transformResults(Query q, Sequence results); + } } From 5b18b8aa5ca7f2abc0dc1e8af455429f5e7d972e Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 07:48:20 +0000 Subject: [PATCH 109/149] x --- .../main/java/org/apache/druid/query/scan/ScanQuery.java | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java b/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java index 037148f50c3a..dd197facd70d 100644 --- a/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java +++ b/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java @@ -39,6 +39,7 @@ import org.apache.druid.query.OrderBy; import org.apache.druid.query.Queries; import org.apache.druid.query.filter.DimFilter; +import org.apache.druid.query.groupby.SupportRowSignature; import org.apache.druid.query.operator.OffsetLimit; import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.segment.VirtualColumn; @@ -48,6 +49,7 @@ import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature.Builder; +import org.apache.druid.segment.column.RowSignature.Finalization; import javax.annotation.Nullable; import java.util.Collections; @@ -57,7 +59,7 @@ import java.util.Objects; import java.util.Set; -public class ScanQuery extends BaseQuery +public class ScanQuery extends BaseQuery implements SupportRowSignature { public enum ResultFormat { @@ -611,6 +613,11 @@ public boolean equals(Object obj) } } + @Override + public RowSignature getResultRowSignature(Finalization finalization) + { + return getRowSignature(); + } /** * Returns the RowSignature. From 9c1e2a1d6e0d0af7855c043572ec1ba19b215caa Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 08:24:26 +0000 Subject: [PATCH 110/149] x --- .../org/apache/druid/java/util/common/guava/Sequences.java | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/java/util/common/guava/Sequences.java b/processing/src/main/java/org/apache/druid/java/util/common/guava/Sequences.java index 9f8169434af6..ccd449c21aa7 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/guava/Sequences.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/guava/Sequences.java @@ -22,7 +22,6 @@ import com.google.common.base.Function; import com.google.common.base.Preconditions; import com.google.common.base.Predicate; - import java.io.Closeable; import java.util.Arrays; import java.util.Collections; @@ -147,4 +146,9 @@ public Yielder toYielder(OutType initValue, YieldingAccumulat return Yielders.done(initValue, null); } } + + public static Sequence singleton(T value) + { + return simple(Collections.singletonList(value)); + } } From 15d1b255d5e301c09fafc234ef50d92d50908f59 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 08:32:58 +0000 Subject: [PATCH 111/149] test not added befgore --- .../java/util/common/guava/Sequences.java | 4 +- .../druid/query/QueryToolChestTestHelper.java | 2 +- .../union/UnionQueryQueryToolChestTest2.java | 160 ++++++++++++++++++ 3 files changed, 163 insertions(+), 3 deletions(-) create mode 100644 processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest2.java diff --git a/processing/src/main/java/org/apache/druid/java/util/common/guava/Sequences.java b/processing/src/main/java/org/apache/druid/java/util/common/guava/Sequences.java index ccd449c21aa7..dab5835568c3 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/guava/Sequences.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/guava/Sequences.java @@ -147,8 +147,8 @@ public Yielder toYielder(OutType initValue, YieldingAccumulat } } - public static Sequence singleton(T value) + public static Sequence of(T ...values) { - return simple(Collections.singletonList(value)); + return simple(Arrays.asList(values)); } } diff --git a/processing/src/test/java/org/apache/druid/query/QueryToolChestTestHelper.java b/processing/src/test/java/org/apache/druid/query/QueryToolChestTestHelper.java index a3bedddc563a..3c0d46242231 100644 --- a/processing/src/test/java/org/apache/druid/query/QueryToolChestTestHelper.java +++ b/processing/src/test/java/org/apache/druid/query/QueryToolChestTestHelper.java @@ -26,7 +26,7 @@ public class QueryToolChestTestHelper { - public static void assertArrayResultsEquals(final List expected, final Sequence actual) + public static void assertArrayResultsEquals(List expected, final Sequence actual) { final List actualList = actual.toList(); Assert.assertEquals("number of results", expected.size(), actualList.size()); diff --git a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest2.java b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest2.java new file mode 100644 index 000000000000..671af0f4d768 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest2.java @@ -0,0 +1,160 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.union; + +import com.google.common.collect.ImmutableList; + +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.java.util.common.Intervals; +import org.apache.druid.java.util.common.guava.Sequence; +import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.query.Druids; +import org.apache.druid.query.Query; +import org.apache.druid.query.QueryRunnerTestHelper; +import org.apache.druid.query.QueryToolChestTestHelper; +import org.apache.druid.query.scan.ScanQuery; +import org.apache.druid.query.scan.ScanResultValue; +import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.segment.column.RowSignature; +import org.junit.Assert; +import org.junit.jupiter.api.BeforeAll; +import org.junit.jupiter.api.Test; + +import java.util.ArrayList; +import java.util.Arrays; +import java.util.List; + +public class UnionQueryQueryToolChestTest2 +{ + private static final String TIMESTAMP_RESULT_FIELD_NAME = "d0"; + private static final UnionQueryQueryToolChest TOOL_CHEST = new UnionQueryQueryToolChest(); + + @BeforeAll + public static void setUpClass() + { + NullHandling.initializeForTests(); + } + + @Test + public void testResultArraySignatureWithTimestampResultField() + { + RowSignature sig = RowSignature.builder() + .add("a", ColumnType.STRING) + .add("b", ColumnType.STRING) + .build(); + + TestScanQuery scan1 = new TestScanQuery("foo", sig) + .appendRow("a", "a") + .appendRow("a", "b"); + TestScanQuery scan2 = new TestScanQuery("bar", sig) + .appendRow("x", "x") + .appendRow("x", "y"); + + List> queries = ImmutableList.of( + scan1.query, + scan2.query + ); + + UnionQuery query = new UnionQuery(queries); + + Assert.assertEquals( + sig, + TOOL_CHEST.resultArraySignature(query) + ); + } + + static class TestScanQuery + { + final ScanQuery query; + final List results = new ArrayList(); + + public TestScanQuery(String sourceName, RowSignature signature) + { + this.query = Druids.newScanQueryBuilder() + .dataSource("bar") + .intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("2000/3000")))) + .columns(signature.getColumnNames()) + .columnTypes(signature.getColumnTypes()) + .build(); + } + + public TestScanQuery appendRow(Object... row) + { + results.add(row); + return this; + } + + public Sequence makeResultSequence() + { + ScanResultValue result = new ScanResultValue( + QueryRunnerTestHelper.SEGMENT_ID.toString(), + query.getColumns(), + convertResultsToListOfLists() + ); + return Sequences.of(result); + } + + private List> convertResultsToListOfLists() + { + List> resultsRows = new ArrayList>(); + for (Object[] objects : results) { + resultsRows.add(Arrays.asList(objects)); + } + return resultsRows; + } + } + + @Test + public void testResultsAsArrays() + { + RowSignature sig = RowSignature.builder() + .add("a", ColumnType.STRING) + .add("b", ColumnType.STRING) + .build(); + + TestScanQuery scan1 = new TestScanQuery("foo", sig) + .appendRow("a", "a") + .appendRow("a", "b"); + TestScanQuery scan2 = new TestScanQuery("bar", sig) + .appendRow("x", "x") + .appendRow("x", "y"); + + UnionQuery query = new UnionQuery( + ImmutableList.of( + scan1.query, + scan2.query + ) + ); + QueryToolChestTestHelper.assertArrayResultsEquals( + ImmutableList.builder() + .addAll(scan1.results) + .addAll(scan2.results) + .build(), + TOOL_CHEST.resultsAsArrays( + query, + Sequences.of( + new RealUnionResult(scan1.makeResultSequence()), + new RealUnionResult(scan2.makeResultSequence()) + ) + ) + ); + } +} From 6bf5cb95e5a1f6f9c4ec1d690d9a07bc70746e5a Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 08:43:13 +0000 Subject: [PATCH 112/149] make some stuff --- .../scan/ScanQueryQueryToolChestTest.java | 9 ++++-- ...java => UnionQueryQueryToolChestTest.java} | 28 +++++++++++++++---- 2 files changed, 28 insertions(+), 9 deletions(-) rename processing/src/test/java/org/apache/druid/query/union/{UnionQueryQueryToolChestTest2.java => UnionQueryQueryToolChestTest.java} (83%) diff --git a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryQueryToolChestTest.java index 76e7f6bdd3ea..2e2158f866ab 100644 --- a/processing/src/test/java/org/apache/druid/query/scan/ScanQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/scan/ScanQueryQueryToolChestTest.java @@ -71,9 +71,12 @@ public class ScanQueryQueryToolChestTest new Object[]{3.5, "str4"} ); - private final ScanQueryQueryToolChest toolChest = new ScanQueryQueryToolChest( - DefaultGenericQueryMetricsFactory.instance() - ); + private final ScanQueryQueryToolChest toolChest = makeTestScanQueryToolChest(); + + public static ScanQueryQueryToolChest makeTestScanQueryToolChest() + { + return new ScanQueryQueryToolChest(DefaultGenericQueryMetricsFactory.instance()); + } @Test public void test_resultArraySignature_columnsNotSpecified() diff --git a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest2.java b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java similarity index 83% rename from processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest2.java rename to processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java index 671af0f4d768..94acd6214928 100644 --- a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest2.java +++ b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java @@ -20,16 +20,20 @@ package org.apache.druid.query.union; import com.google.common.collect.ImmutableList; - +import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.query.Druids; +import org.apache.druid.query.MapQueryToolChestWarehouse; import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunnerTestHelper; +import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.QueryToolChestTestHelper; +import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.scan.ScanQuery; +import org.apache.druid.query.scan.ScanQueryQueryToolChestTest; import org.apache.druid.query.scan.ScanResultValue; import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; import org.apache.druid.segment.column.ColumnType; @@ -42,10 +46,8 @@ import java.util.Arrays; import java.util.List; -public class UnionQueryQueryToolChestTest2 +public class UnionQueryQueryToolChestTest { - private static final String TIMESTAMP_RESULT_FIELD_NAME = "d0"; - private static final UnionQueryQueryToolChest TOOL_CHEST = new UnionQueryQueryToolChest(); @BeforeAll public static void setUpClass() @@ -53,6 +55,19 @@ public static void setUpClass() NullHandling.initializeForTests(); } + final UnionQueryQueryToolChest toolChest; + + public UnionQueryQueryToolChestTest() + { + toolChest = new UnionQueryQueryToolChest(); + QueryToolChestWarehouse warehouse = new MapQueryToolChestWarehouse( + ImmutableMap., QueryToolChest>builder() + .put(ScanQuery.class, ScanQueryQueryToolChestTest.makeTestScanQueryToolChest()) + .build() + ); + toolChest.setWarehouse(warehouse); + } + @Test public void testResultArraySignatureWithTimestampResultField() { @@ -77,7 +92,7 @@ public void testResultArraySignatureWithTimestampResultField() Assert.assertEquals( sig, - TOOL_CHEST.resultArraySignature(query) + toolChest.resultArraySignature(query) ); } @@ -91,6 +106,7 @@ public TestScanQuery(String sourceName, RowSignature signature) this.query = Druids.newScanQueryBuilder() .dataSource("bar") .intervals(new MultipleIntervalSegmentSpec(ImmutableList.of(Intervals.of("2000/3000")))) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) .columns(signature.getColumnNames()) .columnTypes(signature.getColumnTypes()) .build(); @@ -148,7 +164,7 @@ public void testResultsAsArrays() .addAll(scan1.results) .addAll(scan2.results) .build(), - TOOL_CHEST.resultsAsArrays( + toolChest.resultsAsArrays( query, Sequences.of( new RealUnionResult(scan1.makeResultSequence()), From 5327b1cc9ad38658c4869782f3211a1eae52dbba Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 08:48:31 +0000 Subject: [PATCH 113/149] add test for resultsasframes --- .../union/UnionQueryQueryToolChestTest.java | 47 ++++++++++++++++++- 1 file changed, 46 insertions(+), 1 deletion(-) diff --git a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java index 94acd6214928..becf74d4e01f 100644 --- a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java @@ -22,10 +22,14 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.frame.allocation.HeapMemoryAllocator; +import org.apache.druid.frame.allocation.SingleMemoryAllocatorFactory; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.query.Druids; +import org.apache.druid.query.FrameBasedInlineDataSource; +import org.apache.druid.query.FrameSignaturePair; import org.apache.druid.query.MapQueryToolChestWarehouse; import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunnerTestHelper; @@ -48,7 +52,6 @@ public class UnionQueryQueryToolChestTest { - @BeforeAll public static void setUpClass() { @@ -173,4 +176,46 @@ public void testResultsAsArrays() ) ); } + + @Test + public void testResultsAsFrames() + { + RowSignature sig = RowSignature.builder() + .add("a", ColumnType.STRING) + .add("b", ColumnType.STRING) + .build(); + + TestScanQuery scan1 = new TestScanQuery("foo", sig) + .appendRow("a", "a") + .appendRow("a", "b"); + TestScanQuery scan2 = new TestScanQuery("bar", sig) + .appendRow("x", "x") + .appendRow("x", "y"); + + UnionQuery query = new UnionQuery( + ImmutableList.of( + scan1.query, + scan2.query + ) + ); + List frames = toolChest.resultsAsFrames( + query, + Sequences.of( + new RealUnionResult(scan1.makeResultSequence()), + new RealUnionResult(scan2.makeResultSequence()) + ), + new SingleMemoryAllocatorFactory(HeapMemoryAllocator.unlimited()), + true + ).get().toList(); + + Sequence rows = new FrameBasedInlineDataSource(frames, scan1.query.getRowSignature()).getRowsAsSequence(); + + QueryToolChestTestHelper.assertArrayResultsEquals( + ImmutableList.builder() + .addAll(scan1.results) + .addAll(scan2.results) + .build(), + rows + ); + } } From 9aaefc475d49d5356db49d375176bbec9eacd505 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 09:00:42 +0000 Subject: [PATCH 114/149] tesdt isglobal --- .../planning/DataSourceAnalysisTest.java | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/processing/src/test/java/org/apache/druid/query/planning/DataSourceAnalysisTest.java b/processing/src/test/java/org/apache/druid/query/planning/DataSourceAnalysisTest.java index 1240115221d7..fc85b4e3fb6c 100644 --- a/processing/src/test/java/org/apache/druid/query/planning/DataSourceAnalysisTest.java +++ b/processing/src/test/java/org/apache/druid/query/planning/DataSourceAnalysisTest.java @@ -73,6 +73,7 @@ public void testTable() Assert.assertEquals(Optional.empty(), analysis.getBaseQuery()); Assert.assertEquals(Optional.empty(), analysis.getBaseQuerySegmentSpec()); Assert.assertEquals(Collections.emptyList(), analysis.getPreJoinableClauses()); + Assert.assertFalse(analysis.isGlobal()); Assert.assertFalse(analysis.isJoin()); Assert.assertTrue(analysis.isBaseColumn("foo")); } @@ -92,6 +93,7 @@ public void testUnion() Assert.assertEquals(Optional.empty(), analysis.getBaseQuery()); Assert.assertEquals(Optional.empty(), analysis.getBaseQuerySegmentSpec()); Assert.assertEquals(Collections.emptyList(), analysis.getPreJoinableClauses()); + Assert.assertEquals(unionDataSource.isGlobal(), analysis.isGlobal()); Assert.assertFalse(analysis.isJoin()); Assert.assertTrue(analysis.isBaseColumn("foo")); } @@ -114,6 +116,7 @@ public void testQueryOnTable() analysis.getBaseQuerySegmentSpec() ); Assert.assertEquals(Collections.emptyList(), analysis.getPreJoinableClauses()); + Assert.assertFalse(analysis.isGlobal()); Assert.assertFalse(analysis.isJoin()); Assert.assertFalse(analysis.isBaseColumn("foo")); } @@ -137,6 +140,7 @@ public void testQueryOnUnion() analysis.getBaseQuerySegmentSpec() ); Assert.assertEquals(Collections.emptyList(), analysis.getPreJoinableClauses()); + Assert.assertFalse(analysis.isGlobal()); Assert.assertFalse(analysis.isJoin()); Assert.assertFalse(analysis.isBaseColumn("foo")); } @@ -155,6 +159,7 @@ public void testLookup() Assert.assertEquals(Optional.empty(), analysis.getBaseQuery()); Assert.assertEquals(Optional.empty(), analysis.getBaseQuerySegmentSpec()); Assert.assertEquals(Collections.emptyList(), analysis.getPreJoinableClauses()); + Assert.assertTrue(analysis.isGlobal()); Assert.assertFalse(analysis.isJoin()); Assert.assertTrue(analysis.isBaseColumn("foo")); } @@ -177,6 +182,7 @@ public void testQueryOnLookup() analysis.getBaseQuerySegmentSpec() ); Assert.assertEquals(Collections.emptyList(), analysis.getPreJoinableClauses()); + Assert.assertTrue(analysis.isGlobal()); Assert.assertFalse(analysis.isJoin()); Assert.assertFalse(analysis.isBaseColumn("foo")); } @@ -195,6 +201,8 @@ public void testInline() Assert.assertEquals(Optional.empty(), analysis.getBaseQuery()); Assert.assertEquals(Optional.empty(), analysis.getBaseQuerySegmentSpec()); Assert.assertEquals(Collections.emptyList(), analysis.getPreJoinableClauses()); + Assert.assertEquals(INLINE.isGlobal(), analysis.isGlobal()); + Assert.assertTrue(analysis.isGlobal()); Assert.assertFalse(analysis.isJoin()); Assert.assertTrue(analysis.isBaseColumn("foo")); } @@ -243,6 +251,8 @@ public void testJoinSimpleLeftLeaning() ), analysis.getPreJoinableClauses() ); + Assert.assertEquals(joinDataSource.isGlobal(), analysis.isGlobal()); + Assert.assertFalse(analysis.isGlobal()); Assert.assertTrue(analysis.isJoin()); Assert.assertTrue(analysis.isBaseColumn("foo")); Assert.assertFalse(analysis.isBaseColumn("1.foo")); @@ -292,6 +302,8 @@ public void testJoinSimpleLeftLeaningWithLeftFilter() ), analysis.getPreJoinableClauses() ); + Assert.assertEquals(joinDataSource.isGlobal(), analysis.isGlobal()); + Assert.assertFalse(analysis.isGlobal()); Assert.assertTrue(analysis.isJoin()); Assert.assertTrue(analysis.isBaseColumn("foo")); Assert.assertFalse(analysis.isBaseColumn("1.foo")); @@ -345,6 +357,7 @@ public void testJoinSimpleRightLeaning() ), analysis.getPreJoinableClauses() ); + Assert.assertFalse(analysis.isGlobal()); Assert.assertTrue(analysis.isJoin()); Assert.assertTrue(analysis.isBaseColumn("foo")); Assert.assertTrue(analysis.isBaseColumn("1.foo")); @@ -394,6 +407,7 @@ public void testJoinSimpleRightLeaningWithLeftFilter() ), analysis.getPreJoinableClauses() ); + Assert.assertFalse(analysis.isGlobal()); Assert.assertTrue(analysis.isJoin()); Assert.assertTrue(analysis.isBaseColumn("foo")); Assert.assertTrue(analysis.isBaseColumn("1.foo")); @@ -427,6 +441,7 @@ public void testJoinOverTableSubquery() ), analysis.getPreJoinableClauses() ); + Assert.assertFalse(analysis.isGlobal()); Assert.assertTrue(analysis.isJoin()); Assert.assertTrue(analysis.isBaseColumn("foo")); Assert.assertFalse(analysis.isBaseColumn("1.foo")); @@ -460,6 +475,7 @@ public void testJoinTableUnionToLookup() ), analysis.getPreJoinableClauses() ); + Assert.assertFalse(analysis.isGlobal()); Assert.assertTrue(analysis.isJoin()); Assert.assertTrue(analysis.isBaseColumn("foo")); Assert.assertFalse(analysis.isBaseColumn("1.foo")); @@ -514,6 +530,7 @@ public void testJoinUnderTopLevelSubqueries() ), analysis.getPreJoinableClauses() ); + Assert.assertFalse(analysis.isGlobal()); Assert.assertTrue(analysis.isJoin()); Assert.assertFalse(analysis.isBaseColumn("foo")); Assert.assertFalse(analysis.isBaseColumn("1.foo")); @@ -546,6 +563,7 @@ public void testJoinLookupToLookup() ), analysis.getPreJoinableClauses() ); + Assert.assertTrue(analysis.isGlobal()); Assert.assertTrue(analysis.isJoin()); Assert.assertTrue(analysis.isBaseColumn("foo")); Assert.assertFalse(analysis.isBaseColumn("1.foo")); @@ -578,6 +596,7 @@ public void testJoinLookupToTable() ), analysis.getPreJoinableClauses() ); + Assert.assertFalse(analysis.isGlobal()); Assert.assertTrue(analysis.isJoin()); Assert.assertTrue(analysis.isBaseColumn("foo")); Assert.assertFalse(analysis.isBaseColumn("1.foo")); From f45787a2772c8a7aeeeb5099a848f7ca3cc59a37 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 09:01:07 +0000 Subject: [PATCH 115/149] rename --- .../apache/druid/query/union/UnionQuery.java | 18 +++++++-------- .../query/union/UnionQueryQueryToolChest.java | 22 +++++++++---------- .../druid/query/union/UnionQueryRunner.java | 10 ++++----- .../query/union/UnionQueryRunnerFactory.java | 10 ++++----- ...{RealUnionResult.java => UnionResult.java} | 4 ++-- .../union/UnionQueryQueryToolChestTest.java | 8 +++---- 6 files changed, 36 insertions(+), 36 deletions(-) rename processing/src/main/java/org/apache/druid/query/union/{RealUnionResult.java => UnionResult.java} (94%) diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index d1b1091125d8..b6bbc18797e3 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -50,7 +50,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; -public class UnionQuery implements Query +public class UnionQuery implements Query { @JsonProperty("context") protected final Map context; @@ -108,7 +108,7 @@ public String getType() } @Override - public QueryRunner getRunner(QuerySegmentWalker walker) + public QueryRunner getRunner(QuerySegmentWalker walker) { throw DruidException.defensive("Use QueryToolChest to get a Runner"); } @@ -144,26 +144,26 @@ public Map getContext() } @Override - public Ordering getResultOrdering() + public Ordering getResultOrdering() { throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); } @Override - public Query withOverriddenContext(Map contextOverrides) + public Query withOverriddenContext(Map contextOverrides) { List> newQueries = mapQueries(q -> q.withOverriddenContext(contextOverrides)); return new UnionQuery(newQueries, QueryContexts.override(getContext(), contextOverrides)); } @Override - public Query withQuerySegmentSpec(QuerySegmentSpec spec) + public Query withQuerySegmentSpec(QuerySegmentSpec spec) { throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); } @Override - public Query withId(String id) + public Query withId(String id) { return withOverriddenContext(ImmutableMap.of(BaseQuery.QUERY_ID, id)); } @@ -175,7 +175,7 @@ public String getId() } @Override - public Query withSubQueryId(String subQueryId) + public Query withSubQueryId(String subQueryId) { return withOverriddenContext(ImmutableMap.of(BaseQuery.SUB_QUERY_ID, subQueryId)); } @@ -187,13 +187,13 @@ public String getSubQueryId() } @Override - public Query withDataSource(DataSource dataSource) + public Query withDataSource(DataSource dataSource) { throw new RuntimeException("This method is not supported. Use withDataSources instead!"); } @Override - public Query withDataSources(List children) + public Query withDataSources(List children) { Preconditions.checkArgument(queries.size() == children.size(), "Number of children must match number of queries"); List> newQueries = new ArrayList<>(); diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 9ae0ae5923b1..1810eab98464 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -44,11 +44,11 @@ import java.util.List; import java.util.Optional; -public class UnionQueryQueryToolChest extends QueryToolChest - implements QueryExecutor +public class UnionQueryQueryToolChest extends QueryToolChest + implements QueryExecutor { @Override - public QueryRunner makeQueryRunner(Query query, + public QueryRunner makeQueryRunner(Query query, QueryToolChestWarehouse warehouse, QuerySegmentWalker clientQuerySegmentWalker) { return new UnionQueryRunner((UnionQuery) query, clientQuerySegmentWalker); @@ -56,7 +56,7 @@ public QueryRunner makeQueryRunner(Query query @Override @SuppressWarnings("unchecked") - public QueryRunner mergeResults(QueryRunner runner) + public QueryRunner mergeResults(QueryRunner runner) { throw new UnsupportedOperationException("Not supported"); } @@ -68,7 +68,7 @@ public QueryMetrics makeMetrics(UnionQuery query) } @Override - public Function makePreComputeManipulatorFn( + public Function makePreComputeManipulatorFn( UnionQuery query, MetricManipulationFn fn) { @@ -76,7 +76,7 @@ public Function makePreComputeManipulatorFn( } @Override - public TypeReference getResultTypeReference() + public TypeReference getResultTypeReference() { return null; } @@ -96,7 +96,7 @@ public RowSignature resultArraySignature(UnionQuery query) @SuppressWarnings({"unchecked", "rawtypes"}) public Sequence resultsAsArrays( UnionQuery query, - Sequence resultSequence) + Sequence resultSequence) { return new UnionSequenceMaker() { @@ -113,7 +113,7 @@ public Optional> transformResults(Query query, Sequence> resultsAsFrames( UnionQuery query, - Sequence resultSequence, + Sequence resultSequence, MemoryAllocatorFactory memoryAllocatorFactory, boolean useNestedForUnknownTypes) { @@ -132,14 +132,14 @@ abstract static class UnionSequenceMaker { public Optional> transform( UnionQuery query, - Sequence resultSequence) + Sequence resultSequence) { - List results = resultSequence.toList(); + List results = resultSequence.toList(); List> resultSeqs = new ArrayList<>(); for (int i = 0; i < results.size(); i++) { Query q = query.queries.get(i); - RealUnionResult realUnionResult = results.get(i); + UnionResult realUnionResult = results.get(i); Optional> queryResults = transformResults(q, realUnionResult.getResults()); if (!queryResults.isPresent()) { return Optional.empty(); diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java index 4f15e1907f6c..53fbce62da07 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java @@ -30,7 +30,7 @@ import java.util.ArrayList; import java.util.List; -class UnionQueryRunner implements QueryRunner +class UnionQueryRunner implements QueryRunner { private final QuerySegmentWalker walker; private final UnionQuery query; @@ -57,11 +57,11 @@ private List makeSubQueryRunners(UnionQuery unionQuery) } @Override - public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) + public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) { UnionQuery unionQuery = queryPlus.unwrapQuery(UnionQuery.class); - List seqs = new ArrayList(); + List seqs = new ArrayList(); for (int i = 0; i < runners.size(); i++) { Query q = unionQuery.queries.get(i); QueryRunner r = runners.get(i); @@ -70,10 +70,10 @@ public Sequence run(QueryPlus queryPlus, Respo return Sequences.simple(seqs); } - private RealUnionResult makeUnionResult(QueryRunner runner, QueryPlus withQuery, + private UnionResult makeUnionResult(QueryRunner runner, QueryPlus withQuery, ResponseContext responseContext) { Sequence seq = runner.run(withQuery, responseContext); - return new RealUnionResult(seq); + return new UnionResult(seq); } } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunnerFactory.java index d493e52b1144..03876cc72388 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunnerFactory.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunnerFactory.java @@ -27,7 +27,7 @@ import org.apache.druid.query.QueryToolChest; import org.apache.druid.segment.Segment; -public class UnionQueryRunnerFactory implements QueryRunnerFactory +public class UnionQueryRunnerFactory implements QueryRunnerFactory { private UnionQueryQueryToolChest toolChest; @@ -38,20 +38,20 @@ public UnionQueryRunnerFactory(UnionQueryQueryToolChest toolChest) } @Override - public QueryRunner createRunner(Segment segment) + public QueryRunner createRunner(Segment segment) { throw DruidException.defensive("not expected to be used"); } @Override - public QueryRunner mergeRunners(QueryProcessingPool queryProcessingPool, - Iterable> queryRunners) + public QueryRunner mergeRunners(QueryProcessingPool queryProcessingPool, + Iterable> queryRunners) { throw DruidException.defensive("not expected to be used"); } @Override - public QueryToolChest getToolchest() + public QueryToolChest getToolchest() { return toolChest; } diff --git a/processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java b/processing/src/main/java/org/apache/druid/query/union/UnionResult.java similarity index 94% rename from processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java rename to processing/src/main/java/org/apache/druid/query/union/UnionResult.java index 23ec20039d69..e2db2ab643c7 100644 --- a/processing/src/main/java/org/apache/druid/query/union/RealUnionResult.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionResult.java @@ -27,11 +27,11 @@ * Caveat: the index of the ResultUnionResult in the output sequence is in line * with the index of the executed query. */ -public class RealUnionResult +public class UnionResult { private final Sequence seq; - public RealUnionResult(Sequence seq) + public UnionResult(Sequence seq) { this.seq = seq; } diff --git a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java index becf74d4e01f..d4469678373a 100644 --- a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java @@ -170,8 +170,8 @@ public void testResultsAsArrays() toolChest.resultsAsArrays( query, Sequences.of( - new RealUnionResult(scan1.makeResultSequence()), - new RealUnionResult(scan2.makeResultSequence()) + new UnionResult(scan1.makeResultSequence()), + new UnionResult(scan2.makeResultSequence()) ) ) ); @@ -201,8 +201,8 @@ public void testResultsAsFrames() List frames = toolChest.resultsAsFrames( query, Sequences.of( - new RealUnionResult(scan1.makeResultSequence()), - new RealUnionResult(scan2.makeResultSequence()) + new UnionResult(scan1.makeResultSequence()), + new UnionResult(scan2.makeResultSequence()) ), new SingleMemoryAllocatorFactory(HeapMemoryAllocator.unlimited()), true From eb5666ee40881ddf89e91c90c71622a736f70460 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 09:06:10 +0000 Subject: [PATCH 116/149] fix intellij --- .../src/main/java/org/apache/druid/query/QueryExecutor.java | 1 - .../apache/druid/query/union/UnionQueryQueryToolChest.java | 3 +-- .../java/org/apache/druid/query/union/UnionQueryRunner.java | 2 -- .../org/apache/druid/server/ClientQuerySegmentWalker.java | 4 ++-- 4 files changed, 3 insertions(+), 7 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryExecutor.java b/processing/src/main/java/org/apache/druid/query/QueryExecutor.java index 8b11fe6da431..1afa9d8f36ce 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryExecutor.java +++ b/processing/src/main/java/org/apache/druid/query/QueryExecutor.java @@ -28,7 +28,6 @@ public interface QueryExecutor { QueryRunner makeQueryRunner( Query query, - QueryToolChestWarehouse warehouse, QuerySegmentWalker walker ); } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 1810eab98464..415c4c2cd39a 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -34,7 +34,6 @@ import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.QueryToolChest; -import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.aggregation.MetricManipulationFn; import org.apache.druid.query.groupby.SupportRowSignature; import org.apache.druid.segment.column.RowSignature; @@ -49,7 +48,7 @@ public class UnionQueryQueryToolChest extends QueryToolChest makeQueryRunner(Query query, - QueryToolChestWarehouse warehouse, QuerySegmentWalker clientQuerySegmentWalker) + QuerySegmentWalker clientQuerySegmentWalker) { return new UnionQueryRunner((UnionQuery) query, clientQuerySegmentWalker); } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java index 53fbce62da07..fc9cad1381cb 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java @@ -33,7 +33,6 @@ class UnionQueryRunner implements QueryRunner { private final QuerySegmentWalker walker; - private final UnionQuery query; private final List runners; public UnionQueryRunner( @@ -41,7 +40,6 @@ public UnionQueryRunner( QuerySegmentWalker walker ) { - this.query = query; this.walker = walker; this.runners = makeSubQueryRunners(query); } diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 5f5c10afc9e6..1e2f0729e4d7 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -181,7 +181,7 @@ public QueryRunner getQueryRunnerForIntervals(final Query query, final final QueryToolChest> toolChest = warehouse.getToolChest(query); if (toolChest instanceof QueryExecutor) { - return ((QueryExecutor) toolChest).makeQueryRunner(query, warehouse, this); + return ((QueryExecutor) toolChest).makeQueryRunner(query, this); } // transform TableDataSource to GlobalTableDataSource when eligible @@ -454,7 +454,7 @@ private DataSource inlineIfNecessary( final QueryRunner subQueryRunner; if (subQueryToolChest instanceof QueryExecutor) { subQueryRunner = ((QueryExecutor) subQueryToolChest) - .makeQueryRunner(subQueryWithSerialization, warehouse, this); + .makeQueryRunner(subQueryWithSerialization, this); } else { subQueryRunner = subQueryWithSerialization.getRunner(this); } From 1b22d2294a8ce1409605f076f7abc45544dbd67d Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 11:39:47 +0000 Subject: [PATCH 117/149] add conglomerate is warehouse --- .../query/QueryRunnerFactoryConglomerate.java | 15 ++++++++++++++- 1 file changed, 14 insertions(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java index da89d81c8dda..842d2f3d2793 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java @@ -19,9 +19,22 @@ package org.apache.druid.query; +import org.apache.druid.error.DruidException; + /** */ -public interface QueryRunnerFactoryConglomerate +public interface QueryRunnerFactoryConglomerate extends QueryToolChestWarehouse { > QueryRunnerFactory findFactory(QueryType query); + + @Override + default > QueryToolChest getToolChest(QueryType query) + { + QueryRunnerFactory factory = findFactory(query); + if (factory == null) { + throw DruidException + .defensive("QueryRunnerFactory for QueryType [%s] is not registered!", query.getClass().getName()); + } + return factory.getToolchest(); + } } From b49486166c0b20d6f7c8bf222889cf43bb1a0a1f Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 11:42:35 +0000 Subject: [PATCH 118/149] remove CongBackedToolchestWH --- ...lomerateBackedQueryToolChestWarehouse.java | 36 ------------------- ...yRunnerBasedOnClusteredClientTestBase.java | 2 +- .../SegmentMetadataCacheTestBase.java | 3 +- .../apache/druid/server/QueryStackTests.java | 4 +-- .../sql/calcite/util/QueryFrameworkUtils.java | 4 +-- 5 files changed, 6 insertions(+), 43 deletions(-) delete mode 100644 server/src/test/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java diff --git a/server/src/test/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java b/server/src/test/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java deleted file mode 100644 index f80ac15987fd..000000000000 --- a/server/src/test/java/org/apache/druid/query/ConglomerateBackedQueryToolChestWarehouse.java +++ /dev/null @@ -1,36 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query; - -public class ConglomerateBackedQueryToolChestWarehouse implements QueryToolChestWarehouse -{ - private QueryRunnerFactoryConglomerate conglomerate; - - public ConglomerateBackedQueryToolChestWarehouse(QueryRunnerFactoryConglomerate conglomerate) - { - this.conglomerate = conglomerate; - } - - @Override - public > QueryToolChest getToolChest(final QueryType query) - { - return conglomerate.findFactory(query).getToolchest(); - } -} diff --git a/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java b/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java index 68da75759176..4cb081c8e11c 100644 --- a/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java +++ b/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java @@ -109,7 +109,7 @@ protected QueryRunnerBasedOnClusteredClientTestBase() CLOSER, TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD ); - toolChestWarehouse = new ConglomerateBackedQueryToolChestWarehouse(conglomerate); + toolChestWarehouse = conglomerate; } @AfterClass diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java index f08d5942f146..1fcd27be29a5 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java @@ -29,7 +29,6 @@ import org.apache.druid.data.input.impl.TimestampSpec; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.io.Closer; -import org.apache.druid.query.ConglomerateBackedQueryToolChestWarehouse; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; import org.apache.druid.query.QueryRunnerFactoryConglomerate; @@ -111,7 +110,7 @@ public void setUpCommon() { resourceCloser = Closer.create(); conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser); - queryToolChestWarehouse = new ConglomerateBackedQueryToolChestWarehouse(conglomerate); + queryToolChestWarehouse = conglomerate; } public void setUpData() throws Exception diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index 9414f4a4af47..1edddfc02aee 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -33,7 +33,6 @@ import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.java.util.emitter.service.ServiceEmitter; import org.apache.druid.query.BrokerParallelMergeConfig; -import org.apache.druid.query.ConglomerateBackedQueryToolChestWarehouse; import org.apache.druid.query.DataSource; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate; @@ -96,6 +95,7 @@ import org.junit.Assert; import javax.annotation.Nullable; + import java.util.Collections; import java.util.Map; import java.util.Set; @@ -135,7 +135,7 @@ public static ClientQuerySegmentWalker createClientQuerySegmentWalker( emitter, clusterWalker, localWalker, - new ConglomerateBackedQueryToolChestWarehouse(conglomerate), + conglomerate, joinableFactory, new RetryQueryRunnerConfig(), injector.getInstance(ObjectMapper.class), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index d46067639479..974dfbfd0c7c 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -28,7 +28,6 @@ import org.apache.druid.client.InternalQueryConfig; import org.apache.druid.java.util.emitter.core.NoopEmitter; import org.apache.druid.java.util.emitter.service.ServiceEmitter; -import org.apache.druid.query.ConglomerateBackedQueryToolChestWarehouse; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; import org.apache.druid.query.GlobalTableDataSource; @@ -74,6 +73,7 @@ import org.easymock.EasyMock; import javax.annotation.Nullable; + import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; @@ -88,7 +88,7 @@ public static QueryLifecycleFactory createMockQueryLifecycleFactory( ) { return new QueryLifecycleFactory( - new ConglomerateBackedQueryToolChestWarehouse(conglomerate), + conglomerate, walker, new DefaultGenericQueryMetricsFactory(), new ServiceEmitter("dummy", "dummy", new NoopEmitter()), From 5bc29e323ba2ce8afbb19c13137db9a78e5551d4 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 11:53:30 +0000 Subject: [PATCH 119/149] Enhance injector usage during conglomerate builds in tests --- .../query/QueryRunnerFactoryConglomerate.java | 15 +- ...AsBrokerQueryComponentSupplierWrapper.java | 89 +------- .../client/MessageRelayClientImpl.java | 10 +- .../CachingClusteredClientPerfTest.java | 1 - ...yRunnerBasedOnClusteredClientTestBase.java | 12 +- .../SegmentMetadataCacheTestBase.java | 11 +- .../apache/druid/server/QueryStackTests.java | 202 +++++++++--------- .../druid/quidem/DruidAvaticaTestDriver.java | 73 +------ .../sql/calcite/SqlTestFrameworkConfig.java | 3 +- .../sql/calcite/util/QueryFrameworkUtils.java | 13 +- .../sql/calcite/util/SqlTestFramework.java | 184 +++++++++++++--- 11 files changed, 286 insertions(+), 327 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java index da89d81c8dda..842d2f3d2793 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java @@ -19,9 +19,22 @@ package org.apache.druid.query; +import org.apache.druid.error.DruidException; + /** */ -public interface QueryRunnerFactoryConglomerate +public interface QueryRunnerFactoryConglomerate extends QueryToolChestWarehouse { > QueryRunnerFactory findFactory(QueryType query); + + @Override + default > QueryToolChest getToolChest(QueryType query) + { + QueryRunnerFactory factory = findFactory(query); + if (factory == null) { + throw DruidException + .defensive("QueryRunnerFactory for QueryType [%s] is not registered!", query.getClass().getName()); + } + return factory.getToolchest(); + } } diff --git a/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java b/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java index 74149cefcb25..d1fa6a349ba4 100644 --- a/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java +++ b/quidem-ut/src/main/java/org/apache/druid/quidem/ExposedAsBrokerQueryComponentSupplierWrapper.java @@ -19,11 +19,9 @@ package org.apache.druid.quidem; -import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableSet; import com.google.inject.AbstractModule; -import com.google.inject.Injector; import com.google.inject.Key; import com.google.inject.Module; import com.google.inject.Provides; @@ -44,7 +42,6 @@ import org.apache.druid.guice.BrokerProcessingModule; import org.apache.druid.guice.BrokerServiceModule; import org.apache.druid.guice.CoordinatorDiscoveryModule; -import org.apache.druid.guice.DruidInjectorBuilder; import org.apache.druid.guice.ExpressionModule; import org.apache.druid.guice.ExtensionsModule; import org.apache.druid.guice.JacksonConfigManagerModule; @@ -73,20 +70,14 @@ import org.apache.druid.initialization.Log4jShutterDownerModule; import org.apache.druid.initialization.ServerInjectorBuilder; import org.apache.druid.initialization.TombstoneDataStorageModule; -import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.metadata.storage.derby.DerbyMetadataStorageDruidModule; -import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.RetryQueryRunnerConfig; -import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.rpc.guice.ServiceClientModule; -import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.segment.writeout.SegmentWriteOutMediumModule; import org.apache.druid.server.BrokerQueryResource; import org.apache.druid.server.ClientInfoResource; import org.apache.druid.server.DruidNode; -import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.ResponseContextConfig; -import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.server.SubqueryGuardrailHelper; import org.apache.druid.server.SubqueryGuardrailHelperProvider; import org.apache.druid.server.coordination.ServerType; @@ -101,50 +92,33 @@ import org.apache.druid.server.metrics.SubqueryCountStatsProvider; import org.apache.druid.server.router.TieredBrokerConfig; import org.apache.druid.server.security.TLSCertificateCheckerModule; -import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.BrokerSegmentMetadataCache; import org.apache.druid.sql.calcite.schema.DruidSchemaName; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SqlTestFramework; -import org.apache.druid.sql.calcite.util.SqlTestFramework.Builder; -import org.apache.druid.sql.calcite.util.SqlTestFramework.PlannerComponentSupplier; import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplier; +import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplierDelegate; import org.apache.druid.sql.guice.SqlModule; import org.apache.druid.storage.StorageConnectorModule; import org.apache.druid.timeline.PruneLoadSpec; import org.eclipse.jetty.server.Server; -import java.io.IOException; import java.util.List; import java.util.Properties; /** * A wrapper class to expose a {@link QueryComponentSupplier} as a Broker service. */ -public class ExposedAsBrokerQueryComponentSupplierWrapper implements QueryComponentSupplier +public class ExposedAsBrokerQueryComponentSupplierWrapper extends QueryComponentSupplierDelegate { - private QueryComponentSupplier delegate; - public ExposedAsBrokerQueryComponentSupplierWrapper(QueryComponentSupplier delegate) { - this.delegate = delegate; - } - - @Override - public void gatherProperties(Properties properties) - { - delegate.gatherProperties(properties); - } - - @Override - public void configureGuice(DruidInjectorBuilder builder) - { + super(delegate); } @Override public void configureGuice(CoreInjectorBuilder builder, List overrideModules) { - delegate.configureGuice(builder); + super.configureGuice(builder); installForServerModules(builder); builder.add(new QueryRunnerFactoryModule()); @@ -154,55 +128,6 @@ public void configureGuice(CoreInjectorBuilder builder, List overrideMod builder.add(QuidemCaptureModule.class); } - @Override - public QueryRunnerFactoryConglomerate createCongolmerate(Builder builder, Closer closer, ObjectMapper om) - { - return delegate.createCongolmerate(builder, closer, om); - } - - @Override - public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(QueryRunnerFactoryConglomerate conglomerate, - JoinableFactoryWrapper joinableFactory, Injector injector) - { - return delegate.createQuerySegmentWalker(conglomerate, joinableFactory, injector); - } - - @Override - public SqlEngine createEngine(QueryLifecycleFactory qlf, ObjectMapper objectMapper, Injector injector) - { - return delegate.createEngine(qlf, objectMapper, injector); - } - - @Override - public void configureJsonMapper(ObjectMapper mapper) - { - delegate.configureJsonMapper(mapper); - } - - @Override - public JoinableFactoryWrapper createJoinableFactoryWrapper(LookupExtractorFactoryContainerProvider lookupProvider) - { - return delegate.createJoinableFactoryWrapper(lookupProvider); - } - - @Override - public void finalizeTestFramework(SqlTestFramework sqlTestFramework) - { - delegate.finalizeTestFramework(sqlTestFramework); - } - - @Override - public void close() throws IOException - { - delegate.close(); - } - - @Override - public PlannerComponentSupplier getPlannerComponentSupplier() - { - return delegate.getPlannerComponentSupplier(); - } - public static class BrokerTestModule extends AbstractModule { @Override @@ -336,10 +261,4 @@ static List brokerModules() } ); } - - @Override - public Boolean isExplainSupported() - { - return delegate.isExplainSupported(); - } } diff --git a/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java b/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java index 140bd45e1af4..b238cdee0aa7 100644 --- a/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java +++ b/server/src/main/java/org/apache/druid/messages/client/MessageRelayClientImpl.java @@ -26,6 +26,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.jackson.JacksonUtils; import org.apache.druid.java.util.http.client.response.BytesFullResponseHandler; +import org.apache.druid.java.util.http.client.response.BytesFullResponseHolder; import org.apache.druid.messages.MessageBatch; import org.apache.druid.rpc.RequestBuilder; import org.apache.druid.rpc.ServiceClient; @@ -68,11 +69,12 @@ public ListenableFuture> getMessages( startWatermark ); + ListenableFuture asyncRequest = serviceClient.asyncRequest( + new RequestBuilder(HttpMethod.GET, path), + new BytesFullResponseHandler() + ); return FutureUtils.transform( - serviceClient.asyncRequest( - new RequestBuilder(HttpMethod.GET, path), - new BytesFullResponseHandler() - ), + asyncRequest, holder -> { if (holder.getResponse().getStatus().getCode() == HttpStatus.NO_CONTENT_204) { return new MessageBatch<>(Collections.emptyList(), epoch, startWatermark); diff --git a/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java b/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java index b31799cf8b4c..e6cc643fa7a1 100644 --- a/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java +++ b/server/src/test/java/org/apache/druid/client/CachingClusteredClientPerfTest.java @@ -172,7 +172,6 @@ private DataSegment makeDataSegment(String dataSource, Interval interval, String private static class MockQueryToolChestWareHouse implements QueryToolChestWarehouse { - @Override public > QueryToolChest getToolChest(QueryType query) { diff --git a/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java b/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java index 308427d40fa4..4cb081c8e11c 100644 --- a/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java +++ b/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java @@ -107,17 +107,9 @@ protected QueryRunnerBasedOnClusteredClientTestBase() { conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate( CLOSER, - () -> TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD + TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD ); - - toolChestWarehouse = new QueryToolChestWarehouse() - { - @Override - public > QueryToolChest getToolChest(final QueryType query) - { - return conglomerate.findFactory(query).getToolchest(); - } - }; + toolChestWarehouse = conglomerate; } @AfterClass diff --git a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java index c62577e01256..1fcd27be29a5 100644 --- a/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java +++ b/server/src/test/java/org/apache/druid/segment/metadata/SegmentMetadataCacheTestBase.java @@ -31,10 +31,8 @@ import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; -import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; -import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.aggregation.CountAggregatorFactory; import org.apache.druid.query.aggregation.DoubleSumAggregatorFactory; @@ -112,14 +110,7 @@ public void setUpCommon() { resourceCloser = Closer.create(); conglomerate = QueryStackTests.createQueryRunnerFactoryConglomerate(resourceCloser); - queryToolChestWarehouse = new QueryToolChestWarehouse() - { - @Override - public > QueryToolChest getToolChest(final QueryType query) - { - return conglomerate.findFactory(query).getToolchest(); - } - }; + queryToolChestWarehouse = conglomerate; } public void setUpData() throws Exception diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index 041c4654d92e..1edddfc02aee 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -45,8 +45,6 @@ import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QueryRunnerTestHelper; import org.apache.druid.query.QuerySegmentWalker; -import org.apache.druid.query.QueryToolChest; -import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.RetryQueryRunnerConfig; import org.apache.druid.query.TestBufferPool; import org.apache.druid.query.expression.LookupEnabledTestExprMacroTable; @@ -97,10 +95,10 @@ import org.junit.Assert; import javax.annotation.Nullable; + import java.util.Collections; import java.util.Map; import java.util.Set; -import java.util.function.Supplier; /** * Utilities for creating query-stack objects for tests. @@ -137,14 +135,7 @@ public static ClientQuerySegmentWalker createClientQuerySegmentWalker( emitter, clusterWalker, localWalker, - new QueryToolChestWarehouse() - { - @Override - public > QueryToolChest getToolChest(final QueryType query) - { - return conglomerate.findFactory(query).getToolchest(); - } - }, + conglomerate, joinableFactory, new RetryQueryRunnerConfig(), injector.getInstance(ObjectMapper.class), @@ -235,21 +226,12 @@ public int getNumMergeBuffers() */ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate(final Closer closer) { - return createQueryRunnerFactoryConglomerate(closer, () -> TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD); + return createQueryRunnerFactoryConglomerate(closer, TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD); } public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( final Closer closer, - final Supplier minTopNThresholdSupplier - ) - { - return createQueryRunnerFactoryConglomerate(closer, minTopNThresholdSupplier, TestHelper.makeJsonMapper()); - } - - public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( - final Closer closer, - final Supplier minTopNThresholdSupplier, - final ObjectMapper jsonMapper + final Integer minTopNThreshold ) { return createQueryRunnerFactoryConglomerate( @@ -257,8 +239,8 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat getProcessingConfig( DEFAULT_NUM_MERGE_BUFFERS ), - minTopNThresholdSupplier, - jsonMapper + minTopNThreshold, + TestHelper.makeJsonMapper() ); } @@ -270,114 +252,122 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat return createQueryRunnerFactoryConglomerate( closer, processingConfig, - () -> TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD + TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD, + TestHelper.makeJsonMapper() ); } + public static TestBufferPool makeTestBufferPool(final Closer closer) + { + final TestBufferPool testBufferPool = TestBufferPool.offHeap(COMPUTE_BUFFER_SIZE, Integer.MAX_VALUE); + closer.register(() -> { + // Verify that all objects have been returned to the pool. + Assert.assertEquals(0, testBufferPool.getOutstandingObjectCount()); + }); + return testBufferPool; + } + + public static TestGroupByBuffers makeGroupByBuffers(final Closer closer, final DruidProcessingConfig processingConfig) + { + final TestGroupByBuffers groupByBuffers = + closer.register(TestGroupByBuffers.createFromProcessingConfig(processingConfig)); + return groupByBuffers; + } + public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( final Closer closer, final DruidProcessingConfig processingConfig, + final Integer minTopNThreshold, final ObjectMapper jsonMapper ) { + final TestBufferPool testBufferPool = makeTestBufferPool(closer); + final TestGroupByBuffers groupByBuffers = makeGroupByBuffers(closer, processingConfig); + return createQueryRunnerFactoryConglomerate( - closer, processingConfig, - () -> TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD, - jsonMapper - ); + minTopNThreshold, + jsonMapper, + testBufferPool, + groupByBuffers); } + public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( - final Closer closer, final DruidProcessingConfig processingConfig, - final Supplier minTopNThresholdSupplier - ) + final Integer minTopNThreshold, + final ObjectMapper jsonMapper, + final TestBufferPool testBufferPool, + final TestGroupByBuffers groupByBuffers) { - return createQueryRunnerFactoryConglomerate( - closer, - processingConfig, - minTopNThresholdSupplier, - TestHelper.makeJsonMapper() + final QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate( + makeDefaultQueryRunnerFactories(processingConfig, minTopNThreshold, jsonMapper, testBufferPool, groupByBuffers) ); + return conglomerate; } - - public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerate( - final Closer closer, + @SuppressWarnings("rawtypes") + public static ImmutableMap, QueryRunnerFactory> makeDefaultQueryRunnerFactories( final DruidProcessingConfig processingConfig, - final Supplier minTopNThresholdSupplier, - final ObjectMapper jsonMapper - ) + final Integer minTopNThreshold, + final ObjectMapper jsonMapper, + final TestBufferPool testBufferPool, + final TestGroupByBuffers groupByBuffers) { - final TestBufferPool testBufferPool = TestBufferPool.offHeap(COMPUTE_BUFFER_SIZE, Integer.MAX_VALUE); - closer.register(() -> { - // Verify that all objects have been returned to the pool. - Assert.assertEquals(0, testBufferPool.getOutstandingObjectCount()); - }); - - final TestGroupByBuffers groupByBuffers = - closer.register(TestGroupByBuffers.createFromProcessingConfig(processingConfig)); - - final GroupByQueryRunnerFactory groupByQueryRunnerFactory = - GroupByQueryRunnerTest.makeQueryRunnerFactory( - jsonMapper, - new GroupByQueryConfig() - { - }, - groupByBuffers, - processingConfig - ); + final GroupByQueryRunnerFactory groupByQueryRunnerFactory = GroupByQueryRunnerTest.makeQueryRunnerFactory( + jsonMapper, + new GroupByQueryConfig() + { + }, + groupByBuffers, + processingConfig + ); - final QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate( - ImmutableMap., QueryRunnerFactory>builder() - .put( - SegmentMetadataQuery.class, - new SegmentMetadataQueryRunnerFactory( - new SegmentMetadataQueryQueryToolChest( - new SegmentMetadataQueryConfig("P1W") - ), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ) + return ImmutableMap., QueryRunnerFactory>builder() + .put( + SegmentMetadataQuery.class, + new SegmentMetadataQueryRunnerFactory( + new SegmentMetadataQueryQueryToolChest( + new SegmentMetadataQueryConfig("P1W") + ), + QueryRunnerTestHelper.NOOP_QUERYWATCHER ) - .put( - ScanQuery.class, - new ScanQueryRunnerFactory( - new ScanQueryQueryToolChest(DefaultGenericQueryMetricsFactory.instance()), - new ScanQueryEngine(), - new ScanQueryConfig() - ) + ) + .put( + ScanQuery.class, + new ScanQueryRunnerFactory( + new ScanQueryQueryToolChest(DefaultGenericQueryMetricsFactory.instance()), + new ScanQueryEngine(), + new ScanQueryConfig() ) - .put( - TimeseriesQuery.class, - new TimeseriesQueryRunnerFactory( - new TimeseriesQueryQueryToolChest(), - new TimeseriesQueryEngine(), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ) + ) + .put( + TimeseriesQuery.class, + new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER ) - .put( - TopNQuery.class, - new TopNQueryRunnerFactory( - testBufferPool, - new TopNQueryQueryToolChest(new TopNQueryConfig() - { - @Override - public int getMinTopNThreshold() - { - return minTopNThresholdSupplier.get(); - } - }), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ) + ) + .put( + TopNQuery.class, + new TopNQueryRunnerFactory( + testBufferPool, + new TopNQueryQueryToolChest(new TopNQueryConfig() + { + @Override + public int getMinTopNThreshold() + { + return minTopNThreshold; + } + }), + QueryRunnerTestHelper.NOOP_QUERYWATCHER ) - .put(GroupByQuery.class, groupByQueryRunnerFactory) - .put(TimeBoundaryQuery.class, new TimeBoundaryQueryRunnerFactory(QueryRunnerTestHelper.NOOP_QUERYWATCHER)) - .put(WindowOperatorQuery.class, new WindowOperatorQueryQueryRunnerFactory()) - .build() - ); - - return conglomerate; + ) + .put(GroupByQuery.class, groupByQueryRunnerFactory) + .put(TimeBoundaryQuery.class, new TimeBoundaryQueryRunnerFactory(QueryRunnerTestHelper.NOOP_QUERYWATCHER)) + .put(WindowOperatorQuery.class, new WindowOperatorQueryQueryRunnerFactory()) + .build(); } public static JoinableFactory makeJoinableFactoryForLookup( diff --git a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java index debd835fc785..66074055abc5 100644 --- a/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java +++ b/sql/src/test/java/org/apache/druid/quidem/DruidAvaticaTestDriver.java @@ -21,7 +21,6 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.google.inject.Binder; -import com.google.inject.Injector; import com.google.inject.Provides; import com.google.inject.name.Named; import com.google.inject.name.Names; @@ -33,10 +32,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.query.QueryRunnerFactoryConglomerate; -import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; -import org.apache.druid.segment.join.JoinableFactoryWrapper; import org.apache.druid.server.DruidNode; -import org.apache.druid.server.QueryLifecycleFactory; import org.apache.druid.server.SpecificSegmentsQuerySegmentWalker; import org.apache.druid.sql.avatica.AvaticaMonitor; import org.apache.druid.sql.avatica.DruidAvaticaJsonHandler; @@ -45,13 +41,10 @@ import org.apache.druid.sql.calcite.SqlTestFrameworkConfig.ConfigurationInstance; import org.apache.druid.sql.calcite.SqlTestFrameworkConfig.SqlTestFrameworkConfigStore; import org.apache.druid.sql.calcite.planner.PlannerConfig; -import org.apache.druid.sql.calcite.run.SqlEngine; import org.apache.druid.sql.calcite.schema.DruidSchemaCatalog; import org.apache.druid.sql.calcite.util.CalciteTests; -import org.apache.druid.sql.calcite.util.SqlTestFramework; -import org.apache.druid.sql.calcite.util.SqlTestFramework.Builder; -import org.apache.druid.sql.calcite.util.SqlTestFramework.PlannerComponentSupplier; import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplier; +import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplierDelegate; import org.apache.druid.sql.hook.DruidHookDispatcher; import org.apache.http.client.utils.URIBuilder; import org.eclipse.jetty.server.Server; @@ -206,27 +199,20 @@ protected AbstractAvaticaHandler getAvaticaHandler(final DruidMeta druidMeta) } } - static class AvaticaBasedTestConnectionSupplier implements QueryComponentSupplier + static class AvaticaBasedTestConnectionSupplier extends QueryComponentSupplierDelegate { - private QueryComponentSupplier delegate; private AvaticaBasedConnectionModule connectionModule; public AvaticaBasedTestConnectionSupplier(QueryComponentSupplier delegate) { - this.delegate = delegate; + super(delegate); this.connectionModule = new AvaticaBasedConnectionModule(); } - @Override - public void gatherProperties(Properties properties) - { - delegate.gatherProperties(properties); - } - @Override public void configureGuice(DruidInjectorBuilder builder) { - delegate.configureGuice(builder); + super.configureGuice(builder); builder.addModule(connectionModule); builder.addModule( binder -> { @@ -237,60 +223,11 @@ public void configureGuice(DruidInjectorBuilder builder) ); } - @Override - public QueryRunnerFactoryConglomerate createCongolmerate(Builder builder, Closer closer, ObjectMapper jsonMapper) - { - return delegate.createCongolmerate(builder, closer, jsonMapper); - } - - @Override - public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker(QueryRunnerFactoryConglomerate conglomerate, - JoinableFactoryWrapper joinableFactory, Injector injector) - { - return delegate.createQuerySegmentWalker(conglomerate, joinableFactory, injector); - } - - @Override - public SqlEngine createEngine(QueryLifecycleFactory qlf, ObjectMapper objectMapper, Injector injector) - { - return delegate.createEngine(qlf, objectMapper, injector); - } - - @Override - public void configureJsonMapper(ObjectMapper mapper) - { - delegate.configureJsonMapper(mapper); - } - - @Override - public JoinableFactoryWrapper createJoinableFactoryWrapper(LookupExtractorFactoryContainerProvider lookupProvider) - { - return delegate.createJoinableFactoryWrapper(lookupProvider); - } - - @Override - public void finalizeTestFramework(SqlTestFramework sqlTestFramework) - { - delegate.finalizeTestFramework(sqlTestFramework); - } - @Override public void close() throws IOException { connectionModule.close(); - delegate.close(); - } - - @Override - public PlannerComponentSupplier getPlannerComponentSupplier() - { - return delegate.getPlannerComponentSupplier(); - } - - @Override - public Boolean isExplainSupported() - { - return delegate.isExplainSupported(); + super.close(); } } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java b/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java index 1fc39d52ec51..73ad030d9701 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/SqlTestFrameworkConfig.java @@ -31,6 +31,7 @@ import org.apache.druid.java.util.common.StringUtils; import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.quidem.DruidAvaticaTestDriver; +import org.apache.druid.server.QueryStackTests; import org.apache.druid.sql.calcite.util.CacheTestHelperModule.ResultCacheMode; import org.apache.druid.sql.calcite.util.SqlTestFramework; import org.apache.druid.sql.calcite.util.SqlTestFramework.QueryComponentSupplier; @@ -94,7 +95,7 @@ public class SqlTestFrameworkConfig { @Retention(RetentionPolicy.RUNTIME) @Target({ElementType.METHOD, ElementType.TYPE}) - @NumMergeBuffers(0) + @NumMergeBuffers(QueryStackTests.DEFAULT_NUM_MERGE_BUFFERS) public @interface NumMergeBuffers { ConfigOptionProcessor PROCESSOR = new ConfigOptionProcessor(NumMergeBuffers.class) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java index 5463f5623196..974dfbfd0c7c 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/QueryFrameworkUtils.java @@ -31,11 +31,8 @@ import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; import org.apache.druid.query.GlobalTableDataSource; -import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; -import org.apache.druid.query.QueryToolChest; -import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.segment.join.JoinableFactory; import org.apache.druid.segment.loading.SegmentCacheManager; @@ -76,6 +73,7 @@ import org.easymock.EasyMock; import javax.annotation.Nullable; + import java.util.HashSet; import java.util.Set; import java.util.stream.Collectors; @@ -90,14 +88,7 @@ public static QueryLifecycleFactory createMockQueryLifecycleFactory( ) { return new QueryLifecycleFactory( - new QueryToolChestWarehouse() - { - @Override - public > QueryToolChest getToolChest(final QueryType query) - { - return conglomerate.findFactory(query).getToolchest(); - } - }, + conglomerate, walker, new DefaultGenericQueryMetricsFactory(), new ServiceEmitter("dummy", "dummy", new NoopEmitter()), diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index d59078deff9d..de09e97923cd 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -39,9 +39,15 @@ import org.apache.druid.java.util.common.RE; import org.apache.druid.java.util.common.io.Closer; import org.apache.druid.math.expr.ExprMacroTable; +import org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate; +import org.apache.druid.query.DruidProcessingConfig; import org.apache.druid.query.GlobalTableDataSource; +import org.apache.druid.query.Query; +import org.apache.druid.query.QueryRunnerFactory; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.TestBufferPool; +import org.apache.druid.query.groupby.TestGroupByBuffers; import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.quidem.TestSqlModule; @@ -82,6 +88,7 @@ import java.util.ArrayList; import java.util.Arrays; import java.util.List; +import java.util.Map; import java.util.Properties; import java.util.Set; @@ -146,6 +153,7 @@ public interface QueryComponentSupplier extends Closeable */ void gatherProperties(Properties properties); + /** * Configure modules needed for tests. This is the preferred way to configure * Jackson: include the production module in this method that includes the @@ -153,12 +161,6 @@ public interface QueryComponentSupplier extends Closeable */ void configureGuice(DruidInjectorBuilder builder); - QueryRunnerFactoryConglomerate createCongolmerate( - Builder builder, - Closer closer, - ObjectMapper jsonMapper - ); - SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker( QueryRunnerFactoryConglomerate conglomerate, JoinableFactoryWrapper joinableFactory, @@ -204,6 +206,91 @@ default void configureGuice(CoreInjectorBuilder injectorBuilder, List ov * MSQ right now needs a full query run. */ Boolean isExplainSupported(); + + QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate, Closer resourceCloser); + } + + public abstract static class QueryComponentSupplierDelegate implements QueryComponentSupplier + { + private final QueryComponentSupplier delegate; + + public QueryComponentSupplierDelegate(QueryComponentSupplier delegate) + { + this.delegate = delegate; + } + + @Override + public void gatherProperties(Properties properties) + { + delegate.gatherProperties(properties); + } + + @Override + public void configureGuice(DruidInjectorBuilder builder) + { + delegate.configureGuice(builder); + } + + @Override + public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker( + QueryRunnerFactoryConglomerate conglomerate, + JoinableFactoryWrapper joinableFactory, + Injector injector) + { + return delegate.createQuerySegmentWalker(conglomerate, joinableFactory, injector); + } + + @Override + public SqlEngine createEngine( + QueryLifecycleFactory qlf, + ObjectMapper objectMapper, + Injector injector) + { + return delegate.createEngine(qlf, objectMapper, injector); + } + + @Override + public void configureJsonMapper(ObjectMapper mapper) + { + delegate.configureJsonMapper(mapper); + } + + @Override + public JoinableFactoryWrapper createJoinableFactoryWrapper(LookupExtractorFactoryContainerProvider lookupProvider) + { + return delegate.createJoinableFactoryWrapper(lookupProvider); + } + + @Override + public void finalizeTestFramework(SqlTestFramework sqlTestFramework) + { + delegate.finalizeTestFramework(sqlTestFramework); + } + + @Override + public PlannerComponentSupplier getPlannerComponentSupplier() + { + return delegate.getPlannerComponentSupplier(); + } + + @Override + public void close() throws IOException + { + delegate.close(); + } + + @Override + public Boolean isExplainSupported() + { + return delegate.isExplainSupported(); + } + + @Override + public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate, + Closer resourceCloser) + { + return delegate.wrapConglomerate(conglomerate, resourceCloser); + } } public interface PlannerComponentSupplier @@ -258,28 +345,6 @@ public void configureGuice(DruidInjectorBuilder builder) { } - @Override - public QueryRunnerFactoryConglomerate createCongolmerate( - Builder builder, - Closer resourceCloser, - ObjectMapper jsonMapper - ) - { - if (builder.mergeBufferCount == 0) { - return QueryStackTests.createQueryRunnerFactoryConglomerate( - resourceCloser, - () -> builder.minTopNThreshold, - jsonMapper - ); - } else { - return QueryStackTests.createQueryRunnerFactoryConglomerate( - resourceCloser, - QueryStackTests.getProcessingConfig(builder.mergeBufferCount), - jsonMapper - ); - } - } - @Override public SpecificSegmentsQuerySegmentWalker createQuerySegmentWalker( final QueryRunnerFactoryConglomerate conglomerate, @@ -348,6 +413,13 @@ public Boolean isExplainSupported() { return true; } + + @Override + public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate, + Closer resourceCloser) + { + return conglomerate; + } } public static class StandardPlannerComponentSupplier implements PlannerComponentSupplier @@ -548,6 +620,8 @@ public SqlStatementFactory statementFactory() } } + public static final String SQL_TEST_FRAME_WORK = "sqlTestFrameWork"; + /** * Guice module to create the various query framework items. By creating items within * a module, later items can depend on those created earlier by grabbing them from the @@ -592,11 +666,61 @@ public void configure(Binder binder) binder.bind(DefaultColumnFormatConfig.class).toInstance(new DefaultColumnFormatConfig(null, null)); } + + @Provides + public @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> makeRunnerFactories( + ObjectMapper jsonMapper, + final TestBufferPool testBufferPool, + final TestGroupByBuffers groupByBuffers, + @Named(SqlTestFramework.SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig) + { + return ImmutableMap., QueryRunnerFactory>builder() + .putAll( + QueryStackTests + .makeDefaultQueryRunnerFactories( + processingConfig, + builder.minTopNThreshold, + jsonMapper, + testBufferPool, + groupByBuffers + ) + ) + .build(); + + } + + /* + * Ideally this should not have a Named annotation, but it clashes with {@link DruidProcessingModule}. + */ + @Named(SQL_TEST_FRAME_WORK) + @Provides + @LazySingleton + public DruidProcessingConfig makeProcessingConfig() + { + return QueryStackTests.getProcessingConfig(builder.mergeBufferCount); + } + + @Provides + @LazySingleton + public TestBufferPool makeTestBufferPool() + { + return QueryStackTests.makeTestBufferPool(resourceCloser); + } + + @Provides + @LazySingleton + public TestGroupByBuffers makeTestGroupByBuffers(@Named(SQL_TEST_FRAME_WORK) DruidProcessingConfig processingConfig) + { + return QueryStackTests.makeGroupByBuffers(resourceCloser, processingConfig); + } + @Provides @LazySingleton - public QueryRunnerFactoryConglomerate conglomerate() + public QueryRunnerFactoryConglomerate conglomerate( + @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> factories) { - return componentSupplier.createCongolmerate(builder, resourceCloser, queryJsonMapper()); + QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate(factories); + return componentSupplier.wrapConglomerate(conglomerate, resourceCloser); } @Provides From 19583f810085896fe62bf18650bdec21522df256 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 13:25:40 +0000 Subject: [PATCH 120/149] apidoc --- .../org/apache/druid/query/groupby/SupportRowSignature.java | 3 +++ 1 file changed, 3 insertions(+) diff --git a/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java b/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java index 5f0c56c0ae3f..6947c1397163 100644 --- a/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java +++ b/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java @@ -21,6 +21,9 @@ import org.apache.druid.segment.column.RowSignature; +/** + * Interface for queries that support a row signature. + */ public interface SupportRowSignature { RowSignature getResultRowSignature(RowSignature.Finalization finalization); From 835fcb4c3c367153ed1dd3813fc14bd9bebcc117 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 13:27:21 +0000 Subject: [PATCH 121/149] cleanup --- .../java/org/apache/druid/java/util/common/guava/Sequences.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/processing/src/main/java/org/apache/druid/java/util/common/guava/Sequences.java b/processing/src/main/java/org/apache/druid/java/util/common/guava/Sequences.java index dab5835568c3..e66ba29e4d7c 100644 --- a/processing/src/main/java/org/apache/druid/java/util/common/guava/Sequences.java +++ b/processing/src/main/java/org/apache/druid/java/util/common/guava/Sequences.java @@ -147,7 +147,7 @@ public Yielder toYielder(OutType initValue, YieldingAccumulat } } - public static Sequence of(T ...values) + public static Sequence of(T... values) { return simple(Arrays.asList(values)); } From 33aa8c353fa8311c608faeeb6972075cf49cc36a Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 14:12:46 +0000 Subject: [PATCH 122/149] notes --- .../org/apache/druid/query/QueryRunnerFactoryConglomerate.java | 2 ++ .../org/apache/druid/query/groupby/SupportRowSignature.java | 1 + 2 files changed, 3 insertions(+) diff --git a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java index 842d2f3d2793..29e5f7751818 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java @@ -37,4 +37,6 @@ default > QueryToolChest getToolChes } return factory.getToolchest(); } + // FIXME: add method here + } diff --git a/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java b/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java index 6947c1397163..d3d5e5b42676 100644 --- a/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java +++ b/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java @@ -24,6 +24,7 @@ /** * Interface for queries that support a row signature. */ +// FIXME: remove/flatten public interface SupportRowSignature { RowSignature getResultRowSignature(RowSignature.Finalization finalization); From e2aa5077e1154a2d042741836e9448c5d393ce5d Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 14:13:10 +0000 Subject: [PATCH 123/149] add test --- ...ultQueryRunnerFactoryConglomerateTest.java | 45 +++++++++++++++++++ 1 file changed, 45 insertions(+) create mode 100644 processing/src/test/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerateTest.java diff --git a/processing/src/test/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerateTest.java b/processing/src/test/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerateTest.java new file mode 100644 index 000000000000..88217b994720 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerateTest.java @@ -0,0 +1,45 @@ +///* +// * Licensed to the Apache Software Foundation (ASF) under one +// * or more contributor license agreements. See the NOTICE file +// * distributed with this work for additional information +// * regarding copyright ownership. The ASF licenses this file +// * to you under the Apache License, Version 2.0 (the +// * "License"); you may not use this file except in compliance +// * with the License. You may obtain a copy of the License at +// * +// * http://www.apache.org/licenses/LICENSE-2.0 +// * +// * Unless required by applicable law or agreed to in writing, +// * software distributed under the License is distributed on an +// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY +// * KIND, either express or implied. See the License for the +// * specific language governing permissions and limitations +// * under the License. +// */ +// +//package org.apache.druid.query; +// +//import com.google.common.collect.ImmutableMap; +//import org.apache.druid.java.util.common.DateTimes; +//import org.junit.Assert; +//import org.junit.Test; +// +//public class DefaultQueryRunnerFactoryConglomerateTest +//{ +// +// @Test +// public void testCompareNullTimestamp() +// { +// DefaultQueryRunnerFactoryConglomerate a=new DefaultQueryRunnerFactoryConglomerate( +// ImmutableMap.builder() +// .put() +// +// ) +// final Result nullTimestamp = new Result<>(null, null); +// final Result nullTimestamp2 = new Result<>(null, null); +// final Result nonNullTimestamp = new Result<>(DateTimes.nowUtc(), null); +// +// Assert.assertEquals(0, nullTimestamp.compareTo(nullTimestamp2)); +// Assert.assertEquals(1, nullTimestamp.compareTo(nonNullTimestamp)); +// } +//} From 113f13d583eaa73b5f072ee896fbf3a3bece7fd7 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 14:13:19 +0000 Subject: [PATCH 124/149] Revert "add test" This reverts commit e2aa5077e1154a2d042741836e9448c5d393ce5d. --- ...ultQueryRunnerFactoryConglomerateTest.java | 45 ------------------- 1 file changed, 45 deletions(-) delete mode 100644 processing/src/test/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerateTest.java diff --git a/processing/src/test/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerateTest.java b/processing/src/test/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerateTest.java deleted file mode 100644 index 88217b994720..000000000000 --- a/processing/src/test/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerateTest.java +++ /dev/null @@ -1,45 +0,0 @@ -///* -// * Licensed to the Apache Software Foundation (ASF) under one -// * or more contributor license agreements. See the NOTICE file -// * distributed with this work for additional information -// * regarding copyright ownership. The ASF licenses this file -// * to you under the Apache License, Version 2.0 (the -// * "License"); you may not use this file except in compliance -// * with the License. You may obtain a copy of the License at -// * -// * http://www.apache.org/licenses/LICENSE-2.0 -// * -// * Unless required by applicable law or agreed to in writing, -// * software distributed under the License is distributed on an -// * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY -// * KIND, either express or implied. See the License for the -// * specific language governing permissions and limitations -// * under the License. -// */ -// -//package org.apache.druid.query; -// -//import com.google.common.collect.ImmutableMap; -//import org.apache.druid.java.util.common.DateTimes; -//import org.junit.Assert; -//import org.junit.Test; -// -//public class DefaultQueryRunnerFactoryConglomerateTest -//{ -// -// @Test -// public void testCompareNullTimestamp() -// { -// DefaultQueryRunnerFactoryConglomerate a=new DefaultQueryRunnerFactoryConglomerate( -// ImmutableMap.builder() -// .put() -// -// ) -// final Result nullTimestamp = new Result<>(null, null); -// final Result nullTimestamp2 = new Result<>(null, null); -// final Result nonNullTimestamp = new Result<>(DateTimes.nowUtc(), null); -// -// Assert.assertEquals(0, nullTimestamp.compareTo(nullTimestamp2)); -// Assert.assertEquals(1, nullTimestamp.compareTo(nonNullTimestamp)); -// } -//} From b27511f8aee57803c478bd03f40199648e9dfdd5 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 14:35:53 +0000 Subject: [PATCH 125/149] add congl methods/etc --- ...DefaultQueryRunnerFactoryConglomerate.java | 40 ++++++++++++++++--- .../query/QueryRunnerFactoryConglomerate.java | 14 ++----- .../apache/druid/query/QueryToolChest.java | 8 ++-- .../query/union/UnionQueryQueryToolChest.java | 4 +- .../union/UnionQueryQueryToolChestTest.java | 9 +++-- .../coordination/ServerManagerTest.java | 20 ++++------ 6 files changed, 57 insertions(+), 38 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java index 33287b9eebde..5e7fd67a6b87 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java @@ -20,23 +20,26 @@ package org.apache.druid.query; import com.google.inject.Inject; +import org.apache.druid.error.DruidException; +import java.util.Collections; import java.util.IdentityHashMap; import java.util.Map; -/** -*/ public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactoryConglomerate { private final Map, QueryRunnerFactory> factories; - @Inject public DefaultQueryRunnerFactoryConglomerate(Map, QueryRunnerFactory> factories) + { + this(factories, Collections.emptyMap()); + } + + @Inject + public DefaultQueryRunnerFactoryConglomerate(Map, QueryRunnerFactory> factories, + Map, QueryToolChest> toolchests) { this.factories = new IdentityHashMap<>(factories); - for (QueryRunnerFactory factory : factories.values()) { - factory.getToolchest().setWarehouse(this); - } } @Override @@ -46,4 +49,29 @@ public > QueryRunnerFactory findFact return factories.get(query.getClass()); } + @Override + public > QueryToolChest getToolChest(QueryType query) + { + QueryRunnerFactory factory = findFactory(query); + if (factory == null) { + throw DruidException + .defensive("QueryRunnerFactory for QueryType [%s] is not registered!", query.getClass().getName()); + } + return factory.getToolchest(); + } + + public > QueryExecutor getQueryExecutor(QueryType query) + { + QueryRunnerFactory factory = findFactory(query); + if (factory == null) { + throw DruidException + .defensive("QueryRunnerFactory for QueryType [%s] is not registered!", query.getClass().getName()); + } + QueryToolChest toolchest = factory.getToolchest(); + if (toolchest instanceof QueryExecutor) { + return (QueryExecutor) toolchest; + } + return null; + } + } diff --git a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java index 29e5f7751818..f831ffac1fd3 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java @@ -19,8 +19,6 @@ package org.apache.druid.query; -import org.apache.druid.error.DruidException; - /** */ public interface QueryRunnerFactoryConglomerate extends QueryToolChestWarehouse @@ -28,15 +26,9 @@ public interface QueryRunnerFactoryConglomerate extends QueryToolChestWarehouse > QueryRunnerFactory findFactory(QueryType query); @Override - default > QueryToolChest getToolChest(QueryType query) - { - QueryRunnerFactory factory = findFactory(query); - if (factory == null) { - throw DruidException - .defensive("QueryRunnerFactory for QueryType [%s] is not registered!", query.getClass().getName()); - } - return factory.getToolchest(); - } + > QueryToolChest getToolChest(QueryType query); + + // FIXME: add method here } diff --git a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java index 2c814553bfc1..9c7fbee8c034 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java @@ -24,6 +24,7 @@ import com.fasterxml.jackson.databind.ObjectMapper; import com.fasterxml.jackson.databind.type.TypeFactory; import com.google.common.base.Function; +import com.google.inject.Inject; import org.apache.druid.error.DruidException; import org.apache.druid.frame.allocation.MemoryAllocatorFactory; import org.apache.druid.guice.annotations.ExtensionPoint; @@ -47,7 +48,7 @@ public abstract class QueryToolChest resultsAsArrays( @Override public Optional> transformResults(Query query, Sequence results) { - QueryToolChest toolChest = warehouse.getToolChest(query); + QueryToolChest toolChest = conglomerate.getToolChest(query); return Optional.of(toolChest.resultsAsArrays(query, results)); } }.transform(query, resultSequence).get(); @@ -121,7 +121,7 @@ public Optional> resultsAsFrames( @Override public Optional> transformResults(Query query, Sequence results) { - QueryToolChest toolChest = warehouse.getToolChest(query); + QueryToolChest toolChest = conglomerate.getToolChest(query); return toolChest.resultsAsFrames(query, results, memoryAllocatorFactory, useNestedForUnknownTypes); } }.transform(query, resultSequence); diff --git a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java index d4469678373a..dffcfb36b218 100644 --- a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java @@ -27,15 +27,14 @@ import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate; import org.apache.druid.query.Druids; import org.apache.druid.query.FrameBasedInlineDataSource; import org.apache.druid.query.FrameSignaturePair; -import org.apache.druid.query.MapQueryToolChestWarehouse; import org.apache.druid.query.Query; import org.apache.druid.query.QueryRunnerTestHelper; import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.QueryToolChestTestHelper; -import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.scan.ScanQuery; import org.apache.druid.query.scan.ScanQueryQueryToolChestTest; import org.apache.druid.query.scan.ScanResultValue; @@ -48,6 +47,7 @@ import java.util.ArrayList; import java.util.Arrays; +import java.util.Collections; import java.util.List; public class UnionQueryQueryToolChestTest @@ -63,12 +63,13 @@ public static void setUpClass() public UnionQueryQueryToolChestTest() { toolChest = new UnionQueryQueryToolChest(); - QueryToolChestWarehouse warehouse = new MapQueryToolChestWarehouse( + DefaultQueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate( + Collections.emptyMap(), ImmutableMap., QueryToolChest>builder() .put(ScanQuery.class, ScanQueryQueryToolChestTest.makeTestScanQueryToolChest()) .build() ); - toolChest.setWarehouse(warehouse); + toolChest.setWarehouse(conglomerate); } @Test diff --git a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java index 69a7c08f2be9..58ba480c62e8 100644 --- a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java @@ -48,6 +48,7 @@ import org.apache.druid.query.ConcatQueryRunner; import org.apache.druid.query.DataSource; import org.apache.druid.query.DefaultQueryMetrics; +import org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate; import org.apache.druid.query.Druids; import org.apache.druid.query.ForwardingQueryProcessingPool; import org.apache.druid.query.NoopQueryRunner; @@ -187,19 +188,14 @@ public ReferenceCountingSegment getSegment(final DataSegment dataSegment) queryNotifyLatch = new CountDownLatch(1); factory = new MyQueryRunnerFactory(queryWaitLatch, queryWaitYieldLatch, queryNotifyLatch); serverManagerExec = Execs.multiThreaded(2, "ServerManagerTest-%d"); + QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate( + ImmutableMap + ., QueryRunnerFactory>builder() + .put(SearchQuery.class, factory) + .build() + ); serverManager = new ServerManager( - new QueryRunnerFactoryConglomerate() - { - @Override - public > QueryRunnerFactory findFactory(QueryType query) - { - if (query instanceof SearchQuery) { - return (QueryRunnerFactory) factory; - } else { - return null; - } - } - }, + conglomerate, new NoopServiceEmitter(), new ForwardingQueryProcessingPool(serverManagerExec), new ForegroundCachePopulator(new DefaultObjectMapper(), new CachePopulatorStats(), -1), From 8c086495963030a697faf16123662eeb71fefa95 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 14:37:42 +0000 Subject: [PATCH 126/149] get access to the conglomerate instead of warehouse --- .../server/ClientQuerySegmentWalker.java | 26 +++++++++---------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 1e2f0729e4d7..c0096fcb34c1 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -52,9 +52,9 @@ import org.apache.druid.query.QueryExecutor; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.QueryToolChest; -import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.ResourceLimitExceededException; import org.apache.druid.query.ResultLevelCachingQueryRunner; import org.apache.druid.query.ResultSerializationMode; @@ -104,7 +104,7 @@ public class ClientQuerySegmentWalker implements QuerySegmentWalker private final ServiceEmitter emitter; private final QuerySegmentWalker clusterClient; private final QuerySegmentWalker localClient; - private final QueryToolChestWarehouse warehouse; + private final QueryRunnerFactoryConglomerate conglomerate; private final JoinableFactory joinableFactory; private final RetryQueryRunnerConfig retryConfig; private final ObjectMapper objectMapper; @@ -118,7 +118,7 @@ public ClientQuerySegmentWalker( ServiceEmitter emitter, QuerySegmentWalker clusterClient, QuerySegmentWalker localClient, - QueryToolChestWarehouse warehouse, + QueryRunnerFactoryConglomerate conglomerate, JoinableFactory joinableFactory, RetryQueryRunnerConfig retryConfig, ObjectMapper objectMapper, @@ -132,7 +132,7 @@ public ClientQuerySegmentWalker( this.emitter = emitter; this.clusterClient = clusterClient; this.localClient = localClient; - this.warehouse = warehouse; + this.conglomerate = conglomerate; this.joinableFactory = joinableFactory; this.retryConfig = retryConfig; this.objectMapper = objectMapper; @@ -148,7 +148,7 @@ public ClientQuerySegmentWalker( ServiceEmitter emitter, CachingClusteredClient clusterClient, LocalQuerySegmentWalker localClient, - QueryToolChestWarehouse warehouse, + QueryRunnerFactoryConglomerate conglomerate, JoinableFactory joinableFactory, RetryQueryRunnerConfig retryConfig, ObjectMapper objectMapper, @@ -163,7 +163,7 @@ public ClientQuerySegmentWalker( emitter, clusterClient, (QuerySegmentWalker) localClient, - warehouse, + conglomerate, joinableFactory, retryConfig, objectMapper, @@ -178,7 +178,7 @@ public ClientQuerySegmentWalker( @Override public QueryRunner getQueryRunnerForIntervals(final Query query, final Iterable intervals) { - final QueryToolChest> toolChest = warehouse.getToolChest(query); + final QueryToolChest> toolChest = conglomerate.getToolChest(query); if (toolChest instanceof QueryExecutor) { return ((QueryExecutor) toolChest).makeQueryRunner(query, this); @@ -296,7 +296,7 @@ public QueryRunner getQueryRunnerForSegments(final Query query, final private boolean canRunQueryUsingLocalWalker(Query query) { final DataSourceAnalysis analysis = query.getDataSourceAnalysis(); - final QueryToolChest> toolChest = warehouse.getToolChest(query); + final QueryToolChest> toolChest = conglomerate.getToolChest(query); // 1) Must be based on a concrete datasource that is not a table. // 2) Must be based on globally available data (so we have a copy here on the Broker). @@ -312,7 +312,7 @@ private boolean canRunQueryUsingLocalWalker(Query query) */ private boolean canRunQueryUsingClusterWalker(Query query) { - final QueryToolChest> toolChest = warehouse.getToolChest(query); + final QueryToolChest> toolChest = conglomerate.getToolChest(query); if (toolChest instanceof QueryExecutor) { // these type of queries should be able to run return true; @@ -381,7 +381,7 @@ private DataSource inlineIfNecessary( if (dataSource instanceof QueryDataSource) { // This datasource is a subquery. final Query subQuery = ((QueryDataSource) dataSource).getQuery(); - final QueryToolChest toolChest = warehouse.getToolChest(subQuery); + final QueryToolChest toolChest = conglomerate.getToolChest(subQuery); if (toolChestIfOutermost != null && toolChestIfOutermost.canPerformSubquery(subQuery)) { // Strip outer queries that are handleable by the toolchest, and inline subqueries that may be underneath @@ -449,7 +449,7 @@ private DataSource inlineIfNecessary( .toString() ) ); - QueryToolChest subQueryToolChest = warehouse.getToolChest(subQuery); + QueryToolChest subQueryToolChest = conglomerate.getToolChest(subQuery); final QueryRunner subQueryRunner; if (subQueryToolChest instanceof QueryExecutor) { @@ -466,7 +466,7 @@ private DataSource inlineIfNecessary( return toInlineDataSource( subQuery, queryResults, - warehouse.getToolChest(subQuery), + conglomerate.getToolChest(subQuery), subqueryRowLimitAccumulator, subqueryMemoryLimitAccumulator, cannotMaterializeToFrames, @@ -541,7 +541,7 @@ private DataSource inlineIfNecessary( */ private QueryRunner decorateClusterRunner(Query query, QueryRunner baseClusterRunner) { - final QueryToolChest> toolChest = warehouse.getToolChest(query); + final QueryToolChest> toolChest = conglomerate.getToolChest(query); final SetAndVerifyContextQueryRunner baseRunner = new SetAndVerifyContextQueryRunner<>( serverConfig, From 574b33701aab22938b41acc9b741bf052838c880 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 14:42:53 +0000 Subject: [PATCH 127/149] use conglomerate.getQueryExecutor --- .../query/DefaultQueryRunnerFactoryConglomerate.java | 6 +++--- .../druid/query/QueryRunnerFactoryConglomerate.java | 4 +--- .../apache/druid/server/ClientQuerySegmentWalker.java | 9 +++++---- 3 files changed, 9 insertions(+), 10 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java index 5e7fd67a6b87..f62dbc5c57a0 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java @@ -60,7 +60,8 @@ public > QueryToolChest getToolChest return factory.getToolchest(); } - public > QueryExecutor getQueryExecutor(QueryType query) + @Override + public > QueryExecutor getQueryExecutor(QueryType query) { QueryRunnerFactory factory = findFactory(query); if (factory == null) { @@ -69,9 +70,8 @@ public > QueryExecutor getQueryExecutor } QueryToolChest toolchest = factory.getToolchest(); if (toolchest instanceof QueryExecutor) { - return (QueryExecutor) toolchest; + return (QueryExecutor) toolchest; } return null; } - } diff --git a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java index f831ffac1fd3..7b796d2197b4 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java @@ -28,7 +28,5 @@ public interface QueryRunnerFactoryConglomerate extends QueryToolChestWarehouse @Override > QueryToolChest getToolChest(QueryType query); - - // FIXME: add method here - + > QueryExecutor getQueryExecutor(QueryType query); } diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index c0096fcb34c1..e6d6dba83549 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -178,12 +178,13 @@ public ClientQuerySegmentWalker( @Override public QueryRunner getQueryRunnerForIntervals(final Query query, final Iterable intervals) { - final QueryToolChest> toolChest = conglomerate.getToolChest(query); - - if (toolChest instanceof QueryExecutor) { - return ((QueryExecutor) toolChest).makeQueryRunner(query, this); + final QueryExecutor queryExecutor = conglomerate.getQueryExecutor(query); + if (queryExecutor != null) { + return queryExecutor.makeQueryRunner(query, this); } + final QueryToolChest> toolChest = conglomerate.getToolChest(query); + // transform TableDataSource to GlobalTableDataSource when eligible // before further transformation to potentially inline From c17a3efbd7a9dc04d25130bfc62915ffabd10303 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 14:53:53 +0000 Subject: [PATCH 128/149] use executor --- .../java/org/apache/druid/query/QueryExecutor.java | 2 -- .../druid/query/QueryRunnerFactoryConglomerate.java | 3 --- .../apache/druid/server/ClientQuerySegmentWalker.java | 11 +++++------ 3 files changed, 5 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryExecutor.java b/processing/src/main/java/org/apache/druid/query/QueryExecutor.java index 1afa9d8f36ce..08046829829e 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryExecutor.java +++ b/processing/src/main/java/org/apache/druid/query/QueryExecutor.java @@ -21,8 +21,6 @@ /** * Executes the query by utilizing the given walker. - * - * {@link QueryToolChest}-s implementing this interface will get control over the execution of the query. */ public interface QueryExecutor { diff --git a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java index 7b796d2197b4..1b0e20fe077c 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java @@ -25,8 +25,5 @@ public interface QueryRunnerFactoryConglomerate extends QueryToolChestWarehouse { > QueryRunnerFactory findFactory(QueryType query); - @Override - > QueryToolChest getToolChest(QueryType query); - > QueryExecutor getQueryExecutor(QueryType query); } diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index e6d6dba83549..f7451817d67f 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -313,11 +313,11 @@ private boolean canRunQueryUsingLocalWalker(Query query) */ private boolean canRunQueryUsingClusterWalker(Query query) { - final QueryToolChest> toolChest = conglomerate.getToolChest(query); - if (toolChest instanceof QueryExecutor) { + if (conglomerate.getQueryExecutor(query) != null) { // these type of queries should be able to run return true; } + final QueryToolChest> toolChest = conglomerate.getToolChest(query); final DataSourceAnalysis analysis = query.getDataSourceAnalysis(); // 1) Must be based on a concrete table (the only shape the Druid cluster can handle). @@ -450,12 +450,11 @@ private DataSource inlineIfNecessary( .toString() ) ); - QueryToolChest subQueryToolChest = conglomerate.getToolChest(subQuery); + QueryExecutor subQueryExecutor = conglomerate.getQueryExecutor(subQuery); final QueryRunner subQueryRunner; - if (subQueryToolChest instanceof QueryExecutor) { - subQueryRunner = ((QueryExecutor) subQueryToolChest) - .makeQueryRunner(subQueryWithSerialization, this); + if (subQueryExecutor != null) { + subQueryRunner = subQueryExecutor.makeQueryRunner(subQueryWithSerialization, this); } else { subQueryRunner = subQueryWithSerialization.getRunner(this); } From 4a76077005d31b27b7db62eba2d393b1f8279988 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 15:29:03 +0000 Subject: [PATCH 129/149] undo --- .../testGroupByWithLiteralInSubqueryGrouping.iq | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq index 85c1b392c49a..b5d2c682643b 100644 --- a/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq +++ b/sql/src/test/quidem/org.apache.druid.sql.calcite.DecoupledPlanningCalciteQueryTest/testGroupByWithLiteralInSubqueryGrouping.iq @@ -30,8 +30,8 @@ SELECT +-------+----+ | t1 | t2 | +-------+----+ -| dummy | b | | dummy | | +| dummy | b | +-------+----+ (2 rows) From 15b698724a2c990ce0d275648f90a1bd4d076f5f Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 15:43:11 +0000 Subject: [PATCH 130/149] cleanup --- .../query/DefaultQueryRunnerFactoryConglomerate.java | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java index f62dbc5c57a0..583cdc50db8d 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java @@ -29,6 +29,7 @@ public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactoryConglomerate { private final Map, QueryRunnerFactory> factories; + private final Map, QueryToolChest> toolchests; public DefaultQueryRunnerFactoryConglomerate(Map, QueryRunnerFactory> factories) { @@ -40,6 +41,7 @@ public DefaultQueryRunnerFactoryConglomerate(Map, QueryRu Map, QueryToolChest> toolchests) { this.factories = new IdentityHashMap<>(factories); + this.toolchests = new IdentityHashMap<>(toolchests); } @Override @@ -63,12 +65,7 @@ public > QueryToolChest getToolChest @Override public > QueryExecutor getQueryExecutor(QueryType query) { - QueryRunnerFactory factory = findFactory(query); - if (factory == null) { - throw DruidException - .defensive("QueryRunnerFactory for QueryType [%s] is not registered!", query.getClass().getName()); - } - QueryToolChest toolchest = factory.getToolchest(); + QueryToolChest toolchest = getToolChest(query); if (toolchest instanceof QueryExecutor) { return (QueryExecutor) toolchest; } From 51bbf552e83b673ea074b3a15e8c5a4757074393 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 16:01:23 +0000 Subject: [PATCH 131/149] remove SupportRowSignature --- ...DefaultQueryRunnerFactoryConglomerate.java | 17 +++++----- .../java/org/apache/druid/query/Query.java | 6 ++++ .../druid/query/groupby/GroupByQuery.java | 2 +- .../query/groupby/SupportRowSignature.java | 31 ------------------- .../apache/druid/query/scan/ScanQuery.java | 3 +- .../query/timeseries/TimeseriesQuery.java | 3 +- .../apache/druid/query/topn/TopNQuery.java | 3 +- .../query/union/UnionQueryQueryToolChest.java | 6 ++-- 8 files changed, 20 insertions(+), 51 deletions(-) delete mode 100644 processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java index 583cdc50db8d..69be35e3b535 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java @@ -19,10 +19,9 @@ package org.apache.druid.query; +import com.google.common.annotations.VisibleForTesting; +import com.google.common.collect.Maps; import com.google.inject.Inject; -import org.apache.druid.error.DruidException; - -import java.util.Collections; import java.util.IdentityHashMap; import java.util.Map; @@ -31,9 +30,10 @@ public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactory private final Map, QueryRunnerFactory> factories; private final Map, QueryToolChest> toolchests; + @VisibleForTesting public DefaultQueryRunnerFactoryConglomerate(Map, QueryRunnerFactory> factories) { - this(factories, Collections.emptyMap()); + this(factories, Maps.transformValues(factories, f -> f.getToolchest())); } @Inject @@ -52,17 +52,14 @@ public > QueryRunnerFactory findFact } @Override + @SuppressWarnings("unchecked") public > QueryToolChest getToolChest(QueryType query) { - QueryRunnerFactory factory = findFactory(query); - if (factory == null) { - throw DruidException - .defensive("QueryRunnerFactory for QueryType [%s] is not registered!", query.getClass().getName()); - } - return factory.getToolchest(); + return toolchests.get(query.getClass()); } @Override + @SuppressWarnings("unchecked") public > QueryExecutor getQueryExecutor(QueryType query) { QueryToolChest toolchest = getToolChest(query); diff --git a/processing/src/main/java/org/apache/druid/query/Query.java b/processing/src/main/java/org/apache/druid/query/Query.java index 50bcd7db4016..82e38fd48d8f 100644 --- a/processing/src/main/java/org/apache/druid/query/Query.java +++ b/processing/src/main/java/org/apache/druid/query/Query.java @@ -47,6 +47,7 @@ import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.Segment; import org.apache.druid.segment.VirtualColumns; +import org.apache.druid.segment.column.RowSignature; import org.apache.druid.utils.CollectionUtils; import org.joda.time.DateTimeZone; import org.joda.time.Duration; @@ -310,4 +311,9 @@ default DataSourceAnalysis getDataSourceAnalysis() { return getDataSource().getAnalysis().maybeWithBaseQuery(this); } + + + default RowSignature getResultRowSignature(RowSignature.Finalization finalization) { + return null; + } } diff --git a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java index 2a84331fa51f..6d7ba0742482 100644 --- a/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java +++ b/processing/src/main/java/org/apache/druid/query/groupby/GroupByQuery.java @@ -85,7 +85,7 @@ /** * */ -public class GroupByQuery extends BaseQuery implements SupportRowSignature +public class GroupByQuery extends BaseQuery { public static final String CTX_KEY_SORT_BY_DIMS_FIRST = "sortByDimsFirst"; public static final String CTX_TIMESTAMP_RESULT_FIELD = "timestampResultField"; diff --git a/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java b/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java deleted file mode 100644 index d3d5e5b42676..000000000000 --- a/processing/src/main/java/org/apache/druid/query/groupby/SupportRowSignature.java +++ /dev/null @@ -1,31 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.groupby; - -import org.apache.druid.segment.column.RowSignature; - -/** - * Interface for queries that support a row signature. - */ -// FIXME: remove/flatten -public interface SupportRowSignature -{ - RowSignature getResultRowSignature(RowSignature.Finalization finalization); -} diff --git a/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java b/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java index dd197facd70d..be3d4fa2d5ea 100644 --- a/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java +++ b/processing/src/main/java/org/apache/druid/query/scan/ScanQuery.java @@ -39,7 +39,6 @@ import org.apache.druid.query.OrderBy; import org.apache.druid.query.Queries; import org.apache.druid.query.filter.DimFilter; -import org.apache.druid.query.groupby.SupportRowSignature; import org.apache.druid.query.operator.OffsetLimit; import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.segment.VirtualColumn; @@ -59,7 +58,7 @@ import java.util.Objects; import java.util.Set; -public class ScanQuery extends BaseQuery implements SupportRowSignature +public class ScanQuery extends BaseQuery { public enum ResultFormat { diff --git a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java index 0ae7f4f13deb..b5e135968a89 100644 --- a/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java +++ b/processing/src/main/java/org/apache/druid/query/timeseries/TimeseriesQuery.java @@ -38,7 +38,6 @@ import org.apache.druid.query.aggregation.AggregatorFactory; import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.filter.DimFilter; -import org.apache.druid.query.groupby.SupportRowSignature; import org.apache.druid.query.groupby.orderby.DefaultLimitSpec.LimitJsonIncludeFilter; import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.segment.VirtualColumns; @@ -57,7 +56,7 @@ /** */ @JsonTypeName("timeseries") -public class TimeseriesQuery extends BaseQuery> implements SupportRowSignature +public class TimeseriesQuery extends BaseQuery> { public static final String CTX_GRAND_TOTAL = "grandTotal"; public static final String SKIP_EMPTY_BUCKETS = "skipEmptyBuckets"; diff --git a/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java b/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java index d1d0ca390469..aee76989db38 100644 --- a/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java +++ b/processing/src/main/java/org/apache/druid/query/topn/TopNQuery.java @@ -35,7 +35,6 @@ import org.apache.druid.query.aggregation.PostAggregator; import org.apache.druid.query.dimension.DimensionSpec; import org.apache.druid.query.filter.DimFilter; -import org.apache.druid.query.groupby.SupportRowSignature; import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.segment.VirtualColumns; import org.apache.druid.segment.column.RowSignature; @@ -50,7 +49,7 @@ /** */ -public class TopNQuery extends BaseQuery> implements SupportRowSignature +public class TopNQuery extends BaseQuery> { public static final String TOPN = "topN"; diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index f26d585d2b07..6d58c340566f 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -35,7 +35,6 @@ import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.aggregation.MetricManipulationFn; -import org.apache.druid.query.groupby.SupportRowSignature; import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature.Finalization; @@ -84,8 +83,9 @@ public TypeReference getResultTypeReference() public RowSignature resultArraySignature(UnionQuery query) { for (Query q : query.queries) { - if (q instanceof SupportRowSignature) { - return ((SupportRowSignature) q).getResultRowSignature(Finalization.UNKNOWN); + RowSignature sig = q.getResultRowSignature(Finalization.UNKNOWN); + if (sig != null) { + return sig; } } throw DruidException.defensive("None of the subqueries support row signature"); From a3f84454ee8c429c9911a4f57d46fe6097bd6d69 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 16:25:06 +0000 Subject: [PATCH 132/149] make unionqueryrunnerfactory not necessatry --- .../query/union/UnionQueryRunnerFactory.java | 58 ------------------- .../druid/guice/QueryRunnerFactoryModule.java | 3 - .../sql/calcite/util/SqlTestFramework.java | 43 +++++++++++--- 3 files changed, 35 insertions(+), 69 deletions(-) delete mode 100644 processing/src/main/java/org/apache/druid/query/union/UnionQueryRunnerFactory.java diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunnerFactory.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunnerFactory.java deleted file mode 100644 index 03876cc72388..000000000000 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunnerFactory.java +++ /dev/null @@ -1,58 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.union; - -import com.google.inject.Inject; -import org.apache.druid.error.DruidException; -import org.apache.druid.query.QueryProcessingPool; -import org.apache.druid.query.QueryRunner; -import org.apache.druid.query.QueryRunnerFactory; -import org.apache.druid.query.QueryToolChest; -import org.apache.druid.segment.Segment; - -public class UnionQueryRunnerFactory implements QueryRunnerFactory -{ - private UnionQueryQueryToolChest toolChest; - - @Inject - public UnionQueryRunnerFactory(UnionQueryQueryToolChest toolChest) - { - this.toolChest = toolChest; - } - - @Override - public QueryRunner createRunner(Segment segment) - { - throw DruidException.defensive("not expected to be used"); - } - - @Override - public QueryRunner mergeRunners(QueryProcessingPool queryProcessingPool, - Iterable> queryRunners) - { - throw DruidException.defensive("not expected to be used"); - } - - @Override - public QueryToolChest getToolchest() - { - return toolChest; - } -} diff --git a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java index 6d7008944134..5e71a0d1f082 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryRunnerFactoryModule.java @@ -46,8 +46,6 @@ import org.apache.druid.query.timeseries.TimeseriesQueryRunnerFactory; import org.apache.druid.query.topn.TopNQuery; import org.apache.druid.query.topn.TopNQueryRunnerFactory; -import org.apache.druid.query.union.UnionQuery; -import org.apache.druid.query.union.UnionQueryRunnerFactory; import org.apache.druid.server.QueryScheduler; import org.apache.druid.server.QuerySchedulerProvider; @@ -68,7 +66,6 @@ public class QueryRunnerFactoryModule extends QueryToolChestModule .put(TimeseriesQuery.class, TimeseriesQueryRunnerFactory.class) .put(TopNQuery.class, TopNQueryRunnerFactory.class) .put(WindowOperatorQuery.class, WindowOperatorQueryQueryRunnerFactory.class) - .put(UnionQuery.class, UnionQueryRunnerFactory.class) .build(); @Override diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java index 14c7d5679669..1a971044eae7 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/util/SqlTestFramework.java @@ -23,6 +23,7 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import com.google.common.collect.ImmutableSet; +import com.google.common.collect.Maps; import com.google.inject.Binder; import com.google.inject.Injector; import com.google.inject.Module; @@ -46,12 +47,13 @@ import org.apache.druid.query.QueryRunnerFactory; import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.TestBufferPool; import org.apache.druid.query.groupby.TestGroupByBuffers; import org.apache.druid.query.lookup.LookupExtractorFactoryContainerProvider; import org.apache.druid.query.topn.TopNQueryConfig; import org.apache.druid.query.union.UnionQuery; -import org.apache.druid.query.union.UnionQueryRunnerFactory; +import org.apache.druid.query.union.UnionQueryQueryToolChest; import org.apache.druid.quidem.TestSqlModule; import org.apache.druid.segment.DefaultColumnFormatConfig; import org.apache.druid.segment.join.JoinableFactoryWrapper; @@ -89,7 +91,7 @@ import java.net.URI; import java.util.ArrayList; import java.util.Arrays; -import java.util.HashMap; +import java.util.Collections; import java.util.List; import java.util.Map; import java.util.Properties; @@ -213,6 +215,8 @@ default void configureGuice(CoreInjectorBuilder injectorBuilder, List ov QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglomerate conglomerate, Closer resourceCloser); Map, ? extends QueryRunnerFactory> makeRunnerFactories(Injector injector); + + Map, ? extends QueryToolChest> makeToolChests(Injector injector); } public abstract static class QueryComponentSupplierDelegate implements QueryComponentSupplier @@ -302,6 +306,12 @@ public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglom { return delegate.makeRunnerFactories(injector); } + + @Override + public Map, ? extends QueryToolChest> makeToolChests(Injector injector) + { + return delegate.makeToolChests(injector); + } } public interface PlannerComponentSupplier @@ -435,10 +445,15 @@ public QueryRunnerFactoryConglomerate wrapConglomerate(QueryRunnerFactoryConglom @Override public Map, ? extends QueryRunnerFactory> makeRunnerFactories(Injector injector) { - Map map = new HashMap<>(); - UnionQueryRunnerFactory factory = injector.getInstance(UnionQueryRunnerFactory.class); - map.put(UnionQuery.class, factory); - return map; + return Collections.emptyMap(); + } + + @Override + public Map, ? extends QueryToolChest> makeToolChests(Injector injector) + { + return ImmutableMap., QueryToolChest>builder() + .put(UnionQuery.class, injector.getInstance(UnionQueryQueryToolChest.class)) + .build(); } } @@ -688,6 +703,7 @@ public void configure(Binder binder) @Provides + @LazySingleton public @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> makeRunnerFactories( ObjectMapper jsonMapper, final TestBufferPool testBufferPool, @@ -707,7 +723,17 @@ public void configure(Binder binder) ) .putAll(componentSupplier.makeRunnerFactories(injector)) .build(); + } + @Provides + @LazySingleton + public @Named(SQL_TEST_FRAME_WORK) Map, QueryToolChest> makeToolchests( + @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> factories) + { + return ImmutableMap., QueryToolChest>builder() + .putAll(Maps.transformValues(factories, f -> f.getToolchest())) + .putAll(componentSupplier.makeToolChests(injector)) + .build(); } /* @@ -738,9 +764,10 @@ public TestGroupByBuffers makeTestGroupByBuffers(@Named(SQL_TEST_FRAME_WORK) Dru @Provides @LazySingleton public QueryRunnerFactoryConglomerate conglomerate( - @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> factories) + @Named(SQL_TEST_FRAME_WORK) Map, QueryRunnerFactory> factories, + @Named(SQL_TEST_FRAME_WORK) Map, QueryToolChest> toolchests) { - QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate(factories); + QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate(factories, toolchests); return componentSupplier.wrapConglomerate(conglomerate, resourceCloser); } From 069e700b57886cc4761cd4ee555933cbf53e53db Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 16:49:40 +0000 Subject: [PATCH 133/149] fix compile --- .../query/movingaverage/MovingAverageQueryTest.java | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/extensions-contrib/moving-average-query/src/test/java/org/apache/druid/query/movingaverage/MovingAverageQueryTest.java b/extensions-contrib/moving-average-query/src/test/java/org/apache/druid/query/movingaverage/MovingAverageQueryTest.java index b6d787cbc215..68668c9b3aa3 100644 --- a/extensions-contrib/moving-average-query/src/test/java/org/apache/druid/query/movingaverage/MovingAverageQueryTest.java +++ b/extensions-contrib/moving-average-query/src/test/java/org/apache/druid/query/movingaverage/MovingAverageQueryTest.java @@ -56,8 +56,8 @@ import org.apache.druid.query.Query; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; -import org.apache.druid.query.QueryToolChestWarehouse; import org.apache.druid.query.Result; import org.apache.druid.query.RetryQueryRunnerConfig; import org.apache.druid.query.SegmentDescriptor; @@ -104,7 +104,7 @@ public class MovingAverageQueryTest extends InitializedNullHandlingTest { private final ObjectMapper jsonMapper; - private final QueryToolChestWarehouse warehouse; + private final QueryRunnerFactoryConglomerate conglomerate; private final RetryQueryRunnerConfig retryConfig; private final ServerConfig serverConfig; @@ -167,7 +167,7 @@ public QueryRunner getQueryRunnerForSegments(Query query, Iterable res = query.getRunner(walker).run(queryPlus); List actualResults = new ArrayList(); - actualResults = (List) res.accumulate(actualResults, Accumulators.list()); + actualResults = res.accumulate(actualResults, Accumulators.list()); expectedResults = consistentTypeCasting(expectedResults); actualResults = consistentTypeCasting(actualResults); From 66736ddb0aba91993bf8abc5c4c6ac3838d38522 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 17:10:55 +0000 Subject: [PATCH 134/149] fix style --- processing/src/main/java/org/apache/druid/query/Query.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/Query.java b/processing/src/main/java/org/apache/druid/query/Query.java index 82e38fd48d8f..b78f2da05be2 100644 --- a/processing/src/main/java/org/apache/druid/query/Query.java +++ b/processing/src/main/java/org/apache/druid/query/Query.java @@ -312,8 +312,8 @@ default DataSourceAnalysis getDataSourceAnalysis() return getDataSource().getAnalysis().maybeWithBaseQuery(this); } - - default RowSignature getResultRowSignature(RowSignature.Finalization finalization) { + default RowSignature getResultRowSignature(RowSignature.Finalization finalization) + { return null; } } From 98cb67ad72cc1ad927ca62fc6a1256fd927f6d60 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 18:20:56 +0000 Subject: [PATCH 135/149] fix --- ...QueryRunnerTest.java => UnionDataSourceQueryRunnerTest.java} | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) rename processing/src/test/java/org/apache/druid/query/{UnionQueryRunnerTest.java => UnionDataSourceQueryRunnerTest.java} (98%) diff --git a/processing/src/test/java/org/apache/druid/query/UnionQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/UnionDataSourceQueryRunnerTest.java similarity index 98% rename from processing/src/test/java/org/apache/druid/query/UnionQueryRunnerTest.java rename to processing/src/test/java/org/apache/druid/query/UnionDataSourceQueryRunnerTest.java index 332f74022e3f..e88adaa5bc83 100644 --- a/processing/src/test/java/org/apache/druid/query/UnionQueryRunnerTest.java +++ b/processing/src/test/java/org/apache/druid/query/UnionDataSourceQueryRunnerTest.java @@ -30,7 +30,7 @@ import java.util.List; import java.util.concurrent.atomic.AtomicBoolean; -public class UnionQueryRunnerTest +public class UnionDataSourceQueryRunnerTest { @Test public void testUnionQueryRunner() From 7fbc8425e2a73b31b1c2cfdebb4f2f1d29b1b944 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 18:45:01 +0000 Subject: [PATCH 136/149] add test --- .../union/UnionQueryQueryToolChestTest.java | 58 +++++++++++++++++++ .../query/union/UnionQueryRunnerTest.java | 32 ++++++++++ 2 files changed, 90 insertions(+) create mode 100644 processing/src/test/java/org/apache/druid/query/union/UnionQueryRunnerTest.java diff --git a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java index dffcfb36b218..b096a47c65c0 100644 --- a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java @@ -32,7 +32,10 @@ import org.apache.druid.query.FrameBasedInlineDataSource; import org.apache.druid.query.FrameSignaturePair; import org.apache.druid.query.Query; +import org.apache.druid.query.QueryPlus; +import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QueryRunnerTestHelper; +import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.QueryToolChestTestHelper; import org.apache.druid.query.scan.ScanQuery; @@ -44,12 +47,16 @@ import org.junit.Assert; import org.junit.jupiter.api.BeforeAll; import org.junit.jupiter.api.Test; +import org.mockito.Mockito; import java.util.ArrayList; import java.util.Arrays; import java.util.Collections; import java.util.List; +import static org.mockito.ArgumentMatchers.any; +import static org.mockito.ArgumentMatchers.argThat; + public class UnionQueryQueryToolChestTest { @BeforeAll @@ -140,6 +147,11 @@ private List> convertResultsToListOfLists() } return resultsRows; } + + private boolean matchQuery(ScanQuery query) + { + return query != null && this.query.equals(query); + } } @Test @@ -219,4 +231,50 @@ public void testResultsAsFrames() rows ); } + + @Test + void testQueryRunner() + { + RowSignature sig = RowSignature.builder() + .add("a", ColumnType.STRING) + .add("b", ColumnType.STRING) + .build(); + + TestScanQuery scan1 = new TestScanQuery("foo", sig) + .appendRow("a", "a") + .appendRow("a", "b"); + TestScanQuery scan2 = new TestScanQuery("bar", sig) + .appendRow("x", "x") + .appendRow("x", "y"); + + UnionQuery query = new UnionQuery( + ImmutableList.of( + scan1.query, + scan2.query + ) + ); + + QuerySegmentWalker walker = Mockito.mock(QuerySegmentWalker.class); + Mockito.when(walker.getQueryRunnerForIntervals(argThat(scan1::matchQuery), any())) + .thenReturn((q, ctx) -> (Sequence) scan1.makeResultSequence()); + Mockito.when(walker.getQueryRunnerForIntervals(argThat(scan2::matchQuery), any())) + .thenReturn((q, ctx) -> (Sequence) scan2.makeResultSequence()); + + QueryRunner unionRunner = toolChest.makeQueryRunner(query, walker); + Sequence results = unionRunner.run(QueryPlus.wrap(query), null); + + QueryToolChestTestHelper.assertArrayResultsEquals( + ImmutableList.builder() + .addAll(scan1.results) + .addAll(scan2.results) + .build(), + toolChest.resultsAsArrays( + query, + Sequences.of( + new UnionResult(scan1.makeResultSequence()), + new UnionResult(scan2.makeResultSequence()) + ) + ) + ); + } } diff --git a/processing/src/test/java/org/apache/druid/query/union/UnionQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/union/UnionQueryRunnerTest.java new file mode 100644 index 000000000000..6480d03177f8 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/query/union/UnionQueryRunnerTest.java @@ -0,0 +1,32 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.query.union; + +import org.junit.jupiter.api.Test; + +public class UnionQueryRunnerTest +{ + @Test + public void test1() + { + + UnionQueryRunner uqr = new UnionQueryRunner(null, null); + } +} From ab21c403143d4454b35e8e7f982113e65809eb11 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Thu, 17 Oct 2024 18:49:40 +0000 Subject: [PATCH 137/149] service ToolChestWarehouse with DefaultQueryRunnerFactoryConglomerate --- .../java/org/apache/druid/guice/QueryToolChestModule.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java b/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java index 3ef5b7409539..87d579cbd3a6 100644 --- a/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java +++ b/server/src/main/java/org/apache/druid/guice/QueryToolChestModule.java @@ -26,8 +26,8 @@ import com.google.inject.multibindings.MapBinder; import org.apache.druid.query.DefaultGenericQueryMetricsFactory; import org.apache.druid.query.DefaultQueryConfig; +import org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate; import org.apache.druid.query.GenericQueryMetricsFactory; -import org.apache.druid.query.MapQueryToolChestWarehouse; import org.apache.druid.query.Query; import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.QueryToolChestWarehouse; @@ -101,7 +101,7 @@ public void configure(Binder binder) binder.bind(entry.getValue()).in(LazySingleton.class); } - binder.bind(QueryToolChestWarehouse.class).to(MapQueryToolChestWarehouse.class); + binder.bind(QueryToolChestWarehouse.class).to(DefaultQueryRunnerFactoryConglomerate.class); JsonConfigProvider.bind(binder, "druid.query.default", DefaultQueryConfig.class); JsonConfigProvider.bind(binder, "druid.query.groupBy", GroupByQueryConfig.class); From 1bdcc4fadfeb27ceb6650d4c4408c88a69be630d Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Fri, 18 Oct 2024 06:09:36 +0000 Subject: [PATCH 138/149] remove invalid file --- .../query/union/UnionQueryRunnerTest.java | 32 ------------------- 1 file changed, 32 deletions(-) delete mode 100644 processing/src/test/java/org/apache/druid/query/union/UnionQueryRunnerTest.java diff --git a/processing/src/test/java/org/apache/druid/query/union/UnionQueryRunnerTest.java b/processing/src/test/java/org/apache/druid/query/union/UnionQueryRunnerTest.java deleted file mode 100644 index 6480d03177f8..000000000000 --- a/processing/src/test/java/org/apache/druid/query/union/UnionQueryRunnerTest.java +++ /dev/null @@ -1,32 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.union; - -import org.junit.jupiter.api.Test; - -public class UnionQueryRunnerTest -{ - @Test - public void test1() - { - - UnionQueryRunner uqr = new UnionQueryRunner(null, null); - } -} From 863b947f444d6543579ee465a910d8e44b60aaa9 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 23 Oct 2024 11:48:10 +0000 Subject: [PATCH 139/149] add factory method --- .../CachingClusteredClientBenchmark.java | 58 +++++++++---------- .../indexing/kafka/KafkaIndexTaskTest.java | 38 ++++++------ .../kinesis/KinesisIndexTaskTest.java | 20 +++---- ...DefaultQueryRunnerFactoryConglomerate.java | 7 +-- .../StreamAppenderatorTester.java | 48 +++++++-------- ...nifiedIndexerAppenderatorsManagerTest.java | 2 +- .../apache/druid/server/QueryStackTests.java | 4 +- .../coordination/ServerManagerTest.java | 10 ++-- 8 files changed, 86 insertions(+), 101 deletions(-) diff --git a/benchmarks/src/test/java/org/apache/druid/benchmark/query/CachingClusteredClientBenchmark.java b/benchmarks/src/test/java/org/apache/druid/benchmark/query/CachingClusteredClientBenchmark.java index 8e0715e0fe5c..16b87982c4c6 100644 --- a/benchmarks/src/test/java/org/apache/druid/benchmark/query/CachingClusteredClientBenchmark.java +++ b/benchmarks/src/test/java/org/apache/druid/benchmark/query/CachingClusteredClientBenchmark.java @@ -258,39 +258,37 @@ public int getNumThreads() } }; - conglomerate = new DefaultQueryRunnerFactoryConglomerate( - ImmutableMap., QueryRunnerFactory>builder() - .put( - TimeseriesQuery.class, - new TimeseriesQueryRunnerFactory( - new TimeseriesQueryQueryToolChest(), - new TimeseriesQueryEngine(), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ) + conglomerate = DefaultQueryRunnerFactoryConglomerate.buildFromQueryRunnerFactories(ImmutableMap., QueryRunnerFactory>builder() + .put( + TimeseriesQuery.class, + new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER ) - .put( - TopNQuery.class, - new TopNQueryRunnerFactory( - new StupidPool<>( - "TopNQueryRunnerFactory-bufferPool", - () -> ByteBuffer.allocate(PROCESSING_BUFFER_SIZE) - ), - new TopNQueryQueryToolChest(new TopNQueryConfig()), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ) + ) + .put( + TopNQuery.class, + new TopNQueryRunnerFactory( + new StupidPool<>( + "TopNQueryRunnerFactory-bufferPool", + () -> ByteBuffer.allocate(PROCESSING_BUFFER_SIZE) + ), + new TopNQueryQueryToolChest(new TopNQueryConfig()), + QueryRunnerTestHelper.NOOP_QUERYWATCHER ) - .put( - GroupByQuery.class, - makeGroupByQueryRunnerFactory( - GroupByQueryRunnerTest.DEFAULT_MAPPER, - new GroupByQueryConfig() - { - }, - processingConfig - ) + ) + .put( + GroupByQuery.class, + makeGroupByQueryRunnerFactory( + GroupByQueryRunnerTest.DEFAULT_MAPPER, + new GroupByQueryConfig() + { + }, + processingConfig ) - .build() - ); + ) + .build()); toolChestWarehouse = new QueryToolChestWarehouse() { diff --git a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java index 23bdeb14acb8..db8db1fdb38c 100644 --- a/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java +++ b/extensions-core/kafka-indexing-service/src/test/java/org/apache/druid/indexing/kafka/KafkaIndexTaskTest.java @@ -2891,28 +2891,26 @@ private static DataSchema cloneDataSchema(final DataSchema dataSchema) @Override protected QueryRunnerFactoryConglomerate makeQueryRunnerConglomerate() { - return new DefaultQueryRunnerFactoryConglomerate( - ImmutableMap., QueryRunnerFactory>builder() - .put( - TimeseriesQuery.class, - new TimeseriesQueryRunnerFactory( - new TimeseriesQueryQueryToolChest(), - new TimeseriesQueryEngine(), - (query, future) -> { - // do nothing - } - ) + return DefaultQueryRunnerFactoryConglomerate.buildFromQueryRunnerFactories(ImmutableMap., QueryRunnerFactory>builder() + .put( + TimeseriesQuery.class, + new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(), + new TimeseriesQueryEngine(), + (query, future) -> { + // do nothing + } ) - .put( - ScanQuery.class, - new ScanQueryRunnerFactory( - new ScanQueryQueryToolChest(DefaultGenericQueryMetricsFactory.instance()), - new ScanQueryEngine(), - new ScanQueryConfig() - ) + ) + .put( + ScanQuery.class, + new ScanQueryRunnerFactory( + new ScanQueryQueryToolChest(DefaultGenericQueryMetricsFactory.instance()), + new ScanQueryEngine(), + new ScanQueryConfig() ) - .build() - ); + ) + .build()); } private void makeToolboxFactory() throws IOException diff --git a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java index 2ef391484008..088bb177d0e6 100644 --- a/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java +++ b/extensions-core/kinesis-indexing-service/src/test/java/org/apache/druid/indexing/kinesis/KinesisIndexTaskTest.java @@ -2418,18 +2418,16 @@ private static DataSchema cloneDataSchema(final DataSchema dataSchema) @Override protected QueryRunnerFactoryConglomerate makeQueryRunnerConglomerate() { - return new DefaultQueryRunnerFactoryConglomerate( - ImmutableMap.of( - TimeseriesQuery.class, - new TimeseriesQueryRunnerFactory( - new TimeseriesQueryQueryToolChest(), - new TimeseriesQueryEngine(), - (query, future) -> { - // do nothing - } - ) + return DefaultQueryRunnerFactoryConglomerate.buildFromQueryRunnerFactories(ImmutableMap.of( + TimeseriesQuery.class, + new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(), + new TimeseriesQueryEngine(), + (query, future) -> { + // do nothing + } ) - ); + )); } private void makeToolboxFactory() throws IOException diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java index 69be35e3b535..63968d0363be 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java @@ -19,7 +19,6 @@ package org.apache.druid.query; -import com.google.common.annotations.VisibleForTesting; import com.google.common.collect.Maps; import com.google.inject.Inject; import java.util.IdentityHashMap; @@ -30,10 +29,10 @@ public class DefaultQueryRunnerFactoryConglomerate implements QueryRunnerFactory private final Map, QueryRunnerFactory> factories; private final Map, QueryToolChest> toolchests; - @VisibleForTesting - public DefaultQueryRunnerFactoryConglomerate(Map, QueryRunnerFactory> factories) + public static DefaultQueryRunnerFactoryConglomerate buildFromQueryRunnerFactories( + Map, QueryRunnerFactory> factories) { - this(factories, Maps.transformValues(factories, f -> f.getToolchest())); + return new DefaultQueryRunnerFactoryConglomerate(factories, Maps.transformValues(factories, f -> f.getToolchest())); } @Inject diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java index 29d758aaed02..c305ada342ae 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/StreamAppenderatorTester.java @@ -224,20 +224,18 @@ public Map makeLoadSpec(URI uri) objectMapper, indexIO, indexMerger, - new DefaultQueryRunnerFactoryConglomerate( - ImmutableMap.of( - TimeseriesQuery.class, new TimeseriesQueryRunnerFactory( - new TimeseriesQueryQueryToolChest(), - new TimeseriesQueryEngine(), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ), - ScanQuery.class, new ScanQueryRunnerFactory( - new ScanQueryQueryToolChest(DefaultGenericQueryMetricsFactory.instance()), - new ScanQueryEngine(), - new ScanQueryConfig() - ) + DefaultQueryRunnerFactoryConglomerate.buildFromQueryRunnerFactories(ImmutableMap.of( + TimeseriesQuery.class, new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ), + ScanQuery.class, new ScanQueryRunnerFactory( + new ScanQueryQueryToolChest(DefaultGenericQueryMetricsFactory.instance()), + new ScanQueryEngine(), + new ScanQueryConfig() ) - ), + )), announcer, emitter, new ForwardingQueryProcessingPool(queryExecutor), @@ -268,20 +266,18 @@ public int getDropSegmentDelayMillis() objectMapper, indexIO, indexMerger, - new DefaultQueryRunnerFactoryConglomerate( - ImmutableMap.of( - TimeseriesQuery.class, new TimeseriesQueryRunnerFactory( - new TimeseriesQueryQueryToolChest(), - new TimeseriesQueryEngine(), - QueryRunnerTestHelper.NOOP_QUERYWATCHER - ), - ScanQuery.class, new ScanQueryRunnerFactory( - new ScanQueryQueryToolChest(DefaultGenericQueryMetricsFactory.instance()), - new ScanQueryEngine(), - new ScanQueryConfig() - ) + DefaultQueryRunnerFactoryConglomerate.buildFromQueryRunnerFactories(ImmutableMap.of( + TimeseriesQuery.class, new TimeseriesQueryRunnerFactory( + new TimeseriesQueryQueryToolChest(), + new TimeseriesQueryEngine(), + QueryRunnerTestHelper.NOOP_QUERYWATCHER + ), + ScanQuery.class, new ScanQueryRunnerFactory( + new ScanQueryQueryToolChest(DefaultGenericQueryMetricsFactory.instance()), + new ScanQueryEngine(), + new ScanQueryConfig() ) - ), + )), new NoopDataSegmentAnnouncer(), emitter, new ForwardingQueryProcessingPool(queryExecutor), diff --git a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java index 21f627baa085..66ac774e1e9e 100644 --- a/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java +++ b/server/src/test/java/org/apache/druid/segment/realtime/appenderator/UnifiedIndexerAppenderatorsManagerTest.java @@ -83,7 +83,7 @@ public class UnifiedIndexerAppenderatorsManagerTest extends InitializedNullHandl new CachePopulatorStats(), TestHelper.makeJsonMapper(), new NoopServiceEmitter(), - () -> new DefaultQueryRunnerFactoryConglomerate(ImmutableMap.of()) + () -> DefaultQueryRunnerFactoryConglomerate.buildFromQueryRunnerFactories(ImmutableMap.of()) ); private AppenderatorConfig appenderatorConfig; diff --git a/server/src/test/java/org/apache/druid/server/QueryStackTests.java b/server/src/test/java/org/apache/druid/server/QueryStackTests.java index 1edddfc02aee..17ee020e6e97 100644 --- a/server/src/test/java/org/apache/druid/server/QueryStackTests.java +++ b/server/src/test/java/org/apache/druid/server/QueryStackTests.java @@ -300,9 +300,7 @@ public static QueryRunnerFactoryConglomerate createQueryRunnerFactoryConglomerat final TestBufferPool testBufferPool, final TestGroupByBuffers groupByBuffers) { - final QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate( - makeDefaultQueryRunnerFactories(processingConfig, minTopNThreshold, jsonMapper, testBufferPool, groupByBuffers) - ); + final QueryRunnerFactoryConglomerate conglomerate = DefaultQueryRunnerFactoryConglomerate.buildFromQueryRunnerFactories(makeDefaultQueryRunnerFactories(processingConfig, minTopNThreshold, jsonMapper, testBufferPool, groupByBuffers)); return conglomerate; } diff --git a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java index 58ba480c62e8..2e20b907184e 100644 --- a/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java +++ b/server/src/test/java/org/apache/druid/server/coordination/ServerManagerTest.java @@ -188,12 +188,10 @@ public ReferenceCountingSegment getSegment(final DataSegment dataSegment) queryNotifyLatch = new CountDownLatch(1); factory = new MyQueryRunnerFactory(queryWaitLatch, queryWaitYieldLatch, queryNotifyLatch); serverManagerExec = Execs.multiThreaded(2, "ServerManagerTest-%d"); - QueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate( - ImmutableMap - ., QueryRunnerFactory>builder() - .put(SearchQuery.class, factory) - .build() - ); + QueryRunnerFactoryConglomerate conglomerate = DefaultQueryRunnerFactoryConglomerate.buildFromQueryRunnerFactories(ImmutableMap + ., QueryRunnerFactory>builder() + .put(SearchQuery.class, factory) + .build()); serverManager = new ServerManager( conglomerate, new NoopServiceEmitter(), From 61ab9518a7f678e72dfeee27b5aa82204dd838e3 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 23 Oct 2024 12:13:30 +0000 Subject: [PATCH 140/149] rename method --- .../query/DefaultQueryRunnerFactoryConglomerate.java | 6 +++--- .../query/{QueryExecutor.java => QueryLogic.java} | 4 ++-- .../druid/query/QueryRunnerFactoryConglomerate.java | 2 +- .../druid/query/union/UnionQueryQueryToolChest.java | 6 +++--- .../query/union/UnionQueryQueryToolChestTest.java | 2 +- .../apache/druid/server/ClientQuerySegmentWalker.java | 10 +++++----- 6 files changed, 15 insertions(+), 15 deletions(-) rename processing/src/main/java/org/apache/druid/query/{QueryExecutor.java => QueryLogic.java} (93%) diff --git a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java index 63968d0363be..b709a8a7a864 100644 --- a/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/DefaultQueryRunnerFactoryConglomerate.java @@ -59,11 +59,11 @@ public > QueryToolChest getToolChest @Override @SuppressWarnings("unchecked") - public > QueryExecutor getQueryExecutor(QueryType query) + public > QueryLogic getQueryExecutor(QueryType query) { QueryToolChest toolchest = getToolChest(query); - if (toolchest instanceof QueryExecutor) { - return (QueryExecutor) toolchest; + if (toolchest instanceof QueryLogic) { + return (QueryLogic) toolchest; } return null; } diff --git a/processing/src/main/java/org/apache/druid/query/QueryExecutor.java b/processing/src/main/java/org/apache/druid/query/QueryLogic.java similarity index 93% rename from processing/src/main/java/org/apache/druid/query/QueryExecutor.java rename to processing/src/main/java/org/apache/druid/query/QueryLogic.java index 08046829829e..9d7e4cf7e71c 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryExecutor.java +++ b/processing/src/main/java/org/apache/druid/query/QueryLogic.java @@ -22,9 +22,9 @@ /** * Executes the query by utilizing the given walker. */ -public interface QueryExecutor +public interface QueryLogic { - QueryRunner makeQueryRunner( + QueryRunner entryPoint( Query query, QuerySegmentWalker walker ); diff --git a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java index 1b0e20fe077c..01789b883230 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java @@ -25,5 +25,5 @@ public interface QueryRunnerFactoryConglomerate extends QueryToolChestWarehouse { > QueryRunnerFactory findFactory(QueryType query); - > QueryExecutor getQueryExecutor(QueryType query); + > QueryLogic getQueryExecutor(QueryType query); } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 6d58c340566f..635b476b7214 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -29,7 +29,7 @@ import org.apache.druid.query.DefaultQueryMetrics; import org.apache.druid.query.FrameSignaturePair; import org.apache.druid.query.Query; -import org.apache.druid.query.QueryExecutor; +import org.apache.druid.query.QueryLogic; import org.apache.druid.query.QueryMetrics; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QuerySegmentWalker; @@ -43,10 +43,10 @@ import java.util.Optional; public class UnionQueryQueryToolChest extends QueryToolChest - implements QueryExecutor + implements QueryLogic { @Override - public QueryRunner makeQueryRunner(Query query, + public QueryRunner entryPoint(Query query, QuerySegmentWalker clientQuerySegmentWalker) { return new UnionQueryRunner((UnionQuery) query, clientQuerySegmentWalker); diff --git a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java index b096a47c65c0..e6eb7867fb02 100644 --- a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java @@ -260,7 +260,7 @@ void testQueryRunner() Mockito.when(walker.getQueryRunnerForIntervals(argThat(scan2::matchQuery), any())) .thenReturn((q, ctx) -> (Sequence) scan2.makeResultSequence()); - QueryRunner unionRunner = toolChest.makeQueryRunner(query, walker); + QueryRunner unionRunner = toolChest.entryPoint(query, walker); Sequence results = unionRunner.run(QueryPlus.wrap(query), null); QueryToolChestTestHelper.assertArrayResultsEquals( diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index f7451817d67f..339c1eadc216 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -49,7 +49,7 @@ import org.apache.druid.query.Query; import org.apache.druid.query.QueryContexts; import org.apache.druid.query.QueryDataSource; -import org.apache.druid.query.QueryExecutor; +import org.apache.druid.query.QueryLogic; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; import org.apache.druid.query.QueryRunnerFactoryConglomerate; @@ -178,9 +178,9 @@ public ClientQuerySegmentWalker( @Override public QueryRunner getQueryRunnerForIntervals(final Query query, final Iterable intervals) { - final QueryExecutor queryExecutor = conglomerate.getQueryExecutor(query); + final QueryLogic queryExecutor = conglomerate.getQueryExecutor(query); if (queryExecutor != null) { - return queryExecutor.makeQueryRunner(query, this); + return queryExecutor.entryPoint(query, this); } final QueryToolChest> toolChest = conglomerate.getToolChest(query); @@ -451,10 +451,10 @@ private DataSource inlineIfNecessary( ) ); - QueryExecutor subQueryExecutor = conglomerate.getQueryExecutor(subQuery); + QueryLogic subQueryExecutor = conglomerate.getQueryExecutor(subQuery); final QueryRunner subQueryRunner; if (subQueryExecutor != null) { - subQueryRunner = subQueryExecutor.makeQueryRunner(subQueryWithSerialization, this); + subQueryRunner = subQueryExecutor.entryPoint(subQueryWithSerialization, this); } else { subQueryRunner = subQueryWithSerialization.getRunner(this); } From 633adc1f3bbc95045fcb13464b0e2b4043d2ab18 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 23 Oct 2024 12:40:57 +0000 Subject: [PATCH 141/149] remove getDataSources from Query interface --- .../java/org/apache/druid/query/Query.java | 16 ------------- .../apache/druid/query/QueryDataSource.java | 24 ++++++++++++++++--- .../apache/druid/query/union/UnionQuery.java | 2 -- .../sql/calcite/BaseCalciteQueryTest.java | 10 +++++++- 4 files changed, 30 insertions(+), 22 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/Query.java b/processing/src/main/java/org/apache/druid/query/Query.java index b78f2da05be2..dc71f3ba36ea 100644 --- a/processing/src/main/java/org/apache/druid/query/Query.java +++ b/processing/src/main/java/org/apache/druid/query/Query.java @@ -22,14 +22,11 @@ import com.fasterxml.jackson.annotation.JsonSubTypes; import com.fasterxml.jackson.annotation.JsonTypeInfo; import com.google.common.base.Preconditions; -import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; -import com.google.common.collect.Iterables; import com.google.common.collect.Ordering; import org.apache.druid.error.DruidException; import org.apache.druid.guice.annotations.ExtensionPoint; import org.apache.druid.java.util.common.HumanReadableBytes; -import org.apache.druid.java.util.common.IAE; import org.apache.druid.java.util.common.granularity.Granularity; import org.apache.druid.query.datasourcemetadata.DataSourceMetadataQuery; import org.apache.druid.query.filter.DimFilter; @@ -90,11 +87,6 @@ public interface Query DataSource getDataSource(); - default List getDataSources() - { - return ImmutableList.of(getDataSource()); - } - boolean hasFilters(); DimFilter getFilter(); @@ -299,14 +291,6 @@ default Interval getSingleInterval() ); } - default Query withDataSources(List children) - { - if (children.size() != 1) { - throw new IAE("Must have exactly one child"); - } - return withDataSource(Iterables.getOnlyElement(children)); - } - default DataSourceAnalysis getDataSourceAnalysis() { return getDataSource().getAnalysis().maybeWithBaseQuery(this); diff --git a/processing/src/main/java/org/apache/druid/query/QueryDataSource.java b/processing/src/main/java/org/apache/druid/query/QueryDataSource.java index 0be20456e85e..08dc44126fee 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryDataSource.java +++ b/processing/src/main/java/org/apache/druid/query/QueryDataSource.java @@ -23,9 +23,12 @@ import com.fasterxml.jackson.annotation.JsonProperty; import com.fasterxml.jackson.annotation.JsonTypeName; import com.google.common.base.Preconditions; +import org.apache.druid.java.util.common.IAE; import org.apache.druid.query.planning.DataSourceAnalysis; +import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.SegmentReference; +import java.util.Collections; import java.util.HashSet; import java.util.List; import java.util.Set; @@ -48,7 +51,7 @@ public QueryDataSource(@JsonProperty("query") Query query) public Set getTableNames() { Set names = new HashSet<>(); - for (DataSource ds : query.getDataSources()) { + for (DataSource ds : getQueryDataSources()) { names.addAll(ds.getTableNames()); } return names; @@ -63,13 +66,28 @@ public Query getQuery() @Override public List getChildren() { - return query.getDataSources(); + return getQueryDataSources(); + } + + private List getQueryDataSources() + { + if (query instanceof UnionQuery) { + return ((UnionQuery) query).getDataSources(); + } + return Collections.singletonList(query.getDataSource()); } @Override public DataSource withChildren(List children) { - return new QueryDataSource(query.withDataSources(children)); + if (query instanceof UnionQuery) { + return new QueryDataSource(((UnionQuery) query).withDataSources(children)); + } else { + if (children.size() != 1) { + throw new IAE("Must have exactly one child"); + } + return new QueryDataSource(query.withDataSource(children.get(0))); + } } @Override diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index b6bbc18797e3..08771a27f2b8 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -79,7 +79,6 @@ public DataSource getDataSource() throw DruidException.defensive("This is not supported"); } - @Override public List getDataSources() { List dataSources = new ArrayList<>(); @@ -192,7 +191,6 @@ public Query withDataSource(DataSource dataSource) throw new RuntimeException("This method is not supported. Use withDataSources instead!"); } - @Override public Query withDataSources(List children) { Preconditions.checkArgument(queries.size() == children.size(), "Number of children must match number of queries"); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 7eb77eb91f65..9c1cf5d602ef 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -78,6 +78,7 @@ import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.query.timeseries.TimeseriesQuery; +import org.apache.druid.query.union.UnionQuery; import org.apache.druid.segment.column.ColumnHolder; import org.apache.druid.segment.column.ColumnType; import org.apache.druid.segment.column.RowSignature; @@ -1347,7 +1348,14 @@ protected static boolean isRewriteJoinToFilter(final Map queryCo public static Query recursivelyClearContext(final Query query, ObjectMapper queryJsonMapper) { try { - Query newQuery = query.withDataSources(recursivelyClearDatasource(query.getDataSources(), queryJsonMapper)); + Query newQuery; + if(query instanceof UnionQuery) { + UnionQuery unionQuery = (UnionQuery) query; + newQuery = (Query) unionQuery + .withDataSources(recursivelyClearDatasource(unionQuery.getDataSources(), queryJsonMapper)); + } else { + newQuery = query.withDataSource(recursivelyClearContext(query.getDataSource(), queryJsonMapper)); + } final JsonNode newQueryNode = queryJsonMapper.valueToTree(newQuery); ((ObjectNode) newQueryNode).remove("context"); return queryJsonMapper.treeToValue(newQueryNode, Query.class); From de4c2df4d8569b72f8cd0ece0b55aa58e3914f71 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 23 Oct 2024 12:44:03 +0000 Subject: [PATCH 142/149] remove field --- .../query/QueryRunnerBasedOnClusteredClientTestBase.java | 9 +++------ .../druid/query/ResultLevelCachingQueryRunnerTest.java | 2 +- 2 files changed, 4 insertions(+), 7 deletions(-) diff --git a/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java b/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java index 4cb081c8e11c..76a63442e3ee 100644 --- a/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java +++ b/server/src/test/java/org/apache/druid/query/QueryRunnerBasedOnClusteredClientTestBase.java @@ -92,9 +92,7 @@ public abstract class QueryRunnerBasedOnClusteredClientTestBase private static final boolean USE_PARALLEL_MERGE_POOL_CONFIGURED = false; protected final ObjectMapper objectMapper = new DefaultObjectMapper(); - protected final QueryToolChestWarehouse toolChestWarehouse; - - private final QueryRunnerFactoryConglomerate conglomerate; + protected final QueryRunnerFactoryConglomerate conglomerate; protected TestHttpClient httpClient; protected SimpleServerView simpleServerView; @@ -109,7 +107,6 @@ protected QueryRunnerBasedOnClusteredClientTestBase() CLOSER, TopNQueryConfig.DEFAULT_MIN_TOPN_THRESHOLD ); - toolChestWarehouse = conglomerate; } @AfterClass @@ -123,9 +120,9 @@ public void setupTestBase() { segmentGenerator = new SegmentGenerator(); httpClient = new TestHttpClient(objectMapper); - simpleServerView = new SimpleServerView(toolChestWarehouse, objectMapper, httpClient); + simpleServerView = new SimpleServerView(conglomerate, objectMapper, httpClient); cachingClusteredClient = new CachingClusteredClient( - toolChestWarehouse, + conglomerate, simpleServerView, MapCache.create(0), objectMapper, diff --git a/server/src/test/java/org/apache/druid/query/ResultLevelCachingQueryRunnerTest.java b/server/src/test/java/org/apache/druid/query/ResultLevelCachingQueryRunnerTest.java index 412c97ba2e0e..6245509465c1 100644 --- a/server/src/test/java/org/apache/druid/query/ResultLevelCachingQueryRunnerTest.java +++ b/server/src/test/java/org/apache/druid/query/ResultLevelCachingQueryRunnerTest.java @@ -241,7 +241,7 @@ private ResultLevelCachingQueryRunner createQueryRunner( new RetryQueryRunnerConfig(), objectMapper ), - toolChestWarehouse.getToolChest(query), + conglomerate.getToolChest(query), query, objectMapper, cache, From f4d16a6583a6625a5fe94c31e9c33954f30505b6 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 23 Oct 2024 12:54:10 +0000 Subject: [PATCH 143/149] update message --- .../org/apache/druid/sql/calcite/rel/logical/DruidUnion.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/rel/logical/DruidUnion.java b/sql/src/main/java/org/apache/druid/sql/calcite/rel/logical/DruidUnion.java index a737e369aeec..ec50e3b5025c 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/rel/logical/DruidUnion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/rel/logical/DruidUnion.java @@ -111,7 +111,7 @@ public SourceDesc getSourceDesc(PlannerContext plannerContext, List return new SourceDesc(new QueryDataSource(new UnionQuery(queries)), signature); } - throw DruidException.defensive("XXXOnly Table and Values are supported as inputs for Union [%s]", sources); + throw DruidException.defensive("Union with input [%s] is not supported. This should not happen.", sources); } private boolean mayUseUnionQuery(List sources) From d4c98776202f2c60bc30f840774efc0e9bf1f41c Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 23 Oct 2024 13:00:42 +0000 Subject: [PATCH 144/149] simpler exception --- .../org/apache/druid/error/DruidException.java | 5 ++--- .../org/apache/druid/query/union/UnionQuery.java | 16 ++++++++-------- 2 files changed, 10 insertions(+), 11 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/error/DruidException.java b/processing/src/main/java/org/apache/druid/error/DruidException.java index 61ff9ccfe733..84ad5c190f70 100644 --- a/processing/src/main/java/org/apache/druid/error/DruidException.java +++ b/processing/src/main/java/org/apache/druid/error/DruidException.java @@ -524,9 +524,8 @@ public String getErrorCode() protected abstract DruidException makeException(DruidExceptionBuilder bob); } - public static String getCurrentMethodName() + public static DruidException methodNotSupported() { - StackTraceElement stackTraceElement = Thread.currentThread().getStackTrace()[2]; - return stackTraceElement.getClassName() + "#" + stackTraceElement.getMethodName(); + return defensive("Method Not supported."); } } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index 08771a27f2b8..88ca9befbed3 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -121,7 +121,7 @@ public List getIntervals() @Override public Duration getDuration() { - throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); + throw DruidException.methodNotSupported(); } @Override @@ -133,7 +133,7 @@ public Granularity getGranularity() @Override public DateTimeZone getTimezone() { - throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); + throw DruidException.methodNotSupported(); } @Override @@ -145,7 +145,7 @@ public Map getContext() @Override public Ordering getResultOrdering() { - throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); + throw DruidException.methodNotSupported(); } @Override @@ -158,7 +158,7 @@ public Query withOverriddenContext(Map contextOverr @Override public Query withQuerySegmentSpec(QuerySegmentSpec spec) { - throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); + throw DruidException.methodNotSupported(); } @Override @@ -247,7 +247,7 @@ public List getChildren() @Override public DataSource withChildren(List children) { - throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); + throw DruidException.methodNotSupported(); } @Override @@ -271,13 +271,13 @@ public boolean isConcrete() @Override public Function createSegmentMapFunction(Query query, AtomicLong cpuTimeAcc) { - throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); + throw DruidException.methodNotSupported(); } @Override public DataSource withUpdatedDataSource(DataSource newSource) { - throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); + throw DruidException.methodNotSupported(); } @Override @@ -289,7 +289,7 @@ public byte[] getCacheKey() @Override public DataSourceAnalysis getAnalysis() { - throw DruidException.defensive("Method supported. [%s]", DruidException.getCurrentMethodName()); + throw DruidException.methodNotSupported(); } } } From 862493ed6c3de893b2781d912f377b8d775b4699 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 23 Oct 2024 14:41:02 +0000 Subject: [PATCH 145/149] transform results in the runner --- .../ArenaMemoryAllocatorFactory.java | 7 ++ .../query/QueryRunnerFactoryConglomerate.java | 1 + .../query/operator/WindowOperatorQuery.java | 7 ++ .../apache/druid/query/union/UnionQuery.java | 18 +-- .../query/union/UnionQueryQueryToolChest.java | 68 +++-------- .../druid/query/union/UnionQueryRunner.java | 73 +++++++++-- .../apache/druid/query/union/UnionResult.java | 45 ------- .../union/UnionQueryQueryToolChestTest.java | 114 ++++-------------- 8 files changed, 124 insertions(+), 209 deletions(-) delete mode 100644 processing/src/main/java/org/apache/druid/query/union/UnionResult.java diff --git a/processing/src/main/java/org/apache/druid/frame/allocation/ArenaMemoryAllocatorFactory.java b/processing/src/main/java/org/apache/druid/frame/allocation/ArenaMemoryAllocatorFactory.java index 80d4dcdadd70..61a20d965e99 100644 --- a/processing/src/main/java/org/apache/druid/frame/allocation/ArenaMemoryAllocatorFactory.java +++ b/processing/src/main/java/org/apache/druid/frame/allocation/ArenaMemoryAllocatorFactory.java @@ -24,6 +24,8 @@ */ public class ArenaMemoryAllocatorFactory implements MemoryAllocatorFactory { + private static final int FRAME_SIZE = 8_000_000; + private final int capacity; public ArenaMemoryAllocatorFactory(final int capacity) @@ -42,4 +44,9 @@ public long allocatorCapacity() { return capacity; } + + public static MemoryAllocatorFactory makeDefault() + { + return new ArenaMemoryAllocatorFactory(FRAME_SIZE); + } } diff --git a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java index 01789b883230..6a35e5a3296b 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java +++ b/processing/src/main/java/org/apache/druid/query/QueryRunnerFactoryConglomerate.java @@ -25,5 +25,6 @@ public interface QueryRunnerFactoryConglomerate extends QueryToolChestWarehouse { > QueryRunnerFactory findFactory(QueryType query); + // XXX rename > QueryLogic getQueryExecutor(QueryType query); } diff --git a/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java index aed132b745ee..79a8ebdb19d2 100644 --- a/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java +++ b/processing/src/main/java/org/apache/druid/query/operator/WindowOperatorQuery.java @@ -34,6 +34,7 @@ import org.apache.druid.query.scan.ScanQuery; import org.apache.druid.query.spec.QuerySegmentSpec; import org.apache.druid.segment.column.RowSignature; +import org.apache.druid.segment.column.RowSignature.Finalization; import javax.annotation.Nullable; import java.util.ArrayList; @@ -163,6 +164,12 @@ public RowSignature getRowSignature() return rowSignature; } + @Override + public RowSignature getResultRowSignature(Finalization finalization) + { + return getRowSignature(); + } + @Override public boolean hasFilters() { diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index 88ca9befbed3..df586fc36c89 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -50,7 +50,7 @@ import java.util.concurrent.atomic.AtomicLong; import java.util.function.Function; -public class UnionQuery implements Query +public class UnionQuery implements Query { @JsonProperty("context") protected final Map context; @@ -107,7 +107,7 @@ public String getType() } @Override - public QueryRunner getRunner(QuerySegmentWalker walker) + public QueryRunner getRunner(QuerySegmentWalker walker) { throw DruidException.defensive("Use QueryToolChest to get a Runner"); } @@ -143,26 +143,26 @@ public Map getContext() } @Override - public Ordering getResultOrdering() + public Ordering getResultOrdering() { throw DruidException.methodNotSupported(); } @Override - public Query withOverriddenContext(Map contextOverrides) + public Query withOverriddenContext(Map contextOverrides) { List> newQueries = mapQueries(q -> q.withOverriddenContext(contextOverrides)); return new UnionQuery(newQueries, QueryContexts.override(getContext(), contextOverrides)); } @Override - public Query withQuerySegmentSpec(QuerySegmentSpec spec) + public Query withQuerySegmentSpec(QuerySegmentSpec spec) { throw DruidException.methodNotSupported(); } @Override - public Query withId(String id) + public Query withId(String id) { return withOverriddenContext(ImmutableMap.of(BaseQuery.QUERY_ID, id)); } @@ -174,7 +174,7 @@ public String getId() } @Override - public Query withSubQueryId(String subQueryId) + public Query withSubQueryId(String subQueryId) { return withOverriddenContext(ImmutableMap.of(BaseQuery.SUB_QUERY_ID, subQueryId)); } @@ -186,12 +186,12 @@ public String getSubQueryId() } @Override - public Query withDataSource(DataSource dataSource) + public Query withDataSource(DataSource dataSource) { throw new RuntimeException("This method is not supported. Use withDataSources instead!"); } - public Query withDataSources(List children) + public Query withDataSources(List children) { Preconditions.checkArgument(queries.size() == children.size(), "Number of children must match number of queries"); List> newQueries = new ArrayList<>(); diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java index 635b476b7214..36b282fe6042 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryQueryToolChest.java @@ -25,7 +25,6 @@ import org.apache.druid.error.DruidException; import org.apache.druid.frame.allocation.MemoryAllocatorFactory; import org.apache.druid.java.util.common.guava.Sequence; -import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.query.DefaultQueryMetrics; import org.apache.druid.query.FrameSignaturePair; import org.apache.druid.query.Query; @@ -38,23 +37,21 @@ import org.apache.druid.segment.column.RowSignature; import org.apache.druid.segment.column.RowSignature.Finalization; -import java.util.ArrayList; -import java.util.List; import java.util.Optional; -public class UnionQueryQueryToolChest extends QueryToolChest - implements QueryLogic +public class UnionQueryQueryToolChest extends QueryToolChest + implements QueryLogic { @Override - public QueryRunner entryPoint(Query query, + public QueryRunner entryPoint(Query query, QuerySegmentWalker clientQuerySegmentWalker) { - return new UnionQueryRunner((UnionQuery) query, clientQuerySegmentWalker); + return new UnionQueryRunner((UnionQuery) query, clientQuerySegmentWalker, conglomerate); } @Override @SuppressWarnings("unchecked") - public QueryRunner mergeResults(QueryRunner runner) + public QueryRunner mergeResults(QueryRunner runner) { throw new UnsupportedOperationException("Not supported"); } @@ -66,7 +63,7 @@ public QueryMetrics makeMetrics(UnionQuery query) } @Override - public Function makePreComputeManipulatorFn( + public Function makePreComputeManipulatorFn( UnionQuery query, MetricManipulationFn fn) { @@ -74,7 +71,7 @@ public Function makePreComputeManipulatorFn( } @Override - public TypeReference getResultTypeReference() + public TypeReference getResultTypeReference() { return null; } @@ -88,66 +85,29 @@ public RowSignature resultArraySignature(UnionQuery query) return sig; } } - throw DruidException.defensive("None of the subqueries support row signature"); + throw DruidException.defensive("None of the subqueries have a valid row signature"); } @Override @SuppressWarnings({"unchecked", "rawtypes"}) public Sequence resultsAsArrays( UnionQuery query, - Sequence resultSequence) + Sequence resultSequence) { - return new UnionSequenceMaker() - { - @Override - public Optional> transformResults(Query query, Sequence results) - { - QueryToolChest toolChest = conglomerate.getToolChest(query); - return Optional.of(toolChest.resultsAsArrays(query, results)); - } - }.transform(query, resultSequence).get(); + Sequence res = resultSequence; + return (Sequence) res; } @Override @SuppressWarnings({"unchecked", "rawtypes"}) public Optional> resultsAsFrames( UnionQuery query, - Sequence resultSequence, + Sequence resultSequence, MemoryAllocatorFactory memoryAllocatorFactory, boolean useNestedForUnknownTypes) { - return new UnionSequenceMaker() - { - @Override - public Optional> transformResults(Query query, Sequence results) - { - QueryToolChest toolChest = conglomerate.getToolChest(query); - return toolChest.resultsAsFrames(query, results, memoryAllocatorFactory, useNestedForUnknownTypes); - } - }.transform(query, resultSequence); + Sequence res = resultSequence; + return Optional.of((Sequence) res); } - abstract static class UnionSequenceMaker - { - public Optional> transform( - UnionQuery query, - Sequence resultSequence) - { - List results = resultSequence.toList(); - List> resultSeqs = new ArrayList<>(); - - for (int i = 0; i < results.size(); i++) { - Query q = query.queries.get(i); - UnionResult realUnionResult = results.get(i); - Optional> queryResults = transformResults(q, realUnionResult.getResults()); - if (!queryResults.isPresent()) { - return Optional.empty(); - } - resultSeqs.add(queryResults.get()); - } - return Optional.of(Sequences.concat(resultSeqs)); - } - - public abstract Optional> transformResults(Query q, Sequence results); - } } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java index fc9cad1381cb..303709df3920 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java @@ -19,28 +19,43 @@ package org.apache.druid.query.union; +import org.apache.druid.error.DruidException; +import org.apache.druid.frame.allocation.ArenaMemoryAllocatorFactory; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; +import org.apache.druid.query.FrameSignaturePair; import org.apache.druid.query.Query; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; +import org.apache.druid.query.QueryRunnerFactoryConglomerate; import org.apache.druid.query.QuerySegmentWalker; +import org.apache.druid.query.QueryToolChest; +import org.apache.druid.query.ResultSerializationMode; import org.apache.druid.query.context.ResponseContext; import java.util.ArrayList; import java.util.List; +import java.util.Optional; -class UnionQueryRunner implements QueryRunner +class UnionQueryRunner implements QueryRunner { private final QuerySegmentWalker walker; private final List runners; + private QueryRunnerFactoryConglomerate conglomerate; + private ResultSerializationMode serializationMode; + private boolean useNestedForUnknownTypeInSubquery; public UnionQueryRunner( UnionQuery query, - QuerySegmentWalker walker - ) + QuerySegmentWalker walker, + QueryRunnerFactoryConglomerate conglomerate) { this.walker = walker; + this.conglomerate = conglomerate; + + serializationMode = getResultSerializationMode(query); + // FIXME: this was more complicated; it dependend on ServerConfig from the server module + useNestedForUnknownTypeInSubquery = query.context().isUseNestedForUnknownTypeInSubquery(false); this.runners = makeSubQueryRunners(query); } @@ -55,23 +70,61 @@ private List makeSubQueryRunners(UnionQuery unionQuery) } @Override - public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) + public Sequence run(QueryPlus queryPlus, ResponseContext responseContext) { UnionQuery unionQuery = queryPlus.unwrapQuery(UnionQuery.class); - List seqs = new ArrayList(); + List> seqs = new ArrayList<>(); for (int i = 0; i < runners.size(); i++) { Query q = unionQuery.queries.get(i); QueryRunner r = runners.get(i); - seqs.add(makeUnionResult(r, queryPlus.withQuery(q), responseContext)); + seqs.add(makeResultSeq(r, queryPlus.withQuery(q), responseContext, serializationMode)); + } + return Sequences.concat(seqs); + } + + private ResultSerializationMode getResultSerializationMode(Query query) + { + ResultSerializationMode serializationMode = query.context().getEnum( + ResultSerializationMode.CTX_SERIALIZATION_PARAMETER, + ResultSerializationMode.class, + null + ); + if (serializationMode == null) { + throw DruidException.defensive( + "Serialization mode [%s] is not setup correctly!", ResultSerializationMode.CTX_SERIALIZATION_PARAMETER + ); } - return Sequences.simple(seqs); + return serializationMode; } - private UnionResult makeUnionResult(QueryRunner runner, QueryPlus withQuery, - ResponseContext responseContext) + private Sequence makeResultSeq(QueryRunner runner, QueryPlus withQuery, + ResponseContext responseContext, ResultSerializationMode serializationMode) { + Query query = withQuery.getQuery(); + QueryToolChest> toolChest = conglomerate.getToolChest(query); Sequence seq = runner.run(withQuery, responseContext); - return new UnionResult(seq); + Sequence resultSeq; + switch (serializationMode) + { + case ROWS: + resultSeq = toolChest.resultsAsArrays(query, seq); + break; + case FRAMES: + Optional> resultsAsFrames = toolChest.resultsAsFrames( + query, + seq, + ArenaMemoryAllocatorFactory.makeDefault(), + false + ); + if (resultsAsFrames.isEmpty()) { + throw DruidException.defensive("Unable to materialize the results as frames."); + } + resultSeq = resultsAsFrames.get(); + break; + default: + throw DruidException.defensive("Not supported serializationMode [%s].", serializationMode); + } + return (Sequence) resultSeq; } } diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionResult.java b/processing/src/main/java/org/apache/druid/query/union/UnionResult.java deleted file mode 100644 index e2db2ab643c7..000000000000 --- a/processing/src/main/java/org/apache/druid/query/union/UnionResult.java +++ /dev/null @@ -1,45 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.query.union; - -import org.apache.druid.java.util.common.guava.Sequence; - -/** - * Holds the resulting Sequence for a union query branch. - * - * Caveat: the index of the ResultUnionResult in the output sequence is in line - * with the index of the executed query. - */ -public class UnionResult -{ - private final Sequence seq; - - public UnionResult(Sequence seq) - { - this.seq = seq; - } - - public Sequence getResults() - { - return (Sequence) seq; - - } - -} diff --git a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java index e6eb7867fb02..2ed8db09a08b 100644 --- a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java @@ -22,15 +22,11 @@ import com.google.common.collect.ImmutableList; import com.google.common.collect.ImmutableMap; import org.apache.druid.common.config.NullHandling; -import org.apache.druid.frame.allocation.HeapMemoryAllocator; -import org.apache.druid.frame.allocation.SingleMemoryAllocatorFactory; import org.apache.druid.java.util.common.Intervals; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; import org.apache.druid.query.DefaultQueryRunnerFactoryConglomerate; import org.apache.druid.query.Druids; -import org.apache.druid.query.FrameBasedInlineDataSource; -import org.apache.druid.query.FrameSignaturePair; import org.apache.druid.query.Query; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; @@ -38,7 +34,9 @@ import org.apache.druid.query.QuerySegmentWalker; import org.apache.druid.query.QueryToolChest; import org.apache.druid.query.QueryToolChestTestHelper; +import org.apache.druid.query.ResultSerializationMode; import org.apache.druid.query.scan.ScanQuery; +import org.apache.druid.query.scan.ScanQueryQueryToolChest; import org.apache.druid.query.scan.ScanQueryQueryToolChestTest; import org.apache.druid.query.scan.ScanResultValue; import org.apache.druid.query.spec.MultipleIntervalSegmentSpec; @@ -66,14 +64,16 @@ public static void setUpClass() } final UnionQueryQueryToolChest toolChest; + private ScanQueryQueryToolChest scanToolChest; public UnionQueryQueryToolChestTest() { toolChest = new UnionQueryQueryToolChest(); + scanToolChest = ScanQueryQueryToolChestTest.makeTestScanQueryToolChest(); DefaultQueryRunnerFactoryConglomerate conglomerate = new DefaultQueryRunnerFactoryConglomerate( Collections.emptyMap(), ImmutableMap., QueryToolChest>builder() - .put(ScanQuery.class, ScanQueryQueryToolChestTest.makeTestScanQueryToolChest()) + .put(ScanQuery.class, scanToolChest) .build() ); toolChest.setWarehouse(conglomerate); @@ -150,86 +150,14 @@ private List> convertResultsToListOfLists() private boolean matchQuery(ScanQuery query) { - return query != null && this.query.equals(query); + return query != null && serializedAsRows(this.query).equals(serializedAsRows(query)); } - } - - @Test - public void testResultsAsArrays() - { - RowSignature sig = RowSignature.builder() - .add("a", ColumnType.STRING) - .add("b", ColumnType.STRING) - .build(); - - TestScanQuery scan1 = new TestScanQuery("foo", sig) - .appendRow("a", "a") - .appendRow("a", "b"); - TestScanQuery scan2 = new TestScanQuery("bar", sig) - .appendRow("x", "x") - .appendRow("x", "y"); - - UnionQuery query = new UnionQuery( - ImmutableList.of( - scan1.query, - scan2.query - ) - ); - QueryToolChestTestHelper.assertArrayResultsEquals( - ImmutableList.builder() - .addAll(scan1.results) - .addAll(scan2.results) - .build(), - toolChest.resultsAsArrays( - query, - Sequences.of( - new UnionResult(scan1.makeResultSequence()), - new UnionResult(scan2.makeResultSequence()) - ) - ) - ); - } - @Test - public void testResultsAsFrames() - { - RowSignature sig = RowSignature.builder() - .add("a", ColumnType.STRING) - .add("b", ColumnType.STRING) - .build(); - - TestScanQuery scan1 = new TestScanQuery("foo", sig) - .appendRow("a", "a") - .appendRow("a", "b"); - TestScanQuery scan2 = new TestScanQuery("bar", sig) - .appendRow("x", "x") - .appendRow("x", "y"); - - UnionQuery query = new UnionQuery( - ImmutableList.of( - scan1.query, - scan2.query - ) - ); - List frames = toolChest.resultsAsFrames( - query, - Sequences.of( - new UnionResult(scan1.makeResultSequence()), - new UnionResult(scan2.makeResultSequence()) - ), - new SingleMemoryAllocatorFactory(HeapMemoryAllocator.unlimited()), - true - ).get().toList(); - - Sequence rows = new FrameBasedInlineDataSource(frames, scan1.query.getRowSignature()).getRowsAsSequence(); - - QueryToolChestTestHelper.assertArrayResultsEquals( - ImmutableList.builder() - .addAll(scan1.results) - .addAll(scan2.results) - .build(), - rows - ); + public Sequence makeResultsAsArrays() + { + ScanQueryQueryToolChest scanToolChest = ScanQueryQueryToolChestTest.makeTestScanQueryToolChest(); + return scanToolChest.resultsAsArrays(query, makeResultSequence()); + } } @Test @@ -253,6 +181,7 @@ void testQueryRunner() scan2.query ) ); + query = (UnionQuery) serializedAsRows(query); QuerySegmentWalker walker = Mockito.mock(QuerySegmentWalker.class); Mockito.when(walker.getQueryRunnerForIntervals(argThat(scan1::matchQuery), any())) @@ -260,21 +189,24 @@ void testQueryRunner() Mockito.when(walker.getQueryRunnerForIntervals(argThat(scan2::matchQuery), any())) .thenReturn((q, ctx) -> (Sequence) scan2.makeResultSequence()); - QueryRunner unionRunner = toolChest.entryPoint(query, walker); - Sequence results = unionRunner.run(QueryPlus.wrap(query), null); + QueryRunner unionRunner = toolChest.entryPoint(query, walker); + Sequence results = unionRunner.run(QueryPlus.wrap(query), null); QueryToolChestTestHelper.assertArrayResultsEquals( ImmutableList.builder() .addAll(scan1.results) .addAll(scan2.results) .build(), - toolChest.resultsAsArrays( - query, - Sequences.of( - new UnionResult(scan1.makeResultSequence()), - new UnionResult(scan2.makeResultSequence()) - ) + Sequences.concat( + scan1.makeResultsAsArrays(), + scan2.makeResultsAsArrays() ) ); } + + private static Query serializedAsRows(Query query) + { + return query + .withOverriddenContext(ImmutableMap.of(ResultSerializationMode.CTX_SERIALIZATION_PARAMETER, "rows")); + } } From 689b5b5aac35d8e3d1c29612f207c9409e6682ef Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 23 Oct 2024 15:00:05 +0000 Subject: [PATCH 146/149] cleanup/fix style --- .../apache/druid/query/union/UnionQueryRunner.java | 13 +++---------- .../druid/server/ClientQuerySegmentWalker.java | 12 ++++++------ .../druid/sql/calcite/BaseCalciteQueryTest.java | 2 +- 3 files changed, 10 insertions(+), 17 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java index 303709df3920..f92f345f2957 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQueryRunner.java @@ -23,7 +23,6 @@ import org.apache.druid.frame.allocation.ArenaMemoryAllocatorFactory; import org.apache.druid.java.util.common.guava.Sequence; import org.apache.druid.java.util.common.guava.Sequences; -import org.apache.druid.query.FrameSignaturePair; import org.apache.druid.query.Query; import org.apache.druid.query.QueryPlus; import org.apache.druid.query.QueryRunner; @@ -35,7 +34,6 @@ import java.util.ArrayList; import java.util.List; -import java.util.Optional; class UnionQueryRunner implements QueryRunner { @@ -105,22 +103,17 @@ private Sequence makeResultSeq(QueryRunner runner, QueryPlus with QueryToolChest> toolChest = conglomerate.getToolChest(query); Sequence seq = runner.run(withQuery, responseContext); Sequence resultSeq; - switch (serializationMode) - { + switch (serializationMode) { case ROWS: resultSeq = toolChest.resultsAsArrays(query, seq); break; case FRAMES: - Optional> resultsAsFrames = toolChest.resultsAsFrames( + resultSeq = toolChest.resultsAsFrames( query, seq, ArenaMemoryAllocatorFactory.makeDefault(), false - ); - if (resultsAsFrames.isEmpty()) { - throw DruidException.defensive("Unable to materialize the results as frames."); - } - resultSeq = resultsAsFrames.get(); + ).orElseThrow(() -> DruidException.defensive("Unable to materialize the results as frames.")); break; default: throw DruidException.defensive("Not supported serializationMode [%s].", serializationMode); diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index 339c1eadc216..b7fc7f84722d 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -178,9 +178,9 @@ public ClientQuerySegmentWalker( @Override public QueryRunner getQueryRunnerForIntervals(final Query query, final Iterable intervals) { - final QueryLogic queryExecutor = conglomerate.getQueryExecutor(query); - if (queryExecutor != null) { - return queryExecutor.entryPoint(query, this); + final QueryLogic queryLogic = conglomerate.getQueryExecutor(query); + if (queryLogic != null) { + return queryLogic.entryPoint(query, this); } final QueryToolChest> toolChest = conglomerate.getToolChest(query); @@ -451,10 +451,10 @@ private DataSource inlineIfNecessary( ) ); - QueryLogic subQueryExecutor = conglomerate.getQueryExecutor(subQuery); + QueryLogic subQueryLogic = conglomerate.getQueryExecutor(subQuery); final QueryRunner subQueryRunner; - if (subQueryExecutor != null) { - subQueryRunner = subQueryExecutor.entryPoint(subQueryWithSerialization, this); + if (subQueryLogic != null) { + subQueryRunner = subQueryLogic.entryPoint(subQueryWithSerialization, this); } else { subQueryRunner = subQueryWithSerialization.getRunner(this); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java index 9c1cf5d602ef..abd57cfdf9b2 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/BaseCalciteQueryTest.java @@ -1349,7 +1349,7 @@ public static Query recursivelyClearContext(final Query query, ObjectM { try { Query newQuery; - if(query instanceof UnionQuery) { + if (query instanceof UnionQuery) { UnionQuery unionQuery = (UnionQuery) query; newQuery = (Query) unionQuery .withDataSources(recursivelyClearDatasource(unionQuery.getDataSources(), queryJsonMapper)); From 3cd56c97e068405705d899dec2ae110c0a50a418 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 23 Oct 2024 15:09:54 +0000 Subject: [PATCH 147/149] move inlined subquery eval for querylogic --- .../server/ClientQuerySegmentWalker.java | 57 +++++++++++++------ 1 file changed, 41 insertions(+), 16 deletions(-) diff --git a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java index b7fc7f84722d..7390432a3991 100644 --- a/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java +++ b/server/src/main/java/org/apache/druid/server/ClientQuerySegmentWalker.java @@ -178,9 +178,9 @@ public ClientQuerySegmentWalker( @Override public QueryRunner getQueryRunnerForIntervals(final Query query, final Iterable intervals) { - final QueryLogic queryLogic = conglomerate.getQueryExecutor(query); - if (queryLogic != null) { - return queryLogic.entryPoint(query, this); + final QueryLogic queryExecutor = conglomerate.getQueryExecutor(query); + if (queryExecutor != null) { + return queryExecutor.entryPoint(query, this); } final QueryToolChest> toolChest = conglomerate.getToolChest(query); @@ -313,10 +313,6 @@ private boolean canRunQueryUsingLocalWalker(Query query) */ private boolean canRunQueryUsingClusterWalker(Query query) { - if (conglomerate.getQueryExecutor(query) != null) { - // these type of queries should be able to run - return true; - } final QueryToolChest> toolChest = conglomerate.getToolChest(query); final DataSourceAnalysis analysis = query.getDataSourceAnalysis(); @@ -383,6 +379,42 @@ private DataSource inlineIfNecessary( // This datasource is a subquery. final Query subQuery = ((QueryDataSource) dataSource).getQuery(); final QueryToolChest toolChest = conglomerate.getToolChest(subQuery); + final QueryLogic subQueryLogic = conglomerate.getQueryExecutor(subQuery); + + if (subQueryLogic != null) { + final Sequence queryResults; + + if (dryRun) { + queryResults = Sequences.empty(); + } else { + Query subQueryWithSerialization = subQuery.withOverriddenContext( + Collections.singletonMap( + ResultSerializationMode.CTX_SERIALIZATION_PARAMETER, + ClientQuerySegmentWalkerUtils.getLimitType(maxSubqueryMemory, cannotMaterializeToFrames.get()) + .serializationMode() + .toString() + ) + ); + queryResults = subQueryLogic + .entryPoint(subQueryWithSerialization, this) + .run(QueryPlus.wrap(subQueryWithSerialization), DirectDruidClient.makeResponseContextForQuery()); + } + + return toInlineDataSource( + subQuery, + queryResults, + conglomerate.getToolChest(subQuery), + subqueryRowLimitAccumulator, + subqueryMemoryLimitAccumulator, + cannotMaterializeToFrames, + maxSubqueryRows, + maxSubqueryMemory, + useNestedForUnknownTypeInSubquery, + subqueryStatsProvider, + !dryRun, + emitter + ); + } if (toolChestIfOutermost != null && toolChestIfOutermost.canPerformSubquery(subQuery)) { // Strip outer queries that are handleable by the toolchest, and inline subqueries that may be underneath @@ -451,15 +483,8 @@ private DataSource inlineIfNecessary( ) ); - QueryLogic subQueryLogic = conglomerate.getQueryExecutor(subQuery); - final QueryRunner subQueryRunner; - if (subQueryLogic != null) { - subQueryRunner = subQueryLogic.entryPoint(subQueryWithSerialization, this); - } else { - subQueryRunner = subQueryWithSerialization.getRunner(this); - } - - queryResults = subQueryRunner + queryResults = subQueryWithSerialization + .getRunner(this) .run(QueryPlus.wrap(subQueryWithSerialization), DirectDruidClient.makeResponseContextForQuery()); } From 43c10788bb064ce8a1b60b881aa4c0ef42027d96 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 23 Oct 2024 15:12:59 +0000 Subject: [PATCH 148/149] typo --- .../main/java/org/apache/druid/query/union/UnionQuery.java | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java index df586fc36c89..dc77f7c84b61 100644 --- a/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java +++ b/processing/src/main/java/org/apache/druid/query/union/UnionQuery.java @@ -219,15 +219,15 @@ public String toString() @Override public DataSourceAnalysis getDataSourceAnalysis() { - OpagueDataSourceCover ds = new OpagueDataSourceCover(new UnionDataSource(getDataSources())); + OpaqueDataSourceCover ds = new OpaqueDataSourceCover(new UnionDataSource(getDataSources())); return new DataSourceAnalysis(ds, null, null, Collections.emptyList()); } - private static class OpagueDataSourceCover implements DataSource + private static class OpaqueDataSourceCover implements DataSource { private DataSource delegate; - public OpagueDataSourceCover(DataSource delegate) + public OpaqueDataSourceCover(DataSource delegate) { this.delegate = delegate; } From 515d064c2524f14684bd201f7adc029f37b4f950 Mon Sep 17 00:00:00 2001 From: Zoltan Haindrich Date: Wed, 23 Oct 2024 15:13:49 +0000 Subject: [PATCH 149/149] rename --- .../src/main/java/org/apache/druid/query/QueryToolChest.java | 2 +- .../apache/druid/query/union/UnionQueryQueryToolChestTest.java | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java index 9c7fbee8c034..5363b78a641b 100644 --- a/processing/src/main/java/org/apache/druid/query/QueryToolChest.java +++ b/processing/src/main/java/org/apache/druid/query/QueryToolChest.java @@ -74,7 +74,7 @@ protected QueryToolChest() } @Inject - public void setWarehouse(QueryRunnerFactoryConglomerate conglomerate) + public void initialize(QueryRunnerFactoryConglomerate conglomerate) { this.conglomerate = conglomerate; } diff --git a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java index 2ed8db09a08b..9857fc7b23f5 100644 --- a/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java +++ b/processing/src/test/java/org/apache/druid/query/union/UnionQueryQueryToolChestTest.java @@ -76,7 +76,7 @@ public UnionQueryQueryToolChestTest() .put(ScanQuery.class, scanToolChest) .build() ); - toolChest.setWarehouse(conglomerate); + toolChest.initialize(conglomerate); } @Test