diff --git a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java index 00a119388a643..70a64ad615c85 100644 --- a/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java +++ b/processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java @@ -34,6 +34,7 @@ import org.apache.druid.segment.column.ValueType; import org.apache.druid.segment.data.Indexed; import org.apache.druid.segment.data.ListIndexed; +import org.apache.druid.segment.filter.AndFilter; import org.apache.druid.segment.filter.BoundFilter; import org.apache.druid.segment.filter.Filters; import org.apache.druid.segment.filter.LikeFilter; @@ -366,10 +367,10 @@ void addPreFilter(@Nullable final Filter filter) final int origFilterSize; if (queryFilter.getRequiredColumns().contains(outputColumnName)) { // outside filter contains unnested column - // requires check for OR - if (queryFilter instanceof OrFilter) { - origFilterSize = ((OrFilter) queryFilter).getFilters().size(); - for (Filter filter : ((OrFilter) queryFilter).getFilters()) { + // requires check for OR and And filters, disqualify rewrite for non-unnest filters + if (queryFilter instanceof BooleanFilter) { + origFilterSize = ((BooleanFilter) queryFilter).getFilters().size(); + for (Filter filter : ((BooleanFilter) queryFilter).getFilters()) { if (filter.getRequiredColumns().contains(outputColumnName)) { final Filter newFilter = rewriteFilterOnUnnestColumnIfPossible( filter, @@ -460,7 +461,6 @@ private static boolean filterMapsOverMultiValueStrings(final Filter filter) return false; } } - return true; } else if (filter instanceof NotFilter) { return filterMapsOverMultiValueStrings(((NotFilter) filter).getBaseFilter()); diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java index da9eb754c433c..2467303c752b9 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java @@ -2834,6 +2834,153 @@ public void testUnnestTwiceWithFiltersAndExpressions() ); } + @Test + public void testUnnestThriceWithFiltersOnDimAndUnnestCol() + { + cannotVectorize(); + String sql = " SELECT dimZipf, dim3_unnest1, dim3_unnest2, dim3_unnest3 FROM \n" + + " ( SELECT * FROM \n" + + " ( SELECT * FROM lotsocolumns, UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest1) )" + + " ,UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest2) \n" + + " ), UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest3) " + + " WHERE dimZipf=27 AND dim3_unnest1='Baz'"; + testQuery( + sql, + QUERY_CONTEXT_UNNEST, + ImmutableList.of( + Druids.newScanQueryBuilder() + .dataSource( + UnnestDataSource.create( + UnnestDataSource.create( + UnnestDataSource.create( + new TableDataSource(CalciteTests.DATASOURCE5), + expressionVirtualColumn( + "j0.unnest", + "\"dimMultivalEnumerated\"", + ColumnType.STRING + ), + null + ), + expressionVirtualColumn( + "_j0.unnest", + "\"dimMultivalEnumerated\"", + ColumnType.STRING + ), null + ), + expressionVirtualColumn( + "__j0.unnest", + "\"dimMultivalEnumerated\"", + ColumnType.STRING + ), + null + ) + ) + .intervals(querySegmentSpec(Filtration.eternity())) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .filters(and( + bound("dimZipf", "27", "27", false, false, null, StringComparators.NUMERIC), + new SelectorDimFilter("j0.unnest", "Baz", null) + )) + .legacy(false) + .context(QUERY_CONTEXT_UNNEST) + .columns(ImmutableList.of("__j0.unnest", "_j0.unnest", "dimZipf", "j0.unnest")) + .build() + ), + ImmutableList.of( + new Object[]{"27", "Baz", "Baz", "Baz"}, + new Object[]{"27", "Baz", "Baz", "Baz"}, + new Object[]{"27", "Baz", "Baz", "Hello"}, + new Object[]{"27", "Baz", "Baz", "World"}, + new Object[]{"27", "Baz", "Baz", "Baz"}, + new Object[]{"27", "Baz", "Baz", "Baz"}, + new Object[]{"27", "Baz", "Baz", "Hello"}, + new Object[]{"27", "Baz", "Baz", "World"}, + new Object[]{"27", "Baz", "Hello", "Baz"}, + new Object[]{"27", "Baz", "Hello", "Baz"}, + new Object[]{"27", "Baz", "Hello", "Hello"}, + new Object[]{"27", "Baz", "Hello", "World"}, + new Object[]{"27", "Baz", "World", "Baz"}, + new Object[]{"27", "Baz", "World", "Baz"}, + new Object[]{"27", "Baz", "World", "Hello"}, + new Object[]{"27", "Baz", "World", "World"}, + new Object[]{"27", "Baz", "Baz", "Baz"}, + new Object[]{"27", "Baz", "Baz", "Baz"}, + new Object[]{"27", "Baz", "Baz", "Hello"}, + new Object[]{"27", "Baz", "Baz", "World"}, + new Object[]{"27", "Baz", "Baz", "Baz"}, + new Object[]{"27", "Baz", "Baz", "Baz"}, + new Object[]{"27", "Baz", "Baz", "Hello"}, + new Object[]{"27", "Baz", "Baz", "World"}, + new Object[]{"27", "Baz", "Hello", "Baz"}, + new Object[]{"27", "Baz", "Hello", "Baz"}, + new Object[]{"27", "Baz", "Hello", "Hello"}, + new Object[]{"27", "Baz", "Hello", "World"}, + new Object[]{"27", "Baz", "World", "Baz"}, + new Object[]{"27", "Baz", "World", "Baz"}, + new Object[]{"27", "Baz", "World", "Hello"}, + new Object[]{"27", "Baz", "World", "World"} + ) + ); + } + + @Test + public void testUnnestThriceWithFiltersOnDimAndAllUnnestColumns() + { + cannotVectorize(); + String sql = " SELECT dimZipf, dim3_unnest1, dim3_unnest2, dim3_unnest3 FROM \n" + + " ( SELECT * FROM \n" + + " ( SELECT * FROM lotsocolumns, UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest1) )" + + " ,UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest2) \n" + + " ), UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest3) " + + " WHERE dimZipf=27 AND dim3_unnest1='Baz' AND dim3_unnest2='Hello' AND dim3_unnest3='World'"; + testQuery( + sql, + QUERY_CONTEXT_UNNEST, + ImmutableList.of( + Druids.newScanQueryBuilder() + .dataSource( + UnnestDataSource.create( + UnnestDataSource.create( + UnnestDataSource.create( + new TableDataSource(CalciteTests.DATASOURCE5), + expressionVirtualColumn( + "j0.unnest", + "\"dimMultivalEnumerated\"", + ColumnType.STRING + ), + null + ), + expressionVirtualColumn( + "_j0.unnest", + "\"dimMultivalEnumerated\"", + ColumnType.STRING + ), new SelectorDimFilter("_j0.unnest", "Hello", null) + ), + expressionVirtualColumn( + "__j0.unnest", + "\"dimMultivalEnumerated\"", + ColumnType.STRING + ), + new SelectorDimFilter("__j0.unnest", "World", null) + ) + ) + .intervals(querySegmentSpec(Filtration.eternity())) + .resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST) + .filters(and( + bound("dimZipf", "27", "27", false, false, null, StringComparators.NUMERIC), + new SelectorDimFilter("j0.unnest", "Baz", null) + )) + .legacy(false) + .context(QUERY_CONTEXT_UNNEST) + .columns(ImmutableList.of("__j0.unnest", "_j0.unnest", "dimZipf", "j0.unnest")) + .build() + ), + ImmutableList.of( + new Object[]{"27", "Baz", "Hello", "World"}, + new Object[]{"27", "Baz", "Hello", "World"} + ) + ); + } @Test public void testUnnestWithGroupBy() {