Skip to content

Commit

Permalink
fix filtering bug in filtering unnest cols and dim cols
Browse files Browse the repository at this point in the history
  • Loading branch information
@pranavbhole
pranavbhole committed Jul 14, 2023
1 parent 0dcb19f commit a3dc3ac
Show file tree
Hide file tree
Showing 2 changed files with 152 additions and 5 deletions.
10 changes: 5 additions & 5 deletions processing/src/main/java/org/apache/druid/segment/UnnestStorageAdapter.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -34,6 +34,7 @@
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.data.Indexed;
import org.apache.druid.segment.data.ListIndexed;
import org.apache.druid.segment.filter.AndFilter;
import org.apache.druid.segment.filter.BoundFilter;
import org.apache.druid.segment.filter.Filters;
import org.apache.druid.segment.filter.LikeFilter;
Expand Down Expand Up @@ -366,10 +367,10 @@ void addPreFilter(@Nullable final Filter filter)
final int origFilterSize;
if (queryFilter.getRequiredColumns().contains(outputColumnName)) {
// outside filter contains unnested column
// requires check for OR
if (queryFilter instanceof OrFilter) {
origFilterSize = ((OrFilter) queryFilter).getFilters().size();
for (Filter filter : ((OrFilter) queryFilter).getFilters()) {
// requires check for OR and And filters, disqualify rewrite for non-unnest filters
if (queryFilter instanceof BooleanFilter) {
origFilterSize = ((BooleanFilter) queryFilter).getFilters().size();
for (Filter filter : ((BooleanFilter) queryFilter).getFilters()) {
if (filter.getRequiredColumns().contains(outputColumnName)) {
final Filter newFilter = rewriteFilterOnUnnestColumnIfPossible(
filter,
Expand Down Expand Up @@ -460,7 +461,6 @@ private static boolean filterMapsOverMultiValueStrings(final Filter filter)
return false;
}
}

return true;
} else if (filter instanceof NotFilter) {
return filterMapsOverMultiValueStrings(((NotFilter) filter).getBaseFilter());
Expand Down
147 changes: 147 additions & 0 deletions sql/src/test/java/org/apache/druid/sql/calcite/CalciteArraysQueryTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -2834,6 +2834,153 @@ public void testUnnestTwiceWithFiltersAndExpressions()
);
}

@Test
public void testUnnestThriceWithFiltersOnDimAndUnnestCol()
{
cannotVectorize();
String sql = " SELECT dimZipf, dim3_unnest1, dim3_unnest2, dim3_unnest3 FROM \n"
+ " ( SELECT * FROM \n"
+ " ( SELECT * FROM lotsocolumns, UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest1) )"
+ " ,UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest2) \n"
+ " ), UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest3) "
+ " WHERE dimZipf=27 AND dim3_unnest1='Baz'";
testQuery(
sql,
QUERY_CONTEXT_UNNEST,
ImmutableList.of(
Druids.newScanQueryBuilder()
.dataSource(
UnnestDataSource.create(
UnnestDataSource.create(
UnnestDataSource.create(
new TableDataSource(CalciteTests.DATASOURCE5),
expressionVirtualColumn(
"j0.unnest",
"\"dimMultivalEnumerated\"",
ColumnType.STRING
),
null
),
expressionVirtualColumn(
"_j0.unnest",
"\"dimMultivalEnumerated\"",
ColumnType.STRING
), null
),
expressionVirtualColumn(
"__j0.unnest",
"\"dimMultivalEnumerated\"",
ColumnType.STRING
),
null
)
)
.intervals(querySegmentSpec(Filtration.eternity()))
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.filters(and(
bound("dimZipf", "27", "27", false, false, null, StringComparators.NUMERIC),
new SelectorDimFilter("j0.unnest", "Baz", null)
))
.legacy(false)
.context(QUERY_CONTEXT_UNNEST)
.columns(ImmutableList.of("__j0.unnest", "_j0.unnest", "dimZipf", "j0.unnest"))
.build()
),
ImmutableList.of(
new Object[]{"27", "Baz", "Baz", "Baz"},
new Object[]{"27", "Baz", "Baz", "Baz"},
new Object[]{"27", "Baz", "Baz", "Hello"},
new Object[]{"27", "Baz", "Baz", "World"},
new Object[]{"27", "Baz", "Baz", "Baz"},
new Object[]{"27", "Baz", "Baz", "Baz"},
new Object[]{"27", "Baz", "Baz", "Hello"},
new Object[]{"27", "Baz", "Baz", "World"},
new Object[]{"27", "Baz", "Hello", "Baz"},
new Object[]{"27", "Baz", "Hello", "Baz"},
new Object[]{"27", "Baz", "Hello", "Hello"},
new Object[]{"27", "Baz", "Hello", "World"},
new Object[]{"27", "Baz", "World", "Baz"},
new Object[]{"27", "Baz", "World", "Baz"},
new Object[]{"27", "Baz", "World", "Hello"},
new Object[]{"27", "Baz", "World", "World"},
new Object[]{"27", "Baz", "Baz", "Baz"},
new Object[]{"27", "Baz", "Baz", "Baz"},
new Object[]{"27", "Baz", "Baz", "Hello"},
new Object[]{"27", "Baz", "Baz", "World"},
new Object[]{"27", "Baz", "Baz", "Baz"},
new Object[]{"27", "Baz", "Baz", "Baz"},
new Object[]{"27", "Baz", "Baz", "Hello"},
new Object[]{"27", "Baz", "Baz", "World"},
new Object[]{"27", "Baz", "Hello", "Baz"},
new Object[]{"27", "Baz", "Hello", "Baz"},
new Object[]{"27", "Baz", "Hello", "Hello"},
new Object[]{"27", "Baz", "Hello", "World"},
new Object[]{"27", "Baz", "World", "Baz"},
new Object[]{"27", "Baz", "World", "Baz"},
new Object[]{"27", "Baz", "World", "Hello"},
new Object[]{"27", "Baz", "World", "World"}
)
);
}

@Test
public void testUnnestThriceWithFiltersOnDimAndAllUnnestColumns()
{
cannotVectorize();
String sql = " SELECT dimZipf, dim3_unnest1, dim3_unnest2, dim3_unnest3 FROM \n"
+ " ( SELECT * FROM \n"
+ " ( SELECT * FROM lotsocolumns, UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest1) )"
+ " ,UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest2) \n"
+ " ), UNNEST(MV_TO_ARRAY(dimMultivalEnumerated)) as ut(dim3_unnest3) "
+ " WHERE dimZipf=27 AND dim3_unnest1='Baz' AND dim3_unnest2='Hello' AND dim3_unnest3='World'";
testQuery(
sql,
QUERY_CONTEXT_UNNEST,
ImmutableList.of(
Druids.newScanQueryBuilder()
.dataSource(
UnnestDataSource.create(
UnnestDataSource.create(
UnnestDataSource.create(
new TableDataSource(CalciteTests.DATASOURCE5),
expressionVirtualColumn(
"j0.unnest",
"\"dimMultivalEnumerated\"",
ColumnType.STRING
),
null
),
expressionVirtualColumn(
"_j0.unnest",
"\"dimMultivalEnumerated\"",
ColumnType.STRING
), new SelectorDimFilter("_j0.unnest", "Hello", null)
),
expressionVirtualColumn(
"__j0.unnest",
"\"dimMultivalEnumerated\"",
ColumnType.STRING
),
new SelectorDimFilter("__j0.unnest", "World", null)
)
)
.intervals(querySegmentSpec(Filtration.eternity()))
.resultFormat(ScanQuery.ResultFormat.RESULT_FORMAT_COMPACTED_LIST)
.filters(and(
bound("dimZipf", "27", "27", false, false, null, StringComparators.NUMERIC),
new SelectorDimFilter("j0.unnest", "Baz", null)
))
.legacy(false)
.context(QUERY_CONTEXT_UNNEST)
.columns(ImmutableList.of("__j0.unnest", "_j0.unnest", "dimZipf", "j0.unnest"))
.build()
),
ImmutableList.of(
new Object[]{"27", "Baz", "Hello", "World"},
new Object[]{"27", "Baz", "Hello", "World"}
)
);
}
@Test
public void testUnnestWithGroupBy()
{
Expand Down

0 comments on commit a3dc3ac

Please sign in to comment.