From 68fd22169fe7e85b20ab3ef1d5bac2d0e95b9fb9 Mon Sep 17 00:00:00 2001 From: Clint Wylie Date: Wed, 19 Jul 2023 10:37:57 -0700 Subject: [PATCH] remove extractionFn from equality, null, and range filters (#14612) * remove extractionFn from equality, null, and range filters changes: * EqualityFilter, NullFilter, and RangeFilter no longer support extractionFn * SQL planner will use ExpressionFilter in the small number of cases where an extractionFn would have been used if sqlUseBoundsAndSelectors is set to false instead of equality/null/range filters * fix bugs and add tests with serde, equals, and cache key for null, equality, and range filters * test coverage fixes bugs * adjust * adjust again * so persnickety --- .../org/apache/druid/math/expr/ExprEval.java | 2 +- .../druid/query/filter/EqualityFilter.java | 96 +- .../apache/druid/query/filter/NullFilter.java | 32 +- .../druid/query/filter/RangeFilter.java | 72 +- .../segment/filter/EqualityFilterTest.java | 678 -------- .../segment/filter/EqualityFilterTests.java | 726 +++++++++ .../druid/segment/filter/NullFilterTest.java | 92 +- .../druid/segment/filter/RangeFilterTest.java | 1338 ---------------- .../segment/filter/RangeFilterTests.java | 1426 +++++++++++++++++ .../sql/calcite/expression/Expressions.java | 23 +- .../ArrayContainsOperatorConversion.java | 2 - .../ArrayOverlapOperatorConversion.java | 1 - .../builtin/CaseOperatorConversion.java | 6 +- .../filtration/CombineAndSimplifyBounds.java | 2 +- .../filtration/ConvertBoundsToSelectors.java | 1 - .../filtration/ConvertSelectorsToIns.java | 2 +- .../MoveTimeFiltersToIntervals.java | 3 +- .../sql/calcite/filtration/RangeRefKey.java | 24 +- .../druid/sql/calcite/filtration/Ranges.java | 9 - .../sql/calcite/BaseCalciteQueryTest.java | 48 +- .../sql/calcite/CalciteExplainQueryTest.java | 14 +- .../calcite/CalciteNestedDataQueryTest.java | 3 +- .../druid/sql/calcite/CalciteQueryTest.java | 86 +- .../sql/calcite/CalciteSelectQueryTest.java | 15 +- .../sql/calcite/CalciteSubqueryTest.java | 13 +- 25 files changed, 2378 insertions(+), 2336 deletions(-) delete mode 100644 processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTest.java create mode 100644 processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java delete mode 100644 processing/src/test/java/org/apache/druid/segment/filter/RangeFilterTest.java create mode 100644 processing/src/test/java/org/apache/druid/segment/filter/RangeFilterTests.java diff --git a/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java b/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java index 15334ce919be..01d27e9006fe 100644 --- a/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java +++ b/processing/src/main/java/org/apache/druid/math/expr/ExprEval.java @@ -1162,7 +1162,7 @@ public String asString() } else if (value.length == 1) { cacheStringValue(Evals.asString(value[0])); } else { - cacheStringValue(Arrays.toString(value)); + cacheStringValue(Arrays.deepToString(value)); } } diff --git a/processing/src/main/java/org/apache/druid/query/filter/EqualityFilter.java b/processing/src/main/java/org/apache/druid/query/filter/EqualityFilter.java index 4fd9b4aef539..0e57b06141e1 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/EqualityFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/EqualityFilter.java @@ -35,7 +35,6 @@ import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.query.cache.CacheKeyBuilder; -import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcherColumnProcessorFactory; import org.apache.druid.segment.BaseDoubleColumnValueSelector; @@ -54,7 +53,6 @@ import org.apache.druid.segment.column.TypeSignature; import org.apache.druid.segment.column.TypeStrategy; import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.filter.DimensionPredicateFilter; import org.apache.druid.segment.filter.Filters; import org.apache.druid.segment.filter.PredicateValueMatcherFactory; import org.apache.druid.segment.filter.ValueMatchers; @@ -67,6 +65,7 @@ import javax.annotation.Nullable; import java.nio.ByteBuffer; +import java.util.Arrays; import java.util.Comparator; import java.util.Map; import java.util.Objects; @@ -78,8 +77,8 @@ public class EqualityFilter extends AbstractOptimizableDimFilter implements Filt private final String column; private final ColumnType matchValueType; private final Object matchValue; - @Nullable - private final ExtractionFn extractionFn; + private final ExprEval matchValueEval; + @Nullable private final FilterTuning filterTuning; private final DruidPredicateFactory predicateFactory; @@ -89,7 +88,6 @@ public EqualityFilter( @JsonProperty("column") String column, @JsonProperty("matchValueType") ColumnType matchValueType, @JsonProperty("matchValue") Object matchValue, - @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { @@ -105,19 +103,18 @@ public EqualityFilter( throw InvalidInput.exception("Invalid equality filter on column [%s], matchValue cannot be null", column); } this.matchValue = matchValue; - // remove once SQL planner no longer uses extractionFn - this.extractionFn = extractionFn; + this.matchValueEval = ExprEval.ofType(ExpressionType.fromColumnTypeStrict(matchValueType), matchValue); this.filterTuning = filterTuning; - this.predicateFactory = new EqualityPredicateFactory(matchValue, matchValueType); + this.predicateFactory = new EqualityPredicateFactory(matchValueEval); } @Override public byte[] getCacheKey() { - final TypeStrategy typeStrategy = matchValueType.getStrategy(); - final int size = typeStrategy.estimateSizeBytes(matchValue); + final TypeStrategy typeStrategy = matchValueEval.type().getStrategy(); + final int size = typeStrategy.estimateSizeBytes(matchValueEval.value()); final ByteBuffer valueBuffer = ByteBuffer.allocate(size); - typeStrategy.write(valueBuffer, matchValue, size); + typeStrategy.write(valueBuffer, matchValueEval.value(), size); return new CacheKeyBuilder(DimFilterUtils.EQUALS_CACHE_ID) .appendByte(DimFilterUtils.STRING_SEPARATOR) .appendString(column) @@ -125,8 +122,6 @@ public byte[] getCacheKey() .appendString(matchValueType.asTypeString()) .appendByte(DimFilterUtils.STRING_SEPARATOR) .appendByteArray(valueBuffer.array()) - .appendByte(DimFilterUtils.STRING_SEPARATOR) - .appendByteArray(extractionFn == null ? new byte[0] : extractionFn.getCacheKey()) .build(); } @@ -139,11 +134,7 @@ public DimFilter optimize() @Override public Filter toFilter() { - if (extractionFn == null) { - return this; - } else { - return new DimensionPredicateFilter(column, predicateFactory, extractionFn, filterTuning); - } + return this; } @JsonProperty @@ -164,14 +155,6 @@ public Object getMatchValue() return matchValue; } - @Nullable - @JsonProperty - @JsonInclude(JsonInclude.Include.NON_NULL) - public ExtractionFn getExtractionFn() - { - return extractionFn; - } - @Nullable @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) @@ -184,9 +167,9 @@ public FilterTuning getFilterTuning() public String toString() { DimFilter.DimFilterToStringBuilder bob = - new DimFilter.DimFilterToStringBuilder().appendDimension(column, extractionFn) + new DimFilter.DimFilterToStringBuilder().appendDimension(column, null) .append(" = ") - .append(matchValue); + .append(matchValueEval.value()); if (!ColumnType.STRING.equals(matchValueType)) { bob.append(" (" + matchValueType.asTypeString() + ")"); @@ -210,39 +193,34 @@ public boolean equals(Object o) if (!Objects.equals(matchValueType, that.matchValueType)) { return false; } - if (!Objects.equals(extractionFn, that.extractionFn)) { - return false; - } if (!Objects.equals(filterTuning, that.filterTuning)) { return false; } if (matchValueType.isArray()) { - // just use predicate to see if the values are the same - final ExprEval thatValue = ExprEval.ofType( - ExpressionType.fromColumnType(that.matchValueType), - that.matchValue - ); - final Predicate arrayPredicate = predicateFactory.makeArrayPredicate(matchValueType); - return arrayPredicate.apply(thatValue.asArray()); + return Arrays.deepEquals(matchValueEval.asArray(), that.matchValueEval.asArray()); } else { - return Objects.equals(matchValue, that.matchValue); + return Objects.equals(matchValueEval.value(), that.matchValueEval.value()); } } @Override public int hashCode() { - return Objects.hash(column, matchValueType, matchValue, extractionFn, filterTuning); + return Objects.hash(column, matchValueType, matchValueEval.value(), filterTuning); } @Override public RangeSet getDimensionRangeSet(String dimension) { - if (!Objects.equals(getColumn(), dimension) || getExtractionFn() != null) { + if (!Objects.equals(getColumn(), dimension)) { return null; } RangeSet retSet = TreeRangeSet.create(); - retSet.add(Range.singleton(String.valueOf(matchValue))); + if (matchValueEval.isArray()) { + retSet.add(Range.singleton(Arrays.deepToString(matchValueEval.asArray()))); + } else { + retSet.add(Range.singleton(matchValueEval.asString())); + } return retSet; } @@ -261,14 +239,14 @@ public BitmapColumnIndex getBitmapColumnIndex(ColumnIndexSelector selector) final ValueIndexes valueIndexes = indexSupplier.as(ValueIndexes.class); if (valueIndexes != null) { - return valueIndexes.forValue(matchValue, matchValueType); + return valueIndexes.forValue(matchValueEval.value(), matchValueType); } if (matchValueType.isPrimitive()) { final StringValueSetIndexes stringValueSetIndexes = indexSupplier.as(StringValueSetIndexes.class); if (stringValueSetIndexes != null) { - return stringValueSetIndexes.forValue(String.valueOf(matchValue)); + return stringValueSetIndexes.forValue(matchValueEval.asString()); } } // column exists, but has no indexes we can use @@ -280,7 +258,7 @@ public ValueMatcher makeMatcher(ColumnSelectorFactory factory) { return ColumnProcessors.makeProcessor( column, - new TypedConstantValueMatcherFactory(matchValue, matchValueType, predicateFactory), + new TypedConstantValueMatcherFactory(matchValueEval, predicateFactory), factory ); } @@ -295,13 +273,13 @@ public VectorValueMatcher makeVectorMatcher(VectorColumnSelectorFactory factory) column, VectorValueMatcherColumnProcessorFactory.instance(), factory - ).makeMatcher(matchValue, matchValueType); + ).makeMatcher(matchValueEval.value(), matchValueType); } return ColumnProcessors.makeVectorProcessor( column, VectorValueMatcherColumnProcessorFactory.instance(), factory - ).makeMatcher(new EqualityPredicateFactory(matchValue, matchValueType)); + ).makeMatcher(new EqualityPredicateFactory(matchValueEval)); } @Override @@ -345,7 +323,6 @@ public Filter rewriteRequiredColumns(Map columnRewrites) rewriteDimensionTo, matchValueType, matchValue, - extractionFn, filterTuning ); } @@ -353,7 +330,6 @@ public Filter rewriteRequiredColumns(Map columnRewrites) private static class EqualityPredicateFactory implements DruidPredicateFactory { private final ExprEval matchValue; - private final ColumnType matchValueType; private final Supplier> stringPredicateSupplier; private final Supplier longPredicateSupplier; private final Supplier floatPredicateSupplier; @@ -362,10 +338,9 @@ private static class EqualityPredicateFactory implements DruidPredicateFactory private final Supplier> typeDetectingArrayPredicateSupplier; private final Supplier> objectPredicateSupplier; - public EqualityPredicateFactory(Object matchValue, ColumnType matchValueType) + public EqualityPredicateFactory(ExprEval matchValue) { - this.matchValue = ExprEval.ofType(ExpressionType.fromColumnType(matchValueType), matchValue); - this.matchValueType = matchValueType; + this.matchValue = matchValue; this.stringPredicateSupplier = makeStringPredicateSupplier(); this.longPredicateSupplier = makeLongPredicateSupplier(); this.floatPredicateSupplier = makeFloatPredicateSupplier(); @@ -469,7 +444,7 @@ private Supplier makeDoublePredicateSupplier() private Supplier> makeObjectPredicateSupplier() { return Suppliers.memoize(() -> { - if (matchValueType.equals(ColumnType.NESTED_DATA)) { + if (matchValue.type().equals(ExpressionType.NESTED_DATA)) { return input -> Objects.equals(StructuredData.unwrap(input), StructuredData.unwrap(matchValue.value())); } return Predicates.equalTo(matchValue.valueOrDefault()); @@ -503,14 +478,20 @@ public boolean equals(Object o) return false; } EqualityPredicateFactory that = (EqualityPredicateFactory) o; - return Objects.equals(matchValue, that.matchValue) && Objects.equals(matchValueType, that.matchValueType); + if (!Objects.equals(matchValue.type(), that.matchValue.type())) { + return false; + } + if (matchValue.isArray()) { + return Arrays.deepEquals(matchValue.asArray(), that.matchValue.asArray()); + } + return Objects.equals(matchValue.value(), that.matchValue.value()); } @Override public int hashCode() { - return Objects.hash(matchValue, matchValueType); + return Objects.hash(matchValue); } } @@ -520,12 +501,11 @@ private static class TypedConstantValueMatcherFactory implements ColumnProcessor private final PredicateValueMatcherFactory predicateMatcherFactory; public TypedConstantValueMatcherFactory( - Object matchValue, - ColumnType matchValueType, + ExprEval matchValue, DruidPredicateFactory predicateFactory ) { - this.matchValue = ExprEval.ofType(ExpressionType.fromColumnType(matchValueType), matchValue); + this.matchValue = matchValue; this.predicateMatcherFactory = new PredicateValueMatcherFactory(predicateFactory); } diff --git a/processing/src/main/java/org/apache/druid/query/filter/NullFilter.java b/processing/src/main/java/org/apache/druid/query/filter/NullFilter.java index 0d88e3214194..1f504e62d2cc 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/NullFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/NullFilter.java @@ -31,7 +31,6 @@ import org.apache.druid.error.InvalidInput; import org.apache.druid.java.util.common.IAE; import org.apache.druid.query.cache.CacheKeyBuilder; -import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcherColumnProcessorFactory; import org.apache.druid.segment.ColumnInspector; @@ -41,7 +40,6 @@ import org.apache.druid.segment.column.ColumnIndexSupplier; import org.apache.druid.segment.column.TypeSignature; import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.filter.DimensionPredicateFilter; import org.apache.druid.segment.filter.Filters; import org.apache.druid.segment.index.AllTrueBitmapColumnIndex; import org.apache.druid.segment.index.BitmapColumnIndex; @@ -57,19 +55,16 @@ public class NullFilter extends AbstractOptimizableDimFilter implements Filter { public static NullFilter forColumn(String column) { - return new NullFilter(column, null, null); + return new NullFilter(column, null); } private final String column; @Nullable - private final ExtractionFn extractionFn; - @Nullable private final FilterTuning filterTuning; @JsonCreator public NullFilter( @JsonProperty("column") String column, - @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { @@ -77,8 +72,6 @@ public NullFilter( throw InvalidInput.exception("Invalid null filter, column cannot be null"); } this.column = column; - // remove once SQL planner no longer uses extractionFn - this.extractionFn = extractionFn; this.filterTuning = filterTuning; } @@ -88,14 +81,6 @@ public String getColumn() return column; } - @Nullable - @JsonProperty - @JsonInclude(JsonInclude.Include.NON_NULL) - public ExtractionFn getExtractionFn() - { - return extractionFn; - } - @Nullable @JsonProperty @JsonInclude(JsonInclude.Include.NON_NULL) @@ -110,8 +95,6 @@ public byte[] getCacheKey() return new CacheKeyBuilder(DimFilterUtils.NULL_CACHE_ID) .appendByte(DimFilterUtils.STRING_SEPARATOR) .appendString(column) - .appendByte(DimFilterUtils.STRING_SEPARATOR) - .appendByteArray(extractionFn == null ? new byte[0] : extractionFn.getCacheKey()) .build(); } @@ -124,11 +107,7 @@ public DimFilter optimize() @Override public Filter toFilter() { - if (extractionFn == null) { - return this; - } else { - return new DimensionPredicateFilter(column, NullPredicateFactory.INSTANCE, extractionFn, filterTuning); - } + return this; } @Nullable @@ -211,7 +190,7 @@ public Filter rewriteRequiredColumns(Map columnRewrites) columnRewrites ); } - return new NullFilter(rewriteDimensionTo, extractionFn, filterTuning); + return new NullFilter(rewriteDimensionTo, filterTuning); } @Override @@ -225,20 +204,19 @@ public boolean equals(Object o) } NullFilter that = (NullFilter) o; return Objects.equals(column, that.column) && - Objects.equals(extractionFn, that.extractionFn) && Objects.equals(filterTuning, that.filterTuning); } @Override public int hashCode() { - return Objects.hash(column, extractionFn, filterTuning); + return Objects.hash(column, filterTuning); } @Override public String toString() { - return new DimFilterToStringBuilder().appendDimension(column, extractionFn) + return new DimFilterToStringBuilder().appendDimension(column, null) .append(" IS NULL") .appendFilterTuning(filterTuning) .build(); diff --git a/processing/src/main/java/org/apache/druid/query/filter/RangeFilter.java b/processing/src/main/java/org/apache/druid/query/filter/RangeFilter.java index d9c1fe608bdd..1f67edac0cc8 100644 --- a/processing/src/main/java/org/apache/druid/query/filter/RangeFilter.java +++ b/processing/src/main/java/org/apache/druid/query/filter/RangeFilter.java @@ -37,7 +37,6 @@ import org.apache.druid.math.expr.ExprEval; import org.apache.druid.math.expr.ExpressionType; import org.apache.druid.query.cache.CacheKeyBuilder; -import org.apache.druid.query.extraction.ExtractionFn; import org.apache.druid.query.filter.vector.VectorValueMatcher; import org.apache.druid.query.filter.vector.VectorValueMatcherColumnProcessorFactory; import org.apache.druid.query.ordering.StringComparators; @@ -50,7 +49,6 @@ import org.apache.druid.segment.column.TypeSignature; import org.apache.druid.segment.column.TypeStrategy; import org.apache.druid.segment.column.ValueType; -import org.apache.druid.segment.filter.DimensionPredicateFilter; import org.apache.druid.segment.filter.Filters; import org.apache.druid.segment.index.AllFalseBitmapColumnIndex; import org.apache.druid.segment.index.BitmapColumnIndex; @@ -72,7 +70,6 @@ public class RangeFilter extends AbstractOptimizableDimFilter implements Filter { private final String column; private final ColumnType matchValueType; - private final ExpressionType matchValueExpressionType; @Nullable private final Object upper; @@ -83,8 +80,6 @@ public class RangeFilter extends AbstractOptimizableDimFilter implements Filter private final boolean lowerOpen; private final boolean upperOpen; @Nullable - private final ExtractionFn extractionFn; - @Nullable private final FilterTuning filterTuning; private final Supplier> stringPredicateSupplier; private final Supplier longPredicateSupplier; @@ -101,7 +96,6 @@ public RangeFilter( @JsonProperty("upper") @Nullable Object upper, @JsonProperty("lowerOpen") @Nullable Boolean lowerOpen, @JsonProperty("upperOpen") @Nullable Boolean upperOpen, - @JsonProperty("extractionFn") @Nullable ExtractionFn extractionFn, @JsonProperty("filterTuning") @Nullable FilterTuning filterTuning ) { @@ -113,9 +107,9 @@ public RangeFilter( throw InvalidInput.exception("Invalid range filter on column [%s], matchValueType cannot be null", column); } this.matchValueType = matchValueType; - this.matchValueExpressionType = ExpressionType.fromColumnType(matchValueType); this.upper = upper; this.lower = lower; + ExpressionType matchValueExpressionType = ExpressionType.fromColumnTypeStrict(matchValueType); this.upperEval = ExprEval.ofType(matchValueExpressionType, upper); this.lowerEval = ExprEval.ofType(matchValueExpressionType, lower); @@ -125,7 +119,7 @@ public RangeFilter( column ); } - if (matchValueExpressionType.isNumeric()) { + if (this.matchValueType.isNumeric()) { if (lower != null && lowerEval.value() == null) { throw InvalidInput.exception( "Invalid range filter on column [%s], lower bound [%s] cannot be parsed as specified match value type [%s]", @@ -145,8 +139,6 @@ public RangeFilter( } this.lowerOpen = lowerOpen != null && lowerOpen; this.upperOpen = upperOpen != null && upperOpen; - // remove once SQL planner no longer uses extractionFn - this.extractionFn = extractionFn; this.filterTuning = filterTuning; this.stringPredicateSupplier = makeStringPredicateSupplier(); this.longPredicateSupplier = makeLongPredicateSupplier(); @@ -199,14 +191,6 @@ public boolean isUpperOpen() return upperOpen; } - @Nullable - @JsonProperty - @JsonInclude(JsonInclude.Include.NON_NULL) - public ExtractionFn getExtractionFn() - { - return extractionFn; - } - @Nullable @JsonInclude(JsonInclude.Include.NON_NULL) @JsonProperty @@ -231,19 +215,19 @@ public byte[] getCacheKey() final byte[] lowerBytes; final byte[] upperBytes; if (hasLowerBound()) { - final TypeStrategy typeStrategy = matchValueType.getStrategy(); - final int size = typeStrategy.estimateSizeBytes(lower); + final TypeStrategy typeStrategy = lowerEval.type().getStrategy(); + final int size = typeStrategy.estimateSizeBytes(lowerEval.value()); final ByteBuffer valueBuffer = ByteBuffer.allocate(size); - typeStrategy.write(valueBuffer, lower, size); + typeStrategy.write(valueBuffer, lowerEval.value(), size); lowerBytes = valueBuffer.array(); } else { lowerBytes = new byte[0]; } if (hasUpperBound()) { - final TypeStrategy typeStrategy = matchValueType.getStrategy(); - final int size = typeStrategy.estimateSizeBytes(upper); + final TypeStrategy typeStrategy = upperEval.type().getStrategy(); + final int size = typeStrategy.estimateSizeBytes(upperEval.value()); final ByteBuffer valueBuffer = ByteBuffer.allocate(size); - typeStrategy.write(valueBuffer, upper, size); + typeStrategy.write(valueBuffer, upperEval.value(), size); upperBytes = valueBuffer.array(); } else { upperBytes = new byte[0]; @@ -272,8 +256,6 @@ public byte[] getCacheKey() .appendByte(lowerStrictByte) .appendByte(DimFilterUtils.STRING_SEPARATOR) .appendByte(upperStrictByte) - .appendByte(DimFilterUtils.STRING_SEPARATOR) - .appendByteArray(extractionFn == null ? new byte[0] : extractionFn.getCacheKey()) .build(); } @@ -286,16 +268,13 @@ public DimFilter optimize() @Override public Filter toFilter() { - if (extractionFn != null) { - return new DimensionPredicateFilter(column, getPredicateFactory(), extractionFn, filterTuning); - } return this; } @Override public RangeSet getDimensionRangeSet(String dimension) { - if (!(Objects.equals(column, dimension) && getExtractionFn() == null)) { + if (!Objects.equals(column, dimension)) { return null; } @@ -303,19 +282,19 @@ public RangeSet getDimensionRangeSet(String dimension) // converting things to String, but we'd probably be better off adjusting the interface to something that is // more type aware in the future - String lowerString = lowerEval.asString(); - String upperString = upperEval.asString(); + final Supplier lowerString = () -> lowerEval.isArray() ? Arrays.deepToString(lowerEval.asArray()) : lowerEval.asString(); + final Supplier upperString = () -> upperEval.isArray() ? Arrays.deepToString(upperEval.asArray()) : upperEval.asString(); RangeSet retSet = TreeRangeSet.create(); - Range range; - if (getLower() == null) { - range = isUpperOpen() ? Range.lessThan(upperString) : Range.atMost(upperString); - } else if (getUpper() == null) { - range = isLowerOpen() ? Range.greaterThan(lowerString) : Range.atLeast(lowerString); + final Range range; + if (!hasLowerBound()) { + range = isUpperOpen() ? Range.lessThan(upperString.get()) : Range.atMost(upperString.get()); + } else if (!hasUpperBound()) { + range = isLowerOpen() ? Range.greaterThan(lowerString.get()) : Range.atLeast(lowerString.get()); } else { range = Range.range( - lowerString, + lowerString.get(), isLowerOpen() ? BoundType.OPEN : BoundType.CLOSED, - upperString, + upperString.get(), isUpperOpen() ? BoundType.OPEN : BoundType.CLOSED ); } @@ -420,7 +399,6 @@ public Filter rewriteRequiredColumns(Map columnRewrites) upper, lowerOpen, upperOpen, - extractionFn, filterTuning ); } @@ -436,7 +414,7 @@ public boolean isEquality() upperEval.asArray() ); } else { - return Objects.equals(upper, lower); + return Objects.equals(upperEval.value(), lowerEval.value()); } } @@ -463,8 +441,8 @@ public boolean equals(Object o) that.lowerEval.asArray() ); } else { - upperSame = Objects.equals(upper, that.upper); - lowerSame = Objects.equals(lower, that.lower); + upperSame = Objects.equals(upperEval.value(), that.upperEval.value()); + lowerSame = Objects.equals(lowerEval.value(), that.lowerEval.value()); } return lowerOpen == that.lowerOpen && @@ -473,7 +451,6 @@ public boolean equals(Object o) Objects.equals(matchValueType, that.matchValueType) && upperSame && lowerSame && - Objects.equals(extractionFn, that.extractionFn) && Objects.equals(filterTuning, that.filterTuning); } @@ -483,11 +460,10 @@ public int hashCode() return Objects.hash( column, matchValueType, - upper, - lower, + upperEval.value(), + lowerEval.value(), lowerOpen, upperOpen, - extractionFn, filterTuning ); } @@ -506,7 +482,7 @@ public String toString() } } - builder.appendDimension(column, extractionFn); + builder.appendDimension(column, null); builder.append(StringUtils.format(" as %s", matchValueType.toString())); diff --git a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTest.java deleted file mode 100644 index 7a63b3f6999f..000000000000 --- a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTest.java +++ /dev/null @@ -1,678 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.segment.filter; - -import com.google.common.base.Function; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import nl.jqno.equalsverifier.EqualsVerifier; -import org.apache.druid.common.config.NullHandling; -import org.apache.druid.error.DruidException; -import org.apache.druid.java.util.common.Pair; -import org.apache.druid.query.extraction.MapLookupExtractor; -import org.apache.druid.query.extraction.TimeDimExtractionFn; -import org.apache.druid.query.filter.EqualityFilter; -import org.apache.druid.query.lookup.LookupExtractionFn; -import org.apache.druid.query.lookup.LookupExtractor; -import org.apache.druid.segment.IndexBuilder; -import org.apache.druid.segment.StorageAdapter; -import org.apache.druid.segment.column.ColumnType; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.Closeable; -import java.util.Map; - -@RunWith(Parameterized.class) -public class EqualityFilterTest extends BaseFilterTest -{ - public EqualityFilterTest( - String testName, - IndexBuilder indexBuilder, - Function> finisher, - boolean cnf, - boolean optimize - ) - { - super(testName, DEFAULT_ROWS, indexBuilder, finisher, cnf, optimize); - } - - @AfterClass - public static void tearDown() throws Exception - { - BaseFilterTest.tearDown(EqualityFilterTest.class.getName()); - } - - @Test - public void testInvalidParameters() - { - Throwable t = Assert.assertThrows( - DruidException.class, - () -> assertFilterMatches( - new EqualityFilter(null, ColumnType.STRING, null, null, null), ImmutableList.of() - ) - ); - Assert.assertEquals("Invalid equality filter, column cannot be null", t.getMessage()); - t = Assert.assertThrows( - DruidException.class, - () -> assertFilterMatches( - new EqualityFilter("dim0", null, null, null, null), ImmutableList.of() - ) - ); - Assert.assertEquals("Invalid equality filter on column [dim0], matchValueType cannot be null", t.getMessage()); - t = Assert.assertThrows( - DruidException.class, - () -> assertFilterMatches( - new EqualityFilter("dim0", ColumnType.STRING, null, null, null), ImmutableList.of() - ) - ); - Assert.assertEquals("Invalid equality filter on column [dim0], matchValue cannot be null", t.getMessage()); - } - - @Test - public void testSingleValueStringColumnWithoutNulls() - { - if (NullHandling.sqlCompatible()) { - assertFilterMatches(new EqualityFilter("dim0", ColumnType.STRING, "", null, null), ImmutableList.of()); - } - assertFilterMatches(new EqualityFilter("dim0", ColumnType.STRING, "0", null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("dim0", ColumnType.STRING, "1", null, null), ImmutableList.of("1")); - - assertFilterMatches(new EqualityFilter("dim0", ColumnType.LONG, 0L, null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("dim0", ColumnType.LONG, 1L, null, null), ImmutableList.of("1")); - } - - @Test - public void testSingleValueVirtualStringColumnWithoutNulls() - { - if (NullHandling.sqlCompatible()) { - assertFilterMatches(new EqualityFilter("vdim0", ColumnType.STRING, "", null, null), ImmutableList.of()); - } - assertFilterMatches(new EqualityFilter("vdim0", ColumnType.STRING, "0", null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("vdim0", ColumnType.STRING, "1", null, null), ImmutableList.of("1")); - assertFilterMatches(new EqualityFilter("vdim0", ColumnType.LONG, 0L, null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("vdim0", ColumnType.LONG, 1L, null, null), ImmutableList.of("1")); - } - - @Test - public void testListFilteredVirtualColumn() - { - assertFilterMatchesSkipVectorize( - new EqualityFilter("allow-dim0", ColumnType.STRING, "1", null, null), - ImmutableList.of() - ); - assertFilterMatchesSkipVectorize( - new EqualityFilter("allow-dim0", ColumnType.STRING, "4", null, null), - ImmutableList.of("4") - ); - assertFilterMatchesSkipVectorize( - new EqualityFilter("deny-dim0", ColumnType.STRING, "0", null, null), - ImmutableList.of("0") - ); - assertFilterMatchesSkipVectorize( - new EqualityFilter("deny-dim0", ColumnType.STRING, "4", null, null), - ImmutableList.of() - ); - - // auto ingests arrays instead of MVDs which dont work with list filtered virtual column - if (!isAutoSchema()) { - assertFilterMatchesSkipVectorize( - new EqualityFilter("allow-dim2", ColumnType.STRING, "b", null, null), - ImmutableList.of() - ); - assertFilterMatchesSkipVectorize( - new EqualityFilter("allow-dim2", ColumnType.STRING, "a", null, null), - ImmutableList.of("0", "3") - ); - assertFilterMatchesSkipVectorize( - new EqualityFilter("deny-dim2", ColumnType.STRING, "b", null, null), - ImmutableList.of("0") - ); - assertFilterMatchesSkipVectorize( - new EqualityFilter("deny-dim2", ColumnType.STRING, "a", null, null), - ImmutableList.of() - ); - } - } - - @Test - public void testSingleValueStringColumnWithNulls() - { - if (NullHandling.sqlCompatible()) { - assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "", null, null), ImmutableList.of("0")); - } - assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "10", null, null), ImmutableList.of("1")); - assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "2", null, null), ImmutableList.of("2")); - assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "1", null, null), ImmutableList.of("3")); - assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "abdef", null, null), ImmutableList.of("4")); - assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "abc", null, null), ImmutableList.of("5")); - assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "ab", null, null), ImmutableList.of()); - } - - @Test - public void testSingleValueVirtualStringColumnWithNulls() - { - // testSingleValueStringColumnWithNulls but with virtual column selector - if (NullHandling.sqlCompatible()) { - assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "", null, null), ImmutableList.of("0")); - } - assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "10", null, null), ImmutableList.of("1")); - assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "2", null, null), ImmutableList.of("2")); - assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "1", null, null), ImmutableList.of("3")); - assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "abdef", null, null), ImmutableList.of("4")); - assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "abc", null, null), ImmutableList.of("5")); - assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "ab", null, null), ImmutableList.of()); - } - - @Test - public void testMultiValueStringColumn() - { - if (isAutoSchema()) { - // auto ingests arrays instead of strings - // single values are implicitly upcast to single element arrays, so we get some matches here... - if (NullHandling.sqlCompatible()) { - assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "", null, null), ImmutableList.of("2")); - } - assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "a", null, null), ImmutableList.of("3")); - assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "b", null, null), ImmutableList.of()); - assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "c", null, null), ImmutableList.of("4")); - assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "d", null, null), ImmutableList.of()); - - // array matchers can match the whole array - if (NullHandling.sqlCompatible()) { - assertFilterMatches( - new EqualityFilter("dim2", ColumnType.STRING, ImmutableList.of(""), null, null), - ImmutableList.of("2") - ); - } - assertFilterMatches( - new EqualityFilter("dim2", ColumnType.STRING_ARRAY, new Object[]{"a", "b"}, null, null), - ImmutableList.of("0") - ); - assertFilterMatches( - new EqualityFilter("dim2", ColumnType.STRING_ARRAY, ImmutableList.of("a", "b"), null, null), - ImmutableList.of("0") - ); - assertFilterMatches( - new EqualityFilter("dim2", ColumnType.STRING_ARRAY, new Object[]{"a"}, null, null), - ImmutableList.of("3") - ); - assertFilterMatches( - new EqualityFilter("dim2", ColumnType.STRING_ARRAY, new Object[]{"b"}, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new EqualityFilter("dim2", ColumnType.STRING_ARRAY, new Object[]{"c"}, null, null), - ImmutableList.of("4") - ); - assertFilterMatches( - new EqualityFilter("dim2", ColumnType.STRING_ARRAY, new Object[]{"d"}, null, null), - ImmutableList.of() - ); - } else { - if (NullHandling.sqlCompatible()) { - assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "", null, null), ImmutableList.of("2")); - } - assertFilterMatches( - new EqualityFilter("dim2", ColumnType.STRING, "a", null, null), - ImmutableList.of("0", "3") - ); - assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "b", null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "c", null, null), ImmutableList.of("4")); - assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "d", null, null), ImmutableList.of()); - } - } - - @Test - public void testMissingColumnSpecifiedInDimensionList() - { - if (NullHandling.sqlCompatible()) { - assertFilterMatches(new EqualityFilter("dim3", ColumnType.STRING, "", null, null), ImmutableList.of()); - } - assertFilterMatches(new EqualityFilter("dim3", ColumnType.STRING, "a", null, null), ImmutableList.of()); - assertFilterMatches(new EqualityFilter("dim3", ColumnType.STRING, "b", null, null), ImmutableList.of()); - assertFilterMatches(new EqualityFilter("dim3", ColumnType.STRING, "c", null, null), ImmutableList.of()); - } - - @Test - public void testMissingColumnNotSpecifiedInDimensionList() - { - if (NullHandling.sqlCompatible()) { - assertFilterMatches(new EqualityFilter("dim4", ColumnType.STRING, "", null, null), ImmutableList.of()); - } - assertFilterMatches(new EqualityFilter("dim4", ColumnType.STRING, "a", null, null), ImmutableList.of()); - assertFilterMatches(new EqualityFilter("dim4", ColumnType.STRING, "b", null, null), ImmutableList.of()); - assertFilterMatches(new EqualityFilter("dim4", ColumnType.STRING, "c", null, null), ImmutableList.of()); - } - - @Test - public void testExpressionVirtualColumn() - { - assertFilterMatches( - new EqualityFilter("expr", ColumnType.STRING, "1.1", null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5") - ); - assertFilterMatches(new EqualityFilter("expr", ColumnType.STRING, "1.2", null, null), ImmutableList.of()); - - assertFilterMatches( - new EqualityFilter("expr", ColumnType.FLOAT, 1.1f, null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5") - ); - assertFilterMatches(new EqualityFilter("expr", ColumnType.FLOAT, 1.2f, null, null), ImmutableList.of()); - - assertFilterMatches( - new EqualityFilter("expr", ColumnType.DOUBLE, 1.1, null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5") - ); - assertFilterMatches(new EqualityFilter("expr", ColumnType.DOUBLE, 1.2, null, null), ImmutableList.of()); - - // if we accidentally specify it as a string, it works too... - assertFilterMatches( - new EqualityFilter("expr", ColumnType.DOUBLE, "1.1", null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5") - ); - assertFilterMatches(new EqualityFilter("expr", ColumnType.DOUBLE, "1.2", null, null), ImmutableList.of()); - } - - @Test - public void testWithTimeExtractionFnNull() - { - assertFilterMatches(new EqualityFilter( - "timeDim", - ColumnType.STRING, - "2017-07", - new TimeDimExtractionFn("yyyy-MM-dd", "yyyy-MM", true), - null - ), ImmutableList.of("0", "1")); - assertFilterMatches(new EqualityFilter( - "timeDim", - ColumnType.STRING, - "2017-05", - new TimeDimExtractionFn("yyyy-MM-dd", "yyyy-MM", true), - null - ), ImmutableList.of("2")); - - assertFilterMatches(new EqualityFilter( - "timeDim", - ColumnType.STRING, - "2020-01", - new TimeDimExtractionFn("yyyy-MM-dd", "yyyy-MM", true), - null - ), ImmutableList.of("3", "5")); - } - - @Test - public void testSelectorWithLookupExtractionFn() - { - final Map stringMap = ImmutableMap.of( - "1", "HELLO", - "a", "HELLO", - "abdef", "HELLO", - "abc", "UNKNOWN" - ); - LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false); - LookupExtractionFn lookupFn = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true); - - assertFilterMatches(new EqualityFilter("dim0", ColumnType.STRING, "HELLO", lookupFn, null), ImmutableList.of("1")); - assertFilterMatches( - new EqualityFilter("dim0", ColumnType.STRING, "UNKNOWN", lookupFn, null), - ImmutableList.of("0", "2", "3", "4", "5") - ); - - assertFilterMatches( - new EqualityFilter("dim1", ColumnType.STRING, "HELLO", lookupFn, null), - ImmutableList.of("3", "4") - ); - assertFilterMatches( - new EqualityFilter("dim1", ColumnType.STRING, "UNKNOWN", lookupFn, null), - ImmutableList.of("0", "1", "2", "5") - ); - - if (!isAutoSchema()) { - // auto ingests arrays instead of mvds, arrays do not support extractionFn - assertFilterMatches( - new EqualityFilter("dim2", ColumnType.STRING, "HELLO", lookupFn, null), - ImmutableList.of("0", "3") - ); - assertFilterMatches( - new EqualityFilter("dim2", ColumnType.STRING, "UNKNOWN", lookupFn, null), - ImmutableList.of("0", "1", "2", "4", "5") - ); - - assertFilterMatches( - new EqualityFilter("dim3", ColumnType.STRING, "HELLO", lookupFn, null), - ImmutableList.of() - ); - assertFilterMatches( - new EqualityFilter("dim3", ColumnType.STRING, "UNKNOWN", lookupFn, null), - ImmutableList.of("0", "1", "2", "3", "4", "5") - ); - - assertFilterMatches( - new EqualityFilter("dim4", ColumnType.STRING, "HELLO", lookupFn, null), - ImmutableList.of() - ); - assertFilterMatches( - new EqualityFilter("dim4", ColumnType.STRING, "UNKNOWN", lookupFn, null), - ImmutableList.of("0", "1", "2", "3", "4", "5") - ); - } - - final Map stringMap2 = ImmutableMap.of( - "2", "5" - ); - LookupExtractor mapExtractor2 = new MapLookupExtractor(stringMap2, false); - LookupExtractionFn lookupFn2 = new LookupExtractionFn(mapExtractor2, true, null, false, true); - assertFilterMatches( - new EqualityFilter("dim0", ColumnType.STRING, "5", lookupFn2, null), - ImmutableList.of("2", "5") - ); - - if (NullHandling.sqlCompatible()) { - - final Map stringMap3 = ImmutableMap.of( - "1", "" - ); - LookupExtractor mapExtractor3 = new MapLookupExtractor(stringMap3, false); - LookupExtractionFn lookupFn3 = new LookupExtractionFn(mapExtractor3, false, null, false, true); - assertFilterMatches( - new EqualityFilter("dim0", ColumnType.STRING, "", lookupFn3, null), - ImmutableList.of("1") - ); - } - } - - @Test - public void testNumericColumnNullsAndDefaults() - { - if (canTestNumericNullsAsDefaultValues) { - assertFilterMatches(new EqualityFilter("f0", ColumnType.FLOAT, 0f, null, null), ImmutableList.of("0", "4")); - assertFilterMatches(new EqualityFilter("d0", ColumnType.DOUBLE, 0.0, null, null), ImmutableList.of("0", "2")); - assertFilterMatches(new EqualityFilter("l0", ColumnType.LONG, 0L, null, null), ImmutableList.of("0", "3")); - assertFilterMatches(new EqualityFilter("f0", ColumnType.STRING, "0", null, null), ImmutableList.of("0", "4")); - assertFilterMatches(new EqualityFilter("d0", ColumnType.STRING, "0", null, null), ImmutableList.of("0", "2")); - assertFilterMatches(new EqualityFilter("l0", ColumnType.STRING, "0", null, null), ImmutableList.of("0", "3")); - } else { - assertFilterMatches(new EqualityFilter("f0", ColumnType.FLOAT, 0f, null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("d0", ColumnType.DOUBLE, 0.0, null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("l0", ColumnType.LONG, 0L, null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("f0", ColumnType.STRING, "0", null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("d0", ColumnType.STRING, "0", null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("l0", ColumnType.STRING, "0", null, null), ImmutableList.of("0")); - } - } - - @Test - public void testVirtualNumericColumnNullsAndDefaults() - { - if (canTestNumericNullsAsDefaultValues) { - assertFilterMatches(new EqualityFilter("vf0", ColumnType.FLOAT, 0f, null, null), ImmutableList.of("0", "4")); - assertFilterMatches(new EqualityFilter("vd0", ColumnType.DOUBLE, 0.0, null, null), ImmutableList.of("0", "2")); - assertFilterMatches(new EqualityFilter("vl0", ColumnType.LONG, 0L, null, null), ImmutableList.of("0", "3")); - assertFilterMatches(new EqualityFilter("vf0", ColumnType.STRING, "0", null, null), ImmutableList.of("0", "4")); - assertFilterMatches(new EqualityFilter("vd0", ColumnType.STRING, "0", null, null), ImmutableList.of("0", "2")); - assertFilterMatches(new EqualityFilter("vl0", ColumnType.STRING, "0", null, null), ImmutableList.of("0", "3")); - } else { - assertFilterMatches(new EqualityFilter("vf0", ColumnType.FLOAT, 0f, null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("vd0", ColumnType.DOUBLE, 0.0, null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("vl0", ColumnType.LONG, 0L, null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("vf0", ColumnType.STRING, "0", null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("vd0", ColumnType.STRING, "0", null, null), ImmutableList.of("0")); - assertFilterMatches(new EqualityFilter("vl0", ColumnType.STRING, "0", null, null), ImmutableList.of("0")); - } - } - - @Test - public void testNumeric() - { - /* - dim0 d0 f0 l0 - "0" .. 0.0, 0.0f, 0L - "1" .. 10.1, 10.1f, 100L - "2" .. null, 5.5f, 40L - "3" .. 120.0245, 110.0f, null - "4" .. 60.0, null, 9001L - "5" .. 765.432, 123.45f, 12345L - */ - - assertFilterMatches(new EqualityFilter("d0", ColumnType.DOUBLE, 10.1, null, null), ImmutableList.of("1")); - assertFilterMatches(new EqualityFilter("d0", ColumnType.DOUBLE, 120.0245, null, null), ImmutableList.of("3")); - assertFilterMatches(new EqualityFilter("d0", ColumnType.DOUBLE, 765.432, null, null), ImmutableList.of("5")); - assertFilterMatches(new EqualityFilter("d0", ColumnType.DOUBLE, 765.431, null, null), ImmutableList.of()); - - assertFilterMatches(new EqualityFilter("l0", ColumnType.LONG, 100L, null, null), ImmutableList.of("1")); - assertFilterMatches(new EqualityFilter("l0", ColumnType.LONG, 40L, null, null), ImmutableList.of("2")); - assertFilterMatches(new EqualityFilter("l0", ColumnType.LONG, 9001L, null, null), ImmutableList.of("4")); - assertFilterMatches(new EqualityFilter("l0", ColumnType.LONG, 9000L, null, null), ImmutableList.of()); - if (!isAutoSchema()) { - // auto schema doesn't store float columns as floats, rather they are stored as doubles... the predicate matcher - // matches fine, but the string value set index does not match correctly if we expect the input float values - assertFilterMatches(new EqualityFilter("f0", ColumnType.FLOAT, 10.1f, null, null), ImmutableList.of("1")); - assertFilterMatches(new EqualityFilter("f0", ColumnType.FLOAT, 110.0f, null, null), ImmutableList.of("3")); - assertFilterMatches(new EqualityFilter("f0", ColumnType.FLOAT, 123.45f, null, null), ImmutableList.of("5")); - assertFilterMatches(new EqualityFilter("f0", ColumnType.FLOAT, 123.46f, null, null), ImmutableList.of()); - } else { - // .. so we need to cast them instead - assertFilterMatches( - new EqualityFilter("f0", ColumnType.DOUBLE, (double) 10.1f, null, null), - ImmutableList.of("1") - ); - assertFilterMatches( - new EqualityFilter("f0", ColumnType.DOUBLE, (double) 110.0f, null, null), - ImmutableList.of("3") - ); - assertFilterMatches( - new EqualityFilter("f0", ColumnType.DOUBLE, (double) 123.45f, null, null), - ImmutableList.of("5") - ); - assertFilterMatches( - new EqualityFilter("f0", ColumnType.DOUBLE, (double) 123.46f, null, null), - ImmutableList.of() - ); - } - } - - @Test - public void testArrays() - { - if (isAutoSchema()) { - // only auto schema supports array columns... skip other segment types - /* - dim0 .. arrayString arrayLong arrayDouble - "0", .. ["a", "b", "c"], [1L, 2L, 3L], [1.1, 2.2, 3.3] - "1", .. [], [], [1.1, 2.2, 3.3] - "2", .. null, [1L, 2L, 3L], [null] - "3", .. ["a", "b", "c"], null, [] - "4", .. ["c", "d"], [null], [-1.1, -333.3] - "5", .. [null], [123L, 345L], null - */ - - assertFilterMatches( - new EqualityFilter( - "arrayString", - ColumnType.STRING_ARRAY, - ImmutableList.of("a", "b", "c"), - null, - null - ), - ImmutableList.of("0", "3") - ); - assertFilterMatches( - new EqualityFilter( - "arrayString", - ColumnType.STRING_ARRAY, - new Object[]{"a", "b", "c"}, - null, - null - ), - ImmutableList.of("0", "3") - ); - assertFilterMatches( - new EqualityFilter( - "arrayString", - ColumnType.STRING_ARRAY, - ImmutableList.of(), - null, - null - ), - ImmutableList.of("1") - ); - assertFilterMatches( - new EqualityFilter( - "arrayString", - ColumnType.STRING_ARRAY, - new Object[]{null}, - null, - null - ), - ImmutableList.of("5") - ); - assertFilterMatches( - new EqualityFilter( - "arrayString", - ColumnType.STRING_ARRAY, - new Object[]{null, null}, - null, - null - ), - ImmutableList.of() - ); - assertFilterMatches( - new EqualityFilter( - "arrayLong", - ColumnType.LONG_ARRAY, - ImmutableList.of(1L, 2L, 3L), - null, - null - ), - ImmutableList.of("0", "2") - ); - assertFilterMatches( - new EqualityFilter( - "arrayLong", - ColumnType.LONG_ARRAY, - new Object[]{1L, 2L, 3L}, - null, - null - ), - ImmutableList.of("0", "2") - ); - assertFilterMatches( - new EqualityFilter( - "arrayLong", - ColumnType.LONG_ARRAY, - ImmutableList.of(), - null, - null - ), - ImmutableList.of("1") - ); - assertFilterMatches( - new EqualityFilter( - "arrayLong", - ColumnType.LONG_ARRAY, - new Object[]{null}, - null, - null - ), - ImmutableList.of("4") - ); - assertFilterMatches( - new EqualityFilter( - "arrayLong", - ColumnType.LONG_ARRAY, - new Object[]{null, null}, - null, - null - ), - ImmutableList.of() - ); - assertFilterMatches( - new EqualityFilter( - "arrayDouble", - ColumnType.DOUBLE_ARRAY, - ImmutableList.of(1.1, 2.2, 3.3), - null, - null - ), - ImmutableList.of("0", "1") - ); - assertFilterMatches( - new EqualityFilter( - "arrayDouble", - ColumnType.DOUBLE_ARRAY, - new Object[]{1.1, 2.2, 3.3}, - null, - null - ), - ImmutableList.of("0", "1") - ); - assertFilterMatches( - new EqualityFilter( - "arrayDouble", - ColumnType.DOUBLE_ARRAY, - ImmutableList.of(), - null, - null - ), - ImmutableList.of("3") - ); - assertFilterMatches( - new EqualityFilter( - "arrayDouble", - ColumnType.DOUBLE_ARRAY, - new Object[]{null}, - null, - null - ), - ImmutableList.of("2") - ); - assertFilterMatches( - new EqualityFilter( - "arrayDouble", - ColumnType.DOUBLE_ARRAY, - ImmutableList.of(1.1, 2.2, 3.4), - null, - null - ), - ImmutableList.of() - ); - } - } - - @Test - public void test_equals() - { - EqualsVerifier.forClass(EqualityFilter.class).usingGetClass() - .withNonnullFields( - "column", - "matchValueType", - "matchValue", - "predicateFactory", - "cachedOptimizedFilter" - ) - .withPrefabValues(ColumnType.class, ColumnType.STRING, ColumnType.DOUBLE) - .withIgnoredFields("predicateFactory", "cachedOptimizedFilter") - .verify(); - } -} diff --git a/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java new file mode 100644 index 000000000000..87f2c4f1fff5 --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/filter/EqualityFilterTests.java @@ -0,0 +1,726 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.filter; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.error.DruidException; +import org.apache.druid.guice.NestedDataModule; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExpressionType; +import org.apache.druid.query.filter.EqualityFilter; +import org.apache.druid.query.filter.FilterTuning; +import org.apache.druid.segment.IndexBuilder; +import org.apache.druid.segment.StorageAdapter; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.testing.InitializedNullHandlingTest; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.Closeable; +import java.util.Arrays; + +@RunWith(Enclosed.class) +public class EqualityFilterTests +{ + @RunWith(Parameterized.class) + public static class EqualityFilterTest extends BaseFilterTest + { + public EqualityFilterTest( + String testName, + IndexBuilder indexBuilder, + Function> finisher, + boolean cnf, + boolean optimize + ) + { + super(testName, DEFAULT_ROWS, indexBuilder, finisher, cnf, optimize); + } + + @AfterClass + public static void tearDown() throws Exception + { + BaseFilterTest.tearDown(EqualityFilterTest.class.getName()); + } + + + @Test + public void testSingleValueStringColumnWithoutNulls() + { + if (NullHandling.sqlCompatible()) { + assertFilterMatches(new EqualityFilter("dim0", ColumnType.STRING, "", null), ImmutableList.of()); + } + assertFilterMatches(new EqualityFilter("dim0", ColumnType.STRING, "0", null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("dim0", ColumnType.STRING, "1", null), ImmutableList.of("1")); + + assertFilterMatches(new EqualityFilter("dim0", ColumnType.LONG, 0L, null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("dim0", ColumnType.LONG, 1L, null), ImmutableList.of("1")); + } + + @Test + public void testSingleValueVirtualStringColumnWithoutNulls() + { + if (NullHandling.sqlCompatible()) { + assertFilterMatches(new EqualityFilter("vdim0", ColumnType.STRING, "", null), ImmutableList.of()); + } + assertFilterMatches(new EqualityFilter("vdim0", ColumnType.STRING, "0", null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("vdim0", ColumnType.STRING, "1", null), ImmutableList.of("1")); + assertFilterMatches(new EqualityFilter("vdim0", ColumnType.LONG, 0L, null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("vdim0", ColumnType.LONG, 1L, null), ImmutableList.of("1")); + } + + @Test + public void testListFilteredVirtualColumn() + { + assertFilterMatchesSkipVectorize( + new EqualityFilter("allow-dim0", ColumnType.STRING, "1", null), + ImmutableList.of() + ); + assertFilterMatchesSkipVectorize( + new EqualityFilter("allow-dim0", ColumnType.STRING, "4", null), + ImmutableList.of("4") + ); + assertFilterMatchesSkipVectorize( + new EqualityFilter("deny-dim0", ColumnType.STRING, "0", null), + ImmutableList.of("0") + ); + assertFilterMatchesSkipVectorize( + new EqualityFilter("deny-dim0", ColumnType.STRING, "4", null), + ImmutableList.of() + ); + + // auto ingests arrays instead of MVDs which dont work with list filtered virtual column + if (!isAutoSchema()) { + assertFilterMatchesSkipVectorize( + new EqualityFilter("allow-dim2", ColumnType.STRING, "b", null), + ImmutableList.of() + ); + assertFilterMatchesSkipVectorize( + new EqualityFilter("allow-dim2", ColumnType.STRING, "a", null), + ImmutableList.of("0", "3") + ); + assertFilterMatchesSkipVectorize( + new EqualityFilter("deny-dim2", ColumnType.STRING, "b", null), + ImmutableList.of("0") + ); + assertFilterMatchesSkipVectorize( + new EqualityFilter("deny-dim2", ColumnType.STRING, "a", null), + ImmutableList.of() + ); + } + } + + @Test + public void testSingleValueStringColumnWithNulls() + { + if (NullHandling.sqlCompatible()) { + assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "", null), ImmutableList.of("0")); + } + assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "10", null), ImmutableList.of("1")); + assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "2", null), ImmutableList.of("2")); + assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "1", null), ImmutableList.of("3")); + assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "abdef", null), ImmutableList.of("4")); + assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "abc", null), ImmutableList.of("5")); + assertFilterMatches(new EqualityFilter("dim1", ColumnType.STRING, "ab", null), ImmutableList.of()); + } + + @Test + public void testSingleValueVirtualStringColumnWithNulls() + { + // testSingleValueStringColumnWithNulls but with virtual column selector + if (NullHandling.sqlCompatible()) { + assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "", null), ImmutableList.of("0")); + } + assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "10", null), ImmutableList.of("1")); + assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "2", null), ImmutableList.of("2")); + assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "1", null), ImmutableList.of("3")); + assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "abdef", null), ImmutableList.of("4")); + assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "abc", null), ImmutableList.of("5")); + assertFilterMatches(new EqualityFilter("vdim1", ColumnType.STRING, "ab", null), ImmutableList.of()); + } + + @Test + public void testMultiValueStringColumn() + { + if (isAutoSchema()) { + // auto ingests arrays instead of strings + // single values are implicitly upcast to single element arrays, so we get some matches here... + if (NullHandling.sqlCompatible()) { + assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "", null), ImmutableList.of("2")); + } + assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "a", null), ImmutableList.of("3")); + assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "b", null), ImmutableList.of()); + assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "c", null), ImmutableList.of("4")); + assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "d", null), ImmutableList.of()); + + // array matchers can match the whole array + if (NullHandling.sqlCompatible()) { + assertFilterMatches( + new EqualityFilter("dim2", ColumnType.STRING, ImmutableList.of(""), null), + ImmutableList.of("2") + ); + } + assertFilterMatches( + new EqualityFilter("dim2", ColumnType.STRING_ARRAY, new Object[]{"a", "b"}, null), + ImmutableList.of("0") + ); + assertFilterMatches( + new EqualityFilter("dim2", ColumnType.STRING_ARRAY, ImmutableList.of("a", "b"), null), + ImmutableList.of("0") + ); + assertFilterMatches( + new EqualityFilter("dim2", ColumnType.STRING_ARRAY, new Object[]{"a"}, null), + ImmutableList.of("3") + ); + assertFilterMatches( + new EqualityFilter("dim2", ColumnType.STRING_ARRAY, new Object[]{"b"}, null), + ImmutableList.of() + ); + assertFilterMatches( + new EqualityFilter("dim2", ColumnType.STRING_ARRAY, new Object[]{"c"}, null), + ImmutableList.of("4") + ); + assertFilterMatches( + new EqualityFilter("dim2", ColumnType.STRING_ARRAY, new Object[]{"d"}, null), + ImmutableList.of() + ); + } else { + if (NullHandling.sqlCompatible()) { + assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "", null), ImmutableList.of("2")); + } + assertFilterMatches( + new EqualityFilter("dim2", ColumnType.STRING, "a", null), + ImmutableList.of("0", "3") + ); + assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "b", null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "c", null), ImmutableList.of("4")); + assertFilterMatches(new EqualityFilter("dim2", ColumnType.STRING, "d", null), ImmutableList.of()); + } + } + + @Test + public void testMissingColumnSpecifiedInDimensionList() + { + if (NullHandling.sqlCompatible()) { + assertFilterMatches(new EqualityFilter("dim3", ColumnType.STRING, "", null), ImmutableList.of()); + } + assertFilterMatches(new EqualityFilter("dim3", ColumnType.STRING, "a", null), ImmutableList.of()); + assertFilterMatches(new EqualityFilter("dim3", ColumnType.STRING, "b", null), ImmutableList.of()); + assertFilterMatches(new EqualityFilter("dim3", ColumnType.STRING, "c", null), ImmutableList.of()); + } + + @Test + public void testMissingColumnNotSpecifiedInDimensionList() + { + if (NullHandling.sqlCompatible()) { + assertFilterMatches(new EqualityFilter("dim4", ColumnType.STRING, "", null), ImmutableList.of()); + } + assertFilterMatches(new EqualityFilter("dim4", ColumnType.STRING, "a", null), ImmutableList.of()); + assertFilterMatches(new EqualityFilter("dim4", ColumnType.STRING, "b", null), ImmutableList.of()); + assertFilterMatches(new EqualityFilter("dim4", ColumnType.STRING, "c", null), ImmutableList.of()); + } + + @Test + public void testExpressionVirtualColumn() + { + assertFilterMatches( + new EqualityFilter("expr", ColumnType.STRING, "1.1", null), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + assertFilterMatches(new EqualityFilter("expr", ColumnType.STRING, "1.2", null), ImmutableList.of()); + + assertFilterMatches( + new EqualityFilter("expr", ColumnType.FLOAT, 1.1f, null), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + assertFilterMatches(new EqualityFilter("expr", ColumnType.FLOAT, 1.2f, null), ImmutableList.of()); + + assertFilterMatches( + new EqualityFilter("expr", ColumnType.DOUBLE, 1.1, null), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + assertFilterMatches(new EqualityFilter("expr", ColumnType.DOUBLE, 1.2, null), ImmutableList.of()); + + // if we accidentally specify it as a string, it works too... + assertFilterMatches( + new EqualityFilter("expr", ColumnType.DOUBLE, "1.1", null), + ImmutableList.of("0", "1", "2", "3", "4", "5") + ); + assertFilterMatches(new EqualityFilter("expr", ColumnType.DOUBLE, "1.2", null), ImmutableList.of()); + } + + @Test + public void testNumericColumnNullsAndDefaults() + { + if (canTestNumericNullsAsDefaultValues) { + assertFilterMatches(new EqualityFilter("f0", ColumnType.FLOAT, 0f, null), ImmutableList.of("0", "4")); + assertFilterMatches(new EqualityFilter("d0", ColumnType.DOUBLE, 0.0, null), ImmutableList.of("0", "2")); + assertFilterMatches(new EqualityFilter("l0", ColumnType.LONG, 0L, null), ImmutableList.of("0", "3")); + assertFilterMatches(new EqualityFilter("f0", ColumnType.STRING, "0", null), ImmutableList.of("0", "4")); + assertFilterMatches(new EqualityFilter("d0", ColumnType.STRING, "0", null), ImmutableList.of("0", "2")); + assertFilterMatches(new EqualityFilter("l0", ColumnType.STRING, "0", null), ImmutableList.of("0", "3")); + } else { + assertFilterMatches(new EqualityFilter("f0", ColumnType.FLOAT, 0f, null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("d0", ColumnType.DOUBLE, 0.0, null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("l0", ColumnType.LONG, 0L, null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("f0", ColumnType.STRING, "0", null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("d0", ColumnType.STRING, "0", null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("l0", ColumnType.STRING, "0", null), ImmutableList.of("0")); + } + } + + @Test + public void testVirtualNumericColumnNullsAndDefaults() + { + if (canTestNumericNullsAsDefaultValues) { + assertFilterMatches(new EqualityFilter("vf0", ColumnType.FLOAT, 0f, null), ImmutableList.of("0", "4")); + assertFilterMatches(new EqualityFilter("vd0", ColumnType.DOUBLE, 0.0, null), ImmutableList.of("0", "2")); + assertFilterMatches(new EqualityFilter("vl0", ColumnType.LONG, 0L, null), ImmutableList.of("0", "3")); + assertFilterMatches(new EqualityFilter("vf0", ColumnType.STRING, "0", null), ImmutableList.of("0", "4")); + assertFilterMatches(new EqualityFilter("vd0", ColumnType.STRING, "0", null), ImmutableList.of("0", "2")); + assertFilterMatches(new EqualityFilter("vl0", ColumnType.STRING, "0", null), ImmutableList.of("0", "3")); + } else { + assertFilterMatches(new EqualityFilter("vf0", ColumnType.FLOAT, 0f, null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("vd0", ColumnType.DOUBLE, 0.0, null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("vl0", ColumnType.LONG, 0L, null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("vf0", ColumnType.STRING, "0", null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("vd0", ColumnType.STRING, "0", null), ImmutableList.of("0")); + assertFilterMatches(new EqualityFilter("vl0", ColumnType.STRING, "0", null), ImmutableList.of("0")); + } + } + + @Test + public void testNumeric() + { + /* + dim0 d0 f0 l0 + "0" .. 0.0, 0.0f, 0L + "1" .. 10.1, 10.1f, 100L + "2" .. null, 5.5f, 40L + "3" .. 120.0245, 110.0f, null + "4" .. 60.0, null, 9001L + "5" .. 765.432, 123.45f, 12345L + */ + + assertFilterMatches(new EqualityFilter("d0", ColumnType.DOUBLE, 10.1, null), ImmutableList.of("1")); + assertFilterMatches(new EqualityFilter("d0", ColumnType.DOUBLE, 120.0245, null), ImmutableList.of("3")); + assertFilterMatches(new EqualityFilter("d0", ColumnType.DOUBLE, 765.432, null), ImmutableList.of("5")); + assertFilterMatches(new EqualityFilter("d0", ColumnType.DOUBLE, 765.431, null), ImmutableList.of()); + + assertFilterMatches(new EqualityFilter("l0", ColumnType.LONG, 100L, null), ImmutableList.of("1")); + assertFilterMatches(new EqualityFilter("l0", ColumnType.LONG, 40L, null), ImmutableList.of("2")); + assertFilterMatches(new EqualityFilter("l0", ColumnType.LONG, 9001L, null), ImmutableList.of("4")); + assertFilterMatches(new EqualityFilter("l0", ColumnType.LONG, 9000L, null), ImmutableList.of()); + if (!isAutoSchema()) { + // auto schema doesn't store float columns as floats, rather they are stored as doubles... the predicate matcher + // matches fine, but the string value set index does not match correctly if we expect the input float values + assertFilterMatches(new EqualityFilter("f0", ColumnType.FLOAT, 10.1f, null), ImmutableList.of("1")); + assertFilterMatches(new EqualityFilter("f0", ColumnType.FLOAT, 110.0f, null), ImmutableList.of("3")); + assertFilterMatches(new EqualityFilter("f0", ColumnType.FLOAT, 123.45f, null), ImmutableList.of("5")); + assertFilterMatches(new EqualityFilter("f0", ColumnType.FLOAT, 123.46f, null), ImmutableList.of()); + } else { + // .. so we need to cast them instead + assertFilterMatches( + new EqualityFilter("f0", ColumnType.DOUBLE, (double) 10.1f, null), + ImmutableList.of("1") + ); + assertFilterMatches( + new EqualityFilter("f0", ColumnType.DOUBLE, (double) 110.0f, null), + ImmutableList.of("3") + ); + assertFilterMatches( + new EqualityFilter("f0", ColumnType.DOUBLE, (double) 123.45f, null), + ImmutableList.of("5") + ); + assertFilterMatches( + new EqualityFilter("f0", ColumnType.DOUBLE, (double) 123.46f, null), + ImmutableList.of() + ); + } + } + + @Test + public void testArrays() + { + if (isAutoSchema()) { + // only auto schema supports array columns... skip other segment types + /* + dim0 .. arrayString arrayLong arrayDouble + "0", .. ["a", "b", "c"], [1L, 2L, 3L], [1.1, 2.2, 3.3] + "1", .. [], [], [1.1, 2.2, 3.3] + "2", .. null, [1L, 2L, 3L], [null] + "3", .. ["a", "b", "c"], null, [] + "4", .. ["c", "d"], [null], [-1.1, -333.3] + "5", .. [null], [123L, 345L], null + */ + + assertFilterMatches( + new EqualityFilter( + "arrayString", + ColumnType.STRING_ARRAY, + ImmutableList.of("a", "b", "c"), + null + ), + ImmutableList.of("0", "3") + ); + assertFilterMatches( + new EqualityFilter( + "arrayString", + ColumnType.STRING_ARRAY, + new Object[]{"a", "b", "c"}, + null + ), + ImmutableList.of("0", "3") + ); + assertFilterMatches( + new EqualityFilter( + "arrayString", + ColumnType.STRING_ARRAY, + ImmutableList.of(), + null + ), + ImmutableList.of("1") + ); + assertFilterMatches( + new EqualityFilter( + "arrayString", + ColumnType.STRING_ARRAY, + new Object[]{null}, + null + ), + ImmutableList.of("5") + ); + assertFilterMatches( + new EqualityFilter( + "arrayString", + ColumnType.STRING_ARRAY, + new Object[]{null, null}, + null + ), + ImmutableList.of() + ); + assertFilterMatches( + new EqualityFilter( + "arrayLong", + ColumnType.LONG_ARRAY, + ImmutableList.of(1L, 2L, 3L), + null + ), + ImmutableList.of("0", "2") + ); + assertFilterMatches( + new EqualityFilter( + "arrayLong", + ColumnType.LONG_ARRAY, + new Object[]{1L, 2L, 3L}, + null + ), + ImmutableList.of("0", "2") + ); + assertFilterMatches( + new EqualityFilter( + "arrayLong", + ColumnType.LONG_ARRAY, + ImmutableList.of(), + null + ), + ImmutableList.of("1") + ); + assertFilterMatches( + new EqualityFilter( + "arrayLong", + ColumnType.LONG_ARRAY, + new Object[]{null}, + null + ), + ImmutableList.of("4") + ); + assertFilterMatches( + new EqualityFilter( + "arrayLong", + ColumnType.LONG_ARRAY, + new Object[]{null, null}, + null + ), + ImmutableList.of() + ); + assertFilterMatches( + new EqualityFilter( + "arrayDouble", + ColumnType.DOUBLE_ARRAY, + ImmutableList.of(1.1, 2.2, 3.3), + null + ), + ImmutableList.of("0", "1") + ); + assertFilterMatches( + new EqualityFilter( + "arrayDouble", + ColumnType.DOUBLE_ARRAY, + new Object[]{1.1, 2.2, 3.3}, + null + ), + ImmutableList.of("0", "1") + ); + assertFilterMatches( + new EqualityFilter( + "arrayDouble", + ColumnType.DOUBLE_ARRAY, + ImmutableList.of(), + null + ), + ImmutableList.of("3") + ); + assertFilterMatches( + new EqualityFilter( + "arrayDouble", + ColumnType.DOUBLE_ARRAY, + new Object[]{null}, + null + ), + ImmutableList.of("2") + ); + assertFilterMatches( + new EqualityFilter( + "arrayDouble", + ColumnType.DOUBLE_ARRAY, + ImmutableList.of(1.1, 2.2, 3.4), + null + ), + ImmutableList.of() + ); + } + } + } + + public static class EqualityFilterNonParameterizedTests extends InitializedNullHandlingTest + { + @Test + public void testSerde() throws JsonProcessingException + { + ObjectMapper mapper = new DefaultObjectMapper(); + EqualityFilter filter = new EqualityFilter("x", ColumnType.STRING, "hello", null); + String s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, EqualityFilter.class)); + + filter = new EqualityFilter("x", ColumnType.LONG, 1L, null); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, EqualityFilter.class)); + + filter = new EqualityFilter("x", ColumnType.LONG, 1, null); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, EqualityFilter.class)); + + filter = new EqualityFilter("x", ColumnType.DOUBLE, 111.111, null); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, EqualityFilter.class)); + + filter = new EqualityFilter("x", ColumnType.FLOAT, 1234.0f, null); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, EqualityFilter.class)); + + filter = new EqualityFilter("x", ColumnType.STRING_ARRAY, new Object[]{"a", "b", null, "c"}, null); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, EqualityFilter.class)); + + filter = new EqualityFilter("x", ColumnType.STRING_ARRAY, Arrays.asList("a", "b", null, "c"), null); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, EqualityFilter.class)); + + filter = new EqualityFilter("x", ColumnType.LONG_ARRAY, new Object[]{1L, null, 2L, 3L}, null); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, EqualityFilter.class)); + + filter = new EqualityFilter("x", ColumnType.LONG_ARRAY, Arrays.asList(1L, null, 2L, 3L), null); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, EqualityFilter.class)); + + filter = new EqualityFilter("x", ColumnType.DOUBLE_ARRAY, new Object[]{1.1, 2.1, null, 3.1}, null); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, EqualityFilter.class)); + + filter = new EqualityFilter("x", ColumnType.DOUBLE_ARRAY, Arrays.asList(1.1, 2.1, null, 3.1), null); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, EqualityFilter.class)); + + filter = new EqualityFilter("x", ColumnType.NESTED_DATA, ImmutableMap.of("x", ImmutableList.of(1, 2, 3)), null); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, EqualityFilter.class)); + } + + @Test + public void testGetCacheKey() + { + EqualityFilter f1 = new EqualityFilter("x", ColumnType.STRING, "hello", null); + EqualityFilter f1_2 = new EqualityFilter("x", ColumnType.STRING, "hello", null); + EqualityFilter f2 = new EqualityFilter("x", ColumnType.STRING, "world", null); + EqualityFilter f3 = new EqualityFilter("x", ColumnType.STRING, "hello", new FilterTuning(true, null, null)); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + + f1 = new EqualityFilter("x", ColumnType.LONG, 1L, null); + f1_2 = new EqualityFilter("x", ColumnType.LONG, 1, null); + f2 = new EqualityFilter("x", ColumnType.LONG, 2L, null); + f3 = new EqualityFilter("x", ColumnType.LONG, 1L, new FilterTuning(true, null, null)); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + + f1 = new EqualityFilter("x", ColumnType.DOUBLE, 1.1, null); + f1_2 = new EqualityFilter("x", ColumnType.DOUBLE, 1.1, null); + f2 = new EqualityFilter("x", ColumnType.DOUBLE, 2.2, null); + f3 = new EqualityFilter("x", ColumnType.DOUBLE, 1.1, new FilterTuning(true, null, null)); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + + f1 = new EqualityFilter("x", ColumnType.FLOAT, 1.1f, null); + f1_2 = new EqualityFilter("x", ColumnType.FLOAT, 1.1f, null); + f2 = new EqualityFilter("x", ColumnType.FLOAT, 2.2f, null); + f3 = new EqualityFilter("x", ColumnType.FLOAT, 1.1f, new FilterTuning(true, null, null)); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + + f1 = new EqualityFilter("x", ColumnType.STRING_ARRAY, new Object[]{"a", "b", null, "c"}, null); + f1_2 = new EqualityFilter("x", ColumnType.STRING_ARRAY, Arrays.asList("a", "b", null, "c"), null); + f2 = new EqualityFilter("x", ColumnType.STRING_ARRAY, new Object[]{"a", "b", "c"}, null); + f3 = new EqualityFilter( + "x", + ColumnType.STRING_ARRAY, + new Object[]{"a", "b", null, "c"}, + new FilterTuning(true, null, null) + ); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + + f1 = new EqualityFilter("x", ColumnType.LONG_ARRAY, new Object[]{100L, 200L, null, 300L}, null); + f1_2 = new EqualityFilter("x", ColumnType.LONG_ARRAY, Arrays.asList(100L, 200L, null, 300L), null); + f2 = new EqualityFilter("x", ColumnType.LONG_ARRAY, new Object[]{100L, null, 200L, 300L}, null); + f3 = new EqualityFilter( + "x", + ColumnType.LONG_ARRAY, + new Object[]{100L, 200L, null, 300L}, + new FilterTuning(true, null, null) + ); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + + f1 = new EqualityFilter("x", ColumnType.DOUBLE_ARRAY, new Object[]{1.001, null, 20.0002, 300.0003}, null); + f1_2 = new EqualityFilter("x", ColumnType.DOUBLE_ARRAY, Arrays.asList(1.001, null, 20.0002, 300.0003), null); + f2 = new EqualityFilter("x", ColumnType.DOUBLE_ARRAY, new Object[]{1.001, 20.0002, 300.0003, null}, null); + f3 = new EqualityFilter( + "x", + ColumnType.DOUBLE_ARRAY, + new Object[]{1.001, null, 20.0002, 300.0003}, + new FilterTuning(true, null, null) + ); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + + NestedDataModule.registerHandlersAndSerde(); + f1 = new EqualityFilter("x", ColumnType.NESTED_DATA, ImmutableMap.of("x", ImmutableList.of(1, 2, 3)), null); + f1_2 = new EqualityFilter("x", ColumnType.NESTED_DATA, ImmutableMap.of("x", ImmutableList.of(1, 2, 3)), null); + f2 = new EqualityFilter("x", ColumnType.NESTED_DATA, ImmutableMap.of("x", ImmutableList.of(1, 2, 3, 4)), null); + f3 = new EqualityFilter( + "x", + ColumnType.NESTED_DATA, + ImmutableMap.of("x", ImmutableList.of(1, 2, 3)), + new FilterTuning(true, null, null) + ); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + } + + @Test + public void testInvalidParameters() + { + Throwable t = Assert.assertThrows( + DruidException.class, + () -> new EqualityFilter(null, ColumnType.STRING, null, null) + ); + Assert.assertEquals("Invalid equality filter, column cannot be null", t.getMessage()); + t = Assert.assertThrows( + DruidException.class, + () -> new EqualityFilter("dim0", null, null, null) + ); + Assert.assertEquals("Invalid equality filter on column [dim0], matchValueType cannot be null", t.getMessage()); + t = Assert.assertThrows( + DruidException.class, + () -> new EqualityFilter("dim0", ColumnType.STRING, null, null) + ); + Assert.assertEquals("Invalid equality filter on column [dim0], matchValue cannot be null", t.getMessage()); + } + + @Test + public void testGetDimensionRangeSet() + { + EqualityFilter filter = new EqualityFilter("x", ColumnType.STRING, "hello", null); + + RangeSet set = TreeRangeSet.create(); + set.add(Range.singleton("hello")); + Assert.assertEquals(set, filter.getDimensionRangeSet("x")); + Assert.assertNull(filter.getDimensionRangeSet("y")); + + ExprEval eval = ExprEval.ofType(ExpressionType.STRING_ARRAY, new Object[]{"abc", "def"}); + filter = new EqualityFilter("x", ColumnType.STRING_ARRAY, eval.value(), null); + set = TreeRangeSet.create(); + set.add(Range.singleton(Arrays.deepToString(eval.asArray()))); + Assert.assertEquals(set, filter.getDimensionRangeSet("x")); + Assert.assertNull(filter.getDimensionRangeSet("y")); + } + + @Test + public void test_equals() + { + EqualsVerifier.forClass(EqualityFilter.class).usingGetClass() + .withNonnullFields( + "column", + "matchValueType", + "matchValueEval", + "matchValue", + "predicateFactory", + "cachedOptimizedFilter" + ) + .withPrefabValues(ColumnType.class, ColumnType.STRING, ColumnType.DOUBLE) + .withIgnoredFields("predicateFactory", "cachedOptimizedFilter", "matchValue") + .verify(); + } + } +} diff --git a/processing/src/test/java/org/apache/druid/segment/filter/NullFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/NullFilterTest.java index eebba4c75f24..54fcb7696e67 100644 --- a/processing/src/test/java/org/apache/druid/segment/filter/NullFilterTest.java +++ b/processing/src/test/java/org/apache/druid/segment/filter/NullFilterTest.java @@ -19,25 +19,26 @@ package org.apache.druid.segment.filter; +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; import com.google.common.base.Function; import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; import nl.jqno.equalsverifier.EqualsVerifier; import org.apache.druid.common.config.NullHandling; +import org.apache.druid.jackson.DefaultObjectMapper; import org.apache.druid.java.util.common.Pair; -import org.apache.druid.query.extraction.MapLookupExtractor; +import org.apache.druid.query.filter.FilterTuning; import org.apache.druid.query.filter.NullFilter; -import org.apache.druid.query.lookup.LookupExtractionFn; -import org.apache.druid.query.lookup.LookupExtractor; import org.apache.druid.segment.IndexBuilder; import org.apache.druid.segment.StorageAdapter; import org.junit.AfterClass; +import org.junit.Assert; import org.junit.Test; import org.junit.runner.RunWith; import org.junit.runners.Parameterized; import java.io.Closeable; -import java.util.Map; +import java.util.Arrays; @RunWith(Parameterized.class) public class NullFilterTest extends BaseFilterTest @@ -175,60 +176,6 @@ public void testNumericColumnNullsAndDefaults() } } - @Test - public void testSelectorWithLookupExtractionFn() - { - /* - static final List DEFAULT_ROWS = ImmutableList.of( - makeDefaultSchemaRow("0", "", ImmutableList.of("a", "b"), "2017-07-25", 0.0, 0.0f, 0L), - makeDefaultSchemaRow("1", "10", ImmutableList.of(), "2017-07-25", 10.1, 10.1f, 100L), - makeDefaultSchemaRow("2", "2", ImmutableList.of(""), "2017-05-25", null, 5.5f, 40L), - makeDefaultSchemaRow("3", "1", ImmutableList.of("a"), "2020-01-25", 120.0245, 110.0f, null), - makeDefaultSchemaRow("4", "abdef", ImmutableList.of("c"), null, 60.0, null, 9001L), - makeDefaultSchemaRow("5", "abc", null, "2020-01-25", 765.432, 123.45f, 12345L) - ); - */ - final Map stringMap = ImmutableMap.of( - "1", "HELLO", - "a", "HELLO", - "abdef", "HELLO", - "abc", "UNKNOWN" - ); - LookupExtractor mapExtractor = new MapLookupExtractor(stringMap, false); - LookupExtractionFn lookupFn = new LookupExtractionFn(mapExtractor, false, null, false, true); - LookupExtractionFn lookupFnRetain = new LookupExtractionFn(mapExtractor, true, null, false, true); - LookupExtractionFn lookupFnReplace = new LookupExtractionFn(mapExtractor, false, "UNKNOWN", false, true); - - if (NullHandling.replaceWithDefault()) { - assertFilterMatches(new NullFilter("dim0", lookupFn, null), ImmutableList.of("0", "2", "3", "4", "5")); - assertFilterMatches(new NullFilter("dim0", lookupFnRetain, null), ImmutableList.of()); - } else { - assertFilterMatches(new NullFilter("dim0", lookupFn, null), ImmutableList.of("0", "2", "3", "4", "5")); - assertFilterMatches(new NullFilter("dim0", lookupFnRetain, null), ImmutableList.of()); - } - - assertFilterMatches(new NullFilter("dim0", lookupFnReplace, null), ImmutableList.of()); - - - final Map stringMapEmpty = ImmutableMap.of( - "1", "" - ); - LookupExtractor mapExtractoryEmpty = new MapLookupExtractor(stringMapEmpty, false); - LookupExtractionFn lookupFnEmpty = new LookupExtractionFn(mapExtractoryEmpty, false, null, false, true); - if (NullHandling.replaceWithDefault()) { - // Nulls and empty strings are considered equivalent - assertFilterMatches( - new NullFilter("dim0", lookupFnEmpty, null), - ImmutableList.of("0", "1", "2", "3", "4", "5") - ); - } else { - assertFilterMatches( - new NullFilter("dim0", lookupFnEmpty, null), - ImmutableList.of("0", "2", "3", "4", "5") - ); - } - } - @Test public void testArrays() { @@ -244,20 +191,41 @@ public void testArrays() "5", .. [null], [123L, 345L], null */ assertFilterMatches( - new NullFilter("arrayString", null, null), + new NullFilter("arrayString", null), ImmutableList.of("2") ); assertFilterMatches( - new NullFilter("arrayLong", null, null), + new NullFilter("arrayLong", null), ImmutableList.of("3") ); assertFilterMatches( - new NullFilter("arrayDouble", null, null), + new NullFilter("arrayDouble", null), ImmutableList.of("5") ); } } + @Test + public void testSerde() throws JsonProcessingException + { + ObjectMapper mapper = new DefaultObjectMapper(); + NullFilter filter = new NullFilter("x", null); + String s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, NullFilter.class)); + } + + @Test + public void testGetCacheKey() + { + NullFilter f1 = new NullFilter("x", null); + NullFilter f1_2 = new NullFilter("x", null); + NullFilter f2 = new NullFilter("y", null); + NullFilter f3 = new NullFilter("x", new FilterTuning(true, 1234, null)); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + } + @Test public void test_equals() { diff --git a/processing/src/test/java/org/apache/druid/segment/filter/RangeFilterTest.java b/processing/src/test/java/org/apache/druid/segment/filter/RangeFilterTest.java deleted file mode 100644 index 622f2ea8c95a..000000000000 --- a/processing/src/test/java/org/apache/druid/segment/filter/RangeFilterTest.java +++ /dev/null @@ -1,1338 +0,0 @@ -/* - * Licensed to the Apache Software Foundation (ASF) under one - * or more contributor license agreements. See the NOTICE file - * distributed with this work for additional information - * regarding copyright ownership. The ASF licenses this file - * to you under the Apache License, Version 2.0 (the - * "License"); you may not use this file except in compliance - * with the License. You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, - * software distributed under the License is distributed on an - * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY - * KIND, either express or implied. See the License for the - * specific language governing permissions and limitations - * under the License. - */ - -package org.apache.druid.segment.filter; - -import com.google.common.base.Function; -import com.google.common.collect.ImmutableList; -import com.google.common.collect.ImmutableMap; -import nl.jqno.equalsverifier.EqualsVerifier; -import org.apache.druid.common.config.NullHandling; -import org.apache.druid.data.input.InputRow; -import org.apache.druid.error.DruidException; -import org.apache.druid.java.util.common.IAE; -import org.apache.druid.java.util.common.Pair; -import org.apache.druid.js.JavaScriptConfig; -import org.apache.druid.query.extraction.ExtractionFn; -import org.apache.druid.query.extraction.JavaScriptExtractionFn; -import org.apache.druid.query.filter.Filter; -import org.apache.druid.query.filter.RangeFilter; -import org.apache.druid.segment.IndexBuilder; -import org.apache.druid.segment.StorageAdapter; -import org.apache.druid.segment.column.ColumnType; -import org.junit.AfterClass; -import org.junit.Assert; -import org.junit.Test; -import org.junit.runner.RunWith; -import org.junit.runners.Parameterized; - -import java.io.Closeable; -import java.util.List; - -@RunWith(Parameterized.class) -public class RangeFilterTest extends BaseFilterTest -{ - private static final List ROWS = - ImmutableList.builder() - .addAll(DEFAULT_ROWS) - .add(makeDefaultSchemaRow( - "6", - "-1000", - ImmutableList.of("a"), - null, - 6.6, - null, - 10L, - new Object[]{"x", "y"}, - new Object[]{100, 200}, - new Object[]{1.1, null, 3.3} - )) - .add(makeDefaultSchemaRow( - "7", - "-10.012", - ImmutableList.of("d"), - null, - null, - 3.0f, - null, - new Object[]{null, "hello", "world"}, - new Object[]{1234, 3456L, null}, - new Object[]{1.23, 4.56, 6.78} - )) - .build(); - - public RangeFilterTest( - String testName, - IndexBuilder indexBuilder, - Function> finisher, - boolean cnf, - boolean optimize - ) - { - super(testName, ROWS, indexBuilder, finisher, cnf, optimize); - } - - @AfterClass - public static void tearDown() throws Exception - { - BaseFilterTest.tearDown(RangeFilterTest.class.getName()); - } - - @Test - public void testLexicographicalMatch() - { - - assertFilterMatches( - new RangeFilter("dim0", ColumnType.STRING, null, "z", false, false, null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - assertFilterMatches( - new RangeFilter("vdim0", ColumnType.STRING, null, "z", false, false, null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - - if (NullHandling.sqlCompatible()) { - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, null, "z", false, false, null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - assertFilterMatches( - new RangeFilter("vdim0", ColumnType.STRING, null, "z", false, false, null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - } else { - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, null, "z", false, false, null, null), - ImmutableList.of("1", "2", "3", "4", "5", "6", "7") - ); - assertFilterMatches( - new RangeFilter("vdim0", ColumnType.STRING, null, "z", false, false, null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - } - - if (isAutoSchema()) { - // auto schema ingests arrays instead of mvds.. this filter doesn't currently support arrays - } else { - assertFilterMatches( - new RangeFilter("dim2", ColumnType.STRING, null, "z", false, false, null, null), - NullHandling.sqlCompatible() - ? ImmutableList.of("0", "2", "3", "4", "6", "7") - : ImmutableList.of("0", "3", "4", "6", "7") - ); - // vdim2 does not exist... - assertFilterMatches( - new RangeFilter("dim3", ColumnType.STRING, null, "z", false, false, null, null), - ImmutableList.of() - ); - } - } - - @Test - public void testLexicographicMatchWithEmptyString() - { - if (NullHandling.sqlCompatible()) { - assertFilterMatches( - new RangeFilter("dim0", ColumnType.STRING, "", "z", false, false, null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, "", "z", false, false, null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - if (!isAutoSchema()) { - // auto schema ingests arrays which are currently incompatible with the range filter - assertFilterMatches( - new RangeFilter("dim2", ColumnType.STRING, "", "z", false, false, null, null), - ImmutableList.of("0", "2", "3", "4", "6", "7") - ); - } - assertFilterMatches( - new RangeFilter("dim3", ColumnType.STRING, "", "z", false, false, null, null), - ImmutableList.of() - ); - } else { - assertFilterMatches( - new RangeFilter("dim0", ColumnType.STRING, "", "z", false, false, null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, "", "z", false, false, null, null), - ImmutableList.of("1", "2", "3", "4", "5", "6", "7") - ); - if (!isAutoSchema()) { - // auto schema ingests arrays which are currently incompatible with the range filter - assertFilterMatches( - new RangeFilter("dim2", ColumnType.STRING, "", "z", false, false, null, null), - ImmutableList.of("0", "3", "4", "6", "7") - ); - } - assertFilterMatches( - new RangeFilter("dim3", ColumnType.STRING, "", "z", false, false, null, null), - ImmutableList.of() - ); - } - } - - @Test - public void testLexicographicMatchNull() - { - - if (NullHandling.replaceWithDefault()) { - // in default value mode this is null on both ends... - Throwable t = Assert.assertThrows( - DruidException.class, - () -> assertFilterMatches( - new RangeFilter("dim0", ColumnType.STRING, "", "", false, false, null, null), - ImmutableList.of() - ) - ); - Assert.assertEquals( - "Invalid range filter on column [dim0], lower and upper cannot be null at the same time", - t.getMessage() - ); - } else { - assertFilterMatches( - new RangeFilter("dim0", ColumnType.STRING, "", "", false, false, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, "", "", false, false, null, null), - ImmutableList.of("0") - ); - // still matches even with auto-schema because match-values are upcast to array types - assertFilterMatches( - new RangeFilter("dim2", ColumnType.STRING, "", "", false, false, null, null), - ImmutableList.of("2") - ); - } - } - - @Test - public void testLexicographicMatchMissingColumn() - { - assertFilterMatches( - new RangeFilter("dim3", ColumnType.STRING, "", "z", false, false, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("dim3", ColumnType.STRING, "a", null, false, true, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("dim3", ColumnType.STRING, null, "z", false, true, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("dim3", ColumnType.STRING, "", "z", true, false, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("dim3", ColumnType.STRING, "", "z", false, true, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("dim3", ColumnType.STRING, null, "z", false, false, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("dim3", ColumnType.STRING, null, "z", false, true, null, null), - ImmutableList.of() - ); - } - - - @Test - public void testLexicographicMatchTooStrict() - { - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, "abc", "abc", true, false, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, "abc", "abc", true, true, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, "abc", "abc", false, true, null, null), - ImmutableList.of() - ); - } - - @Test - public void testLexicographicMatchExactlySingleValue() - { - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, "abc", "abc", false, false, null, null), - ImmutableList.of("5") - ); - } - - @Test - public void testLexicographicMatchSurroundingSingleValue() - { - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, "ab", "abd", true, true, null, null), - ImmutableList.of("5") - ); - } - - @Test - public void testLexicographicMatchNoUpperLimit() - { - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, "ab", null, true, true, null, null), - ImmutableList.of("4", "5") - ); - } - - @Test - public void testLexicographicMatchNoLowerLimit() - { - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, null, "abd", true, true, null, null), - NullHandling.replaceWithDefault() - ? ImmutableList.of("1", "2", "3", "5", "6", "7") - : ImmutableList.of("0", "1", "2", "3", "5", "6", "7") - ); - } - - @Test - public void testLexicographicMatchNumbers() - { - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, "1", "3", false, false, null, null), - ImmutableList.of("1", "2", "3") - ); - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, "1", "3", true, true, null, null), - ImmutableList.of("1", "2") - ); - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, "-1", "3", true, true, null, null), - ImmutableList.of("1", "2", "3", "6", "7") - ); - } - - - @Test - public void testNumericMatchBadParameters() - { - Throwable t = Assert.assertThrows( - DruidException.class, - () -> assertFilterMatches( - new RangeFilter(null, ColumnType.DOUBLE, "1234", "", false, false, null, null), - ImmutableList.of() - ) - ); - Assert.assertEquals( - "Invalid range filter, column cannot be null", - t.getMessage() - ); - t = Assert.assertThrows( - DruidException.class, - () -> assertFilterMatches( - new RangeFilter("dim0", null, "1234", "", false, false, null, null), - ImmutableList.of() - ) - ); - Assert.assertEquals( - "Invalid range filter on column [dim0], matchValueType cannot be null", - t.getMessage() - ); - t = Assert.assertThrows( - DruidException.class, - () -> assertFilterMatches( - new RangeFilter("dim0", ColumnType.DOUBLE, null, null, false, false, null, null), - ImmutableList.of() - ) - ); - Assert.assertEquals( - "Invalid range filter on column [dim0], lower and upper cannot be null at the same time", - t.getMessage() - ); - - t = Assert.assertThrows( - DruidException.class, - () -> assertFilterMatches( - new RangeFilter("dim0", ColumnType.DOUBLE, "1234", "", false, false, null, null), - ImmutableList.of() - ) - ); - Assert.assertEquals( - "Invalid range filter on column [dim0], upper bound [] cannot be parsed as specified match value type [DOUBLE]", - t.getMessage() - ); - - t = Assert.assertThrows( - DruidException.class, - () -> assertFilterMatches( - new RangeFilter("dim0", ColumnType.DOUBLE, "abc", "1234", false, false, null, null), - ImmutableList.of() - ) - ); - Assert.assertEquals( - "Invalid range filter on column [dim0], lower bound [abc] cannot be parsed as specified match value type [DOUBLE]", - t.getMessage() - ); - } - - @Test - public void testNumericMatchTooStrict() - { - assertFilterMatches( - new RangeFilter("dim1", ColumnType.LONG, 2L, 2L, true, false, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("dim1", ColumnType.LONG, 2L, 2L, true, true, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("dim1", ColumnType.LONG, 2L, 2L, false, true, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("d0", ColumnType.LONG, 2L, 3L, false, true, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("d0", ColumnType.DOUBLE, 2L, 3L, false, true, null, null), - ImmutableList.of() - ); - - assertFilterMatches( - new RangeFilter("f0", ColumnType.LONG, 2L, 3L, false, true, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("f0", ColumnType.DOUBLE, 2L, 3L, false, true, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("l0", ColumnType.LONG, 2L, 3L, false, true, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("l0", ColumnType.DOUBLE, 2L, 3L, false, true, null, null), - ImmutableList.of() - ); - } - - @Test - public void testNumericMatchVirtualColumn() - { - assertFilterMatches( - new RangeFilter("expr", ColumnType.LONG, 1L, 2L, false, false, null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - assertFilterMatches( - new RangeFilter("expr", ColumnType.DOUBLE, 1.1, 2.0, false, false, null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - assertFilterMatches( - new RangeFilter("expr", ColumnType.FLOAT, 1.1f, 2.0f, false, false, null, null), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - - assertFilterMatches( - new RangeFilter("expr", ColumnType.LONG, 2L, 3L, false, false, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("expr", ColumnType.DOUBLE, 2.0, 3.0, false, false, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("expr", ColumnType.FLOAT, 2.0f, 3.0f, false, false, null, null), - ImmutableList.of() - ); - } - - @Test - public void testNumericMatchExactlySingleValue() - { - assertFilterMatches( - new RangeFilter("dim1", ColumnType.LONG, 2L, 2L, false, false, null, null), - ImmutableList.of("2") - ); - - assertFilterMatches( - new RangeFilter("dim1", ColumnType.DOUBLE, -10.012, -10.012, false, false, null, null), - ImmutableList.of("7") - ); - assertFilterMatches( - new RangeFilter("d0", ColumnType.DOUBLE, 120.0245, 120.0245, false, false, null, null), - ImmutableList.of("3") - ); - assertFilterMatches( - new RangeFilter("d0", ColumnType.FLOAT, 120.0245f, 120.0245f, false, false, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("d0", ColumnType.FLOAT, 60.0f, 60.0f, false, false, null, null), - ImmutableList.of("4") - ); - assertFilterMatches( - new RangeFilter("f0", ColumnType.DOUBLE, 10.1, 10.1, false, false, null, null), - ImmutableList.of() - ); - assertFilterMatches( - new RangeFilter("f0", ColumnType.FLOAT, 10.1f, 10.1f, false, false, null, null), - ImmutableList.of("1") - ); - assertFilterMatches( - new RangeFilter("l0", ColumnType.LONG, 12345L, 12345L, false, false, null, null), - ImmutableList.of("5") - ); - assertFilterMatches( - new RangeFilter("l0", ColumnType.DOUBLE, 12345.0, 12345.0, false, false, null, null), - ImmutableList.of("5") - ); - assertFilterMatches( - new RangeFilter("l0", ColumnType.FLOAT, 12345.0f, 12345.0f, false, false, null, null), - ImmutableList.of("5") - ); - } - - @Test - public void testNumericMatchSurroundingSingleValue() - { - assertFilterMatches( - new RangeFilter("dim1", ColumnType.LONG, 1L, 3L, true, true, null, null), - ImmutableList.of("2") - ); - - assertFilterMatches( - new RangeFilter("dim1", ColumnType.LONG, -11L, -10L, false, false, null, null), - ImmutableList.of("7") - ); - assertFilterMatches( - new RangeFilter("d0", ColumnType.DOUBLE, 120.0, 120.03, false, false, null, null), - ImmutableList.of("3") - ); - assertFilterMatches( - new RangeFilter("d0", ColumnType.FLOAT, 120.02f, 120.03f, false, false, null, null), - ImmutableList.of("3") - ); - assertFilterMatches( - new RangeFilter("d0", ColumnType.FLOAT, 59.5f, 60.01f, false, false, null, null), - ImmutableList.of("4") - ); - assertFilterMatches( - new RangeFilter("f0", ColumnType.DOUBLE, 10.0, 10.2, false, false, null, null), - ImmutableList.of("1") - ); - assertFilterMatches( - new RangeFilter("f0", ColumnType.FLOAT, 10.05f, 10.11f, false, false, null, null), - ImmutableList.of("1") - ); - assertFilterMatches( - new RangeFilter("l0", ColumnType.LONG, 12344L, 12346L, false, false, null, null), - ImmutableList.of("5") - ); - assertFilterMatches( - new RangeFilter("l0", ColumnType.DOUBLE, 12344.0, 12345.5, false, false, null, null), - ImmutableList.of("5") - ); - assertFilterMatches( - new RangeFilter("l0", ColumnType.FLOAT, 12344.0f, 12345.5f, false, false, null, null), - ImmutableList.of("5") - ); - } - - @Test - public void testNumericMatchNoUpperLimit() - { - assertFilterMatches( - new RangeFilter("dim1", ColumnType.LONG, 1L, null, true, true, null, null), - ImmutableList.of("1", "2") - ); - assertFilterMatches( - new RangeFilter("d0", ColumnType.DOUBLE, 1.0, null, true, true, null, null), - ImmutableList.of("1", "3", "4", "5", "6") - ); - assertFilterMatches( - new RangeFilter("f0", ColumnType.FLOAT, 1.0f, null, true, true, null, null), - ImmutableList.of("1", "2", "3", "5", "7") - ); - assertFilterMatches( - new RangeFilter("l0", ColumnType.LONG, 1L, null, true, true, null, null), - ImmutableList.of("1", "2", "4", "5", "6") - ); - } - - @Test - public void testNumericMatchNoLowerLimit() - { - // strings are wierd... - assertFilterMatches( - new RangeFilter("dim1", ColumnType.LONG, null, 2L, false, true, null, null), - NullHandling.replaceWithDefault() - ? ImmutableList.of("3", "4", "5", "6", "7") - : ImmutableList.of("0", "3", "4", "5", "6", "7") - ); - // numbers are sane though - assertFilterMatches( - new RangeFilter("d0", ColumnType.DOUBLE, null, 10.0, false, true, null, null), - canTestNumericNullsAsDefaultValues ? ImmutableList.of("0", "2", "6", "7") : ImmutableList.of("0", "6") - ); - assertFilterMatches( - new RangeFilter("f0", ColumnType.FLOAT, null, 50.5, false, true, null, null), - canTestNumericNullsAsDefaultValues - ? ImmutableList.of("0", "1", "2", "4", "6", "7") - : ImmutableList.of("0", "1", "2", "7") - ); - assertFilterMatches( - new RangeFilter("l0", ColumnType.LONG, null, 100L, false, true, null, null), - canTestNumericNullsAsDefaultValues ? ImmutableList.of("0", "2", "3", "6", "7") : ImmutableList.of("0", "2", "6") - ); - } - - @Test - public void testNumericMatchWithNegatives() - { - assertFilterMatches( - new RangeFilter("dim1", ColumnType.LONG, -2000L, 3L, true, true, null, null), - ImmutableList.of("2", "3", "6", "7") - ); - } - - @Test - public void testNumericNullsAndZeros() - { - assertFilterMatches( - new RangeFilter( - "d0", - ColumnType.DOUBLE, - 0.0, - 1.1, - false, - false, - null, - null - ), - canTestNumericNullsAsDefaultValues ? ImmutableList.of("0", "2", "7") : ImmutableList.of("0") - ); - - assertFilterMatches( - new RangeFilter( - "f0", - ColumnType.FLOAT, - 0.0, - 1.0, - false, - false, - null, - null - ), - canTestNumericNullsAsDefaultValues ? ImmutableList.of("0", "4", "6") : ImmutableList.of("0") - ); - - assertFilterMatches( - new RangeFilter( - "l0", - ColumnType.LONG, - 0L, - 1L, - false, - false, - null, - null - ), - NullHandling.replaceWithDefault() && canTestNumericNullsAsDefaultValues - ? ImmutableList.of("0", "3", "7") - : ImmutableList.of("0") - ); - } - - @Test - public void testVirtualNumericNullsAndZeros() - { - assertFilterMatches( - new RangeFilter( - "vd0", - ColumnType.DOUBLE, - 0.0, - 1.0, - false, - false, - null, - null - ), - canTestNumericNullsAsDefaultValues ? ImmutableList.of("0", "2", "7") : ImmutableList.of("0") - ); - - assertFilterMatches( - new RangeFilter( - "vf0", - ColumnType.FLOAT, - 0.0, - 1.0, - false, - false, - null, - null - ), - canTestNumericNullsAsDefaultValues ? ImmutableList.of("0", "4", "6") : ImmutableList.of("0") - ); - - assertFilterMatches( - new RangeFilter( - "vl0", - ColumnType.LONG, - 0L, - 1L, - false, - false, - null, - null - ), - NullHandling.replaceWithDefault() && canTestNumericNullsAsDefaultValues - ? ImmutableList.of("0", "3", "7") - : ImmutableList.of("0") - ); - } - - @Test - public void testNumericNulls() - { - assertFilterMatches( - new RangeFilter( - "f0", - ColumnType.FLOAT, - 1.0, - null, - false, - false, - null, - null - ), - ImmutableList.of("1", "2", "3", "5", "7") - ); - assertFilterMatches( - new RangeFilter( - "d0", - ColumnType.DOUBLE, - 1.0, - null, - false, - false, - null, - null - ), - ImmutableList.of("1", "3", "4", "5", "6") - ); - assertFilterMatches( - new RangeFilter( - "l0", - ColumnType.LONG, - 1L, - null, - false, - false, - null, - null - ), - ImmutableList.of("1", "2", "4", "5", "6") - ); - } - - @Test - public void testMatchWithExtractionFn() - { - String extractionJsFn = "function(str) { return 'super-' + str; }"; - ExtractionFn superFn = new JavaScriptExtractionFn(extractionJsFn, false, JavaScriptConfig.getEnabledInstance()); - - String nullJsFn = "function(str) { return null; }"; - ExtractionFn makeNullFn = new JavaScriptExtractionFn(nullJsFn, false, JavaScriptConfig.getEnabledInstance()); - - assertFilterMatches( - new RangeFilter("dim0", ColumnType.STRING, "", "z", false, false, makeNullFn, null), - ImmutableList.of() - ); - - assertFilterMatches( - new RangeFilter( - "dim1", - ColumnType.STRING, - "super-ab", - "super-abd", - true, - true, - superFn, - null - ), - ImmutableList.of("5") - ); - - assertFilterMatches( - new RangeFilter("dim1", ColumnType.STRING, "super-0", "super-10", false, false, superFn, null), - ImmutableList.of("1", "3") - ); - - // auto schema ingests arrays instead of MVDs which aren't compatible with list filtered virtual column - if (!isAutoSchema()) { - assertFilterMatches( - new RangeFilter( - "dim2", - ColumnType.STRING, - "super-", - "super-zzzzzz", - false, - false, - superFn, - null - ), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - - if (NullHandling.replaceWithDefault()) { - assertFilterMatches( - new RangeFilter( - "dim2", - ColumnType.STRING, - "super-null", - "super-null", - false, - false, - superFn, - null - ), - ImmutableList.of("1", "2", "5") - ); - assertFilterMatches( - new RangeFilter( - "dim2", - ColumnType.STRING, - "super-null", - "super-null", - false, - false, - superFn, - null - ), - ImmutableList.of("1", "2", "5") - ); - } else { - assertFilterMatches( - new RangeFilter( - "dim2", - ColumnType.STRING, - "super-null", - "super-null", - false, - false, - superFn, - null - ), - ImmutableList.of("1", "5") - ); - assertFilterMatches( - new RangeFilter("dim2", ColumnType.STRING, "super-", "super-", false, false, superFn, null), - ImmutableList.of("2") - ); - assertFilterMatches( - new RangeFilter( - "dim2", - ColumnType.STRING, - "super-null", - "super-null", - false, - false, - superFn, - null - ), - ImmutableList.of("1", "5") - ); - assertFilterMatches( - new RangeFilter("dim2", ColumnType.STRING, "super-", "super-", false, false, superFn, null), - ImmutableList.of("2") - ); - } - } - - assertFilterMatches( - new RangeFilter( - "dim3", - ColumnType.STRING, - "super-null", - "super-null", - false, - false, - superFn, - null - ), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - - assertFilterMatches( - new RangeFilter( - "dim4", - ColumnType.STRING, - "super-null", - "super-null", - false, - false, - superFn, - null - ), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - - assertFilterMatches( - new RangeFilter("dim4", ColumnType.STRING, "super-null", "super-null", false, false, superFn, null), - ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") - ); - } - - @Test - public void testListFilteredVirtualColumn() - { - assertFilterMatchesSkipVectorize( - new RangeFilter("allow-dim0", ColumnType.STRING, "0", "2", false, false, null, null), - ImmutableList.of() - ); - assertFilterMatchesSkipVectorize( - new RangeFilter("allow-dim0", ColumnType.STRING, "0", "6", false, false, null, null), - ImmutableList.of("3", "4") - ); - assertFilterMatchesSkipVectorize( - new RangeFilter("allow-dim0", ColumnType.STRING, null, "6", false, false, null, null), - ImmutableList.of("3", "4") - ); - - assertFilterMatchesSkipVectorize( - new RangeFilter("deny-dim0", ColumnType.STRING, "0", "6", false, false, null, null), - ImmutableList.of("0", "1", "2", "5", "6") - ); - assertFilterMatchesSkipVectorize( - new RangeFilter("deny-dim0", ColumnType.STRING, "3", "4", false, false, null, null), - ImmutableList.of() - ); - assertFilterMatchesSkipVectorize( - new RangeFilter("deny-dim0", ColumnType.STRING, null, "6", false, false, null, null), - ImmutableList.of("0", "1", "2", "5", "6") - ); - - if (isAutoSchema()) { - // bail out, auto ingests arrays instead of mvds and this virtual column is for mvd stuff - return; - } - - assertFilterMatchesSkipVectorize( - new RangeFilter("allow-dim2", ColumnType.STRING, "a", "c", false, false, null, null), - ImmutableList.of("0", "3", "6") - ); - assertFilterMatchesSkipVectorize( - new RangeFilter("allow-dim2", ColumnType.STRING, "c", "z", false, false, null, null), - ImmutableList.of() - ); - assertFilterMatchesSkipVectorize( - new RangeFilter("allow-dim2", ColumnType.STRING, null, "z", false, false, null, null), - ImmutableList.of("0", "3", "6") - ); - - assertFilterMatchesSkipVectorize( - new RangeFilter("deny-dim2", ColumnType.STRING, "a", "b", false, true, null, null), - ImmutableList.of() - ); - assertFilterMatchesSkipVectorize( - new RangeFilter("deny-dim2", ColumnType.STRING, "c", "z", false, false, null, null), - ImmutableList.of("4", "7") - ); - - assertFilterMatchesSkipVectorize( - new RangeFilter("deny-dim2", ColumnType.STRING, null, "z", false, false, null, null), - NullHandling.replaceWithDefault() ? ImmutableList.of("0", "4", "7") : ImmutableList.of("0", "2", "4", "7") - ); - } - - @Test - public void testRequiredColumnRewrite() - { - RangeFilter filter = new RangeFilter("dim0", ColumnType.STRING, "abc", "def", false, false, null, null); - RangeFilter filter2 = new RangeFilter("dim1", ColumnType.STRING, "abc", "def", false, false, null, null); - Assert.assertTrue(filter.supportsRequiredColumnRewrite()); - Assert.assertTrue(filter2.supportsRequiredColumnRewrite()); - - Filter rewrittenFilter = filter.rewriteRequiredColumns(ImmutableMap.of("dim0", "dim1")); - Assert.assertEquals(filter2, rewrittenFilter); - - Throwable t = Assert.assertThrows( - IAE.class, - () -> filter.rewriteRequiredColumns(ImmutableMap.of("invalidName", "dim1")) - ); - - Assert.assertEquals( - "Received a non-applicable rewrite: {invalidName=dim1}, filter's dimension: dim0", - t.getMessage() - ); - } - - @Test - public void testArrayRanges() - { - if (isAutoSchema()) { - // only auto schema supports array columns currently, this means the match value will need to be coerceable to - // the column value type... - - /* dim0 .. arrayString arrayLong arrayDouble - "0", .. ["a", "b", "c"], [1L, 2L, 3L], [1.1, 2.2, 3.3] - "1", .. [], [], [1.1, 2.2, 3.3] - "2", .. null, [1L, 2L, 3L], [null] - "3", .. ["a", "b", "c"], null, [] - "4", .. ["c", "d"], [null], [-1.1, -333.3] - "5", .. [null], [123L, 345L], null - "6", .. ["x", "y"], [100, 200], [1.1, null, 3.3] - "7", .. [null, "hello", "world"], [1234, 3456L, null], [1.23, 4.56, 6.78] - */ - assertFilterMatches( - new RangeFilter( - "arrayString", - ColumnType.STRING_ARRAY, - new Object[]{"a", "b", "c"}, - new Object[]{"a", "b", "c"}, - false, - false, - null, - null - ), - ImmutableList.of("0", "3") - ); - assertFilterMatches( - new RangeFilter( - "arrayString", - ColumnType.STRING_ARRAY, - null, - new Object[]{"a", "b", "c"}, - false, - false, - null, - null - ), - ImmutableList.of("0", "1", "3", "5", "7") - ); - - assertFilterMatches( - new RangeFilter( - "arrayString", - ColumnType.STRING_ARRAY, - new Object[]{"a", "b", "c"}, - null, - true, - false, - null, - null - ), - ImmutableList.of("4", "6") - ); - - assertFilterMatches( - new RangeFilter( - "arrayString", - ColumnType.STRING_ARRAY, - null, - new Object[]{"a", "b", "c"}, - false, - true, - null, - null - ), - ImmutableList.of("1", "5", "7") - ); - - assertFilterMatches( - new RangeFilter( - "arrayString", - ColumnType.STRING_ARRAY, - new Object[]{"a", "b"}, - new Object[]{"a", "b", "c", "d"}, - true, - true, - null, - null - ), - ImmutableList.of("0", "3") - ); - - assertFilterMatches( - new RangeFilter( - "arrayString", - ColumnType.STRING_ARRAY, - new Object[]{"c", "d"}, - new Object[]{"c", "d", "e"}, - false, - true, - null, - null - ), - ImmutableList.of("4") - ); - - assertFilterMatches( - new RangeFilter( - "arrayString", - ColumnType.STRING_ARRAY, - null, - new Object[]{}, - false, - false, - null, - null - ), - ImmutableList.of("1") - ); - - assertFilterMatches( - new RangeFilter( - "arrayString", - ColumnType.STRING_ARRAY, - null, - new Object[]{null}, - false, - false, - null, - null - ), - ImmutableList.of("1", "5") - ); - - assertFilterMatches( - new RangeFilter( - "arrayLong", - ColumnType.LONG_ARRAY, - null, - new Object[]{}, - false, - false, - null, - null - ), - ImmutableList.of("1") - ); - - assertFilterMatches( - new RangeFilter( - "arrayLong", - ColumnType.LONG_ARRAY, - new Object[]{}, - null, - true, - false, - null, - null - ), - ImmutableList.of("0", "2", "4", "5", "6", "7") - ); - - assertFilterMatches( - new RangeFilter( - "arrayLong", - ColumnType.LONG_ARRAY, - null, - new Object[]{null}, - false, - false, - null, - null - ), - ImmutableList.of("1", "4") - ); - - assertFilterMatches( - new RangeFilter( - "arrayLong", - ColumnType.LONG_ARRAY, - new Object[]{1L, 2L, 3L}, - new Object[]{1L, 2L, 3L}, - false, - false, - null, - null - ), - ImmutableList.of("0", "2") - ); - - - assertFilterMatches( - new RangeFilter( - "arrayLong", - ColumnType.LONG_ARRAY, - null, - new Object[]{1L, 2L, 3L}, - false, - true, - null, - null - ), - ImmutableList.of("1", "4") - ); - - assertFilterMatches( - new RangeFilter( - "arrayLong", - ColumnType.LONG_ARRAY, - new Object[]{1L, 2L, 3L}, - null, - true, - false, - null, - null - ), - ImmutableList.of("5", "6", "7") - ); - - // empties and nulls still sort before numbers - assertFilterMatches( - new RangeFilter( - "arrayLong", - ColumnType.LONG_ARRAY, - null, - new Object[]{-1L}, - false, - false, - null, - null - ), - ImmutableList.of("1", "4") - ); - - assertFilterMatches( - new RangeFilter( - "arrayDouble", - ColumnType.DOUBLE_ARRAY, - null, - new Object[]{}, - false, - false, - null, - null - ), - ImmutableList.of("3") - ); - - assertFilterMatches( - new RangeFilter( - "arrayDouble", - ColumnType.DOUBLE_ARRAY, - new Object[]{}, - null, - true, - false, - null, - null - ), - ImmutableList.of("0", "1", "2", "4", "6", "7") - ); - - assertFilterMatches( - new RangeFilter( - "arrayDouble", - ColumnType.DOUBLE_ARRAY, - null, - new Object[]{null}, - false, - false, - null, - null - ), - ImmutableList.of("2", "3") - ); - - assertFilterMatches( - new RangeFilter( - "arrayDouble", - ColumnType.DOUBLE_ARRAY, - new Object[]{1.1, 2.2, 3.3}, - new Object[]{1.1, 2.2, 3.3}, - false, - false, - null, - null - ), - ImmutableList.of("0", "1") - ); - assertFilterMatches( - new RangeFilter( - "arrayDouble", - ColumnType.DOUBLE_ARRAY, - new Object[]{1.1, 2.2, 3.3}, - null, - true, - false, - null, - null - ), - ImmutableList.of("7") - ); - - assertFilterMatches( - new RangeFilter( - "arrayDouble", - ColumnType.DOUBLE_ARRAY, - null, - new Object[]{1.1, 2.2, 3.3}, - true, - false, - null, - null - ), - ImmutableList.of("0", "1", "2", "3", "4", "6") - ); - - // empties and nulls sort before numbers - assertFilterMatches( - new RangeFilter( - "arrayDouble", - ColumnType.DOUBLE_ARRAY, - null, - new Object[]{0.0}, - true, - false, - null, - null - ), - ImmutableList.of("2", "3", "4") - ); - } - } - - @Test - public void test_equals() - { - EqualsVerifier.forClass(RangeFilter.class) - .withNonnullFields("column", "matchValueType") - .withIgnoredFields( - "matchValueExpressionType", - "lowerEval", - "upperEval", - "cachedOptimizedFilter", - "stringPredicateSupplier", - "longPredicateSupplier", - "floatPredicateSupplier", - "doublePredicateSupplier", - "arrayPredicates", - "typeDetectingArrayPredicateSupplier" - ) - .withPrefabValues(ColumnType.class, ColumnType.STRING, ColumnType.DOUBLE) - .usingGetClass() - .verify(); - } -} diff --git a/processing/src/test/java/org/apache/druid/segment/filter/RangeFilterTests.java b/processing/src/test/java/org/apache/druid/segment/filter/RangeFilterTests.java new file mode 100644 index 000000000000..6313ad93908b --- /dev/null +++ b/processing/src/test/java/org/apache/druid/segment/filter/RangeFilterTests.java @@ -0,0 +1,1426 @@ +/* + * Licensed to the Apache Software Foundation (ASF) under one + * or more contributor license agreements. See the NOTICE file + * distributed with this work for additional information + * regarding copyright ownership. The ASF licenses this file + * to you under the Apache License, Version 2.0 (the + * "License"); you may not use this file except in compliance + * with the License. You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, + * software distributed under the License is distributed on an + * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY + * KIND, either express or implied. See the License for the + * specific language governing permissions and limitations + * under the License. + */ + +package org.apache.druid.segment.filter; + +import com.fasterxml.jackson.core.JsonProcessingException; +import com.fasterxml.jackson.databind.ObjectMapper; +import com.google.common.base.Function; +import com.google.common.collect.BoundType; +import com.google.common.collect.ImmutableList; +import com.google.common.collect.ImmutableMap; +import com.google.common.collect.Range; +import com.google.common.collect.RangeSet; +import com.google.common.collect.TreeRangeSet; +import nl.jqno.equalsverifier.EqualsVerifier; +import org.apache.druid.common.config.NullHandling; +import org.apache.druid.data.input.InputRow; +import org.apache.druid.error.DruidException; +import org.apache.druid.jackson.DefaultObjectMapper; +import org.apache.druid.java.util.common.IAE; +import org.apache.druid.java.util.common.Pair; +import org.apache.druid.math.expr.ExprEval; +import org.apache.druid.math.expr.ExpressionType; +import org.apache.druid.query.filter.Filter; +import org.apache.druid.query.filter.FilterTuning; +import org.apache.druid.query.filter.RangeFilter; +import org.apache.druid.segment.IndexBuilder; +import org.apache.druid.segment.StorageAdapter; +import org.apache.druid.segment.column.ColumnType; +import org.apache.druid.testing.InitializedNullHandlingTest; +import org.junit.AfterClass; +import org.junit.Assert; +import org.junit.Test; +import org.junit.experimental.runners.Enclosed; +import org.junit.runner.RunWith; +import org.junit.runners.Parameterized; + +import java.io.Closeable; +import java.util.Arrays; +import java.util.List; + +@RunWith(Enclosed.class) +public class RangeFilterTests +{ + @RunWith(Parameterized.class) + public static class RangeFilterTest extends BaseFilterTest + { + private static final List ROWS = + ImmutableList.builder() + .addAll(DEFAULT_ROWS) + .add(makeDefaultSchemaRow( + "6", + "-1000", + ImmutableList.of("a"), + null, + 6.6, + null, + 10L, + new Object[]{"x", "y"}, + new Object[]{100, 200}, + new Object[]{1.1, null, 3.3} + )) + .add(makeDefaultSchemaRow( + "7", + "-10.012", + ImmutableList.of("d"), + null, + null, + 3.0f, + null, + new Object[]{null, "hello", "world"}, + new Object[]{1234, 3456L, null}, + new Object[]{1.23, 4.56, 6.78} + )) + .build(); + + public RangeFilterTest( + String testName, + IndexBuilder indexBuilder, + Function> finisher, + boolean cnf, + boolean optimize + ) + { + super(testName, ROWS, indexBuilder, finisher, cnf, optimize); + } + + @AfterClass + public static void tearDown() throws Exception + { + BaseFilterTest.tearDown(RangeFilterTest.class.getName()); + } + + @Test + public void testLexicographicalMatch() + { + + assertFilterMatches( + new RangeFilter("dim0", ColumnType.STRING, null, "z", false, false, null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + assertFilterMatches( + new RangeFilter("vdim0", ColumnType.STRING, null, "z", false, false, null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + + if (NullHandling.sqlCompatible()) { + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, null, "z", false, false, null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + assertFilterMatches( + new RangeFilter("vdim0", ColumnType.STRING, null, "z", false, false, null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } else { + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, null, "z", false, false, null), + ImmutableList.of("1", "2", "3", "4", "5", "6", "7") + ); + assertFilterMatches( + new RangeFilter("vdim0", ColumnType.STRING, null, "z", false, false, null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + } + + if (isAutoSchema()) { + // auto schema ingests arrays instead of mvds.. this filter doesn't currently support arrays + } else { + assertFilterMatches( + new RangeFilter("dim2", ColumnType.STRING, null, "z", false, false, null), + NullHandling.sqlCompatible() + ? ImmutableList.of("0", "2", "3", "4", "6", "7") + : ImmutableList.of("0", "3", "4", "6", "7") + ); + // vdim2 does not exist... + assertFilterMatches( + new RangeFilter("dim3", ColumnType.STRING, null, "z", false, false, null), + ImmutableList.of() + ); + } + } + + @Test + public void testLexicographicMatchWithEmptyString() + { + if (NullHandling.sqlCompatible()) { + assertFilterMatches( + new RangeFilter("dim0", ColumnType.STRING, "", "z", false, false, null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, "", "z", false, false, null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + if (!isAutoSchema()) { + // auto schema ingests arrays which are currently incompatible with the range filter + assertFilterMatches( + new RangeFilter("dim2", ColumnType.STRING, "", "z", false, false, null), + ImmutableList.of("0", "2", "3", "4", "6", "7") + ); + } + assertFilterMatches( + new RangeFilter("dim3", ColumnType.STRING, "", "z", false, false, null), + ImmutableList.of() + ); + } else { + assertFilterMatches( + new RangeFilter("dim0", ColumnType.STRING, "", "z", false, false, null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, "", "z", false, false, null), + ImmutableList.of("1", "2", "3", "4", "5", "6", "7") + ); + if (!isAutoSchema()) { + // auto schema ingests arrays which are currently incompatible with the range filter + assertFilterMatches( + new RangeFilter("dim2", ColumnType.STRING, "", "z", false, false, null), + ImmutableList.of("0", "3", "4", "6", "7") + ); + } + assertFilterMatches( + new RangeFilter("dim3", ColumnType.STRING, "", "z", false, false, null), + ImmutableList.of() + ); + } + } + + @Test + public void testLexicographicMatchNull() + { + + if (NullHandling.replaceWithDefault()) { + // in default value mode this is null on both ends... + Throwable t = Assert.assertThrows( + DruidException.class, + () -> assertFilterMatches( + new RangeFilter("dim0", ColumnType.STRING, "", "", false, false, null), + ImmutableList.of() + ) + ); + Assert.assertEquals( + "Invalid range filter on column [dim0], lower and upper cannot be null at the same time", + t.getMessage() + ); + } else { + assertFilterMatches( + new RangeFilter("dim0", ColumnType.STRING, "", "", false, false, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, "", "", false, false, null), + ImmutableList.of("0") + ); + // still matches even with auto-schema because match-values are upcast to array types + assertFilterMatches( + new RangeFilter("dim2", ColumnType.STRING, "", "", false, false, null), + ImmutableList.of("2") + ); + } + } + + @Test + public void testLexicographicMatchMissingColumn() + { + assertFilterMatches( + new RangeFilter("dim3", ColumnType.STRING, "", "z", false, false, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("dim3", ColumnType.STRING, "a", null, false, true, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("dim3", ColumnType.STRING, null, "z", false, true, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("dim3", ColumnType.STRING, "", "z", true, false, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("dim3", ColumnType.STRING, "", "z", false, true, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("dim3", ColumnType.STRING, null, "z", false, false, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("dim3", ColumnType.STRING, null, "z", false, true, null), + ImmutableList.of() + ); + } + + + @Test + public void testLexicographicMatchTooStrict() + { + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, "abc", "abc", true, false, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, "abc", "abc", true, true, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, "abc", "abc", false, true, null), + ImmutableList.of() + ); + } + + @Test + public void testLexicographicMatchExactlySingleValue() + { + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, "abc", "abc", false, false, null), + ImmutableList.of("5") + ); + } + + @Test + public void testLexicographicMatchSurroundingSingleValue() + { + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, "ab", "abd", true, true, null), + ImmutableList.of("5") + ); + } + + @Test + public void testLexicographicMatchNoUpperLimit() + { + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, "ab", null, true, true, null), + ImmutableList.of("4", "5") + ); + } + + @Test + public void testLexicographicMatchNoLowerLimit() + { + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, null, "abd", true, true, null), + NullHandling.replaceWithDefault() + ? ImmutableList.of("1", "2", "3", "5", "6", "7") + : ImmutableList.of("0", "1", "2", "3", "5", "6", "7") + ); + } + + @Test + public void testLexicographicMatchNumbers() + { + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, "1", "3", false, false, null), + ImmutableList.of("1", "2", "3") + ); + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, "1", "3", true, true, null), + ImmutableList.of("1", "2") + ); + assertFilterMatches( + new RangeFilter("dim1", ColumnType.STRING, "-1", "3", true, true, null), + ImmutableList.of("1", "2", "3", "6", "7") + ); + } + + @Test + public void testNumericMatchTooStrict() + { + assertFilterMatches( + new RangeFilter("dim1", ColumnType.LONG, 2L, 2L, true, false, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("dim1", ColumnType.LONG, 2L, 2L, true, true, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("dim1", ColumnType.LONG, 2L, 2L, false, true, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("d0", ColumnType.LONG, 2L, 3L, false, true, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("d0", ColumnType.DOUBLE, 2L, 3L, false, true, null), + ImmutableList.of() + ); + + assertFilterMatches( + new RangeFilter("f0", ColumnType.LONG, 2L, 3L, false, true, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("f0", ColumnType.DOUBLE, 2L, 3L, false, true, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("l0", ColumnType.LONG, 2L, 3L, false, true, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("l0", ColumnType.DOUBLE, 2L, 3L, false, true, null), + ImmutableList.of() + ); + } + + @Test + public void testNumericMatchVirtualColumn() + { + assertFilterMatches( + new RangeFilter("expr", ColumnType.LONG, 1L, 2L, false, false, null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + assertFilterMatches( + new RangeFilter("expr", ColumnType.DOUBLE, 1.1, 2.0, false, false, null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + assertFilterMatches( + new RangeFilter("expr", ColumnType.FLOAT, 1.1f, 2.0f, false, false, null), + ImmutableList.of("0", "1", "2", "3", "4", "5", "6", "7") + ); + + assertFilterMatches( + new RangeFilter("expr", ColumnType.LONG, 2L, 3L, false, false, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("expr", ColumnType.DOUBLE, 2.0, 3.0, false, false, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("expr", ColumnType.FLOAT, 2.0f, 3.0f, false, false, null), + ImmutableList.of() + ); + } + + @Test + public void testNumericMatchExactlySingleValue() + { + assertFilterMatches( + new RangeFilter("dim1", ColumnType.LONG, 2L, 2L, false, false, null), + ImmutableList.of("2") + ); + + assertFilterMatches( + new RangeFilter("dim1", ColumnType.DOUBLE, -10.012, -10.012, false, false, null), + ImmutableList.of("7") + ); + assertFilterMatches( + new RangeFilter("d0", ColumnType.DOUBLE, 120.0245, 120.0245, false, false, null), + ImmutableList.of("3") + ); + assertFilterMatches( + new RangeFilter("d0", ColumnType.FLOAT, 120.0245f, 120.0245f, false, false, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("d0", ColumnType.FLOAT, 60.0f, 60.0f, false, false, null), + ImmutableList.of("4") + ); + assertFilterMatches( + new RangeFilter("f0", ColumnType.DOUBLE, 10.1, 10.1, false, false, null), + ImmutableList.of() + ); + assertFilterMatches( + new RangeFilter("f0", ColumnType.FLOAT, 10.1f, 10.1f, false, false, null), + ImmutableList.of("1") + ); + assertFilterMatches( + new RangeFilter("l0", ColumnType.LONG, 12345L, 12345L, false, false, null), + ImmutableList.of("5") + ); + assertFilterMatches( + new RangeFilter("l0", ColumnType.DOUBLE, 12345.0, 12345.0, false, false, null), + ImmutableList.of("5") + ); + assertFilterMatches( + new RangeFilter("l0", ColumnType.FLOAT, 12345.0f, 12345.0f, false, false, null), + ImmutableList.of("5") + ); + } + + @Test + public void testNumericMatchSurroundingSingleValue() + { + assertFilterMatches( + new RangeFilter("dim1", ColumnType.LONG, 1L, 3L, true, true, null), + ImmutableList.of("2") + ); + + assertFilterMatches( + new RangeFilter("dim1", ColumnType.LONG, -11L, -10L, false, false, null), + ImmutableList.of("7") + ); + assertFilterMatches( + new RangeFilter("d0", ColumnType.DOUBLE, 120.0, 120.03, false, false, null), + ImmutableList.of("3") + ); + assertFilterMatches( + new RangeFilter("d0", ColumnType.FLOAT, 120.02f, 120.03f, false, false, null), + ImmutableList.of("3") + ); + assertFilterMatches( + new RangeFilter("d0", ColumnType.FLOAT, 59.5f, 60.01f, false, false, null), + ImmutableList.of("4") + ); + assertFilterMatches( + new RangeFilter("f0", ColumnType.DOUBLE, 10.0, 10.2, false, false, null), + ImmutableList.of("1") + ); + assertFilterMatches( + new RangeFilter("f0", ColumnType.FLOAT, 10.05f, 10.11f, false, false, null), + ImmutableList.of("1") + ); + assertFilterMatches( + new RangeFilter("l0", ColumnType.LONG, 12344L, 12346L, false, false, null), + ImmutableList.of("5") + ); + assertFilterMatches( + new RangeFilter("l0", ColumnType.DOUBLE, 12344.0, 12345.5, false, false, null), + ImmutableList.of("5") + ); + assertFilterMatches( + new RangeFilter("l0", ColumnType.FLOAT, 12344.0f, 12345.5f, false, false, null), + ImmutableList.of("5") + ); + } + + @Test + public void testNumericMatchNoUpperLimit() + { + assertFilterMatches( + new RangeFilter("dim1", ColumnType.LONG, 1L, null, true, true, null), + ImmutableList.of("1", "2") + ); + assertFilterMatches( + new RangeFilter("d0", ColumnType.DOUBLE, 1.0, null, true, true, null), + ImmutableList.of("1", "3", "4", "5", "6") + ); + assertFilterMatches( + new RangeFilter("f0", ColumnType.FLOAT, 1.0f, null, true, true, null), + ImmutableList.of("1", "2", "3", "5", "7") + ); + assertFilterMatches( + new RangeFilter("l0", ColumnType.LONG, 1L, null, true, true, null), + ImmutableList.of("1", "2", "4", "5", "6") + ); + } + + @Test + public void testNumericMatchNoLowerLimit() + { + // strings are wierd... + assertFilterMatches( + new RangeFilter("dim1", ColumnType.LONG, null, 2L, false, true, null), + NullHandling.replaceWithDefault() + ? ImmutableList.of("3", "4", "5", "6", "7") + : ImmutableList.of("0", "3", "4", "5", "6", "7") + ); + // numbers are sane though + assertFilterMatches( + new RangeFilter("d0", ColumnType.DOUBLE, null, 10.0, false, true, null), + canTestNumericNullsAsDefaultValues ? ImmutableList.of("0", "2", "6", "7") : ImmutableList.of("0", "6") + ); + assertFilterMatches( + new RangeFilter("f0", ColumnType.FLOAT, null, 50.5, false, true, null), + canTestNumericNullsAsDefaultValues + ? ImmutableList.of("0", "1", "2", "4", "6", "7") + : ImmutableList.of("0", "1", "2", "7") + ); + assertFilterMatches( + new RangeFilter("l0", ColumnType.LONG, null, 100L, false, true, null), + canTestNumericNullsAsDefaultValues + ? ImmutableList.of("0", "2", "3", "6", "7") + : ImmutableList.of("0", "2", "6") + ); + } + + @Test + public void testNumericMatchWithNegatives() + { + assertFilterMatches( + new RangeFilter("dim1", ColumnType.LONG, -2000L, 3L, true, true, null), + ImmutableList.of("2", "3", "6", "7") + ); + } + + @Test + public void testNumericNullsAndZeros() + { + assertFilterMatches( + new RangeFilter( + "d0", + ColumnType.DOUBLE, + 0.0, + 1.1, + false, + false, + null + ), + canTestNumericNullsAsDefaultValues ? ImmutableList.of("0", "2", "7") : ImmutableList.of("0") + ); + + assertFilterMatches( + new RangeFilter( + "f0", + ColumnType.FLOAT, + 0.0, + 1.0, + false, + false, + null + ), + canTestNumericNullsAsDefaultValues ? ImmutableList.of("0", "4", "6") : ImmutableList.of("0") + ); + + assertFilterMatches( + new RangeFilter( + "l0", + ColumnType.LONG, + 0L, + 1L, + false, + false, + null + ), + NullHandling.replaceWithDefault() && canTestNumericNullsAsDefaultValues + ? ImmutableList.of("0", "3", "7") + : ImmutableList.of("0") + ); + } + + @Test + public void testVirtualNumericNullsAndZeros() + { + assertFilterMatches( + new RangeFilter( + "vd0", + ColumnType.DOUBLE, + 0.0, + 1.0, + false, + false, + null + ), + canTestNumericNullsAsDefaultValues ? ImmutableList.of("0", "2", "7") : ImmutableList.of("0") + ); + + assertFilterMatches( + new RangeFilter( + "vf0", + ColumnType.FLOAT, + 0.0, + 1.0, + false, + false, + null + ), + canTestNumericNullsAsDefaultValues ? ImmutableList.of("0", "4", "6") : ImmutableList.of("0") + ); + + assertFilterMatches( + new RangeFilter( + "vl0", + ColumnType.LONG, + 0L, + 1L, + false, + false, + null + ), + NullHandling.replaceWithDefault() && canTestNumericNullsAsDefaultValues + ? ImmutableList.of("0", "3", "7") + : ImmutableList.of("0") + ); + } + + @Test + public void testNumericNulls() + { + assertFilterMatches( + new RangeFilter( + "f0", + ColumnType.FLOAT, + 1.0, + null, + false, + false, + null + ), + ImmutableList.of("1", "2", "3", "5", "7") + ); + assertFilterMatches( + new RangeFilter( + "d0", + ColumnType.DOUBLE, + 1.0, + null, + false, + false, + null + ), + ImmutableList.of("1", "3", "4", "5", "6") + ); + assertFilterMatches( + new RangeFilter( + "l0", + ColumnType.LONG, + 1L, + null, + false, + false, + null + ), + ImmutableList.of("1", "2", "4", "5", "6") + ); + } + + @Test + public void testListFilteredVirtualColumn() + { + assertFilterMatchesSkipVectorize( + new RangeFilter("allow-dim0", ColumnType.STRING, "0", "2", false, false, null), + ImmutableList.of() + ); + assertFilterMatchesSkipVectorize( + new RangeFilter("allow-dim0", ColumnType.STRING, "0", "6", false, false, null), + ImmutableList.of("3", "4") + ); + assertFilterMatchesSkipVectorize( + new RangeFilter("allow-dim0", ColumnType.STRING, null, "6", false, false, null), + ImmutableList.of("3", "4") + ); + + assertFilterMatchesSkipVectorize( + new RangeFilter("deny-dim0", ColumnType.STRING, "0", "6", false, false, null), + ImmutableList.of("0", "1", "2", "5", "6") + ); + assertFilterMatchesSkipVectorize( + new RangeFilter("deny-dim0", ColumnType.STRING, "3", "4", false, false, null), + ImmutableList.of() + ); + assertFilterMatchesSkipVectorize( + new RangeFilter("deny-dim0", ColumnType.STRING, null, "6", false, false, null), + ImmutableList.of("0", "1", "2", "5", "6") + ); + + if (isAutoSchema()) { + // bail out, auto ingests arrays instead of mvds and this virtual column is for mvd stuff + return; + } + + assertFilterMatchesSkipVectorize( + new RangeFilter("allow-dim2", ColumnType.STRING, "a", "c", false, false, null), + ImmutableList.of("0", "3", "6") + ); + assertFilterMatchesSkipVectorize( + new RangeFilter("allow-dim2", ColumnType.STRING, "c", "z", false, false, null), + ImmutableList.of() + ); + assertFilterMatchesSkipVectorize( + new RangeFilter("allow-dim2", ColumnType.STRING, null, "z", false, false, null), + ImmutableList.of("0", "3", "6") + ); + + assertFilterMatchesSkipVectorize( + new RangeFilter("deny-dim2", ColumnType.STRING, "a", "b", false, true, null), + ImmutableList.of() + ); + assertFilterMatchesSkipVectorize( + new RangeFilter("deny-dim2", ColumnType.STRING, "c", "z", false, false, null), + ImmutableList.of("4", "7") + ); + + assertFilterMatchesSkipVectorize( + new RangeFilter("deny-dim2", ColumnType.STRING, null, "z", false, false, null), + NullHandling.replaceWithDefault() ? ImmutableList.of("0", "4", "7") : ImmutableList.of("0", "2", "4", "7") + ); + } + + + @Test + public void testArrayRanges() + { + if (isAutoSchema()) { + // only auto schema supports array columns currently, this means the match value will need to be coerceable to + // the column value type... + + /* dim0 .. arrayString arrayLong arrayDouble + "0", .. ["a", "b", "c"], [1L, 2L, 3L], [1.1, 2.2, 3.3] + "1", .. [], [], [1.1, 2.2, 3.3] + "2", .. null, [1L, 2L, 3L], [null] + "3", .. ["a", "b", "c"], null, [] + "4", .. ["c", "d"], [null], [-1.1, -333.3] + "5", .. [null], [123L, 345L], null + "6", .. ["x", "y"], [100, 200], [1.1, null, 3.3] + "7", .. [null, "hello", "world"], [1234, 3456L, null], [1.23, 4.56, 6.78] + */ + assertFilterMatches( + new RangeFilter( + "arrayString", + ColumnType.STRING_ARRAY, + new Object[]{"a", "b", "c"}, + new Object[]{"a", "b", "c"}, + false, + false, + null + ), + ImmutableList.of("0", "3") + ); + assertFilterMatches( + new RangeFilter( + "arrayString", + ColumnType.STRING_ARRAY, + null, + new Object[]{"a", "b", "c"}, + false, + false, + null + ), + ImmutableList.of("0", "1", "3", "5", "7") + ); + + assertFilterMatches( + new RangeFilter( + "arrayString", + ColumnType.STRING_ARRAY, + new Object[]{"a", "b", "c"}, + null, + true, + false, + null + ), + ImmutableList.of("4", "6") + ); + + assertFilterMatches( + new RangeFilter( + "arrayString", + ColumnType.STRING_ARRAY, + null, + new Object[]{"a", "b", "c"}, + false, + true, + null + ), + ImmutableList.of("1", "5", "7") + ); + + assertFilterMatches( + new RangeFilter( + "arrayString", + ColumnType.STRING_ARRAY, + new Object[]{"a", "b"}, + new Object[]{"a", "b", "c", "d"}, + true, + true, + null + ), + ImmutableList.of("0", "3") + ); + + assertFilterMatches( + new RangeFilter( + "arrayString", + ColumnType.STRING_ARRAY, + new Object[]{"c", "d"}, + new Object[]{"c", "d", "e"}, + false, + true, + null + ), + ImmutableList.of("4") + ); + + assertFilterMatches( + new RangeFilter( + "arrayString", + ColumnType.STRING_ARRAY, + null, + new Object[]{}, + false, + false, + null + ), + ImmutableList.of("1") + ); + + assertFilterMatches( + new RangeFilter( + "arrayString", + ColumnType.STRING_ARRAY, + null, + new Object[]{null}, + false, + false, + null + ), + ImmutableList.of("1", "5") + ); + + assertFilterMatches( + new RangeFilter( + "arrayLong", + ColumnType.LONG_ARRAY, + null, + new Object[]{}, + false, + false, + null + ), + ImmutableList.of("1") + ); + + assertFilterMatches( + new RangeFilter( + "arrayLong", + ColumnType.LONG_ARRAY, + new Object[]{}, + null, + true, + false, + null + ), + ImmutableList.of("0", "2", "4", "5", "6", "7") + ); + + assertFilterMatches( + new RangeFilter( + "arrayLong", + ColumnType.LONG_ARRAY, + null, + new Object[]{null}, + false, + false, + null + ), + ImmutableList.of("1", "4") + ); + + assertFilterMatches( + new RangeFilter( + "arrayLong", + ColumnType.LONG_ARRAY, + new Object[]{1L, 2L, 3L}, + new Object[]{1L, 2L, 3L}, + false, + false, + null + ), + ImmutableList.of("0", "2") + ); + + + assertFilterMatches( + new RangeFilter( + "arrayLong", + ColumnType.LONG_ARRAY, + null, + new Object[]{1L, 2L, 3L}, + false, + true, + null + ), + ImmutableList.of("1", "4") + ); + + assertFilterMatches( + new RangeFilter( + "arrayLong", + ColumnType.LONG_ARRAY, + new Object[]{1L, 2L, 3L}, + null, + true, + false, + null + ), + ImmutableList.of("5", "6", "7") + ); + + // empties and nulls still sort before numbers + assertFilterMatches( + new RangeFilter( + "arrayLong", + ColumnType.LONG_ARRAY, + null, + new Object[]{-1L}, + false, + false, + null + ), + ImmutableList.of("1", "4") + ); + + assertFilterMatches( + new RangeFilter( + "arrayDouble", + ColumnType.DOUBLE_ARRAY, + null, + new Object[]{}, + false, + false, + null + ), + ImmutableList.of("3") + ); + + assertFilterMatches( + new RangeFilter( + "arrayDouble", + ColumnType.DOUBLE_ARRAY, + new Object[]{}, + null, + true, + false, + null + ), + ImmutableList.of("0", "1", "2", "4", "6", "7") + ); + + assertFilterMatches( + new RangeFilter( + "arrayDouble", + ColumnType.DOUBLE_ARRAY, + null, + new Object[]{null}, + false, + false, + null + ), + ImmutableList.of("2", "3") + ); + + assertFilterMatches( + new RangeFilter( + "arrayDouble", + ColumnType.DOUBLE_ARRAY, + new Object[]{1.1, 2.2, 3.3}, + new Object[]{1.1, 2.2, 3.3}, + false, + false, + null + ), + ImmutableList.of("0", "1") + ); + assertFilterMatches( + new RangeFilter( + "arrayDouble", + ColumnType.DOUBLE_ARRAY, + new Object[]{1.1, 2.2, 3.3}, + null, + true, + false, + null + ), + ImmutableList.of("7") + ); + + assertFilterMatches( + new RangeFilter( + "arrayDouble", + ColumnType.DOUBLE_ARRAY, + null, + new Object[]{1.1, 2.2, 3.3}, + true, + false, + null + ), + ImmutableList.of("0", "1", "2", "3", "4", "6") + ); + + // empties and nulls sort before numbers + assertFilterMatches( + new RangeFilter( + "arrayDouble", + ColumnType.DOUBLE_ARRAY, + null, + new Object[]{0.0}, + true, + false, + null + ), + ImmutableList.of("2", "3", "4") + ); + } + } + } + + public static class RangeFilterNonParameterizedTests extends InitializedNullHandlingTest + { + @Test + public void testSerde() throws JsonProcessingException + { + ObjectMapper mapper = new DefaultObjectMapper(); + RangeFilter filter = new RangeFilter("x", ColumnType.STRING, "abc", "xyz", true, true, null); + String s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, RangeFilter.class)); + + filter = new RangeFilter("x", ColumnType.STRING, "abc", "xyz", false, false, null); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, RangeFilter.class)); + + filter = new RangeFilter("x", ColumnType.LONG, 100L, null, true, false, null); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, RangeFilter.class)); + + filter = new RangeFilter( + "x", + ColumnType.STRING_ARRAY, + new Object[]{"a", "b"}, + new Object[]{"x", null, "z"}, + true, + true, + null + ); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, RangeFilter.class)); + + filter = new RangeFilter( + "x", + ColumnType.LONG_ARRAY, + new Object[]{101L, 1L}, + new Object[]{101L, 100L}, + true, + true, + null + ); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, RangeFilter.class)); + + filter = new RangeFilter( + "x", + ColumnType.DOUBLE_ARRAY, + new Object[]{1.1, null, 1.2}, + new Object[]{2.02, 3.03}, + true, + true, + null + ); + s = mapper.writeValueAsString(filter); + Assert.assertEquals(filter, mapper.readValue(s, RangeFilter.class)); + } + + @Test + public void testGetCacheKey() + { + RangeFilter f1 = new RangeFilter("x", ColumnType.STRING, "abc", "xyz", true, true, null); + RangeFilter f1_2 = new RangeFilter("x", ColumnType.STRING, "abc", "xyz", true, true, null); + RangeFilter f2 = new RangeFilter("x", ColumnType.STRING, "abc", "xyz", false, true, null); + RangeFilter f2_2 = new RangeFilter("x", ColumnType.STRING, "abc", "xyzz", true, true, null); + RangeFilter f3 = new RangeFilter( + "x", + ColumnType.STRING, + "abc", + "xyz", + true, + true, + new FilterTuning(true, null, null) + ); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2_2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + + f1 = new RangeFilter("x", ColumnType.LONG, 100L, 300L, true, true, null); + f1_2 = new RangeFilter("x", ColumnType.LONG, 100, 300, true, true, null); + f2 = new RangeFilter("x", ColumnType.LONG, 100L, 300L, false, true, null); + f2_2 = new RangeFilter("x", ColumnType.LONG, 101L, 300L, true, true, null); + f3 = new RangeFilter("x", ColumnType.LONG, 100L, 300L, true, true, new FilterTuning(true, null, null)); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2_2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + + f1 = new RangeFilter("x", ColumnType.DOUBLE, -1.1, 1.1, true, true, null); + f1_2 = new RangeFilter("x", ColumnType.DOUBLE, -1.1, 1.1, true, true, null); + f2 = new RangeFilter("x", ColumnType.DOUBLE, -1.1, 1.1, false, true, null); + f2_2 = new RangeFilter("x", ColumnType.DOUBLE, -1.1000000001, 1.1, true, true, null); + f3 = new RangeFilter("x", ColumnType.DOUBLE, -1.1, 1.1, true, true, new FilterTuning(true, null, null)); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2_2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + + f1 = new RangeFilter( + "x", + ColumnType.STRING_ARRAY, + new Object[]{"a", "b"}, + new Object[]{"x", null, "z"}, + true, + true, + null + ); + f1_2 = new RangeFilter( + "x", + ColumnType.STRING_ARRAY, + Arrays.asList("a", "b"), + Arrays.asList("x", null, "z"), + true, + true, + null + ); + f2 = new RangeFilter( + "x", + ColumnType.STRING_ARRAY, + new Object[]{"a", "b"}, + new Object[]{"x", null, "z"}, + false, + true, + null + ); + f2_2 = new RangeFilter( + "x", + ColumnType.STRING_ARRAY, + new Object[]{"a", "b"}, + new Object[]{"x", "z"}, + true, + true, + null + ); + f3 = new RangeFilter( + "x", + ColumnType.STRING_ARRAY, + new Object[]{"a", "b"}, + new Object[]{"x", null, "z"}, + true, + true, + new FilterTuning(true, null, null) + ); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2_2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + + f1 = new RangeFilter( + "x", + ColumnType.LONG_ARRAY, + new Object[]{101L, 1L}, + new Object[]{101L, 100L}, + true, + true, + null + ); + f1_2 = new RangeFilter( + "x", + ColumnType.LONG_ARRAY, + Arrays.asList(101L, 1L), + Arrays.asList(101L, 100L), + true, + true, + null + ); + f2 = new RangeFilter( + "x", + ColumnType.LONG_ARRAY, + new Object[]{101L, 1L}, + new Object[]{101L, 100L}, + false, + true, + null + ); + f2_2 = new RangeFilter( + "x", + ColumnType.LONG_ARRAY, + new Object[]{101L, 1L}, + new Object[]{101L, 99L}, + true, + true, + null + ); + f3 = new RangeFilter( + "x", + ColumnType.LONG_ARRAY, + new Object[]{101L, 1L}, + new Object[]{101L, 100L}, + true, + true, + new FilterTuning(true, null, null) + ); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2_2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + + f1 = new RangeFilter( + "x", + ColumnType.DOUBLE_ARRAY, + new Object[]{1.1, null, 1.2}, + new Object[]{2.02, 3.03}, + true, + true, + null + ); + f1_2 = new RangeFilter( + "x", + ColumnType.DOUBLE_ARRAY, + Arrays.asList(1.1, null, 1.2), + Arrays.asList(2.02, 3.03), + true, + true, + null + ); + f2 = new RangeFilter( + "x", + ColumnType.DOUBLE_ARRAY, + new Object[]{1.1, null, 1.2}, + new Object[]{2.02, 3.03}, + false, + true, + null + ); + f2_2 = new RangeFilter( + "x", + ColumnType.DOUBLE_ARRAY, + new Object[]{1.1, 1.2}, + new Object[]{2.02, 3.03}, + true, + true, + null + ); + f3 = new RangeFilter( + "x", + ColumnType.DOUBLE_ARRAY, + new Object[]{1.1, null, 1.2}, + new Object[]{2.02, 3.03}, + true, + true, + new FilterTuning(true, null, null) + ); + Assert.assertArrayEquals(f1.getCacheKey(), f1_2.getCacheKey()); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2.getCacheKey())); + Assert.assertFalse(Arrays.equals(f1.getCacheKey(), f2_2.getCacheKey())); + Assert.assertArrayEquals(f1.getCacheKey(), f3.getCacheKey()); + } + + @Test + public void testRequiredColumnRewrite() + { + RangeFilter filter = new RangeFilter("dim0", ColumnType.STRING, "abc", "def", false, false, null); + RangeFilter filter2 = new RangeFilter("dim1", ColumnType.STRING, "abc", "def", false, false, null); + Assert.assertTrue(filter.supportsRequiredColumnRewrite()); + Assert.assertTrue(filter2.supportsRequiredColumnRewrite()); + + Filter rewrittenFilter = filter.rewriteRequiredColumns(ImmutableMap.of("dim0", "dim1")); + Assert.assertEquals(filter2, rewrittenFilter); + + Throwable t = Assert.assertThrows( + IAE.class, + () -> filter.rewriteRequiredColumns(ImmutableMap.of("invalidName", "dim1")) + ); + + Assert.assertEquals( + "Received a non-applicable rewrite: {invalidName=dim1}, filter's dimension: dim0", + t.getMessage() + ); + } + + @Test + public void testNumericMatchBadParameters() + { + Throwable t = Assert.assertThrows( + DruidException.class, + () -> new RangeFilter(null, ColumnType.DOUBLE, "1234", "", false, false, null) + ); + Assert.assertEquals( + "Invalid range filter, column cannot be null", + t.getMessage() + ); + t = Assert.assertThrows( + DruidException.class, + () -> new RangeFilter("dim0", null, "1234", "", false, false, null) + ); + Assert.assertEquals( + "Invalid range filter on column [dim0], matchValueType cannot be null", + t.getMessage() + ); + t = Assert.assertThrows( + DruidException.class, + () -> new RangeFilter("dim0", ColumnType.DOUBLE, null, null, false, false, null) + ); + Assert.assertEquals( + "Invalid range filter on column [dim0], lower and upper cannot be null at the same time", + t.getMessage() + ); + + t = Assert.assertThrows( + DruidException.class, + () -> new RangeFilter("dim0", ColumnType.DOUBLE, "1234", "", false, false, null) + ); + Assert.assertEquals( + "Invalid range filter on column [dim0], upper bound [] cannot be parsed as specified match value type [DOUBLE]", + t.getMessage() + ); + + t = Assert.assertThrows( + DruidException.class, + () -> new RangeFilter("dim0", ColumnType.DOUBLE, "abc", "1234", false, false, null) + ); + Assert.assertEquals( + "Invalid range filter on column [dim0], lower bound [abc] cannot be parsed as specified match value type [DOUBLE]", + t.getMessage() + ); + } + + + @Test + public void testGetDimensionRangeSet() + { + RangeFilter filter = new RangeFilter("x", ColumnType.STRING, "abc", "xyz", true, true, null); + + RangeSet set = TreeRangeSet.create(); + set.add(Range.range("abc", BoundType.OPEN, "xyz", BoundType.OPEN)); + Assert.assertEquals(set, filter.getDimensionRangeSet("x")); + Assert.assertNull(filter.getDimensionRangeSet("y")); + + ExprEval evalLower = ExprEval.ofType(ExpressionType.STRING_ARRAY, new Object[]{"abc", "def"}); + filter = new RangeFilter("x", ColumnType.STRING_ARRAY, evalLower.value(), null, true, false, null); + set = TreeRangeSet.create(); + set.add(Range.greaterThan(Arrays.deepToString(evalLower.asArray()))); + Assert.assertEquals(set, filter.getDimensionRangeSet("x")); + Assert.assertNull(filter.getDimensionRangeSet("y")); + } + + @Test + public void test_equals() + { + EqualsVerifier.forClass(RangeFilter.class) + .withNonnullFields("column", "matchValueType", "lowerEval", "upperEval") + .withIgnoredFields( + "lower", + "upper", + "cachedOptimizedFilter", + "stringPredicateSupplier", + "longPredicateSupplier", + "floatPredicateSupplier", + "doublePredicateSupplier", + "arrayPredicates", + "typeDetectingArrayPredicateSupplier" + ) + .withPrefabValues(ColumnType.class, ColumnType.STRING, ColumnType.DOUBLE) + .usingGetClass() + .verify(); + } + } +} diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java index c0f32aa5744f..64db457f2085 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/Expressions.java @@ -520,11 +520,11 @@ private static DimFilter toSimpleLeafFilter( if (druidExpression.isSimpleExtraction() && !(isOutputNumeric && !rowSignature.isNumeric(druidExpression.getDirectColumn()))) { if (!plannerContext.isUseBoundsAndSelectors()) { - equalFilter = new NullFilter( - druidExpression.getSimpleExtraction().getColumn(), - druidExpression.getSimpleExtraction().getExtractionFn(), - null - ); + if (druidExpression.getSimpleExtraction().getExtractionFn() != null) { + // return null to fallback to using an expression filter + return null; + } + equalFilter = NullFilter.forColumn(druidExpression.getDirectColumn()); } else { equalFilter = new SelectorDimFilter( druidExpression.getSimpleExtraction().getColumn(), @@ -539,7 +539,7 @@ private static DimFilter toSimpleLeafFilter( ); if (!plannerContext.isUseBoundsAndSelectors()) { - equalFilter = new NullFilter(virtualColumn, null, null); + equalFilter = NullFilter.forColumn(virtualColumn); } else { equalFilter = new SelectorDimFilter( virtualColumn, @@ -664,7 +664,7 @@ private static DimFilter toSimpleLeafFilter( return getBoundTimeDimFilter(flippedKind, boundRefKey, rhsInterval, rhsAligned); } else { - final RangeRefKey rangeRefKey = new RangeRefKey(column, ColumnType.LONG, null); + final RangeRefKey rangeRefKey = new RangeRefKey(column, ColumnType.LONG); return getRangeTimeDimFilter(flippedKind, rangeRefKey, rhsInterval, rhsAligned); } } @@ -719,6 +719,11 @@ private static DimFilter toSimpleLeafFilter( return filter; } else { + //noinspection VariableNotUsedInsideIf + if (extractionFn != null) { + // fall back to expression filter + return null; + } final Object val; if (parsedRhsExpression != null && parsedRhsExpression.isLiteral()) { val = parsedRhsExpression.getLiteralValue(); @@ -728,7 +733,7 @@ private static DimFilter toSimpleLeafFilter( } final ColumnType matchValueType = Calcites.getColumnTypeForRelDataType(rhs.getType()); - final RangeRefKey rangeRefKey = new RangeRefKey(column, matchValueType, extractionFn); + final RangeRefKey rangeRefKey = new RangeRefKey(column, matchValueType); final DimFilter filter; // Always use RangeFilter, to simplify filter optimization later @@ -854,7 +859,7 @@ private static DimFilter buildTimeFloorFilter( return getBoundTimeDimFilter(operatorKind, boundRefKey, rhsInterval, rhsAligned); } else { - final RangeRefKey rangeRefKey = new RangeRefKey(column, ColumnType.LONG, null); + final RangeRefKey rangeRefKey = new RangeRefKey(column, ColumnType.LONG); final Interval rhsInterval = granularity.bucket(DateTimes.utc(rhsMillis)); // Is rhs aligned on granularity boundaries? diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayContainsOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayContainsOperatorConversion.java index b53a96d45716..15819d1e155e 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayContainsOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayContainsOperatorConversion.java @@ -123,7 +123,6 @@ public DimFilter toDruidFilter( leftExpr.getSimpleExtraction().getColumn(), ExpressionType.toColumnType(ExpressionType.elementType(exprEval.type())), arrayElements[0], - leftExpr.getSimpleExtraction().getExtractionFn(), null ); } @@ -138,7 +137,6 @@ public DimFilter toDruidFilter( leftExpr.getSimpleExtraction().getColumn(), ExpressionType.toColumnType(ExpressionType.elementType(exprEval.type())), val, - leftExpr.getSimpleExtraction().getExtractionFn(), null ); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOverlapOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOverlapOperatorConversion.java index 68a3d9ff7e93..ea2c590a2f97 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOverlapOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/ArrayOverlapOperatorConversion.java @@ -133,7 +133,6 @@ public DimFilter toDruidFilter( simpleExtractionExpr.getSimpleExtraction().getColumn(), ExpressionType.toColumnType(exprEval.type()), arrayElements[0], - simpleExtractionExpr.getSimpleExtraction().getExtractionFn(), null ); } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CaseOperatorConversion.java b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CaseOperatorConversion.java index beee936b1947..e980ce91c6e4 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CaseOperatorConversion.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/expression/builtin/CaseOperatorConversion.java @@ -125,11 +125,7 @@ public DimFilter toDruidFilter( null ); } else { - isNull = new NullFilter( - condition.getArguments().get(0).getDirectColumn(), - null, - null - ); + isNull = NullFilter.forColumn(condition.getArguments().get(0).getDirectColumn()); } if (call.getOperands().get(1) instanceof RexCall) { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/CombineAndSimplifyBounds.java b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/CombineAndSimplifyBounds.java index a11ffd64ed8a..bcf136ef423b 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/CombineAndSimplifyBounds.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/CombineAndSimplifyBounds.java @@ -212,7 +212,7 @@ private static DimFilter doSimplify(final List children, boolean disj RangeRefKey refKey = entry.getKey(); if (entry.getKey().getMatchValueType().isNumeric()) { ColumnType numericTypeToUse = leastRestrictiveNumericTypes.get(refKey.getColumn()); - refKey = new RangeRefKey(refKey.getColumn(), numericTypeToUse, refKey.getExtractionFn()); + refKey = new RangeRefKey(refKey.getColumn(), numericTypeToUse); } final List filterList = consolidatedNumericRanges.computeIfAbsent(refKey, k -> new ArrayList<>()); for (RangeFilter filter : entry.getValue()) { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertBoundsToSelectors.java b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertBoundsToSelectors.java index 6c5109693beb..b5694b4b4813 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertBoundsToSelectors.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertBoundsToSelectors.java @@ -76,7 +76,6 @@ public DimFilter process(DimFilter filter) bound.getColumn(), bound.getMatchValueType(), bound.getUpper(), - bound.getExtractionFn(), null ); } else { diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java index 16769892443e..c57a4f3f7c8f 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/ConvertSelectorsToIns.java @@ -130,7 +130,7 @@ public DimFilter process(DimFilter filter) } } - children.add(new InDimFilter(entry.getKey().getColumn(), values, entry.getKey().getExtractionFn(), null)); + children.add(new InDimFilter(entry.getKey().getColumn(), values, null, null)); } } diff --git a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/MoveTimeFiltersToIntervals.java b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/MoveTimeFiltersToIntervals.java index dc93dc1bb20c..1bee138c97a1 100644 --- a/sql/src/main/java/org/apache/druid/sql/calcite/filtration/MoveTimeFiltersToIntervals.java +++ b/sql/src/main/java/org/apache/druid/sql/calcite/filtration/MoveTimeFiltersToIntervals.java @@ -49,8 +49,7 @@ public class MoveTimeFiltersToIntervals implements Function 'z'"; final String legacyExplanation = NullHandling.replaceWithDefault() ? "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"selector\",\"dimension\":\"dim2\",\"value\":\"a\"},{\"type\":\"not\",\"field\":{\"type\":\"selector\",\"dimension\":\"dim1\",\"value\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]},\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{a0:LONG}])\n" - : "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"equals\",\"column\":\"dim2\",\"matchValueType\":\"STRING\",\"matchValue\":\"a\"},{\"type\":\"not\",\"field\":{\"type\":\"equals\",\"column\":\"dim1\",\"matchValueType\":\"STRING\",\"matchValue\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]},\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{a0:LONG}])\n"; + : "DruidQueryRel(query=[{\"queryType\":\"timeseries\",\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"},\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]},\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"equals\",\"column\":\"dim2\",\"matchValueType\":\"STRING\",\"matchValue\":\"a\"},{\"type\":\"expression\",\"expression\":\"(substring(\\\"dim1\\\", 0, 1) != 'z')\"}]},\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}],\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}], signature=[{a0:LONG}])\n"; final String explanation = NullHandling.replaceWithDefault() ? "[{" + "\"query\":{\"queryType\":\"timeseries\"," @@ -55,14 +55,16 @@ public void testExplainCountStarOnView() + "\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]," + "\"columnMappings\":[{\"queryColumn\":\"a0\",\"outputColumn\":\"EXPR$0\"}]" + "}]" - : "[{\"query\":{\"queryType\":\"timeseries\"," + : "[{" + + "\"query\":{\"queryType\":\"timeseries\"," + "\"dataSource\":{\"type\":\"table\",\"name\":\"foo\"}," + "\"intervals\":{\"type\":\"intervals\",\"intervals\":[\"-146136543-09-08T08:23:32.096Z/146140482-04-24T15:36:27.903Z\"]}," - + "\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"equals\",\"column\":\"dim2\",\"matchValueType\":\"STRING\",\"matchValue\":\"a\"},{\"type\":\"not\",\"field\":{\"type\":\"equals\",\"column\":\"dim1\",\"matchValueType\":\"STRING\",\"matchValue\":\"z\",\"extractionFn\":{\"type\":\"substring\",\"index\":0,\"length\":1}}}]}," - + "\"granularity\":{\"type\":\"all\"},\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}]," + + "\"filter\":{\"type\":\"and\",\"fields\":[{\"type\":\"equals\",\"column\":\"dim2\",\"matchValueType\":\"STRING\",\"matchValue\":\"a\"},{\"type\":\"expression\",\"expression\":\"(substring(\\\"dim1\\\", 0, 1) != 'z')\"}]}," + + "\"granularity\":{\"type\":\"all\"}," + + "\"aggregations\":[{\"type\":\"count\",\"name\":\"a0\"}]," + "\"context\":{\"defaultTimeout\":300000,\"maxScatterGatherBytes\":9223372036854775807,\"sqlCurrentTimestamp\":\"2000-01-01T00:00:00Z\",\"sqlQueryId\":\"dummy\",\"vectorize\":\"false\",\"vectorizeVirtualColumns\":\"false\"}}," - + "\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}]," - + "\"columnMappings\":[{\"queryColumn\":\"a0\",\"outputColumn\":\"EXPR$0\"}]}]"; + + "\"signature\":[{\"name\":\"a0\",\"type\":\"LONG\"}],\"columnMappings\":[{\"queryColumn\":\"a0\",\"outputColumn\":\"EXPR$0\"}]" + + "}]"; final String resources = "[{\"name\":\"aview\",\"type\":\"VIEW\"}]"; final String attributes = "{\"statementType\":\"SELECT\"}"; diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java index db37a763ab7f..5cd593947ea3 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteNestedDataQueryTest.java @@ -1243,7 +1243,6 @@ public void testGroupByRootSingleTypeArrayLongFilteredArrayEquality() "arrayLong", ColumnType.LONG_ARRAY, new Object[]{1L, 2L, 3L}, - null, null ) ) @@ -2286,7 +2285,7 @@ public void testGroupByRootSingleTypeArrayDoubleElementFiltered() ) .setDimFilter( // dont use static function since context flag indicates to always use equality - new EqualityFilter("v0", ColumnType.DOUBLE, 5.5, null, null) + new EqualityFilter("v0", ColumnType.DOUBLE, 5.5, null) ) .setVirtualColumns( new NestedFieldVirtualColumn("arrayDoubleNulls", "$[2]", "v0", ColumnType.DOUBLE) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java index 876ade4271ff..0e920a56379f 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteQueryTest.java @@ -3957,11 +3957,11 @@ public void testCoalesceColumnsFilterWithEquality() .setDimFilter( or( and( - new EqualityFilter("dim1", ColumnType.STRING, "a", null, null), + new EqualityFilter("dim1", ColumnType.STRING, "a", null), NullFilter.forColumn("dim2") ), and( - new EqualityFilter("dim1", ColumnType.STRING, "abc", null, null), + new EqualityFilter("dim1", ColumnType.STRING, "abc", null), NullFilter.forColumn("dim2") ), in( @@ -4364,6 +4364,10 @@ public void testCountStar() @Test public void testCountStarOnCommonTableExpression() { + if (NullHandling.sqlCompatible()) { + // cannot vectorize due to substring expression + cannotVectorize(); + } testQuery( "WITH beep (dim1_firstchar) AS (SELECT SUBSTRING(dim1, 1, 1) FROM foo WHERE dim2 = 'a')\n" + "SELECT COUNT(*) FROM beep WHERE dim1_firstchar <> 'z'", @@ -4373,7 +4377,9 @@ public void testCountStarOnCommonTableExpression() .intervals(querySegmentSpec(Filtration.eternity())) .filters(and( equality("dim2", "a", ColumnType.STRING), - not(equality("dim1", "z", new SubstringDimExtractionFn(0, 1), ColumnType.STRING)) + NullHandling.replaceWithDefault() + ? not(selector("dim1", "z", new SubstringDimExtractionFn(0, 1))) + : expressionFilter("(substring(\"dim1\", 0, 1) != 'z')") )) .granularity(Granularities.ALL) .aggregators(aggregators(new CountAggregatorFactory("a0"))) @@ -4381,7 +4387,8 @@ public void testCountStarOnCommonTableExpression() .build() ), ImmutableList.of( - new Object[]{2L} + // in sql compatible mode, the expression filter causes us to correctly not match null + new Object[]{NullHandling.replaceWithDefault() ? 2L : 1L} ) ); } @@ -4389,6 +4396,10 @@ public void testCountStarOnCommonTableExpression() @Test public void testCountStarOnView() { + if (NullHandling.sqlCompatible()) { + // cannot vectorize due to substring expression + cannotVectorize(); + } testQuery( "SELECT COUNT(*) FROM view.aview WHERE dim1_firstchar <> 'z'", ImmutableList.of( @@ -4397,7 +4408,9 @@ public void testCountStarOnView() .intervals(querySegmentSpec(Filtration.eternity())) .filters(and( equality("dim2", "a", ColumnType.STRING), - not(equality("dim1", "z", new SubstringDimExtractionFn(0, 1), ColumnType.STRING)) + NullHandling.replaceWithDefault() + ? not(selector("dim1", "z", new SubstringDimExtractionFn(0, 1))) + : expressionFilter("(substring(\"dim1\", 0, 1) != 'z')") )) .granularity(Granularities.ALL) .aggregators(aggregators(new CountAggregatorFactory("a0"))) @@ -4405,7 +4418,8 @@ public void testCountStarOnView() .build() ), ImmutableList.of( - new Object[]{2L} + // in sql compatible mode, the expression filter causes us to correctly not match null + new Object[]{NullHandling.replaceWithDefault() ? 2L : 1L} ) ); } @@ -4413,6 +4427,10 @@ public void testCountStarOnView() @Test public void testConfusedView() { + if (NullHandling.sqlCompatible()) { + // cannot vectorize due to substring expression + cannotVectorize(); + } testQuery( "SELECT COUNT(*) FROM view.dview as druid WHERE druid.numfoo <> 'z'", ImmutableList.of( @@ -4421,7 +4439,9 @@ public void testConfusedView() .intervals(querySegmentSpec(Filtration.eternity())) .filters(and( equality("dim2", "a", ColumnType.STRING), - not(equality("dim1", "z", new SubstringDimExtractionFn(0, 1), ColumnType.STRING)) + NullHandling.replaceWithDefault() + ? not(selector("dim1", "z", new SubstringDimExtractionFn(0, 1))) + : expressionFilter("(substring(\"dim1\", 0, 1) != 'z')") )) .granularity(Granularities.ALL) .aggregators(aggregators(new CountAggregatorFactory("a0"))) @@ -4429,7 +4449,8 @@ public void testConfusedView() .build() ), ImmutableList.of( - new Object[]{2L} + // in sql compatible mode, the expression filter causes us to correctly not match null + new Object[]{NullHandling.replaceWithDefault() ? 2L : 1L} ) ); } @@ -4501,8 +4522,8 @@ public void testCountStarWithLongColumnFiltersForceRange() .granularity(Granularities.ALL) .filters( or( - new RangeFilter("cnt", ColumnType.LONG, 3L, null, false, false, null, null), - new EqualityFilter("cnt", ColumnType.LONG, 1L, null, null) + new RangeFilter("cnt", ColumnType.LONG, 3L, null, false, false, null), + new EqualityFilter("cnt", ColumnType.LONG, 1L, null) ) ) .aggregators(aggregators(new CountAggregatorFactory("a0"))) @@ -4996,6 +5017,10 @@ public void testGroupByWithSortOnPostAggregationNoTopNContext() @Test public void testFilteredAggregations() { + if (NullHandling.sqlCompatible()) { + // cannot vectorize due to expression filter + cannotVectorize(); + } testQuery( "SELECT " + "SUM(case dim1 when 'abc' then cnt end), " @@ -5027,7 +5052,10 @@ public void testFilteredAggregations() ), new FilteredAggregatorFactory( new LongSumAggregatorFactory("a2", "cnt"), - equality("dim1", "a", new SubstringDimExtractionFn(0, 1), ColumnType.STRING) + NullHandling.replaceWithDefault() + ? selector("dim1", "a", new SubstringDimExtractionFn(0, 1)) + : expressionFilter("(substring(\"dim1\", 0, 1) == 'a')") + ), new FilteredAggregatorFactory( new CountAggregatorFactory("a3"), @@ -7633,10 +7661,9 @@ public void testRegexpExtract() .setInterval(querySegmentSpec(Filtration.eternity())) .setGranularity(Granularities.ALL) .setDimFilter( - not(equality( - "dim1", - "x", new RegexDimExtractionFn("^(.)", 1, true, null), ColumnType.STRING - )) + NullHandling.replaceWithDefault() + ? not(selector("dim1", "x", new RegexDimExtractionFn("^(.)", 1, true, null))) + : expressionFilter("(regexp_extract(\"dim1\",'^(.)',1) != 'x')") ) .setDimensions( dimensions( @@ -7655,13 +7682,21 @@ public void testRegexpExtract() .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - ImmutableList.of( + NullHandling.replaceWithDefault() + ? ImmutableList.of( new Object[]{NULL_STRING, NULL_STRING}, new Object[]{"1", "1"}, new Object[]{"2", "2"}, new Object[]{"a", "a"}, new Object[]{"d", "d"} ) + // sql compatible mode expression filter (correctly) leaves out nulls + : ImmutableList.of( + new Object[]{"1", "1"}, + new Object[]{"2", "2"}, + new Object[]{"a", "a"}, + new Object[]{"d", "d"} + ) ); } @@ -7691,7 +7726,6 @@ public void testRegexpExtractFilterViaNotNullCheck() { // Cannot vectorize due to extractionFn in dimension spec. cannotVectorize(); - testQuery( "SELECT COUNT(*)\n" + "FROM foo\n" @@ -7706,7 +7740,9 @@ public void testRegexpExtractFilterViaNotNullCheck() ) .filters( or( - not(isNull("dim1", new RegexDimExtractionFn("^1", 0, true, null))), + NullHandling.replaceWithDefault() + ? not(selector("dim1", NullHandling.defaultStringValue(), new RegexDimExtractionFn("^1", 0, true, null))) + : expressionFilter("notnull(regexp_extract(\"dim1\",'^1'))"), notNull("v0") ) ) @@ -8635,10 +8671,9 @@ public void testFilterAndGroupByLookup() .setInterval(querySegmentSpec(Filtration.eternity())) .setGranularity(Granularities.ALL) .setDimFilter( - not(equality( - "dim1", - "xxx", extractionFn, ColumnType.STRING - )) + NullHandling.replaceWithDefault() + ? not(selector("dim1", "xxx", extractionFn)) + : expressionFilter("(lookup(\"dim1\",'lookyloo') != 'xxx')") ) .setDimensions( dimensions( @@ -8658,10 +8693,15 @@ public void testFilterAndGroupByLookup() .setContext(QUERY_CONTEXT_DEFAULT) .build() ), - ImmutableList.of( + NullHandling.replaceWithDefault() + ? ImmutableList.of( new Object[]{NULL_STRING, 5L}, new Object[]{"xabc", 1L} ) + // sql compatible mode expression filter (correctly) leaves out null values + : ImmutableList.of( + new Object[]{"xabc", 1L} + ) ); } diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java index be05b46c3716..661f5cd6ec9e 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSelectQueryTest.java @@ -612,6 +612,10 @@ public void testSelectStarWithDimFilter() @Test public void testSelectDistinctWithCascadeExtractionFilter() { + if (NullHandling.sqlCompatible()) { + // cannot vectorize due to expression filter + cannotVectorize(); + } testQuery( "SELECT distinct dim1 FROM druid.foo WHERE substring(substring(dim1, 2), 1, 1) = 'e' OR dim2 = 'a'", ImmutableList.of( @@ -622,13 +626,16 @@ public void testSelectDistinctWithCascadeExtractionFilter() .setDimensions(dimensions(new DefaultDimensionSpec("dim1", "d0"))) .setDimFilter( or( - equality( + NullHandling.replaceWithDefault() + ? selector( "dim1", - "e", cascade( + "e", + cascade( new SubstringDimExtractionFn(1, null), new SubstringDimExtractionFn(0, 1) - ), ColumnType.STRING - ), + ) + ) + : expressionFilter("(substring(substring(\"dim1\", 1, -1), 0, 1) == 'e')"), equality("dim2", "a", ColumnType.STRING) ) ) diff --git a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java index 21853f3c3942..7b21210904a4 100644 --- a/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java +++ b/sql/src/test/java/org/apache/druid/sql/calcite/CalciteSubqueryTest.java @@ -344,15 +344,24 @@ public void testViewAndJoin() ) ) .intervals(querySegmentSpec(Filtration.eternity())) - .filters(not(equality("dim1", "z", new SubstringDimExtractionFn(0, 1), ColumnType.STRING))) + .filters( + NullHandling.replaceWithDefault() + ? not(selector("dim1", "z", new SubstringDimExtractionFn(0, 1))) + : expressionFilter("(substring(\"dim1\", 0, 1) != 'z')") + ) .granularity(Granularities.ALL) .aggregators(aggregators(new CountAggregatorFactory("a0"))) .context(queryContextModified) .build() ), - ImmutableList.of( + NullHandling.replaceWithDefault() + ? ImmutableList.of( new Object[]{8L} ) + // in sql compatible mode, expression filter correctly does not match null values... + : ImmutableList.of( + new Object[]{4L} + ) ); }