Skip to content

Commit

Permalink
Fix non-sqlcompat validation in CalciteWindowQueryTest (apache#15086)
Browse files Browse the repository at this point in the history 
* fixes

* check for latest rewrite place

* Revert "check for latest rewrite place"

This reverts commit 5cf1e2c.

* some stuff

(cherry picked from commit ab346d4373ea888eb8ef6115e018e7fb0d27407f)

* update test output

* updates to test ouptuts

* some stuff

* move validator

* cleanup

* fix

* change test slightly

* add apidoc cleanup warnings

* cleanup/etc

* instead of telling the story; add a fail with some reason whats the issue

* lead-lag fix

* add test

* remove unnecessary throw

* druidexception-trial

* Revert "druidexception-trial"

This reverts commit 8fa0664.

* undo changes to no_grouping; add no_grouping2

* add missing assert on resultcount

* rename method; update

* introduce enum/etc

* make resultmatchmode accessible from TestBuilder#expectedResults

* fix dump results to use log

* fix

* handle null correctly

* disable feature type based things for MSQ

* fix varianssqlaggtest

* use eps in other test

* fix intellij error

* add final

* addrss review

* update test/string/etc

* write concat in 3 lines :D
  • Loading branch information
@kgyrtkirk
kgyrtkirk authored Oct 11, 2023
1 parent c6ca990 commit ae88f2c
Show file tree
Hide file tree
Showing 20 changed files with 3,694 additions and 3,477 deletions.
31 changes: 10 additions & 21 deletions .../org/apache/druid/query/aggregation/tdigestsketch/sql/TDigestSketchSqlAggregatorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -53,7 +53,6 @@
import org.apache.druid.sql.calcite.util.TestDataBuilder;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.partition.LinearShardSpec;
import org.junit.Assert;
import org.junit.Test;

import java.io.IOException;
Expand Down Expand Up @@ -130,6 +129,7 @@ public void testComputingSketchOnNumericValues()
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
ImmutableList.of(
new String[]{
"\"AAAAAT/wAAAAAAAAQBgAAAAAAABAaQAAAAAAAAAAAAY/8AAAAAAAAD/wAAAAAAAAP/AAAAAAAABAAAAAAAAAAD/wAAAAAAAAQAgAAAAAAAA/8AAAAAAAAEAQAAAAAAAAP/AAAAAAAABAFAAAAAAAAD/wAAAAAAAAQBgAAAAAAAA=\""
Expand Down Expand Up @@ -166,6 +166,7 @@ public void testComputingSketchOnCastedString()
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
ImmutableList.of(
NullHandling.replaceWithDefault()
? new String[]{
Expand Down Expand Up @@ -198,6 +199,7 @@ public void testDefaultCompressionForTDigestGenerateSketchAgg()
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
ImmutableList.of(
new Object[]{"\"AAAAAT/wAAAAAAAAQBgAAAAAAABAWQAAAAAAAAAAAAY/8AAAAAAAAD/wAAAAAAAAP/AAAAAAAABAAAAAAAAAAD/wAAAAAAAAQAgAAAAAAAA/8AAAAAAAAEAQAAAAAAAAP/AAAAAAAABAFAAAAAAAAD/wAAAAAAAAQBgAAAAAAAA=\""}
)
Expand Down Expand Up @@ -242,6 +244,7 @@ public void testComputingQuantileOnPreAggregatedSketch()
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
expectedResults
);
}
Expand Down Expand Up @@ -297,6 +300,7 @@ public void testGeneratingSketchAndComputingQuantileOnFly()
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
expectedResults
);
}
Expand Down Expand Up @@ -334,6 +338,7 @@ public void testQuantileOnNumericValues()
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
expectedResults
);
}
Expand Down Expand Up @@ -370,6 +375,7 @@ public void testCompressionParamForTDigestQuantileAgg()
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
ImmutableList.of(
new Object[]{1.0, 3.5, 6.0}
)
Expand Down Expand Up @@ -421,6 +427,7 @@ public void testQuantileOnCastedString()
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
ImmutableList.of(
NullHandling.replaceWithDefault()
? new Object[]{0.0, 0.5, 10.1}
Expand Down Expand Up @@ -455,6 +462,7 @@ public void testEmptyTimeseriesResults()
.context(QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
ImmutableList.of(
new Object[]{"\"AAAAAX/wAAAAAAAA//AAAAAAAABAWQAAAAAAAAAAAAA=\"", Double.NaN}
)
Expand Down Expand Up @@ -499,32 +507,13 @@ public void testGroupByAggregatorDefaultValues()
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
ImmutableList.of(
new Object[]{"a", "\"AAAAAX/wAAAAAAAA//AAAAAAAABAWQAAAAAAAAAAAAA=\"", Double.NaN}
)
);
}

@Override
public void assertResultsEquals(String sql, List<Object[]> expectedResults, List<Object[]> results)
{
Assert.assertEquals(expectedResults.size(), results.size());
for (int i = 0; i < expectedResults.size(); i++) {
Object[] expectedResult = expectedResults.get(i);
Object[] result = results.get(i);
Assert.assertEquals(expectedResult.length, result.length);
for (int j = 0; j < expectedResult.length; j++) {
if (expectedResult[j] instanceof Float) {
Assert.assertEquals((Float) expectedResult[j], (Float) result[j], 0.000001);
} else if (expectedResult[j] instanceof Double) {
Assert.assertEquals((Double) expectedResult[j], (Double) result[j], 0.000001);
} else {
Assert.assertEquals(expectedResult[j], result[j]);
}
}
}
}

private static PostAggregator makeFieldAccessPostAgg(String name)
{
return new FieldAccessPostAggregator(name, name);
Expand Down
31 changes: 10 additions & 21 deletions .../test/java/org/apache/druid/query/aggregation/variance/sql/VarianceSqlAggregatorTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -62,7 +62,6 @@
import org.apache.druid.sql.calcite.util.TestDataBuilder;
import org.apache.druid.timeline.DataSegment;
import org.apache.druid.timeline.partition.LinearShardSpec;
import org.junit.Assert;
import org.junit.Test;

import java.io.IOException;
Expand Down Expand Up @@ -196,6 +195,7 @@ public void testVarPop()
.context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
expectedResults
);
}
Expand Down Expand Up @@ -243,6 +243,7 @@ public void testVarSamp()
.context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
expectedResults
);
}
Expand Down Expand Up @@ -298,6 +299,7 @@ public void testStdDevPop()
.context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
expectedResults
);
}
Expand Down Expand Up @@ -351,6 +353,7 @@ public void testStdDevSamp()
.context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
expectedResults
);
}
Expand Down Expand Up @@ -409,6 +412,7 @@ public void testStdDevWithVirtualColumns()
.context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
expectedResults
);
}
Expand Down Expand Up @@ -455,6 +459,7 @@ public void testVarianceOrderBy()
.setContext(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
expectedResults
);
}
Expand Down Expand Up @@ -491,6 +496,7 @@ public void testVariancesOnCastedString()
.context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
ImmutableList.of(
NullHandling.replaceWithDefault()
? new Object[]{3.61497656362466, 3.960008417499471, 3.960008417499471, 15.681666666666667}
Expand Down Expand Up @@ -541,6 +547,7 @@ public void testEmptyTimeseriesResults()
.context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
ImmutableList.of(
NullHandling.replaceWithDefault()
? new Object[]{0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}
Expand Down Expand Up @@ -621,6 +628,7 @@ public void testGroupByAggregatorDefaultValues()
.setContext(QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
ImmutableList.of(
NullHandling.replaceWithDefault()
? new Object[]{"a", 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0, 0.0}
Expand Down Expand Up @@ -674,27 +682,8 @@ public void testVarianceAggAsInput()
.context(BaseCalciteQueryTest.QUERY_CONTEXT_DEFAULT)
.build()
),
ResultMatchMode.EQUALS_EPS,
expectedResults
);
}

@Override
public void assertResultsEquals(String sql, List<Object[]> expectedResults, List<Object[]> results)
{
Assert.assertEquals(expectedResults.size(), results.size());
for (int i = 0; i < expectedResults.size(); i++) {
Object[] expectedResult = expectedResults.get(i);
Object[] result = results.get(i);
Assert.assertEquals(expectedResult.length, result.length);
for (int j = 0; j < expectedResult.length; j++) {
if (expectedResult[j] instanceof Float) {
Assert.assertEquals((Float) expectedResult[j], (Float) result[j], 1e-5);
} else if (expectedResult[j] instanceof Double) {
Assert.assertEquals((Double) expectedResult[j], (Double) result[j], 1e-5);
} else {
Assert.assertEquals(expectedResult[j], result[j]);
}
}
}
}
}
23 changes: 23 additions & 0 deletions processing/src/main/java/org/apache/druid/common/config/NullHandling.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -22,6 +22,7 @@
import com.google.common.annotations.VisibleForTesting;
import com.google.common.base.Strings;
import com.google.inject.Inject;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.data.Indexed;

import javax.annotation.Nullable;
Expand Down Expand Up @@ -161,6 +162,28 @@ public static <T> T defaultValueForClass(final Class<T> clazz)
}
}

/**
* Returns the default value for the given {@link ValueType}.
*
* May be null or non-null based on the current SQL-compatible null handling mode.
*/
@Nullable
@SuppressWarnings("unchecked")
public static Object defaultValueForType(ValueType type)
{
if (type == ValueType.FLOAT) {
return defaultFloatValue();
} else if (type == ValueType.DOUBLE) {
return defaultDoubleValue();
} else if (type == ValueType.LONG) {
return defaultLongValue();
} else if (type == ValueType.STRING) {
return defaultStringValue();
} else {
return null;
}
}

public static boolean isNullOrEquivalent(@Nullable String value)
{
return replaceWithDefault() ? Strings.isNullOrEmpty(value) : value == null;
Expand Down
14 changes: 14 additions & 0 deletions processing/src/test/java/org/apache/druid/common/config/NullHandlingTest.java
View file
Original file line number Diff line number Diff line change
Expand Up @@ -20,13 +20,16 @@
package org.apache.druid.common.config;

import org.apache.druid.java.util.common.StringUtils;
import org.apache.druid.segment.column.ValueType;
import org.apache.druid.segment.data.ListIndexed;
import org.apache.druid.testing.InitializedNullHandlingTest;
import org.junit.Assert;
import org.junit.Test;

import java.util.Collections;

import static org.apache.druid.common.config.NullHandling.defaultValueForClass;
import static org.apache.druid.common.config.NullHandling.defaultValueForType;
import static org.apache.druid.common.config.NullHandling.replaceWithDefault;
import static org.junit.Assert.assertEquals;

Expand Down Expand Up @@ -89,6 +92,17 @@ public void test_defaultValueForClass_object()
Assert.assertNull(NullHandling.defaultValueForClass(Object.class));
}

@Test
public void test_defaultValueForType()
{
assertEquals(defaultValueForClass(Float.class), defaultValueForType(ValueType.FLOAT));
assertEquals(defaultValueForClass(Double.class), defaultValueForType(ValueType.DOUBLE));
assertEquals(defaultValueForClass(Long.class), defaultValueForType(ValueType.LONG));
assertEquals(defaultValueForClass(String.class), defaultValueForType(ValueType.STRING));
assertEquals(defaultValueForClass(Object.class), defaultValueForType(ValueType.COMPLEX));
assertEquals(defaultValueForClass(Object.class), defaultValueForType(ValueType.ARRAY));
}

@Test
public void test_ignoreNullsStrings()
{
Expand Down
Loading

0 comments on commit ae88f2c

Please sign in to comment.