From b150f2fc8328eed5196146b5a43febac77d3d744 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Thu, 4 Apr 2024 23:26:04 -0700 Subject: [PATCH 01/21] Change comparator expression to lazy evaluation --- .../delta/kernel/data/VectorComparator.java | 27 ++++++ .../DefaultExpressionEvaluator.java | 71 +++++++++------- .../expressions/DefaultExpressionUtils.java | 83 +++++++++++++++++++ 3 files changed, 150 insertions(+), 31 deletions(-) create mode 100644 kernel/kernel-api/src/main/java/io/delta/kernel/data/VectorComparator.java diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/data/VectorComparator.java b/kernel/kernel-api/src/main/java/io/delta/kernel/data/VectorComparator.java new file mode 100644 index 00000000000..e7ff9ead485 --- /dev/null +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/data/VectorComparator.java @@ -0,0 +1,27 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.kernel.data; + +import io.delta.kernel.annotation.Evolving; + +/** + * Comparator interface for lazy evaluating comparator + */ +@Evolving +public interface VectorComparator { + boolean compare(int compareResult); +} \ No newline at end of file diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java index 0257bbddc3a..bef619302be 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java @@ -18,12 +18,15 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; + +import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.*; import static java.lang.String.format; import static java.util.Objects.requireNonNull; import io.delta.kernel.client.ExpressionHandler; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; +import io.delta.kernel.data.VectorComparator; import io.delta.kernel.expressions.*; import io.delta.kernel.types.*; @@ -35,10 +38,7 @@ import io.delta.kernel.defaults.internal.data.vector.DefaultBooleanVector; import io.delta.kernel.defaults.internal.data.vector.DefaultConstantVector; -import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.booleanWrapperVector; -import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.childAt; -import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.compare; -import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.evalNullability; + import static io.delta.kernel.defaults.internal.expressions.ImplicitCastExpression.canCastTo; /** @@ -405,44 +405,53 @@ ColumnVector visitAlwaysFalse(AlwaysFalse alwaysFalse) { @Override ColumnVector visitComparator(Predicate predicate) { PredicateChildrenEvalResult argResults = evalBinaryExpressionChildren(predicate); - - int numRows = argResults.rowCount; - boolean[] result = new boolean[numRows]; - boolean[] nullability = evalNullability(argResults.leftResult, argResults.rightResult); - int[] compareResult = compare(argResults.leftResult, argResults.rightResult); + VectorComparator v; switch (predicate.getName()) { case "=": - for (int rowId = 0; rowId < numRows; rowId++) { - result[rowId] = compareResult[rowId] == 0; - } - break; + v = new VectorComparator() { + @Override + public boolean compare(int compareResult) { + return compareResult == 0; + } + }; + return comparatorVector(argResults.leftResult, argResults.rightResult, v); case ">": - for (int rowId = 0; rowId < numRows; rowId++) { - result[rowId] = compareResult[rowId] > 0; - } - break; + v = new VectorComparator() { + @Override + public boolean compare(int compareResult) { + return compareResult > 0; + } + }; + return comparatorVector(argResults.leftResult, argResults.rightResult, v); case ">=": - for (int rowId = 0; rowId < numRows; rowId++) { - result[rowId] = compareResult[rowId] >= 0; - } - break; + v = new VectorComparator() { + @Override + public boolean compare(int compareResult) { + return compareResult >= 0; + } + }; + return comparatorVector(argResults.leftResult, argResults.rightResult, v); case "<": - for (int rowId = 0; rowId < numRows; rowId++) { - result[rowId] = compareResult[rowId] < 0; - } - break; + v = new VectorComparator() { + @Override + public boolean compare(int compareResult) { + return compareResult < 0; + } + }; + return comparatorVector(argResults.leftResult, argResults.rightResult, v); case "<=": - for (int rowId = 0; rowId < numRows; rowId++) { - result[rowId] = compareResult[rowId] <= 0; - } - break; + v = new VectorComparator() { + @Override + public boolean compare(int compareResult) { + return compareResult <= 0; + } + }; + return comparatorVector(argResults.leftResult, argResults.rightResult, v); default: throw DeltaErrors.unsupportedExpression( predicate, Optional.of("unsupported expression encountered")); } - - return new DefaultBooleanVector(numRows, Optional.of(nullability), result); } @Override diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index e078a7a7cd9..864f865635b 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -24,6 +24,7 @@ import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; +import io.delta.kernel.data.VectorComparator; import io.delta.kernel.expressions.Expression; import io.delta.kernel.types.*; import io.delta.kernel.internal.util.Utils; @@ -33,6 +34,20 @@ * Utility methods used by the default expression evaluator. */ class DefaultExpressionUtils { + + static final Comparator BIGDECIMAL_COMPARATOR = Comparator.naturalOrder(); + static final Comparator STRING_COMPARATOR = Comparator.naturalOrder(); + static final Comparator BINARY_COMPARTOR = (leftOp, rightOp) -> { + int i = 0; + while (i < leftOp.length && i < rightOp.length) { + if (leftOp[i] != rightOp[i]) { + return Byte.compare(leftOp[i], rightOp[i]); + } + i++; + } + return Integer.compare(leftOp.length, rightOp.length); + }; + private DefaultExpressionUtils() {} /** @@ -86,6 +101,74 @@ public boolean getBoolean(int rowId) { }; } + static ColumnVector comparatorVector( + ColumnVector left, + ColumnVector right, VectorComparator vectorComparator) { + checkArgument( + left.getSize() == right.getSize(), + "Left and right operand have different vector sizes."); + return new ColumnVector() { + @Override + public DataType getDataType() { return BooleanType.BOOLEAN;} + + @Override + public void close() { + Utils.closeCloseables(left, right); + } + + @Override + public int getSize() { return left.getSize(); } + @Override + public boolean isNullAt(int rowId) { + return left.isNullAt(rowId) || right.isNullAt(rowId); + } + + @Override + public boolean getBoolean(int rowId) { + if (isNullAt(rowId)) { + return false; + } + DataType dataType = left.getDataType(); + if (dataType instanceof BooleanType) { + return vectorComparator.compare( + Boolean.compare(left.getBoolean(rowId), right.getBoolean(rowId))); + } else if (dataType instanceof ByteType) { + return vectorComparator.compare( + Byte.compare(left.getByte(rowId), right.getByte(rowId))); + } else if (dataType instanceof ShortType) { + return vectorComparator.compare( + Short.compare(left.getShort(rowId), right.getShort(rowId))); + } else if (dataType instanceof IntegerType || dataType instanceof DateType) { + return vectorComparator.compare( + Integer.compare(left.getInt(rowId), right.getInt(rowId))); + } else if (dataType instanceof LongType || dataType instanceof TimestampType) { + return vectorComparator.compare( + Long.compare(left.getLong(rowId), right.getLong(rowId))); + } else if (dataType instanceof FloatType) { + return vectorComparator.compare( + Float.compare(left.getFloat(rowId), right.getFloat(rowId))); + } else if (dataType instanceof DoubleType) { + return vectorComparator.compare( + Double.compare(left.getDouble(rowId), right.getDouble(rowId))); + } else if (dataType instanceof DecimalType) { + return vectorComparator.compare( + BIGDECIMAL_COMPARATOR.compare( + left.getDecimal(rowId), right.getDecimal(rowId))); + } else if (dataType instanceof StringType) { + return vectorComparator.compare( + STRING_COMPARATOR.compare( + left.getString(rowId), right.getString(rowId))); + } else if (dataType instanceof BinaryType) { + return vectorComparator.compare( + BINARY_COMPARTOR.compare( + left.getBinary(rowId), right.getBinary(rowId))); + } else { + throw new UnsupportedOperationException(dataType + " can not be compared."); + } + } + }; + } + /** * Utility method to compare the left and right according to the natural ordering * and return an integer array where each row contains the comparison result (-1, 0, 1) for From 4b7ecba24ec7bc9382580fea0c0db6033f0331ff Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Thu, 4 Apr 2024 23:30:28 -0700 Subject: [PATCH 02/21] fix style --- .../src/main/java/io/delta/kernel/data/VectorComparator.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/data/VectorComparator.java b/kernel/kernel-api/src/main/java/io/delta/kernel/data/VectorComparator.java index e7ff9ead485..86f625b3de1 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/data/VectorComparator.java +++ b/kernel/kernel-api/src/main/java/io/delta/kernel/data/VectorComparator.java @@ -23,5 +23,5 @@ */ @Evolving public interface VectorComparator { - boolean compare(int compareResult); -} \ No newline at end of file + boolean compare(int compareResult); +} From 851fc75add45e291ab86ebe1931a2cf5584a4339 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Thu, 4 Apr 2024 23:42:35 -0700 Subject: [PATCH 03/21] fix style --- .../internal/expressions/DefaultExpressionEvaluator.java | 4 ++-- .../defaults/internal/expressions/DefaultExpressionUtils.java | 2 ++ 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java index bef619302be..b68c0fdfd40 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java @@ -19,7 +19,6 @@ import java.util.Optional; import java.util.stream.Collectors; -import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.*; import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -31,6 +30,7 @@ import io.delta.kernel.types.*; import io.delta.kernel.internal.DeltaErrors; + import static io.delta.kernel.internal.util.ExpressionUtils.getLeft; import static io.delta.kernel.internal.util.ExpressionUtils.getRight; import static io.delta.kernel.internal.util.ExpressionUtils.getUnaryChild; @@ -38,8 +38,8 @@ import io.delta.kernel.defaults.internal.data.vector.DefaultBooleanVector; import io.delta.kernel.defaults.internal.data.vector.DefaultConstantVector; - import static io.delta.kernel.defaults.internal.expressions.ImplicitCastExpression.canCastTo; +import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.*; /** * Implementation of {@link ExpressionEvaluator} for default {@link ExpressionHandler}. diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index 864f865635b..6410c77c59d 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -108,6 +108,7 @@ static ColumnVector comparatorVector( left.getSize() == right.getSize(), "Left and right operand have different vector sizes."); return new ColumnVector() { + @Override public DataType getDataType() { return BooleanType.BOOLEAN;} @@ -118,6 +119,7 @@ public void close() { @Override public int getSize() { return left.getSize(); } + @Override public boolean isNullAt(int rowId) { return left.isNullAt(rowId) || right.isNullAt(rowId); From 4641e81eda13956970134f38ec8f55931db8be9e Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Thu, 4 Apr 2024 23:50:57 -0700 Subject: [PATCH 04/21] fix style --- .../internal/expressions/DefaultExpressionEvaluator.java | 6 ++---- .../internal/expressions/DefaultExpressionUtils.java | 8 ++++++-- 2 files changed, 8 insertions(+), 6 deletions(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java index b68c0fdfd40..ea9a8a606d6 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java @@ -18,7 +18,6 @@ import java.util.List; import java.util.Optional; import java.util.stream.Collectors; - import static java.lang.String.format; import static java.util.Objects.requireNonNull; @@ -30,7 +29,6 @@ import io.delta.kernel.types.*; import io.delta.kernel.internal.DeltaErrors; - import static io.delta.kernel.internal.util.ExpressionUtils.getLeft; import static io.delta.kernel.internal.util.ExpressionUtils.getRight; import static io.delta.kernel.internal.util.ExpressionUtils.getUnaryChild; @@ -38,8 +36,8 @@ import io.delta.kernel.defaults.internal.data.vector.DefaultBooleanVector; import io.delta.kernel.defaults.internal.data.vector.DefaultConstantVector; -import static io.delta.kernel.defaults.internal.expressions.ImplicitCastExpression.canCastTo; import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.*; +import static io.delta.kernel.defaults.internal.expressions.ImplicitCastExpression.canCastTo; /** * Implementation of {@link ExpressionEvaluator} for default {@link ExpressionHandler}. @@ -408,7 +406,7 @@ ColumnVector visitComparator(Predicate predicate) { VectorComparator v; switch (predicate.getName()) { case "=": - v = new VectorComparator() { + v = new VectorComparator() { @Override public boolean compare(int compareResult) { return compareResult == 0; diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index 6410c77c59d..bcbb1d30893 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -110,7 +110,9 @@ static ColumnVector comparatorVector( return new ColumnVector() { @Override - public DataType getDataType() { return BooleanType.BOOLEAN;} + public DataType getDataType() { + return BooleanType.BOOLEAN; + } @Override public void close() { @@ -118,7 +120,9 @@ public void close() { } @Override - public int getSize() { return left.getSize(); } + public int getSize() { + return left.getSize(); + } @Override public boolean isNullAt(int rowId) { From 844e2775c4e618a140a15ee1081dd2b50531527b Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Mon, 8 Apr 2024 21:32:50 -0700 Subject: [PATCH 05/21] Fix code review --- .../DefaultExpressionEvaluator.java | 55 +++++++------------ 1 file changed, 20 insertions(+), 35 deletions(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java index ea9a8a606d6..9c367a7aa2e 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java @@ -406,45 +406,30 @@ ColumnVector visitComparator(Predicate predicate) { VectorComparator v; switch (predicate.getName()) { case "=": - v = new VectorComparator() { - @Override - public boolean compare(int compareResult) { - return compareResult == 0; - } - }; - return comparatorVector(argResults.leftResult, argResults.rightResult, v); + return comparatorVector( + argResults.leftResult, + argResults.rightResult, + (compareResult) -> (compareResult == 0)); case ">": - v = new VectorComparator() { - @Override - public boolean compare(int compareResult) { - return compareResult > 0; - } - }; - return comparatorVector(argResults.leftResult, argResults.rightResult, v); + return comparatorVector( + argResults.leftResult, + argResults.rightResult, + (compareResult) -> (compareResult > 0)); case ">=": - v = new VectorComparator() { - @Override - public boolean compare(int compareResult) { - return compareResult >= 0; - } - }; - return comparatorVector(argResults.leftResult, argResults.rightResult, v); + return comparatorVector( + argResults.leftResult, + argResults.rightResult, + (compareResult) -> (compareResult >= 0)); case "<": - v = new VectorComparator() { - @Override - public boolean compare(int compareResult) { - return compareResult < 0; - } - }; - return comparatorVector(argResults.leftResult, argResults.rightResult, v); + return comparatorVector( + argResults.leftResult, + argResults.rightResult, + (compareResult) -> (compareResult < 0)); case "<=": - v = new VectorComparator() { - @Override - public boolean compare(int compareResult) { - return compareResult <= 0; - } - }; - return comparatorVector(argResults.leftResult, argResults.rightResult, v); + return comparatorVector( + argResults.leftResult, + argResults.rightResult, + (compareResult) -> (compareResult <= 0)); default: throw DeltaErrors.unsupportedExpression( predicate, From daffb8087218e56d408ca1d66d92f9f075558578 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Mon, 8 Apr 2024 21:34:37 -0700 Subject: [PATCH 06/21] fix code review --- .../internal/expressions/DefaultExpressionEvaluator.java | 1 - .../defaults/internal/expressions/DefaultExpressionUtils.java | 1 - .../kernel/defaults/internal/expressions}/VectorComparator.java | 2 +- 3 files changed, 1 insertion(+), 3 deletions(-) rename kernel/{kernel-api/src/main/java/io/delta/kernel/data => kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions}/VectorComparator.java (93%) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java index 9c367a7aa2e..a6f2f5b46b3 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java @@ -24,7 +24,6 @@ import io.delta.kernel.client.ExpressionHandler; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.ColumnarBatch; -import io.delta.kernel.data.VectorComparator; import io.delta.kernel.expressions.*; import io.delta.kernel.types.*; diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index bcbb1d30893..6264609c13e 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -24,7 +24,6 @@ import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; -import io.delta.kernel.data.VectorComparator; import io.delta.kernel.expressions.Expression; import io.delta.kernel.types.*; import io.delta.kernel.internal.util.Utils; diff --git a/kernel/kernel-api/src/main/java/io/delta/kernel/data/VectorComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/VectorComparator.java similarity index 93% rename from kernel/kernel-api/src/main/java/io/delta/kernel/data/VectorComparator.java rename to kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/VectorComparator.java index 86f625b3de1..40365988090 100644 --- a/kernel/kernel-api/src/main/java/io/delta/kernel/data/VectorComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/VectorComparator.java @@ -14,7 +14,7 @@ * limitations under the License. */ -package io.delta.kernel.data; +package io.delta.kernel.defaults.internal.expressions; import io.delta.kernel.annotation.Evolving; From 70e829c515a5b9faaaac94b4a05dff4d0fec10c4 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Mon, 8 Apr 2024 21:54:58 -0700 Subject: [PATCH 07/21] fix cr --- .../expressions/DefaultExpressionEvaluator.java | 1 - .../internal/expressions/DefaultExpressionUtils.java | 10 +++++++++- 2 files changed, 9 insertions(+), 2 deletions(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java index a6f2f5b46b3..6729a2a11b9 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java @@ -402,7 +402,6 @@ ColumnVector visitAlwaysFalse(AlwaysFalse alwaysFalse) { @Override ColumnVector visitComparator(Predicate predicate) { PredicateChildrenEvalResult argResults = evalBinaryExpressionChildren(predicate); - VectorComparator v; switch (predicate.getName()) { case "=": return comparatorVector( diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index 6264609c13e..7fc8d702d88 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -100,9 +100,17 @@ public boolean getBoolean(int rowId) { }; } + /** + * Utility method to create a column vector that lazily evaluate the + * comparator ex. vectorComparator (ie. ==, >=, <=......) for left and right + * column vector according to the natural ordering of numbers + *

+ * Only primitive data types are supported. + */ static ColumnVector comparatorVector( ColumnVector left, - ColumnVector right, VectorComparator vectorComparator) { + ColumnVector right, + VectorComparator vectorComparator) { checkArgument( left.getSize() == right.getSize(), "Left and right operand have different vector sizes."); From 833b04bd6b246eaf51cc55ec50793d8f4ba1b95a Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Mon, 8 Apr 2024 21:57:04 -0700 Subject: [PATCH 08/21] fix ntz test --- .../defaults/internal/expressions/DefaultExpressionUtils.java | 4 +++- 1 file changed, 3 insertions(+), 1 deletion(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index 5f70e77ab7d..167aca2ae79 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -154,7 +154,9 @@ public boolean getBoolean(int rowId) { } else if (dataType instanceof IntegerType || dataType instanceof DateType) { return vectorComparator.compare( Integer.compare(left.getInt(rowId), right.getInt(rowId))); - } else if (dataType instanceof LongType || dataType instanceof TimestampType) { + } else if (dataType instanceof LongType || + dataType instanceof TimestampType || + dataType instanceof TimestampNTZType) { return vectorComparator.compare( Long.compare(left.getLong(rowId), right.getLong(rowId))); } else if (dataType instanceof FloatType) { From 1b9e5ddbb480127a592b3e68f396989773172c7e Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Mon, 8 Apr 2024 22:22:59 -0700 Subject: [PATCH 09/21] code review --- .../expressions/DefaultExpressionUtils.java | 43 ------------------- 1 file changed, 43 deletions(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index 167aca2ae79..517d8826f2f 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -184,49 +184,6 @@ public boolean getBoolean(int rowId) { }; } - /** - * Utility method to compare the left and right according to the natural ordering - * and return an integer array where each row contains the comparison result (-1, 0, 1) for - * corresponding rows in the input vectors compared. - *

- * Only primitive data types are supported. - */ - static int[] compare(ColumnVector left, ColumnVector right) { - checkArgument( - left.getSize() == right.getSize(), - "Left and right operand have different vector sizes."); - DataType dataType = left.getDataType(); - - int numRows = left.getSize(); - int[] result = new int[numRows]; - if (dataType instanceof BooleanType) { - compareBoolean(left, right, result); - } else if (dataType instanceof ByteType) { - compareByte(left, right, result); - } else if (dataType instanceof ShortType) { - compareShort(left, right, result); - } else if (dataType instanceof IntegerType || dataType instanceof DateType) { - compareInt(left, right, result); - } else if (dataType instanceof LongType || - dataType instanceof TimestampType || - dataType instanceof TimestampNTZType) { - compareLong(left, right, result); - } else if (dataType instanceof FloatType) { - compareFloat(left, right, result); - } else if (dataType instanceof DoubleType) { - compareDouble(left, right, result); - } else if (dataType instanceof DecimalType) { - compareDecimal(left, right, result); - } else if (dataType instanceof StringType) { - compareString(left, right, result); - } else if (dataType instanceof BinaryType) { - compareBinary(left, right, result); - } else { - throw new UnsupportedOperationException(dataType + " can not be compared."); - } - return result; - } - static void compareBoolean(ColumnVector left, ColumnVector right, int[] result) { for (int rowId = 0; rowId < left.getSize(); rowId++) { if (!left.isNullAt(rowId) && !right.isNullAt(rowId)) { From ee078b7502b035e9b80f2b7107c7beda00d24d89 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Wed, 1 May 2024 00:56:39 -0700 Subject: [PATCH 10/21] Fix CR --- .../internal/data/ValueComparator.java | 27 ++++++ .../data/vector/BinaryValueComparator.java | 47 +++++++++++ .../data/vector/BooleanValueComparator.java | 36 ++++++++ .../data/vector/ByteValueComparator.java | 36 ++++++++ .../data/vector/DecimalValueComparator.java | 41 +++++++++ .../data/vector/DoubleValueComparator.java | 36 ++++++++ .../data/vector/FloatValueComparator.java | 36 ++++++++ .../data/vector/IntegerValueComparator.java | 36 ++++++++ .../data/vector/LongValueComparator.java | 36 ++++++++ .../data/vector/ShortValueComparator.java | 36 ++++++++ .../data/vector/StringValueComparator.java | 39 +++++++++ .../expressions/DefaultExpressionUtils.java | 84 +++++++------------ 12 files changed, 438 insertions(+), 52 deletions(-) create mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/ValueComparator.java create mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java create mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BooleanValueComparator.java create mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ByteValueComparator.java create mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java create mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DoubleValueComparator.java create mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/FloatValueComparator.java create mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/IntegerValueComparator.java create mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/LongValueComparator.java create mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ShortValueComparator.java create mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/ValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/ValueComparator.java new file mode 100644 index 00000000000..d78e0b92aee --- /dev/null +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/ValueComparator.java @@ -0,0 +1,27 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ + +package io.delta.kernel.defaults.internal.data; + +import io.delta.kernel.annotation.Evolving; + +/** + * Comparator interface for comparing values of column vector + */ +@Evolving +public interface ValueComparator { + boolean getCompareResult(int rowId); +} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java new file mode 100644 index 00000000000..d65ea519736 --- /dev/null +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java @@ -0,0 +1,47 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults.internal.data.vector; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.defaults.internal.data.ValueComparator; +import io.delta.kernel.defaults.internal.expressions.VectorComparator; +import java.util.Comparator; + +public class BinaryValueComparator implements ValueComparator { + static final Comparator BINARY_COMPARTOR = (leftOp, rightOp) -> { + int i = 0; + while (i < leftOp.length && i < rightOp.length) { + if (leftOp[i] != rightOp[i]) { + return Byte.compare(leftOp[i], rightOp[i]); + } + i++; + } + return Integer.compare(leftOp.length, rightOp.length); + }; + ColumnVector v1; + ColumnVector v2; + VectorComparator vectorComparator; + + public BinaryValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + this.v1 = v1; + this.v2 = v2; + this.vectorComparator = vectorComparator; + } + @Override + public boolean getCompareResult(int rowId) { + return vectorComparator.compare(BINARY_COMPARTOR.compare(v1.getBinary(rowId), v2.getBinary(rowId))); + } +} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BooleanValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BooleanValueComparator.java new file mode 100644 index 00000000000..d31e09f7b79 --- /dev/null +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BooleanValueComparator.java @@ -0,0 +1,36 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults.internal.data.vector; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.defaults.internal.data.ValueComparator; +import io.delta.kernel.defaults.internal.expressions.VectorComparator; + +public class BooleanValueComparator implements ValueComparator { + ColumnVector v1; + ColumnVector v2; + VectorComparator vectorComparator; + + public BooleanValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + this.v1 = v1; + this.v2 = v2; + this.vectorComparator = vectorComparator; + } + @Override + public boolean getCompareResult(int rowId) { + return vectorComparator.compare(Boolean.compare(v1.getBoolean(rowId), v2.getBoolean(rowId))); + } +} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ByteValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ByteValueComparator.java new file mode 100644 index 00000000000..24451e9ff28 --- /dev/null +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ByteValueComparator.java @@ -0,0 +1,36 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults.internal.data.vector; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.defaults.internal.data.ValueComparator; +import io.delta.kernel.defaults.internal.expressions.VectorComparator; + +public class ByteValueComparator implements ValueComparator { + ColumnVector v1; + ColumnVector v2; + VectorComparator vectorComparator; + + public ByteValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + this.v1 = v1; + this.v2 = v2; + this.vectorComparator = vectorComparator; + } + @Override + public boolean getCompareResult(int rowId) { + return vectorComparator.compare(Byte.compare(v1.getByte(rowId), v2.getByte(rowId))); + } +} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java new file mode 100644 index 00000000000..7abeb3e9c39 --- /dev/null +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java @@ -0,0 +1,41 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults.internal.data.vector; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.defaults.internal.data.ValueComparator; +import io.delta.kernel.defaults.internal.expressions.VectorComparator; + +import java.math.BigDecimal; +import java.util.Comparator; + +public class DecimalValueComparator implements ValueComparator { + static final Comparator BIGDECIMAL_COMPARATOR = Comparator.naturalOrder(); + + ColumnVector v1; + ColumnVector v2; + VectorComparator vectorComparator; + + public DecimalValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + this.v1 = v1; + this.v2 = v2; + this.vectorComparator = vectorComparator; + } + @Override + public boolean getCompareResult(int rowId) { + return vectorComparator.compare(BIGDECIMAL_COMPARATOR.compare(v1.getDecimal(rowId), v2.getDecimal(rowId))); + } +} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DoubleValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DoubleValueComparator.java new file mode 100644 index 00000000000..acb59778702 --- /dev/null +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DoubleValueComparator.java @@ -0,0 +1,36 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults.internal.data.vector; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.defaults.internal.data.ValueComparator; +import io.delta.kernel.defaults.internal.expressions.VectorComparator; + +public class DoubleValueComparator implements ValueComparator { + ColumnVector v1; + ColumnVector v2; + VectorComparator vectorComparator; + + public DoubleValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + this.v1 = v1; + this.v2 = v2; + this.vectorComparator = vectorComparator; + } + @Override + public boolean getCompareResult(int rowId) { + return vectorComparator.compare(Double.compare(v1.getDouble(rowId), v2.getDouble(rowId))); + } +} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/FloatValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/FloatValueComparator.java new file mode 100644 index 00000000000..8af9e2f04f7 --- /dev/null +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/FloatValueComparator.java @@ -0,0 +1,36 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults.internal.data.vector; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.defaults.internal.data.ValueComparator; +import io.delta.kernel.defaults.internal.expressions.VectorComparator; + +public class FloatValueComparator implements ValueComparator { + ColumnVector v1; + ColumnVector v2; + VectorComparator vectorComparator; + + public FloatValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + this.v1 = v1; + this.v2 = v2; + this.vectorComparator = vectorComparator; + } + @Override + public boolean getCompareResult(int rowId) { + return vectorComparator.compare(Float.compare(v1.getFloat(rowId), v2.getFloat(rowId))); + } +} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/IntegerValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/IntegerValueComparator.java new file mode 100644 index 00000000000..e1a44d5eecb --- /dev/null +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/IntegerValueComparator.java @@ -0,0 +1,36 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults.internal.data.vector; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.defaults.internal.data.ValueComparator; +import io.delta.kernel.defaults.internal.expressions.VectorComparator; + +public class IntegerValueComparator implements ValueComparator { + ColumnVector v1; + ColumnVector v2; + VectorComparator vectorComparator; + + public IntegerValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + this.v1 = v1; + this.v2 = v2; + this.vectorComparator = vectorComparator; + } + @Override + public boolean getCompareResult(int rowId) { + return vectorComparator.compare(Integer.compare(v1.getInt(rowId), v2.getInt(rowId))); + } +} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/LongValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/LongValueComparator.java new file mode 100644 index 00000000000..2c5ce7f1c96 --- /dev/null +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/LongValueComparator.java @@ -0,0 +1,36 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults.internal.data.vector; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.defaults.internal.data.ValueComparator; +import io.delta.kernel.defaults.internal.expressions.VectorComparator; + +public class LongValueComparator implements ValueComparator { + ColumnVector v1; + ColumnVector v2; + VectorComparator vectorComparator; + + public LongValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + this.v1 = v1; + this.v2 = v2; + this.vectorComparator = vectorComparator; + } + @Override + public boolean getCompareResult(int rowId) { + return vectorComparator.compare(Long.compare(v1.getLong(rowId), v2.getLong(rowId))); + } +} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ShortValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ShortValueComparator.java new file mode 100644 index 00000000000..a1dc6add535 --- /dev/null +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ShortValueComparator.java @@ -0,0 +1,36 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults.internal.data.vector; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.defaults.internal.data.ValueComparator; +import io.delta.kernel.defaults.internal.expressions.VectorComparator; + +public class ShortValueComparator implements ValueComparator { + ColumnVector v1; + ColumnVector v2; + VectorComparator vectorComparator; + + public ShortValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + this.v1 = v1; + this.v2 = v2; + this.vectorComparator = vectorComparator; + } + @Override + public boolean getCompareResult(int rowId) { + return vectorComparator.compare(Short.compare(v1.getShort(rowId), v2.getShort(rowId))); + } +} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java new file mode 100644 index 00000000000..e39220edb15 --- /dev/null +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java @@ -0,0 +1,39 @@ +/* + * Copyright (2023) The Delta Lake Project Authors. + * + * Licensed under the Apache License, Version 2.0 (the "License"); + * you may not use this file except in compliance with the License. + * You may obtain a copy of the License at + * + * http://www.apache.org/licenses/LICENSE-2.0 + * + * Unless required by applicable law or agreed to in writing, software + * distributed under the License is distributed on an "AS IS" BASIS, + * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. + * See the License for the specific language governing permissions and + * limitations under the License. + */ +package io.delta.kernel.defaults.internal.data.vector; + +import io.delta.kernel.data.ColumnVector; +import io.delta.kernel.defaults.internal.data.ValueComparator; +import io.delta.kernel.defaults.internal.expressions.VectorComparator; +import java.util.Comparator; + +public class StringValueComparator implements ValueComparator { + static final Comparator STRING_COMPARATOR = Comparator.naturalOrder(); + + ColumnVector v1; + ColumnVector v2; + VectorComparator vectorComparator; + + public StringValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + this.v1 = v1; + this.v2 = v2; + this.vectorComparator = vectorComparator; + } + @Override + public boolean getCompareResult(int rowId) { + return vectorComparator.compare(STRING_COMPARATOR.compare(v1.getString(rowId), v2.getString(rowId))); + } +} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index 517d8826f2f..4faaca00fd3 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -24,6 +24,8 @@ import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; +import io.delta.kernel.defaults.internal.data.ValueComparator; +import io.delta.kernel.defaults.internal.data.vector.*; import io.delta.kernel.expressions.Expression; import io.delta.kernel.types.*; import io.delta.kernel.internal.util.Utils; @@ -34,19 +36,6 @@ */ class DefaultExpressionUtils { - static final Comparator BIGDECIMAL_COMPARATOR = Comparator.naturalOrder(); - static final Comparator STRING_COMPARATOR = Comparator.naturalOrder(); - static final Comparator BINARY_COMPARTOR = (leftOp, rightOp) -> { - int i = 0; - while (i < leftOp.length && i < rightOp.length) { - if (leftOp[i] != rightOp[i]) { - return Byte.compare(leftOp[i], rightOp[i]); - } - i++; - } - return Integer.compare(leftOp.length, rightOp.length); - }; - private DefaultExpressionUtils() {} /** @@ -114,6 +103,35 @@ static ColumnVector comparatorVector( checkArgument( left.getSize() == right.getSize(), "Left and right operand have different vector sizes."); + + DataType dataType = left.getDataType(); + ValueComparator valueComparator; + if (dataType instanceof BooleanType) { + valueComparator = new BooleanValueComparator(left, right, vectorComparator); + } else if (dataType instanceof ByteType) { + valueComparator = new ByteValueComparator(left, right, vectorComparator); + } else if (dataType instanceof ShortType) { + valueComparator = new ShortValueComparator(left, right, vectorComparator); + } else if (dataType instanceof IntegerType || dataType instanceof DateType) { + valueComparator = new IntegerValueComparator(left, right, vectorComparator); + } else if (dataType instanceof LongType || + dataType instanceof TimestampType || + dataType instanceof TimestampNTZType) { + valueComparator = new LongValueComparator(left, right, vectorComparator); + } else if (dataType instanceof FloatType) { + valueComparator = new FloatValueComparator(left, right, vectorComparator); + } else if (dataType instanceof DoubleType) { + valueComparator = new DoubleValueComparator(left, right, vectorComparator); + } else if (dataType instanceof DecimalType) { + valueComparator = new DecimalValueComparator(left, right, vectorComparator); + } else if (dataType instanceof StringType) { + valueComparator = new StringValueComparator(left, right, vectorComparator); + } else if (dataType instanceof BinaryType) { + valueComparator = new BinaryValueComparator(left, right, vectorComparator); + } else { + throw new UnsupportedOperationException(dataType + " can not be compared."); + } + return new ColumnVector() { @Override @@ -141,45 +159,7 @@ public boolean getBoolean(int rowId) { if (isNullAt(rowId)) { return false; } - DataType dataType = left.getDataType(); - if (dataType instanceof BooleanType) { - return vectorComparator.compare( - Boolean.compare(left.getBoolean(rowId), right.getBoolean(rowId))); - } else if (dataType instanceof ByteType) { - return vectorComparator.compare( - Byte.compare(left.getByte(rowId), right.getByte(rowId))); - } else if (dataType instanceof ShortType) { - return vectorComparator.compare( - Short.compare(left.getShort(rowId), right.getShort(rowId))); - } else if (dataType instanceof IntegerType || dataType instanceof DateType) { - return vectorComparator.compare( - Integer.compare(left.getInt(rowId), right.getInt(rowId))); - } else if (dataType instanceof LongType || - dataType instanceof TimestampType || - dataType instanceof TimestampNTZType) { - return vectorComparator.compare( - Long.compare(left.getLong(rowId), right.getLong(rowId))); - } else if (dataType instanceof FloatType) { - return vectorComparator.compare( - Float.compare(left.getFloat(rowId), right.getFloat(rowId))); - } else if (dataType instanceof DoubleType) { - return vectorComparator.compare( - Double.compare(left.getDouble(rowId), right.getDouble(rowId))); - } else if (dataType instanceof DecimalType) { - return vectorComparator.compare( - BIGDECIMAL_COMPARATOR.compare( - left.getDecimal(rowId), right.getDecimal(rowId))); - } else if (dataType instanceof StringType) { - return vectorComparator.compare( - STRING_COMPARATOR.compare( - left.getString(rowId), right.getString(rowId))); - } else if (dataType instanceof BinaryType) { - return vectorComparator.compare( - BINARY_COMPARTOR.compare( - left.getBinary(rowId), right.getBinary(rowId))); - } else { - throw new UnsupportedOperationException(dataType + " can not be compared."); - } + return valueComparator.getCompareResult(rowId); } }; } From 840f58685a4d7d0a2bbbbe7c61361349270c1e24 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Wed, 1 May 2024 01:06:45 -0700 Subject: [PATCH 11/21] fix cr --- .../defaults/internal/expressions/DefaultExpressionUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index 4faaca00fd3..543e47a1318 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -26,9 +26,9 @@ import io.delta.kernel.data.MapValue; import io.delta.kernel.defaults.internal.data.ValueComparator; import io.delta.kernel.defaults.internal.data.vector.*; -import io.delta.kernel.expressions.Expression; import io.delta.kernel.types.*; import io.delta.kernel.internal.util.Utils; +import io.delta.kernel.expressions.Expression; import static io.delta.kernel.internal.util.Preconditions.checkArgument; /** From 4b0818e3800079ba04f834998cbb53970952593e Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Wed, 1 May 2024 01:11:23 -0700 Subject: [PATCH 12/21] fix cr --- .../internal/data/vector/BinaryValueComparator.java | 6 ++++-- .../internal/data/vector/BooleanValueComparator.java | 6 ++++-- .../defaults/internal/data/vector/ByteValueComparator.java | 3 ++- .../internal/data/vector/DecimalValueComparator.java | 6 ++++-- .../internal/data/vector/DoubleValueComparator.java | 6 ++++-- .../defaults/internal/data/vector/FloatValueComparator.java | 6 ++++-- .../internal/data/vector/IntegerValueComparator.java | 6 ++++-- .../defaults/internal/data/vector/LongValueComparator.java | 3 ++- .../defaults/internal/data/vector/ShortValueComparator.java | 3 ++- .../internal/data/vector/StringValueComparator.java | 6 ++++-- .../internal/expressions/DefaultExpressionUtils.java | 2 +- 11 files changed, 35 insertions(+), 18 deletions(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java index d65ea519736..02f5c95f750 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java @@ -35,13 +35,15 @@ public class BinaryValueComparator implements ValueComparator { ColumnVector v2; VectorComparator vectorComparator; - public BinaryValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + public BinaryValueComparator(ColumnVector v1, ColumnVector v2, + VectorComparator vectorComparator) { this.v1 = v1; this.v2 = v2; this.vectorComparator = vectorComparator; } @Override public boolean getCompareResult(int rowId) { - return vectorComparator.compare(BINARY_COMPARTOR.compare(v1.getBinary(rowId), v2.getBinary(rowId))); + return vectorComparator.compare( + BINARY_COMPARTOR.compare(v1.getBinary(rowId), v2.getBinary(rowId))); } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BooleanValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BooleanValueComparator.java index d31e09f7b79..3fb4e0dd60c 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BooleanValueComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BooleanValueComparator.java @@ -24,13 +24,15 @@ public class BooleanValueComparator implements ValueComparator { ColumnVector v2; VectorComparator vectorComparator; - public BooleanValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + public BooleanValueComparator(ColumnVector v1, ColumnVector v2, + VectorComparator vectorComparator) { this.v1 = v1; this.v2 = v2; this.vectorComparator = vectorComparator; } @Override public boolean getCompareResult(int rowId) { - return vectorComparator.compare(Boolean.compare(v1.getBoolean(rowId), v2.getBoolean(rowId))); + return vectorComparator.compare( + Boolean.compare(v1.getBoolean(rowId), v2.getBoolean(rowId))); } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ByteValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ByteValueComparator.java index 24451e9ff28..0ee66a4c688 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ByteValueComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ByteValueComparator.java @@ -24,7 +24,8 @@ public class ByteValueComparator implements ValueComparator { ColumnVector v2; VectorComparator vectorComparator; - public ByteValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + public ByteValueComparator(ColumnVector v1, ColumnVector v2, + VectorComparator vectorComparator) { this.v1 = v1; this.v2 = v2; this.vectorComparator = vectorComparator; diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java index 7abeb3e9c39..04bd915b5cb 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java @@ -29,13 +29,15 @@ public class DecimalValueComparator implements ValueComparator { ColumnVector v2; VectorComparator vectorComparator; - public DecimalValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + public DecimalValueComparator(ColumnVector v1, ColumnVector v2, + VectorComparator vectorComparator) { this.v1 = v1; this.v2 = v2; this.vectorComparator = vectorComparator; } @Override public boolean getCompareResult(int rowId) { - return vectorComparator.compare(BIGDECIMAL_COMPARATOR.compare(v1.getDecimal(rowId), v2.getDecimal(rowId))); + return vectorComparator.compare( + BIGDECIMAL_COMPARATOR.compare(v1.getDecimal(rowId), v2.getDecimal(rowId))); } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DoubleValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DoubleValueComparator.java index acb59778702..4ad09421727 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DoubleValueComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DoubleValueComparator.java @@ -24,13 +24,15 @@ public class DoubleValueComparator implements ValueComparator { ColumnVector v2; VectorComparator vectorComparator; - public DoubleValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + public DoubleValueComparator(ColumnVector v1, ColumnVector v2, + VectorComparator vectorComparator) { this.v1 = v1; this.v2 = v2; this.vectorComparator = vectorComparator; } @Override public boolean getCompareResult(int rowId) { - return vectorComparator.compare(Double.compare(v1.getDouble(rowId), v2.getDouble(rowId))); + return vectorComparator.compare( + Double.compare(v1.getDouble(rowId), v2.getDouble(rowId))); } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/FloatValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/FloatValueComparator.java index 8af9e2f04f7..8bc405ffd60 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/FloatValueComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/FloatValueComparator.java @@ -24,13 +24,15 @@ public class FloatValueComparator implements ValueComparator { ColumnVector v2; VectorComparator vectorComparator; - public FloatValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + public FloatValueComparator(ColumnVector v1, ColumnVector v2, + VectorComparator vectorComparator) { this.v1 = v1; this.v2 = v2; this.vectorComparator = vectorComparator; } @Override public boolean getCompareResult(int rowId) { - return vectorComparator.compare(Float.compare(v1.getFloat(rowId), v2.getFloat(rowId))); + return vectorComparator.compare( + Float.compare(v1.getFloat(rowId), v2.getFloat(rowId))); } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/IntegerValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/IntegerValueComparator.java index e1a44d5eecb..7a1be0c9d1e 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/IntegerValueComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/IntegerValueComparator.java @@ -24,13 +24,15 @@ public class IntegerValueComparator implements ValueComparator { ColumnVector v2; VectorComparator vectorComparator; - public IntegerValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + public IntegerValueComparator(ColumnVector v1, ColumnVector v2, + VectorComparator vectorComparator) { this.v1 = v1; this.v2 = v2; this.vectorComparator = vectorComparator; } @Override public boolean getCompareResult(int rowId) { - return vectorComparator.compare(Integer.compare(v1.getInt(rowId), v2.getInt(rowId))); + return vectorComparator.compare( + Integer.compare(v1.getInt(rowId), v2.getInt(rowId))); } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/LongValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/LongValueComparator.java index 2c5ce7f1c96..b4271bd5f06 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/LongValueComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/LongValueComparator.java @@ -24,7 +24,8 @@ public class LongValueComparator implements ValueComparator { ColumnVector v2; VectorComparator vectorComparator; - public LongValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + public LongValueComparator(ColumnVector v1, ColumnVector v2, + VectorComparator vectorComparator) { this.v1 = v1; this.v2 = v2; this.vectorComparator = vectorComparator; diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ShortValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ShortValueComparator.java index a1dc6add535..fa2f6c7a852 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ShortValueComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ShortValueComparator.java @@ -24,7 +24,8 @@ public class ShortValueComparator implements ValueComparator { ColumnVector v2; VectorComparator vectorComparator; - public ShortValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + public ShortValueComparator(ColumnVector v1, ColumnVector v2, + VectorComparator vectorComparator) { this.v1 = v1; this.v2 = v2; this.vectorComparator = vectorComparator; diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java index e39220edb15..e1b244009d6 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java @@ -27,13 +27,15 @@ public class StringValueComparator implements ValueComparator { ColumnVector v2; VectorComparator vectorComparator; - public StringValueComparator(ColumnVector v1, ColumnVector v2, VectorComparator vectorComparator) { + public StringValueComparator(ColumnVector v1, ColumnVector v2, + VectorComparator vectorComparator) { this.v1 = v1; this.v2 = v2; this.vectorComparator = vectorComparator; } @Override public boolean getCompareResult(int rowId) { - return vectorComparator.compare(STRING_COMPARATOR.compare(v1.getString(rowId), v2.getString(rowId))); + return vectorComparator.compare( + STRING_COMPARATOR.compare(v1.getString(rowId), v2.getString(rowId))); } } diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index 543e47a1318..097a0fd9b3b 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -27,8 +27,8 @@ import io.delta.kernel.defaults.internal.data.ValueComparator; import io.delta.kernel.defaults.internal.data.vector.*; import io.delta.kernel.types.*; -import io.delta.kernel.internal.util.Utils; import io.delta.kernel.expressions.Expression; +import io.delta.kernel.internal.util.Utils; import static io.delta.kernel.internal.util.Preconditions.checkArgument; /** From be97175d5ba9d9ea1bae5a03bcd9392c989eb467 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Wed, 1 May 2024 01:15:17 -0700 Subject: [PATCH 13/21] fix cr --- .../defaults/internal/expressions/DefaultExpressionUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index 097a0fd9b3b..b69157da9f4 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -21,13 +21,13 @@ import java.util.function.Function; import java.util.stream.Collectors; +import io.delta.kernel.expressions.Expression; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; import io.delta.kernel.defaults.internal.data.ValueComparator; import io.delta.kernel.defaults.internal.data.vector.*; import io.delta.kernel.types.*; -import io.delta.kernel.expressions.Expression; import io.delta.kernel.internal.util.Utils; import static io.delta.kernel.internal.util.Preconditions.checkArgument; From 8afbc835143e0d8af5b0a65ae4ad79d182b40c13 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Wed, 1 May 2024 01:18:36 -0700 Subject: [PATCH 14/21] fix cr --- .../defaults/internal/expressions/DefaultExpressionUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index b69157da9f4..d481c6332f8 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -21,10 +21,10 @@ import java.util.function.Function; import java.util.stream.Collectors; -import io.delta.kernel.expressions.Expression; import io.delta.kernel.data.ArrayValue; import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; +import io.delta.kernel.expressions.Expression; import io.delta.kernel.defaults.internal.data.ValueComparator; import io.delta.kernel.defaults.internal.data.vector.*; import io.delta.kernel.types.*; From d01c35d37e3f5ab6623aaedcf79da6722e3e7e44 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Wed, 1 May 2024 01:24:16 -0700 Subject: [PATCH 15/21] fix cr --- .../defaults/internal/expressions/DefaultExpressionUtils.java | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index d481c6332f8..20c46025408 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -25,10 +25,10 @@ import io.delta.kernel.data.ColumnVector; import io.delta.kernel.data.MapValue; import io.delta.kernel.expressions.Expression; -import io.delta.kernel.defaults.internal.data.ValueComparator; -import io.delta.kernel.defaults.internal.data.vector.*; import io.delta.kernel.types.*; import io.delta.kernel.internal.util.Utils; +import io.delta.kernel.defaults.internal.data.ValueComparator; +import io.delta.kernel.defaults.internal.data.vector.*; import static io.delta.kernel.internal.util.Preconditions.checkArgument; /** From 63dc5df847ed747427a895719713b2acb39cd9cf Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Wed, 1 May 2024 01:25:52 -0700 Subject: [PATCH 16/21] fix cr --- .../internal/data/vector/BinaryValueComparator.java | 3 ++- .../internal/data/vector/DecimalValueComparator.java | 6 +++--- .../internal/data/vector/StringValueComparator.java | 3 ++- 3 files changed, 7 insertions(+), 5 deletions(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java index 02f5c95f750..68483ced97e 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java @@ -15,10 +15,11 @@ */ package io.delta.kernel.defaults.internal.data.vector; +import java.util.Comparator; + import io.delta.kernel.data.ColumnVector; import io.delta.kernel.defaults.internal.data.ValueComparator; import io.delta.kernel.defaults.internal.expressions.VectorComparator; -import java.util.Comparator; public class BinaryValueComparator implements ValueComparator { static final Comparator BINARY_COMPARTOR = (leftOp, rightOp) -> { diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java index 04bd915b5cb..27aa4d96f2d 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java @@ -15,13 +15,13 @@ */ package io.delta.kernel.defaults.internal.data.vector; +import java.math.BigDecimal; +import java.util.Comparator; + import io.delta.kernel.data.ColumnVector; import io.delta.kernel.defaults.internal.data.ValueComparator; import io.delta.kernel.defaults.internal.expressions.VectorComparator; -import java.math.BigDecimal; -import java.util.Comparator; - public class DecimalValueComparator implements ValueComparator { static final Comparator BIGDECIMAL_COMPARATOR = Comparator.naturalOrder(); diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java index e1b244009d6..3871286c939 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java @@ -15,10 +15,11 @@ */ package io.delta.kernel.defaults.internal.data.vector; +import java.util.Comparator; + import io.delta.kernel.data.ColumnVector; import io.delta.kernel.defaults.internal.data.ValueComparator; import io.delta.kernel.defaults.internal.expressions.VectorComparator; -import java.util.Comparator; public class StringValueComparator implements ValueComparator { static final Comparator STRING_COMPARATOR = Comparator.naturalOrder(); From eb309613455e2112366124d86f98e3e11b4c8247 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Wed, 1 May 2024 01:27:57 -0700 Subject: [PATCH 17/21] fix cr --- .../defaults/internal/expressions/DefaultExpressionUtils.java | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index 20c46025408..350eb43b967 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -27,9 +27,9 @@ import io.delta.kernel.expressions.Expression; import io.delta.kernel.types.*; import io.delta.kernel.internal.util.Utils; +import static io.delta.kernel.internal.util.Preconditions.checkArgument; import io.delta.kernel.defaults.internal.data.ValueComparator; import io.delta.kernel.defaults.internal.data.vector.*; -import static io.delta.kernel.internal.util.Preconditions.checkArgument; /** * Utility methods used by the default expression evaluator. From a086174caa45461739f624b3fbc9e49d10d94a31 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Wed, 15 May 2024 22:26:16 -0700 Subject: [PATCH 18/21] cr --- .../internal/expressions/DefaultExpressionEvaluator.java | 2 -- 1 file changed, 2 deletions(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java index a5c9f1e9f14..c444bb7bea0 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionEvaluator.java @@ -39,8 +39,6 @@ import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.*; import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.booleanWrapperVector; import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.childAt; -import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.compare; -import static io.delta.kernel.defaults.internal.expressions.DefaultExpressionUtils.evalNullability; import static io.delta.kernel.defaults.internal.expressions.ImplicitCastExpression.canCastTo; /** From 843ad18e9560cafcc6805f4f58f115b65d0e9fd9 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Tue, 28 May 2024 01:23:24 -0700 Subject: [PATCH 19/21] fix cr --- .../internal/data/ValueComparator.java | 27 --------- .../data/vector/BinaryValueComparator.java | 50 ----------------- .../data/vector/BooleanValueComparator.java | 38 ------------- .../data/vector/ByteValueComparator.java | 37 ------------- .../data/vector/DecimalValueComparator.java | 43 --------------- .../data/vector/DoubleValueComparator.java | 38 ------------- .../data/vector/FloatValueComparator.java | 38 ------------- .../data/vector/IntegerValueComparator.java | 38 ------------- .../data/vector/LongValueComparator.java | 37 ------------- .../data/vector/ShortValueComparator.java | 37 ------------- .../data/vector/StringValueComparator.java | 42 -------------- .../expressions/DefaultExpressionUtils.java | 55 ++++++++++++++----- 12 files changed, 40 insertions(+), 440 deletions(-) delete mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/ValueComparator.java delete mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java delete mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BooleanValueComparator.java delete mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ByteValueComparator.java delete mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java delete mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DoubleValueComparator.java delete mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/FloatValueComparator.java delete mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/IntegerValueComparator.java delete mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/LongValueComparator.java delete mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ShortValueComparator.java delete mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/ValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/ValueComparator.java deleted file mode 100644 index d78e0b92aee..00000000000 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/ValueComparator.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (2023) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.kernel.defaults.internal.data; - -import io.delta.kernel.annotation.Evolving; - -/** - * Comparator interface for comparing values of column vector - */ -@Evolving -public interface ValueComparator { - boolean getCompareResult(int rowId); -} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java deleted file mode 100644 index 68483ced97e..00000000000 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BinaryValueComparator.java +++ /dev/null @@ -1,50 +0,0 @@ -/* - * Copyright (2023) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.defaults.internal.data.vector; - -import java.util.Comparator; - -import io.delta.kernel.data.ColumnVector; -import io.delta.kernel.defaults.internal.data.ValueComparator; -import io.delta.kernel.defaults.internal.expressions.VectorComparator; - -public class BinaryValueComparator implements ValueComparator { - static final Comparator BINARY_COMPARTOR = (leftOp, rightOp) -> { - int i = 0; - while (i < leftOp.length && i < rightOp.length) { - if (leftOp[i] != rightOp[i]) { - return Byte.compare(leftOp[i], rightOp[i]); - } - i++; - } - return Integer.compare(leftOp.length, rightOp.length); - }; - ColumnVector v1; - ColumnVector v2; - VectorComparator vectorComparator; - - public BinaryValueComparator(ColumnVector v1, ColumnVector v2, - VectorComparator vectorComparator) { - this.v1 = v1; - this.v2 = v2; - this.vectorComparator = vectorComparator; - } - @Override - public boolean getCompareResult(int rowId) { - return vectorComparator.compare( - BINARY_COMPARTOR.compare(v1.getBinary(rowId), v2.getBinary(rowId))); - } -} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BooleanValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BooleanValueComparator.java deleted file mode 100644 index 3fb4e0dd60c..00000000000 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/BooleanValueComparator.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (2023) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.defaults.internal.data.vector; - -import io.delta.kernel.data.ColumnVector; -import io.delta.kernel.defaults.internal.data.ValueComparator; -import io.delta.kernel.defaults.internal.expressions.VectorComparator; - -public class BooleanValueComparator implements ValueComparator { - ColumnVector v1; - ColumnVector v2; - VectorComparator vectorComparator; - - public BooleanValueComparator(ColumnVector v1, ColumnVector v2, - VectorComparator vectorComparator) { - this.v1 = v1; - this.v2 = v2; - this.vectorComparator = vectorComparator; - } - @Override - public boolean getCompareResult(int rowId) { - return vectorComparator.compare( - Boolean.compare(v1.getBoolean(rowId), v2.getBoolean(rowId))); - } -} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ByteValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ByteValueComparator.java deleted file mode 100644 index 0ee66a4c688..00000000000 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ByteValueComparator.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (2023) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.defaults.internal.data.vector; - -import io.delta.kernel.data.ColumnVector; -import io.delta.kernel.defaults.internal.data.ValueComparator; -import io.delta.kernel.defaults.internal.expressions.VectorComparator; - -public class ByteValueComparator implements ValueComparator { - ColumnVector v1; - ColumnVector v2; - VectorComparator vectorComparator; - - public ByteValueComparator(ColumnVector v1, ColumnVector v2, - VectorComparator vectorComparator) { - this.v1 = v1; - this.v2 = v2; - this.vectorComparator = vectorComparator; - } - @Override - public boolean getCompareResult(int rowId) { - return vectorComparator.compare(Byte.compare(v1.getByte(rowId), v2.getByte(rowId))); - } -} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java deleted file mode 100644 index 27aa4d96f2d..00000000000 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DecimalValueComparator.java +++ /dev/null @@ -1,43 +0,0 @@ -/* - * Copyright (2023) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.defaults.internal.data.vector; - -import java.math.BigDecimal; -import java.util.Comparator; - -import io.delta.kernel.data.ColumnVector; -import io.delta.kernel.defaults.internal.data.ValueComparator; -import io.delta.kernel.defaults.internal.expressions.VectorComparator; - -public class DecimalValueComparator implements ValueComparator { - static final Comparator BIGDECIMAL_COMPARATOR = Comparator.naturalOrder(); - - ColumnVector v1; - ColumnVector v2; - VectorComparator vectorComparator; - - public DecimalValueComparator(ColumnVector v1, ColumnVector v2, - VectorComparator vectorComparator) { - this.v1 = v1; - this.v2 = v2; - this.vectorComparator = vectorComparator; - } - @Override - public boolean getCompareResult(int rowId) { - return vectorComparator.compare( - BIGDECIMAL_COMPARATOR.compare(v1.getDecimal(rowId), v2.getDecimal(rowId))); - } -} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DoubleValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DoubleValueComparator.java deleted file mode 100644 index 4ad09421727..00000000000 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/DoubleValueComparator.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (2023) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.defaults.internal.data.vector; - -import io.delta.kernel.data.ColumnVector; -import io.delta.kernel.defaults.internal.data.ValueComparator; -import io.delta.kernel.defaults.internal.expressions.VectorComparator; - -public class DoubleValueComparator implements ValueComparator { - ColumnVector v1; - ColumnVector v2; - VectorComparator vectorComparator; - - public DoubleValueComparator(ColumnVector v1, ColumnVector v2, - VectorComparator vectorComparator) { - this.v1 = v1; - this.v2 = v2; - this.vectorComparator = vectorComparator; - } - @Override - public boolean getCompareResult(int rowId) { - return vectorComparator.compare( - Double.compare(v1.getDouble(rowId), v2.getDouble(rowId))); - } -} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/FloatValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/FloatValueComparator.java deleted file mode 100644 index 8bc405ffd60..00000000000 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/FloatValueComparator.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (2023) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.defaults.internal.data.vector; - -import io.delta.kernel.data.ColumnVector; -import io.delta.kernel.defaults.internal.data.ValueComparator; -import io.delta.kernel.defaults.internal.expressions.VectorComparator; - -public class FloatValueComparator implements ValueComparator { - ColumnVector v1; - ColumnVector v2; - VectorComparator vectorComparator; - - public FloatValueComparator(ColumnVector v1, ColumnVector v2, - VectorComparator vectorComparator) { - this.v1 = v1; - this.v2 = v2; - this.vectorComparator = vectorComparator; - } - @Override - public boolean getCompareResult(int rowId) { - return vectorComparator.compare( - Float.compare(v1.getFloat(rowId), v2.getFloat(rowId))); - } -} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/IntegerValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/IntegerValueComparator.java deleted file mode 100644 index 7a1be0c9d1e..00000000000 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/IntegerValueComparator.java +++ /dev/null @@ -1,38 +0,0 @@ -/* - * Copyright (2023) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.defaults.internal.data.vector; - -import io.delta.kernel.data.ColumnVector; -import io.delta.kernel.defaults.internal.data.ValueComparator; -import io.delta.kernel.defaults.internal.expressions.VectorComparator; - -public class IntegerValueComparator implements ValueComparator { - ColumnVector v1; - ColumnVector v2; - VectorComparator vectorComparator; - - public IntegerValueComparator(ColumnVector v1, ColumnVector v2, - VectorComparator vectorComparator) { - this.v1 = v1; - this.v2 = v2; - this.vectorComparator = vectorComparator; - } - @Override - public boolean getCompareResult(int rowId) { - return vectorComparator.compare( - Integer.compare(v1.getInt(rowId), v2.getInt(rowId))); - } -} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/LongValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/LongValueComparator.java deleted file mode 100644 index b4271bd5f06..00000000000 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/LongValueComparator.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (2023) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.defaults.internal.data.vector; - -import io.delta.kernel.data.ColumnVector; -import io.delta.kernel.defaults.internal.data.ValueComparator; -import io.delta.kernel.defaults.internal.expressions.VectorComparator; - -public class LongValueComparator implements ValueComparator { - ColumnVector v1; - ColumnVector v2; - VectorComparator vectorComparator; - - public LongValueComparator(ColumnVector v1, ColumnVector v2, - VectorComparator vectorComparator) { - this.v1 = v1; - this.v2 = v2; - this.vectorComparator = vectorComparator; - } - @Override - public boolean getCompareResult(int rowId) { - return vectorComparator.compare(Long.compare(v1.getLong(rowId), v2.getLong(rowId))); - } -} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ShortValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ShortValueComparator.java deleted file mode 100644 index fa2f6c7a852..00000000000 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/ShortValueComparator.java +++ /dev/null @@ -1,37 +0,0 @@ -/* - * Copyright (2023) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.defaults.internal.data.vector; - -import io.delta.kernel.data.ColumnVector; -import io.delta.kernel.defaults.internal.data.ValueComparator; -import io.delta.kernel.defaults.internal.expressions.VectorComparator; - -public class ShortValueComparator implements ValueComparator { - ColumnVector v1; - ColumnVector v2; - VectorComparator vectorComparator; - - public ShortValueComparator(ColumnVector v1, ColumnVector v2, - VectorComparator vectorComparator) { - this.v1 = v1; - this.v2 = v2; - this.vectorComparator = vectorComparator; - } - @Override - public boolean getCompareResult(int rowId) { - return vectorComparator.compare(Short.compare(v1.getShort(rowId), v2.getShort(rowId))); - } -} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java deleted file mode 100644 index 3871286c939..00000000000 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/data/vector/StringValueComparator.java +++ /dev/null @@ -1,42 +0,0 @@ -/* - * Copyright (2023) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ -package io.delta.kernel.defaults.internal.data.vector; - -import java.util.Comparator; - -import io.delta.kernel.data.ColumnVector; -import io.delta.kernel.defaults.internal.data.ValueComparator; -import io.delta.kernel.defaults.internal.expressions.VectorComparator; - -public class StringValueComparator implements ValueComparator { - static final Comparator STRING_COMPARATOR = Comparator.naturalOrder(); - - ColumnVector v1; - ColumnVector v2; - VectorComparator vectorComparator; - - public StringValueComparator(ColumnVector v1, ColumnVector v2, - VectorComparator vectorComparator) { - this.v1 = v1; - this.v2 = v2; - this.vectorComparator = vectorComparator; - } - @Override - public boolean getCompareResult(int rowId) { - return vectorComparator.compare( - STRING_COMPARATOR.compare(v1.getString(rowId), v2.getString(rowId))); - } -} diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index 350eb43b967..1c5dff4098d 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -19,6 +19,7 @@ import java.util.Comparator; import java.util.List; import java.util.function.Function; +import java.util.function.IntPredicate; import java.util.stream.Collectors; import io.delta.kernel.data.ArrayValue; @@ -28,14 +29,25 @@ import io.delta.kernel.types.*; import io.delta.kernel.internal.util.Utils; import static io.delta.kernel.internal.util.Preconditions.checkArgument; -import io.delta.kernel.defaults.internal.data.ValueComparator; -import io.delta.kernel.defaults.internal.data.vector.*; /** * Utility methods used by the default expression evaluator. */ class DefaultExpressionUtils { + static final Comparator BIGDECIMAL_COMPARATOR = Comparator.naturalOrder(); + static final Comparator STRING_COMPARATOR = Comparator.naturalOrder(); + static final Comparator BINARY_COMPARTOR = (leftOp, rightOp) -> { + int i = 0; + while (i < leftOp.length && i < rightOp.length) { + if (leftOp[i] != rightOp[i]) { + return Byte.compare(leftOp[i], rightOp[i]); + } + i++; + } + return Integer.compare(leftOp.length, rightOp.length); + }; + private DefaultExpressionUtils() {} /** @@ -99,35 +111,48 @@ public boolean getBoolean(int rowId) { static ColumnVector comparatorVector( ColumnVector left, ColumnVector right, - VectorComparator vectorComparator) { + IntPredicate booleanComparator) { checkArgument( left.getSize() == right.getSize(), "Left and right operand have different vector sizes."); DataType dataType = left.getDataType(); - ValueComparator valueComparator; + IntPredicate vectorValueComparator; if (dataType instanceof BooleanType) { - valueComparator = new BooleanValueComparator(left, right, vectorComparator); + vectorValueComparator = rowId -> booleanComparator.test( + Boolean.compare(left.getBoolean(rowId), right.getBoolean(rowId))); } else if (dataType instanceof ByteType) { - valueComparator = new ByteValueComparator(left, right, vectorComparator); + vectorValueComparator = rowId -> booleanComparator.test( + Byte.compare(left.getByte(rowId), right.getByte(rowId))); } else if (dataType instanceof ShortType) { - valueComparator = new ShortValueComparator(left, right, vectorComparator); + vectorValueComparator = rowId -> booleanComparator.test( + Short.compare(left.getShort(rowId), right.getShort(rowId))); } else if (dataType instanceof IntegerType || dataType instanceof DateType) { - valueComparator = new IntegerValueComparator(left, right, vectorComparator); + vectorValueComparator = rowId -> booleanComparator.test( + Integer.compare(left.getInt(rowId), right.getInt(rowId))); } else if (dataType instanceof LongType || dataType instanceof TimestampType || dataType instanceof TimestampNTZType) { - valueComparator = new LongValueComparator(left, right, vectorComparator); + vectorValueComparator = rowId -> booleanComparator.test( + Long.compare(left.getLong(rowId), right.getLong(rowId))); } else if (dataType instanceof FloatType) { - valueComparator = new FloatValueComparator(left, right, vectorComparator); + vectorValueComparator = rowId -> booleanComparator.test( + Float.compare(left.getFloat(rowId), right.getFloat(rowId))); } else if (dataType instanceof DoubleType) { - valueComparator = new DoubleValueComparator(left, right, vectorComparator); + vectorValueComparator = rowId -> booleanComparator.test( + Double.compare(left.getDouble(rowId), right.getDouble(rowId))); } else if (dataType instanceof DecimalType) { - valueComparator = new DecimalValueComparator(left, right, vectorComparator); + vectorValueComparator = rowId -> booleanComparator.test( + BIGDECIMAL_COMPARATOR.compare( + left.getDecimal(rowId), right.getDecimal(rowId))); } else if (dataType instanceof StringType) { - valueComparator = new StringValueComparator(left, right, vectorComparator); + vectorValueComparator = rowId -> booleanComparator.test( + STRING_COMPARATOR.compare( + left.getString(rowId), right.getString(rowId))); } else if (dataType instanceof BinaryType) { - valueComparator = new BinaryValueComparator(left, right, vectorComparator); + vectorValueComparator = rowId -> booleanComparator.test( + BINARY_COMPARTOR.compare( + left.getBinary(rowId), right.getBinary(rowId))); } else { throw new UnsupportedOperationException(dataType + " can not be compared."); } @@ -159,7 +184,7 @@ public boolean getBoolean(int rowId) { if (isNullAt(rowId)) { return false; } - return valueComparator.getCompareResult(rowId); + return vectorValueComparator.test(rowId); } }; } From a73803bbb300c2dc1862fa7f427d6c05671ba988 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Tue, 28 May 2024 01:28:01 -0700 Subject: [PATCH 20/21] fix cr --- .../expressions/DefaultExpressionUtils.java | 92 ------------------- 1 file changed, 92 deletions(-) diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index 1c5dff4098d..675d9dd8718 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -189,98 +189,6 @@ public boolean getBoolean(int rowId) { }; } - static void compareBoolean(ColumnVector left, ColumnVector right, int[] result) { - for (int rowId = 0; rowId < left.getSize(); rowId++) { - if (!left.isNullAt(rowId) && !right.isNullAt(rowId)) { - result[rowId] = Boolean.compare(left.getBoolean(rowId), right.getBoolean(rowId)); - } - } - } - - static void compareByte(ColumnVector left, ColumnVector right, int[] result) { - for (int rowId = 0; rowId < left.getSize(); rowId++) { - if (!left.isNullAt(rowId) && !right.isNullAt(rowId)) { - result[rowId] = Byte.compare(left.getByte(rowId), right.getByte(rowId)); - } - } - } - - static void compareShort(ColumnVector left, ColumnVector right, int[] result) { - for (int rowId = 0; rowId < left.getSize(); rowId++) { - if (!left.isNullAt(rowId) && !right.isNullAt(rowId)) { - result[rowId] = Short.compare(left.getShort(rowId), right.getShort(rowId)); - } - } - } - - static void compareInt(ColumnVector left, ColumnVector right, int[] result) { - for (int rowId = 0; rowId < left.getSize(); rowId++) { - if (!left.isNullAt(rowId) && !right.isNullAt(rowId)) { - result[rowId] = Integer.compare(left.getInt(rowId), right.getInt(rowId)); - } - } - } - - static void compareLong(ColumnVector left, ColumnVector right, int[] result) { - for (int rowId = 0; rowId < left.getSize(); rowId++) { - if (!left.isNullAt(rowId) && !right.isNullAt(rowId)) { - result[rowId] = Long.compare(left.getLong(rowId), right.getLong(rowId)); - } - } - } - - static void compareFloat(ColumnVector left, ColumnVector right, int[] result) { - for (int rowId = 0; rowId < left.getSize(); rowId++) { - if (!left.isNullAt(rowId) && !right.isNullAt(rowId)) { - result[rowId] = Float.compare(left.getFloat(rowId), right.getFloat(rowId)); - } - } - } - - static void compareDouble(ColumnVector left, ColumnVector right, int[] result) { - for (int rowId = 0; rowId < left.getSize(); rowId++) { - if (!left.isNullAt(rowId) && !right.isNullAt(rowId)) { - result[rowId] = Double.compare(left.getDouble(rowId), right.getDouble(rowId)); - } - } - } - - static void compareString(ColumnVector left, ColumnVector right, int[] result) { - Comparator comparator = Comparator.naturalOrder(); - for (int rowId = 0; rowId < left.getSize(); rowId++) { - if (!left.isNullAt(rowId) && !right.isNullAt(rowId)) { - result[rowId] = comparator.compare(left.getString(rowId), right.getString(rowId)); - } - } - } - - static void compareDecimal(ColumnVector left, ColumnVector right, int[] result) { - Comparator comparator = Comparator.naturalOrder(); - for (int rowId = 0; rowId < left.getSize(); rowId++) { - if (!left.isNullAt(rowId) && !right.isNullAt(rowId)) { - result[rowId] = comparator.compare(left.getDecimal(rowId), right.getDecimal(rowId)); - } - } - } - - static void compareBinary(ColumnVector left, ColumnVector right, int[] result) { - Comparator comparator = (leftOp, rightOp) -> { - int i = 0; - while (i < leftOp.length && i < rightOp.length) { - if (leftOp[i] != rightOp[i]) { - return Byte.compare(leftOp[i], rightOp[i]); - } - i++; - } - return Integer.compare(leftOp.length, rightOp.length); - }; - for (int rowId = 0; rowId < left.getSize(); rowId++) { - if (!left.isNullAt(rowId) && !right.isNullAt(rowId)) { - result[rowId] = comparator.compare(left.getBinary(rowId), right.getBinary(rowId)); - } - } - } - static Expression childAt(Expression expression, int index) { return expression.getChildren().get(index); } From 8d237d55018bb10c0d6c16ea095d43ef8efb6766 Mon Sep 17 00:00:00 2001 From: zzkevl7 Date: Wed, 29 May 2024 19:37:46 -0700 Subject: [PATCH 21/21] fix cr --- .../expressions/DefaultExpressionUtils.java | 2 +- .../expressions/VectorComparator.java | 27 ------------------- 2 files changed, 1 insertion(+), 28 deletions(-) delete mode 100644 kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/VectorComparator.java diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java index 675d9dd8718..46047ed989d 100644 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java +++ b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/DefaultExpressionUtils.java @@ -103,7 +103,7 @@ public boolean getBoolean(int rowId) { /** * Utility method to create a column vector that lazily evaluate the - * comparator ex. vectorComparator (ie. ==, >=, <=......) for left and right + * comparator ex. (ie. ==, >=, <=......) for left and right * column vector according to the natural ordering of numbers *

* Only primitive data types are supported. diff --git a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/VectorComparator.java b/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/VectorComparator.java deleted file mode 100644 index 40365988090..00000000000 --- a/kernel/kernel-defaults/src/main/java/io/delta/kernel/defaults/internal/expressions/VectorComparator.java +++ /dev/null @@ -1,27 +0,0 @@ -/* - * Copyright (2023) The Delta Lake Project Authors. - * - * Licensed under the Apache License, Version 2.0 (the "License"); - * you may not use this file except in compliance with the License. - * You may obtain a copy of the License at - * - * http://www.apache.org/licenses/LICENSE-2.0 - * - * Unless required by applicable law or agreed to in writing, software - * distributed under the License is distributed on an "AS IS" BASIS, - * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. - * See the License for the specific language governing permissions and - * limitations under the License. - */ - -package io.delta.kernel.defaults.internal.expressions; - -import io.delta.kernel.annotation.Evolving; - -/** - * Comparator interface for lazy evaluating comparator - */ -@Evolving -public interface VectorComparator { - boolean compare(int compareResult); -}