You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2022/06/03 23:13:18 UTC
[iceberg] branch master updated: API: Add expression equivalence testing (#4947)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new ff7c21738 API: Add expression equivalence testing (#4947)
ff7c21738 is described below
commit ff7c21738d8e2daae5e69341f80ec6832ce209d1
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Fri Jun 3 16:13:12 2022 -0700
API: Add expression equivalence testing (#4947)
---
.../java/org/apache/iceberg/expressions/And.java | 11 ++
.../iceberg/expressions/BoundLiteralPredicate.java | 57 +++++++
.../apache/iceberg/expressions/BoundReference.java | 13 ++
.../iceberg/expressions/BoundSetPredicate.java | 11 ++
.../org/apache/iceberg/expressions/BoundTerm.java | 7 +
.../apache/iceberg/expressions/BoundTransform.java | 12 ++
.../iceberg/expressions/BoundUnaryPredicate.java | 9 +
.../org/apache/iceberg/expressions/Expression.java | 17 ++
.../apache/iceberg/expressions/ExpressionUtil.java | 36 ++++
.../java/org/apache/iceberg/expressions/False.java | 5 +
.../java/org/apache/iceberg/expressions/Or.java | 11 ++
.../java/org/apache/iceberg/expressions/True.java | 5 +
.../iceberg/expressions/UnboundPredicate.java | 2 +-
.../iceberg/expressions/TestExpressionUtil.java | 190 +++++++++++++++++++++
14 files changed, 385 insertions(+), 1 deletion(-)
diff --git a/api/src/main/java/org/apache/iceberg/expressions/And.java b/api/src/main/java/org/apache/iceberg/expressions/And.java
index 945a1c253..2fa899ba1 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/And.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/And.java
@@ -41,6 +41,17 @@ public class And implements Expression {
return Expression.Operation.AND;
}
+ @Override
+ public boolean isEquivalentTo(Expression expr) {
+ if (expr.op() == Operation.AND) {
+ And other = (And) expr;
+ return (left.isEquivalentTo(other.left()) && right.isEquivalentTo(other.right())) ||
+ (left.isEquivalentTo(other.right()) && right.isEquivalentTo(other.left()));
+ }
+
+ return false;
+ }
+
@Override
public Expression negate() {
// not(and(a, b)) => or(not(a), not(b))
diff --git a/api/src/main/java/org/apache/iceberg/expressions/BoundLiteralPredicate.java b/api/src/main/java/org/apache/iceberg/expressions/BoundLiteralPredicate.java
index 17b043a3b..3c89895f3 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/BoundLiteralPredicate.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/BoundLiteralPredicate.java
@@ -20,9 +20,19 @@
package org.apache.iceberg.expressions;
import java.util.Comparator;
+import java.util.Set;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
+import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.types.Type;
public class BoundLiteralPredicate<T> extends BoundPredicate<T> {
+ private static final Set<Type.TypeID> INTEGRAL_TYPES = Sets.newHashSet(
+ Type.TypeID.INTEGER, Type.TypeID.LONG, Type.TypeID.DATE, Type.TypeID.TIME, Type.TypeID.TIMESTAMP);
+
+ private static long toLong(Literal<?> lit) {
+ return ((Number) lit.value()).longValue();
+ }
+
private final Literal<T> literal;
BoundLiteralPredicate(Operation op, BoundTerm<T> term, Literal<T> lit) {
@@ -76,6 +86,53 @@ public class BoundLiteralPredicate<T> extends BoundPredicate<T> {
}
}
+ @Override
+ @SuppressWarnings("unchecked")
+ public boolean isEquivalentTo(Expression expr) {
+ if (op() == expr.op()) {
+ BoundLiteralPredicate<?> other = (BoundLiteralPredicate<?>) expr;
+ if (term().isEquivalentTo(other.term())) {
+ // because the term is equivalent, the literal must have the same type, T
+ Literal<T> otherLiteral = (Literal<T>) other.literal();
+ Comparator<T> cmp = literal.comparator();
+ return cmp.compare(literal.value(), otherLiteral.value()) == 0;
+ }
+
+ } else if (expr instanceof BoundLiteralPredicate) {
+ BoundLiteralPredicate<?> other = (BoundLiteralPredicate<?>) expr;
+ if (INTEGRAL_TYPES.contains(term().type().typeId()) && term().isEquivalentTo(other.term())) {
+ switch (op()) {
+ case LT:
+ if (other.op() == Operation.LT_EQ) {
+ // < 6 is equivalent to <= 5
+ return toLong(literal()) == toLong(other.literal()) + 1L;
+ }
+ break;
+ case LT_EQ:
+ if (other.op() == Operation.LT) {
+ // <= 5 is equivalent to < 6
+ return toLong(literal()) == toLong(other.literal()) - 1L;
+ }
+ break;
+ case GT:
+ if (other.op() == Operation.GT_EQ) {
+ // > 5 is equivalent to >= 6
+ return toLong(literal()) == toLong(other.literal()) - 1L;
+ }
+ break;
+ case GT_EQ:
+ if (other.op() == Operation.GT) {
+ // >= 5 is equivalent to > 4
+ return toLong(literal()) == toLong(other.literal()) + 1L;
+ }
+ break;
+ }
+ }
+ }
+
+ return false;
+ }
+
@Override
public String toString() {
switch (op()) {
diff --git a/api/src/main/java/org/apache/iceberg/expressions/BoundReference.java b/api/src/main/java/org/apache/iceberg/expressions/BoundReference.java
index be380b263..c2be9b653 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/BoundReference.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/BoundReference.java
@@ -53,6 +53,19 @@ public class BoundReference<T> implements BoundTerm<T>, Reference<T> {
return field.type();
}
+ @Override
+ public boolean isEquivalentTo(BoundTerm<?> other) {
+ if (other instanceof BoundReference) {
+ Types.NestedField otherField = ((BoundReference<?>) other).field();
+ // equivalence only depends on the field ID, type, and optional. name and accessor are ignored
+ return field.fieldId() == otherField.fieldId() &&
+ field.type().equals(otherField.type()) &&
+ field.isOptional() == otherField.isOptional();
+ }
+
+ return other.isEquivalentTo(this);
+ }
+
public int fieldId() {
return field.fieldId();
}
diff --git a/api/src/main/java/org/apache/iceberg/expressions/BoundSetPredicate.java b/api/src/main/java/org/apache/iceberg/expressions/BoundSetPredicate.java
index aa2ae268b..3c1ad2893 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/BoundSetPredicate.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/BoundSetPredicate.java
@@ -65,6 +65,17 @@ public class BoundSetPredicate<T> extends BoundPredicate<T> {
}
}
+ @Override
+ public boolean isEquivalentTo(Expression other) {
+ // only check bound set predicate; binding will convert sets of a single item to a literal predicate
+ if (op() == other.op()) {
+ BoundSetPredicate<?> pred = (BoundSetPredicate<?>) other;
+ return literalSet().equals(pred.literalSet());
+ }
+
+ return false;
+ }
+
@Override
public String toString() {
switch (op()) {
diff --git a/api/src/main/java/org/apache/iceberg/expressions/BoundTerm.java b/api/src/main/java/org/apache/iceberg/expressions/BoundTerm.java
index 940a6917b..3691bbf5a 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/BoundTerm.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/BoundTerm.java
@@ -40,4 +40,11 @@ public interface BoundTerm<T> extends Bound<T>, Term {
default Comparator<T> comparator() {
return Comparators.forType(type().asPrimitiveType());
}
+
+ /**
+ * Returns whether this term is equivalent to another.
+ * @param other a term
+ * @return true if this term returns the same values as the other, false otherwise
+ */
+ boolean isEquivalentTo(BoundTerm<?> other);
}
diff --git a/api/src/main/java/org/apache/iceberg/expressions/BoundTransform.java b/api/src/main/java/org/apache/iceberg/expressions/BoundTransform.java
index 8e2727c61..2e4e21c2b 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/BoundTransform.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/BoundTransform.java
@@ -57,6 +57,18 @@ public class BoundTransform<S, T> implements BoundTerm<T> {
return transform.getResultType(ref.type());
}
+ @Override
+ public boolean isEquivalentTo(BoundTerm<?> other) {
+ if (other instanceof BoundTransform) {
+ BoundTransform<?, ?> bound = (BoundTransform<?, ?>) other;
+ return ref.isEquivalentTo(bound.ref()) && transform.equals(bound.transform());
+ } else if (transform.isIdentity() && other instanceof BoundReference) {
+ return ref.isEquivalentTo(other);
+ }
+
+ return false;
+ }
+
@Override
public String toString() {
return transform + "(" + ref + ")";
diff --git a/api/src/main/java/org/apache/iceberg/expressions/BoundUnaryPredicate.java b/api/src/main/java/org/apache/iceberg/expressions/BoundUnaryPredicate.java
index 3989c49a3..c5f0c0d9f 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/BoundUnaryPredicate.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/BoundUnaryPredicate.java
@@ -57,6 +57,15 @@ public class BoundUnaryPredicate<T> extends BoundPredicate<T> {
}
}
+ @Override
+ public boolean isEquivalentTo(Expression other) {
+ if (op() == other.op()) {
+ return term().isEquivalentTo(((BoundUnaryPredicate<?>) other).term());
+ }
+
+ return false;
+ }
+
@Override
public String toString() {
switch (op()) {
diff --git a/api/src/main/java/org/apache/iceberg/expressions/Expression.java b/api/src/main/java/org/apache/iceberg/expressions/Expression.java
index 4ff852dd5..887385935 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/Expression.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/Expression.java
@@ -124,4 +124,21 @@ public interface Expression extends Serializable {
default Expression negate() {
throw new UnsupportedOperationException(String.format("%s cannot be negated", this));
}
+
+ /**
+ * Returns whether this expression will accept the same values as another.
+ * <p>
+ * If this returns true, the expressions are guaranteed to return the same evaluation for the same input. However, if
+ * this returns false the expressions may return the same evaluation for the same input. That is, expressions may
+ * be equivalent even if this returns false.
+ * <p>
+ * For best results, rewrite not and bind expressions before calling this method.
+ *
+ * @param other another expression
+ * @return true if the expressions are equivalent
+ */
+ default boolean isEquivalentTo(Expression other) {
+ // only bound predicates can be equivalent
+ return false;
+ }
}
diff --git a/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java b/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java
index 97a3ad067..2e9a0bb2f 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java
@@ -21,6 +21,7 @@ package org.apache.iceberg.expressions;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
+import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Types;
@@ -66,6 +67,41 @@ public class ExpressionUtil {
return ExpressionVisitors.visit(expr, StringSanitizer.INSTANCE);
}
+ /**
+ * Returns whether two unbound expressions will accept the same inputs.
+ * <p>
+ * If this returns true, the expressions are guaranteed to return the same evaluation for the same input. However, if
+ * this returns false the expressions may return the same evaluation for the same input. That is, expressions may
+ * be equivalent even if this returns false.
+ *
+ * @param left an unbound expression
+ * @param right an unbound expression
+ * @param struct a struct type for binding
+ * @param caseSensitive whether to bind expressions using case-sensitive matching
+ * @return true if the expressions are equivalent
+ */
+ public static boolean equivalent(Expression left, Expression right, Types.StructType struct, boolean caseSensitive) {
+ return Binder.bind(struct, Expressions.rewriteNot(left), caseSensitive)
+ .isEquivalentTo(Binder.bind(struct, Expressions.rewriteNot(right), caseSensitive));
+ }
+
+ /**
+ * Returns whether an expression selects whole partitions for a partition spec.
+ * <p>
+ * For example, ts < '2021-03-09T10:00:00.000' selects whole partitions in an hourly spec, [hours(ts)], but does
+ * not select whole partitions in a daily spec, [days(ts)].
+ *
+ * @param expr an unbound expression
+ * @param spec a partition spec
+ * @return true if the expression will select whole partitions in the given spec
+ */
+ public static boolean selectsPartitions(Expression expr, PartitionSpec spec, boolean caseSensitive) {
+ return equivalent(
+ Projections.inclusive(spec, caseSensitive).project(expr),
+ Projections.strict(spec, caseSensitive).project(expr),
+ spec.partitionType(), caseSensitive);
+ }
+
private static class ExpressionSanitizer extends ExpressionVisitors.ExpressionVisitor<Expression> {
private static final ExpressionSanitizer INSTANCE = new ExpressionSanitizer();
diff --git a/api/src/main/java/org/apache/iceberg/expressions/False.java b/api/src/main/java/org/apache/iceberg/expressions/False.java
index 940750a67..21d81cd31 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/False.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/False.java
@@ -40,6 +40,11 @@ public class False implements Expression {
return True.INSTANCE;
}
+ @Override
+ public boolean isEquivalentTo(Expression other) {
+ return other.op() == Operation.FALSE;
+ }
+
@Override
public String toString() {
return "false";
diff --git a/api/src/main/java/org/apache/iceberg/expressions/Or.java b/api/src/main/java/org/apache/iceberg/expressions/Or.java
index b41ef7f67..20a213a5d 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/Or.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/Or.java
@@ -41,6 +41,17 @@ public class Or implements Expression {
return Expression.Operation.OR;
}
+ @Override
+ public boolean isEquivalentTo(Expression expr) {
+ if (expr.op() == Operation.OR) {
+ Or other = (Or) expr;
+ return (left.isEquivalentTo(other.left()) && right.isEquivalentTo(other.right())) ||
+ (left.isEquivalentTo(other.right()) && right.isEquivalentTo(other.left()));
+ }
+
+ return false;
+ }
+
@Override
public Expression negate() {
// not(or(a, b)) => and(not(a), not(b))
diff --git a/api/src/main/java/org/apache/iceberg/expressions/True.java b/api/src/main/java/org/apache/iceberg/expressions/True.java
index 8f43bc075..dc24a2bcb 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/True.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/True.java
@@ -40,6 +40,11 @@ public class True implements Expression {
return False.INSTANCE;
}
+ @Override
+ public boolean isEquivalentTo(Expression other) {
+ return other.op() == Operation.TRUE;
+ }
+
@Override
public String toString() {
return "true";
diff --git a/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java b/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java
index 8143f25c7..10a459e78 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/UnboundPredicate.java
@@ -259,7 +259,7 @@ public class UnboundPredicate<T> extends Predicate<T, UnboundTerm<T>> implements
@SuppressWarnings("unchecked")
static <T> Set<T> setOf(Iterable<Literal<T>> literals) {
Literal<T> lit = Iterables.get(literals, 0);
- if (lit instanceof Literals.StringLiteral && lit.value() instanceof CharSequence) {
+ if (lit instanceof Literals.StringLiteral) {
Iterable<T> values = Iterables.transform(literals, Literal::value);
Iterable<CharSequence> charSeqs = Iterables.transform(values, val -> (CharSequence) val);
return (Set<T>) CharSequenceSet.of(charSeqs);
diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java
index b4e72e977..e35a73acf 100644
--- a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java
+++ b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionUtil.java
@@ -19,11 +19,26 @@
package org.apache.iceberg.expressions;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Types;
import org.junit.Assert;
import org.junit.Test;
public class TestExpressionUtil {
+ private static final Schema SCHEMA = new Schema(
+ Types.NestedField.required(1, "id", Types.LongType.get()),
+ Types.NestedField.required(2, "val", Types.IntegerType.get()),
+ Types.NestedField.required(3, "val2", Types.IntegerType.get()),
+ Types.NestedField.required(4, "ts", Types.TimestampType.withoutZone()),
+ Types.NestedField.required(5, "date", Types.DateType.get()),
+ Types.NestedField.required(6, "time", Types.DateType.get()),
+ Types.NestedField.optional(7, "data", Types.StringType.get()),
+ Types.NestedField.optional(8, "measurement", Types.DoubleType.get()));
+
+ private static final Types.StructType STRUCT = SCHEMA.asStruct();
+
@Test
public void testUnchangedUnaryPredicates() {
for (Expression unary : Lists.newArrayList(
@@ -223,6 +238,181 @@ public class TestExpressionUtil {
}
}
+ @Test
+ public void testIdenticalExpressionIsEquivalent() {
+ Expression[] exprs = new Expression[] {
+ Expressions.isNull("data"),
+ Expressions.notNull("data"),
+ Expressions.isNaN("measurement"),
+ Expressions.notNaN("measurement"),
+ Expressions.lessThan("id", 5),
+ Expressions.lessThanOrEqual("id", 5),
+ Expressions.greaterThan("id", 5),
+ Expressions.greaterThanOrEqual("id", 5),
+ Expressions.equal("id", 5),
+ Expressions.notEqual("id", 5),
+ Expressions.in("id", 5, 6),
+ Expressions.notIn("id", 5, 6),
+ Expressions.startsWith("data", "aaa"),
+ Expressions.notStartsWith("data", "aaa"),
+ Expressions.alwaysTrue(),
+ Expressions.alwaysFalse(),
+ Expressions.and(Expressions.lessThan("id", 5), Expressions.notNull("data")),
+ Expressions.or(Expressions.lessThan("id", 5), Expressions.notNull("data")),
+ };
+
+ for (Expression expr : exprs) {
+ Assert.assertTrue("Should accept identical expression: " + expr,
+ ExpressionUtil.equivalent(expr, expr, STRUCT, true));
+
+ for (Expression other : exprs) {
+ if (expr != other) {
+ Assert.assertFalse(ExpressionUtil.equivalent(expr, other, STRUCT, true));
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testIdenticalTermIsEquivalent() {
+ UnboundTerm<?>[] terms = new UnboundTerm<?>[] {
+ Expressions.ref("id"),
+ Expressions.truncate("id", 2),
+ Expressions.bucket("id", 16),
+ Expressions.year("ts"),
+ Expressions.month("ts"),
+ Expressions.day("ts"),
+ Expressions.hour("ts"),
+ };
+
+ for (UnboundTerm<?> term : terms) {
+ BoundTerm<?> bound = term.bind(STRUCT, true);
+ Assert.assertTrue("Should accept identical expression: " + term, bound.isEquivalentTo(bound));
+
+ for (UnboundTerm<?> other : terms) {
+ if (term != other) {
+ Assert.assertFalse(bound.isEquivalentTo(other.bind(STRUCT, true)));
+ }
+ }
+ }
+ }
+
+ @Test
+ public void testRefEquivalence() {
+ Assert.assertFalse("Should not find different refs equivalent",
+ Expressions.ref("val").bind(STRUCT, true).isEquivalentTo(Expressions.ref("val2").bind(STRUCT, true)));
+ }
+
+ @Test
+ public void testInEquivalence() {
+ Assert.assertTrue("Should ignore duplicate longs (in)",
+ ExpressionUtil.equivalent(Expressions.in("id", 1, 2, 1), Expressions.in("id", 2, 1, 2), STRUCT, true));
+ Assert.assertTrue("Should ignore duplicate longs (notIn)",
+ ExpressionUtil.equivalent(Expressions.notIn("id", 1, 2, 1), Expressions.notIn("id", 2, 1, 2), STRUCT, true));
+
+ Assert.assertTrue("Should ignore duplicate strings (in)",
+ ExpressionUtil.equivalent(
+ Expressions.in("data", "a", "b", "a"),
+ Expressions.in("data", "b", "a"),
+ STRUCT, true));
+ Assert.assertTrue("Should ignore duplicate strings (notIn)",
+ ExpressionUtil.equivalent(Expressions.notIn("data", "b", "b"), Expressions.notIn("data", "b"), STRUCT, true));
+
+ Assert.assertTrue("Should detect equivalence with equal (in, string)",
+ ExpressionUtil.equivalent(Expressions.in("data", "a"), Expressions.equal("data", "a"), STRUCT, true));
+ Assert.assertTrue("Should detect equivalence with notEqual (notIn, long)",
+ ExpressionUtil.equivalent(Expressions.notIn("id", 1), Expressions.notEqual("id", 1), STRUCT, true));
+
+ Assert.assertFalse("Should detect different sets (in, long)",
+ ExpressionUtil.equivalent(Expressions.in("id", 1, 2, 3), Expressions.in("id", 1, 2), STRUCT, true));
+ Assert.assertFalse("Should detect different sets (notIn, string)",
+ ExpressionUtil.equivalent(Expressions.notIn("data", "a", "b"), Expressions.notIn("data", "a"), STRUCT, true));
+ }
+
+ @Test
+ public void testInequalityEquivalence() {
+ String[] cols = new String[] {"id", "val", "ts", "date", "time"};
+
+ for (String col : cols) {
+ Assert.assertTrue("Should detect < to <= equivalence: " + col,
+ ExpressionUtil.equivalent(
+ Expressions.lessThan(col, 34L),
+ Expressions.lessThanOrEqual(col, 33L),
+ STRUCT, true));
+ Assert.assertTrue("Should detect <= to < equivalence: " + col,
+ ExpressionUtil.equivalent(
+ Expressions.lessThanOrEqual(col, 34L),
+ Expressions.lessThan(col, 35L),
+ STRUCT, true));
+ Assert.assertTrue("Should detect > to >= equivalence: " + col,
+ ExpressionUtil.equivalent(
+ Expressions.greaterThan(col, 34L),
+ Expressions.greaterThanOrEqual(col, 35L),
+ STRUCT, true));
+ Assert.assertTrue("Should detect >= to > equivalence: " + col,
+ ExpressionUtil.equivalent(
+ Expressions.greaterThanOrEqual(col, 34L),
+ Expressions.greaterThan(col, 33L),
+ STRUCT, true));
+ }
+
+ Assert.assertFalse("Should not detect equivalence for different columns",
+ ExpressionUtil.equivalent(
+ Expressions.lessThan("val", 34L),
+ Expressions.lessThanOrEqual("val2", 33L),
+ STRUCT, true));
+ Assert.assertFalse("Should not detect equivalence for different types",
+ ExpressionUtil.equivalent(
+ Expressions.lessThan("val", 34L),
+ Expressions.lessThanOrEqual("id", 33L),
+ STRUCT, true));
+ }
+
+ @Test
+ public void testAndEquivalence() {
+ Assert.assertTrue("Should detect and equivalence in any order",
+ ExpressionUtil.equivalent(
+ Expressions.and(Expressions.lessThan("id", 34), Expressions.greaterThanOrEqual("id", 20)),
+ Expressions.and(Expressions.greaterThan("id", 19L), Expressions.lessThanOrEqual("id", 33L)),
+ STRUCT, true));
+ }
+
+ @Test
+ public void testOrEquivalence() {
+ Assert.assertTrue("Should detect or equivalence in any order",
+ ExpressionUtil.equivalent(
+ Expressions.or(Expressions.lessThan("id", 20), Expressions.greaterThanOrEqual("id", 34)),
+ Expressions.or(Expressions.greaterThan("id", 33L), Expressions.lessThanOrEqual("id", 19L)),
+ STRUCT, true));
+ }
+
+ @Test
+ public void testNotEquivalence() {
+ Assert.assertTrue("Should detect not equivalence by rewriting",
+ ExpressionUtil.equivalent(
+ Expressions.not(Expressions.or(Expressions.in("data", "a"), Expressions.greaterThanOrEqual("id", 34))),
+ Expressions.and(Expressions.lessThan("id", 34L), Expressions.notEqual("data", "a")),
+ STRUCT, true));
+ }
+
+ @Test
+ public void testSelectsPartitions() {
+ Assert.assertTrue("Should select partitions, on boundary",
+ ExpressionUtil.selectsPartitions(
+ Expressions.lessThan("ts", "2021-03-09T10:00:00.000000"),
+ PartitionSpec.builderFor(SCHEMA).hour("ts").build(), true));
+
+ Assert.assertFalse("Should not select partitions, 1 ms off boundary",
+ ExpressionUtil.selectsPartitions(
+ Expressions.lessThanOrEqual("ts", "2021-03-09T10:00:00.000000"),
+ PartitionSpec.builderFor(SCHEMA).hour("ts").build(), true));
+
+ Assert.assertFalse("Should not select partitions, on hour not day boundary",
+ ExpressionUtil.selectsPartitions(
+ Expressions.lessThan("ts", "2021-03-09T10:00:00.000000"),
+ PartitionSpec.builderFor(SCHEMA).day("ts").build(), true));
+ }
+
private void assertEquals(Expression expected, Expression actual) {
if (expected instanceof UnboundPredicate) {
Assert.assertTrue("Should be an UnboundPredicate", actual instanceof UnboundPredicate);