You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2019/08/02 17:52:32 UTC
[incubator-iceberg] branch master updated: Add strict projections
for Truncate transformations (#332)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 1a9bb2c Add strict projections for Truncate transformations (#332)
1a9bb2c is described below
commit 1a9bb2c4863ebf363e5aa347543125d3c184a32d
Author: moulimukherjee <mo...@gmail.com>
AuthorDate: Fri Aug 2 10:52:27 2019 -0700
Add strict projections for Truncate transformations (#332)
This also includes new tests for residual evaluation.
---
.../apache/iceberg/transforms/ProjectionUtil.java | 116 ++++----
.../org/apache/iceberg/transforms/Truncate.java | 70 ++---
.../iceberg/transforms/TestDatesProjection.java | 40 +--
.../transforms/TestTimestampsProjection.java | 64 ++---
.../transforms/TestTruncatesProjection.java | 315 +++++++++++++++++++++
.../iceberg/transforms/TestTruncatesResiduals.java | 177 ++++++++++++
6 files changed, 632 insertions(+), 150 deletions(-)
diff --git a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java
index ef1e0c3..84f604c 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java
@@ -52,38 +52,18 @@ class ProjectionUtil {
}
}
- static UnboundPredicate<Integer> truncateIntegerStrict(
- String name, BoundPredicate<Integer> pred, Transform<Integer, Integer> transform) {
+ static <T> UnboundPredicate<T> truncateIntegerStrict(
+ String name, BoundPredicate<Integer> pred, Transform<Integer, T> transform) {
int boundary = pred.literal().value();
switch (pred.op()) {
case LT:
- // predicate would be <= the previous partition
- return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary) - 1);
+ return predicate(Expression.Operation.LT, name, transform.apply(boundary));
case LT_EQ:
- // Checking if the literal is at the upper partition boundary
- if (transform.apply(boundary + 1).equals(transform.apply(boundary))) {
- // Literal is not at upper boundary, for eg: 2019-07-02T02:12:34.0000
- // the predicate can be < 2019-07-01
- return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary) - 1);
- } else {
- // Literal is not at upper boundary, for eg: 2019-07-02T23:59:59.99999
- // the predicate can be <= 2019-07-02
- return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary));
- }
+ return predicate(Expression.Operation.LT, name, transform.apply(boundary + 1));
case GT:
- // predicate would be >= the next partition
- return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary) + 1);
+ return predicate(Expression.Operation.GT, name, transform.apply(boundary));
case GT_EQ:
- // Checking if the literal is at the lower partition boundary
- if (transform.apply(boundary - 1).equals(transform.apply(boundary))) {
- // Literal is not at lower boundary, for eg: 2019-07-02T02:12:34.0000
- // the predicate can be >= 2019-07-03
- return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary) + 1);
- } else {
- // Literal was at the lower boundary, for eg: 2019-07-02T00:00:00.0000
- // the predicate can be >= 2019-07-02
- return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary));
- }
+ return predicate(Expression.Operation.GT, name, transform.apply(boundary - 1));
case NOT_EQ:
return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary));
case EQ:
@@ -94,38 +74,18 @@ class ProjectionUtil {
}
}
- static UnboundPredicate<Integer> truncateLongStrict(
- String name, BoundPredicate<Long> pred, Transform<Long, Integer> transform) {
+ static <T> UnboundPredicate<T> truncateLongStrict(
+ String name, BoundPredicate<Long> pred, Transform<Long, T> transform) {
long boundary = pred.literal().value();
switch (pred.op()) {
case LT:
- // predicate would be <= the previous partition
- return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary) - 1);
+ return predicate(Expression.Operation.LT, name, transform.apply(boundary));
case LT_EQ:
- // Checking if the literal is at the upper partition boundary
- if (transform.apply(boundary + 1L).equals(transform.apply(boundary))) {
- // Literal is not at upper boundary, for eg: 2019-07-02T02:12:34.0000
- // the predicate can be <= 2019-07-01
- return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary) - 1);
- } else {
- // Literal is not at upper boundary, for eg: 2019-07-02T23:59:59.99999
- // the predicate can be <= 2019-07-02
- return predicate(Expression.Operation.LT_EQ, name, transform.apply(boundary));
- }
+ return predicate(Expression.Operation.LT, name, transform.apply(boundary + 1L));
case GT:
- // predicate would be >= the next partition
- return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary) + 1);
+ return predicate(Expression.Operation.GT, name, transform.apply(boundary));
case GT_EQ:
- // Checking if the literal is at the lower partition boundary
- if (transform.apply(boundary - 1L).equals(transform.apply(boundary))) {
- // Literal is not at lower boundary, for eg: 2019-07-02T02:12:34.0000
- // the predicate can be >= 2019-07-03
- return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary) + 1);
- } else {
- // Literal was at the lower boundary, for eg: 2019-07-02T00:00:00.0000
- // the predicate can be >= 2019-07-02
- return predicate(Expression.Operation.GT_EQ, name, transform.apply(boundary));
- }
+ return predicate(Expression.Operation.GT, name, transform.apply(boundary - 1L));
case NOT_EQ:
return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary));
case EQ:
@@ -185,6 +145,38 @@ class ProjectionUtil {
}
}
+ static <T> UnboundPredicate<T> truncateDecimalStrict(
+ String name, BoundPredicate<BigDecimal> pred,
+ Transform<BigDecimal, T> transform) {
+ BigDecimal boundary = pred.literal().value();
+
+ BigDecimal minusOne = new BigDecimal(
+ boundary.unscaledValue().subtract(BigInteger.ONE),
+ boundary.scale());
+
+ BigDecimal plusOne = new BigDecimal(
+ boundary.unscaledValue().add(BigInteger.ONE),
+ boundary.scale());
+
+ switch (pred.op()) {
+ case LT:
+ return predicate(Expression.Operation.LT, name, transform.apply(boundary));
+ case LT_EQ:
+ return predicate(Expression.Operation.LT, name, transform.apply(plusOne));
+ case GT:
+ return predicate(Expression.Operation.GT, name, transform.apply(boundary));
+ case GT_EQ:
+ return predicate(Expression.Operation.GT, name, transform.apply(minusOne));
+ case NOT_EQ:
+ return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary));
+ case EQ:
+ // there is no predicate that guarantees equality because adjacent decimals transform to the same value
+ return null;
+ default:
+ return null;
+ }
+ }
+
static <S, T> UnboundPredicate<T> truncateArray(
String name, BoundPredicate<S> pred, Transform<S, T> transform) {
S boundary = pred.literal().value();
@@ -203,4 +195,24 @@ class ProjectionUtil {
return null;
}
}
+
+ static <S, T> UnboundPredicate<T> truncateArrayStrict(
+ String name, BoundPredicate<S> pred, Transform<S, T> transform) {
+ S boundary = pred.literal().value();
+ switch (pred.op()) {
+ case LT:
+ case LT_EQ:
+ return predicate(Expression.Operation.LT, name, transform.apply(boundary));
+ case GT:
+ case GT_EQ:
+ return predicate(Expression.Operation.GT, name, transform.apply(boundary));
+ case NOT_EQ:
+ return predicate(Expression.Operation.NOT_EQ, name, transform.apply(boundary));
+ case EQ:
+ // there is no predicate that guarantees equality because adjacent values transform to the same partition
+ return null;
+ default:
+ return null;
+ }
+ }
}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/Truncate.java b/api/src/main/java/org/apache/iceberg/transforms/Truncate.java
index 2eacaa2..d7d55dd 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/Truncate.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/Truncate.java
@@ -30,8 +30,6 @@ import org.apache.iceberg.types.Type;
import org.apache.iceberg.util.UnicodeUtil;
import static org.apache.iceberg.expressions.Expression.Operation.IS_NULL;
-import static org.apache.iceberg.expressions.Expression.Operation.LT;
-import static org.apache.iceberg.expressions.Expression.Operation.LT_EQ;
import static org.apache.iceberg.expressions.Expression.Operation.NOT_NULL;
abstract class Truncate<T> implements Transform<T, T> {
@@ -95,46 +93,14 @@ abstract class Truncate<T> implements Transform<T, T> {
}
@Override
- public UnboundPredicate<Integer> projectStrict(String name, BoundPredicate<Integer> predicate) {
+ public UnboundPredicate<Integer> projectStrict(String name, BoundPredicate<Integer> pred) {
// TODO: for integers, can this return the original predicate?
// No. the predicate needs to be in terms of the applied value. For all x, apply(x) <= x.
// Therefore, the lower bound can be transformed outside of a greater-than bound.
- int in;
- int out;
- int inImage;
- int outImage;
- switch (predicate.op()) {
- case LT:
- in = predicate.literal().value() - 1;
- out = predicate.literal().value();
- inImage = apply(in);
- outImage = apply(out);
- if (inImage != outImage) {
- return Expressions.predicate(LT_EQ, name, inImage);
- } else {
- return Expressions.predicate(LT, name, inImage);
- }
- case LT_EQ:
- in = predicate.literal().value();
- out = predicate.literal().value() + 1;
- inImage = apply(in);
- outImage = apply(out);
- if (inImage != outImage) {
- return Expressions.predicate(LT_EQ, name, inImage);
- } else {
- return Expressions.predicate(LT, name, inImage);
- }
- case GT:
- case GT_EQ:
- case EQ:
- case NOT_EQ:
-// case IN:
-// break;
-// case NOT_IN:
-// break;
- default:
- return null;
+ if (pred.op() == NOT_NULL || pred.op() == IS_NULL) {
+ return Expressions.predicate(pred.op(), name);
}
+ return ProjectionUtil.truncateIntegerStrict(name, pred, this);
}
@Override
@@ -192,8 +158,11 @@ abstract class Truncate<T> implements Transform<T, T> {
}
@Override
- public UnboundPredicate<Long> projectStrict(String name, BoundPredicate<Long> predicate) {
- return null;
+ public UnboundPredicate<Long> projectStrict(String name, BoundPredicate<Long> pred) {
+ if (pred.op() == NOT_NULL || pred.op() == IS_NULL) {
+ return Expressions.predicate(pred.op(), name);
+ }
+ return ProjectionUtil.truncateLongStrict(name, pred, this);
}
@Override
@@ -253,8 +222,11 @@ abstract class Truncate<T> implements Transform<T, T> {
@Override
public UnboundPredicate<CharSequence> projectStrict(String name,
- BoundPredicate<CharSequence> predicate) {
- return null;
+ BoundPredicate<CharSequence> pred) {
+ if (pred.op() == NOT_NULL || pred.op() == IS_NULL) {
+ return Expressions.predicate(pred.op(), name);
+ }
+ return ProjectionUtil.truncateArrayStrict(name, pred, this);
}
@Override
@@ -316,8 +288,11 @@ abstract class Truncate<T> implements Transform<T, T> {
@Override
public UnboundPredicate<ByteBuffer> projectStrict(String name,
- BoundPredicate<ByteBuffer> predicate) {
- return null;
+ BoundPredicate<ByteBuffer> pred) {
+ if (pred.op() == NOT_NULL || pred.op() == IS_NULL) {
+ return Expressions.predicate(pred.op(), name);
+ }
+ return ProjectionUtil.truncateArrayStrict(name, pred, this);
}
@Override
@@ -388,8 +363,11 @@ abstract class Truncate<T> implements Transform<T, T> {
@Override
public UnboundPredicate<BigDecimal> projectStrict(String name,
- BoundPredicate<BigDecimal> predicate) {
- return null;
+ BoundPredicate<BigDecimal> pred) {
+ if (pred.op() == NOT_NULL || pred.op() == IS_NULL) {
+ return Expressions.predicate(pred.op(), name);
+ }
+ return ProjectionUtil.truncateDecimalStrict(name, pred, this);
}
@Override
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java
index 7b4ee78..b6602a6 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java
@@ -88,10 +88,10 @@ public class TestDatesProjection {
Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value();
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build();
- assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2016-12");
- assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2016-12");
- assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2017-02");
- assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017-01");
+ assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "2017-01");
+ assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "2017-01");
+ assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "2017-01");
+ assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "2016-12");
assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-01");
assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE);
}
@@ -101,10 +101,10 @@ public class TestDatesProjection {
Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value();
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("date").build();
- assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2017-11");
- assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017-12");
- assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018-01");
- assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2018-01");
+ assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "2017-12");
+ assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "2018-01");
+ assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "2017-12");
+ assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "2017-12");
assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-12");
assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE);
}
@@ -140,12 +140,12 @@ public class TestDatesProjection {
Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value();
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("date").build();
- assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2016-12-31");
+ assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "2017-01-01");
// should be the same date for <=
- assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017-01-01");
- assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2017-01-02");
+ assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "2017-01-02");
+ assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "2017-01-01");
// should be the same date for >=
- assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017-01-01");
+ assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "2016-12-31");
assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017-01-01");
assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE);
}
@@ -168,10 +168,10 @@ public class TestDatesProjection {
Integer date = (Integer) Literal.of("2017-01-01").to(TYPE).value();
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build();
- assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2016");
- assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2016");
- assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018");
- assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2017");
+ assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "2017");
+ assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "2017");
+ assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "2017");
+ assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "2016");
assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017");
assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE);
}
@@ -181,10 +181,10 @@ public class TestDatesProjection {
Integer date = (Integer) Literal.of("2017-12-31").to(TYPE).value();
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("date").build();
- assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT_EQ, "2016");
- assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT_EQ, "2017");
- assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT_EQ, "2018");
- assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT_EQ, "2018");
+ assertProjectionStrict(spec, lessThan("date", date), Expression.Operation.LT, "2017");
+ assertProjectionStrict(spec, lessThanOrEqual("date", date), Expression.Operation.LT, "2018");
+ assertProjectionStrict(spec, greaterThan("date", date), Expression.Operation.GT, "2017");
+ assertProjectionStrict(spec, greaterThanOrEqual("date", date), Expression.Operation.GT, "2017");
assertProjectionStrict(spec, notEqual("date", date), Expression.Operation.NOT_EQ, "2017");
assertProjectionStrictValue(spec, equal("date", date), Expression.Operation.FALSE);
}
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java
index 5d3c5b0..3fec404 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java
@@ -88,10 +88,10 @@ public class TestTimestampsProjection {
Long date = (long) Literal.of("2017-12-01T00:00:00.00000").to(TYPE).value();
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build();
- assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11");
- assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-11");
- assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018-01");
- assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12");
+ assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT, "2017-12");
+ assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT, "2017-12");
+ assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "2017-12");
+ assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "2017-11");
assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12");
assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE);
}
@@ -101,10 +101,10 @@ public class TestTimestampsProjection {
Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value();
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).month("timestamp").build();
- assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11");
- assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12");
- assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018-01");
- assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2018-01");
+ assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT, "2017-12");
+ assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT, "2018-01");
+ assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "2017-12");
+ assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "2017-12");
assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12");
assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE);
}
@@ -140,10 +140,10 @@ public class TestTimestampsProjection {
Long date = (long) Literal.of("2017-12-01T00:00:00.00000").to(TYPE).value();
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build();
- assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30");
- assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30");
- assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02");
- assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01");
+ assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT, "2017-12-01");
+ assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT, "2017-12-01");
+ assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "2017-12-01");
+ assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "2017-11-30");
assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01");
assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE);
}
@@ -153,10 +153,10 @@ public class TestTimestampsProjection {
Long date = (long) Literal.of("2017-12-01T23:59:59.999999").to(TYPE).value();
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).day("timestamp").build();
- assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-11-30");
- assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01");
- assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02");
- assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-02");
+ assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT, "2017-12-01");
+ assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT, "2017-12-02");
+ assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "2017-12-01");
+ assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "2017-12-01");
assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01");
assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE);
}
@@ -192,10 +192,10 @@ public class TestTimestampsProjection {
Long date = (long) Literal.of("2017-01-01T00:00:00.00000").to(TYPE).value();
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build();
- assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2016");
- assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2016");
- assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018");
- assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017");
+ assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT, "2017");
+ assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT, "2017");
+ assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "2017");
+ assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "2016");
assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017");
assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE);
}
@@ -205,10 +205,10 @@ public class TestTimestampsProjection {
Long date = (long) Literal.of("2017-12-31T23:59:59.999999").to(TYPE).value();
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).year("timestamp").build();
- assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2016");
- assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017");
- assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2018");
- assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2018");
+ assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT, "2017");
+ assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT, "2018");
+ assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "2017");
+ assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "2017");
assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017");
assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE);
}
@@ -244,10 +244,10 @@ public class TestTimestampsProjection {
Long date = (long) Literal.of("2017-12-01T10:00:00.00000").to(TYPE).value();
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build();
- assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09");
- assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09");
- assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11");
- assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-10");
+ assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT, "2017-12-01-10");
+ assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT, "2017-12-01-10");
+ assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "2017-12-01-10");
+ assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "2017-12-01-09");
assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01-10");
assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE);
}
@@ -257,10 +257,10 @@ public class TestTimestampsProjection {
Long date = (long) Literal.of("2017-12-01T10:59:59.999999").to(TYPE).value();
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("timestamp").build();
- assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-09");
- assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT_EQ, "2017-12-01-10");
- assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11");
- assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT_EQ, "2017-12-01-11");
+ assertProjectionStrict(spec, lessThan("timestamp", date), Expression.Operation.LT, "2017-12-01-10");
+ assertProjectionStrict(spec, lessThanOrEqual("timestamp", date), Expression.Operation.LT, "2017-12-01-11");
+ assertProjectionStrict(spec, greaterThan("timestamp", date), Expression.Operation.GT, "2017-12-01-10");
+ assertProjectionStrict(spec, greaterThanOrEqual("timestamp", date), Expression.Operation.GT, "2017-12-01-10");
assertProjectionStrict(spec, notEqual("timestamp", date), Expression.Operation.NOT_EQ, "2017-12-01-10");
assertProjectionStrictValue(spec, equal("timestamp", date), Expression.Operation.FALSE);
}
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesProjection.java
new file mode 100644
index 0000000..a10b649
--- /dev/null
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesProjection.java
@@ -0,0 +1,315 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.transforms;
+
+import java.math.BigDecimal;
+import java.nio.ByteBuffer;
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.Literal;
+import org.apache.iceberg.expressions.Projections;
+import org.apache.iceberg.expressions.UnboundPredicate;
+import org.apache.iceberg.types.Types;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.iceberg.TestHelpers.assertAndUnwrapUnbound;
+import static org.apache.iceberg.expressions.Expressions.equal;
+import static org.apache.iceberg.expressions.Expressions.greaterThan;
+import static org.apache.iceberg.expressions.Expressions.greaterThanOrEqual;
+import static org.apache.iceberg.expressions.Expressions.lessThan;
+import static org.apache.iceberg.expressions.Expressions.lessThanOrEqual;
+import static org.apache.iceberg.expressions.Expressions.notEqual;
+import static org.apache.iceberg.types.Types.NestedField.optional;
+
+public class TestTruncatesProjection {
+
+ public void assertProjectionStrict(PartitionSpec spec, UnboundPredicate<?> filter,
+ Expression.Operation expectedOp, String expectedLiteral) {
+
+ Expression projection = Projections.strict(spec).project(filter);
+ UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);
+
+ Assert.assertEquals(expectedOp, predicate.op());
+
+ Literal literal = predicate.literal();
+ Truncate transform = (Truncate) spec.getFieldsBySourceId(1).get(0).transform();
+ String output = transform.toHumanString(literal.value());
+ Assert.assertEquals(expectedLiteral, output);
+ }
+
+ public void assertProjectionStrictValue(PartitionSpec spec, UnboundPredicate<?> filter,
+ Expression.Operation expectedOp) {
+
+ Expression projection = Projections.strict(spec).project(filter);
+ Assert.assertEquals(projection.op(), expectedOp);
+ }
+
+ public void assertProjectionInclusiveValue(PartitionSpec spec, UnboundPredicate<?> filter,
+ Expression.Operation expectedOp) {
+
+ Expression projection = Projections.inclusive(spec).project(filter);
+ Assert.assertEquals(projection.op(), expectedOp);
+ }
+
+ public void assertProjectionInclusive(PartitionSpec spec, UnboundPredicate<?> filter,
+ Expression.Operation expectedOp, String expectedLiteral) {
+ Expression projection = Projections.inclusive(spec).project(filter);
+ UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);
+
+ Assert.assertEquals(predicate.op(), expectedOp);
+
+ Literal literal = predicate.literal();
+ Truncate transform = (Truncate) spec.getFieldsBySourceId(1).get(0).transform();
+ String output = transform.toHumanString(literal.value());
+ Assert.assertEquals(expectedLiteral, output);
+ }
+
+ @Test
+ public void testIntegerStrictLowerBound() {
+ Integer value = 100;
+ Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();
+
+ assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "100");
+ assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100");
+ assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "100");
+ assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90");
+ assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "100");
+ assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
+ }
+
+ @Test
+ public void testIntegerStrictUpperBound() {
+ Integer value = 99;
+ Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();
+
+ assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "90");
+ assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100");
+ assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "90");
+ assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90");
+ assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "90");
+ assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
+ }
+
+ @Test
+ public void testIntegerInclusiveLowerBound() {
+ Integer value = 100;
+ Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();
+
+ assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "90");
+ assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "100");
+ assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100");
+ assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "100");
+ assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "100");
+ assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
+ }
+
+ @Test
+ public void testIntegerInclusiveUpperBound() {
+ Integer value = 99;
+ Schema schema = new Schema(optional(1, "value", Types.IntegerType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();
+
+ assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "90");
+ assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "90");
+ assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100");
+ assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "90");
+ assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "90");
+ assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
+ }
+
+ @Test
+ public void testLongStrictLowerBound() {
+ Long value = 100L;
+ Schema schema = new Schema(optional(1, "value", Types.LongType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();
+
+ assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "100");
+ assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100");
+ assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "100");
+ assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90");
+ assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "100");
+ assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
+ }
+
+ @Test
+ public void testLongStrictUpperBound() {
+ Long value = 99L;
+ Schema schema = new Schema(optional(1, "value", Types.LongType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();
+
+ assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "90");
+ assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100");
+ assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "90");
+ assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "90");
+ assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "90");
+ assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
+ }
+
+ @Test
+ public void testLongInclusiveLowerBound() {
+ Long value = 100L;
+ Schema schema = new Schema(optional(1, "value", Types.LongType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();
+
+ assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "90");
+ assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "100");
+ assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100");
+ assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "100");
+ assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "100");
+ assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
+ }
+
+ @Test
+ public void testLongInclusiveUpperBound() {
+ Long value = 99L;
+ Schema schema = new Schema(optional(1, "value", Types.LongType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();
+
+ assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "90");
+ assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "90");
+ assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100");
+ assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "90");
+ assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "90");
+ assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
+ }
+
+ @Test
+ public void testDecimalStrictLowerBound() {
+ Types.DecimalType type = Types.DecimalType.of(9, 2);
+ BigDecimal value = (BigDecimal) Literal.of("100.00").to(type).value();
+ Schema schema = new Schema(optional(1, "value", type));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();
+
+ assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "100.00");
+ assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100.00");
+ assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "100.00");
+ assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "99.90");
+ assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "100.00");
+ assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
+ }
+
+ @Test
+ public void testDecimalStrictUpperBound() {
+ Types.DecimalType type = Types.DecimalType.of(9, 2);
+ BigDecimal value = (BigDecimal) Literal.of("99.99").to(type).value();
+ Schema schema = new Schema(optional(1, "value", type));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();
+
+ assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "99.90");
+ assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "100.00");
+ assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "99.90");
+ assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "99.90");
+ assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "99.90");
+ assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
+ }
+
+ @Test
+ public void testDecimalInclusiveLowerBound() {
+ Types.DecimalType type = Types.DecimalType.of(9, 2);
+ BigDecimal value = (BigDecimal) Literal.of("100.00").to(type).value();
+ Schema schema = new Schema(optional(1, "value", type));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();
+
+ assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "99.90");
+ assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "100.00");
+ assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100.00");
+ assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "100.00");
+ assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "100.00");
+ assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
+ }
+
+ @Test
+ public void testDecimalInclusiveUpperBound() {
+ Types.DecimalType type = Types.DecimalType.of(9, 2);
+ BigDecimal value = (BigDecimal) Literal.of("99.99").to(type).value();
+ Schema schema = new Schema(optional(1, "value", type));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();
+
+ assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "99.90");
+ assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "99.90");
+ assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "100.00");
+ assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "99.90");
+ assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "99.90");
+ assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
+ }
+
+ @Test
+ public void testStringStrict() {
+ String value = "abcdefg";
+ Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build();
+
+ assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, "abcde");
+ assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, "abcde");
+ assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, "abcde");
+ assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, "abcde");
+ assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, "abcde");
+ assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
+ }
+
+ @Test
+ public void testStringInclusive() {
+ String value = "abcdefg";
+ Schema schema = new Schema(optional(1, "value", Types.StringType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build();
+
+ assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, "abcde");
+ assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, "abcde");
+ assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, "abcde");
+ assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, "abcde");
+ assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, "abcde");
+ assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
+ }
+
+ @Test
+ public void testBinaryStrict() throws Exception {
+ ByteBuffer value = ByteBuffer.wrap("abcdefg".getBytes("UTF-8"));
+ Schema schema = new Schema(optional(1, "value", Types.BinaryType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build();
+ String expectedValue = TransformUtil.base64encode(ByteBuffer.wrap("abcde".getBytes("UTF-8")));
+
+ assertProjectionStrict(spec, lessThan("value", value), Expression.Operation.LT, expectedValue);
+ assertProjectionStrict(spec, lessThanOrEqual("value", value), Expression.Operation.LT, expectedValue);
+ assertProjectionStrict(spec, greaterThan("value", value), Expression.Operation.GT, expectedValue);
+ assertProjectionStrict(spec, greaterThanOrEqual("value", value), Expression.Operation.GT, expectedValue);
+ assertProjectionStrict(spec, notEqual("value", value), Expression.Operation.NOT_EQ, expectedValue);
+ assertProjectionStrictValue(spec, equal("value", value), Expression.Operation.FALSE);
+ }
+
+ @Test
+ public void testBinaryInclusive() throws Exception {
+ ByteBuffer value = ByteBuffer.wrap("abcdefg".getBytes("UTF-8"));
+ Schema schema = new Schema(optional(1, "value", Types.BinaryType.get()));
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 5).build();
+ String expectedValue = TransformUtil.base64encode(ByteBuffer.wrap("abcde".getBytes("UTF-8")));
+
+ assertProjectionInclusive(spec, lessThan("value", value), Expression.Operation.LT_EQ, expectedValue);
+ assertProjectionInclusive(spec, lessThanOrEqual("value", value), Expression.Operation.LT_EQ, expectedValue);
+ assertProjectionInclusive(spec, greaterThan("value", value), Expression.Operation.GT_EQ, expectedValue);
+ assertProjectionInclusive(spec, greaterThanOrEqual("value", value), Expression.Operation.GT_EQ, expectedValue);
+ assertProjectionInclusive(spec, equal("value", value), Expression.Operation.EQ, expectedValue);
+ assertProjectionInclusiveValue(spec, notEqual("value", value), Expression.Operation.TRUE);
+ }
+}
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesResiduals.java b/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesResiduals.java
new file mode 100644
index 0000000..1ed1f4c
--- /dev/null
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesResiduals.java
@@ -0,0 +1,177 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.iceberg.transforms;
+
+import org.apache.iceberg.PartitionSpec;
+import org.apache.iceberg.Schema;
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.expressions.Expression;
+import org.apache.iceberg.expressions.ResidualEvaluator;
+import org.apache.iceberg.expressions.UnboundPredicate;
+import org.apache.iceberg.types.Types;
+import org.junit.Assert;
+import org.junit.Test;
+
+import static org.apache.iceberg.TestHelpers.assertAndUnwrapUnbound;
+import static org.apache.iceberg.expressions.Expressions.equal;
+import static org.apache.iceberg.expressions.Expressions.greaterThan;
+import static org.apache.iceberg.expressions.Expressions.greaterThanOrEqual;
+import static org.apache.iceberg.expressions.Expressions.lessThan;
+import static org.apache.iceberg.expressions.Expressions.lessThanOrEqual;
+import static org.apache.iceberg.expressions.Expressions.notEqual;
+
+public class TestTruncatesResiduals {
+
+ /**
+ * Test helper method to compute residual for a given partitionValue against a predicate
+ * and assert the resulting residual expression is same as the exprectedOp
+ *
+ * @param spec the partition spec
+ * @param predicate predicate to calculate the residual against
+ * @param partitionValue value of the partition to check the residual for
+ * @param expectedOp expected operation to assert against
+ * @param <T> Type parameter of partitionValue
+ */
+ public <T> void assertResidualValue(PartitionSpec spec, UnboundPredicate<?> predicate,
+ T partitionValue, Expression.Operation expectedOp) {
+ ResidualEvaluator resEval = ResidualEvaluator.of(spec, predicate, true);
+ Expression residual = resEval.residualFor(TestHelpers.Row.of(partitionValue));
+
+ Assert.assertEquals(expectedOp, residual.op());
+ }
+
+ /**
+ * Test helper method to compute residual for a given partitionValue against a predicate
+ * and assert that the resulting expression is same as the original predicate
+ *
+ * @param spec the partition spec
+ * @param predicate predicate to calculate the residual against
+ * @param partitionValue value of the partition to check the residual for
+ * @param <T> Type parameter of partitionValue
+ */
+ public <T> void assertResidualPredicate(PartitionSpec spec,
+ UnboundPredicate<?> predicate, T partitionValue) {
+ ResidualEvaluator resEval = ResidualEvaluator.of(spec, predicate, true);
+ Expression residual = resEval.residualFor(TestHelpers.Row.of(partitionValue));
+
+ UnboundPredicate<?> unbound = assertAndUnwrapUnbound(residual);
+ Assert.assertEquals(predicate.op(), unbound.op());
+ Assert.assertEquals(predicate.ref().name(), unbound.ref().name());
+ Assert.assertEquals(predicate.literal().value(), unbound.literal().value());
+ }
+
+ @Test
+ public void testIntegerTruncateTransformResiduals() {
+ Schema schema = new Schema(Types.NestedField.optional(50, "value", Types.IntegerType.get()));
+ // valid partitions would be 0, 10, 20...90, 100 etc.
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 10).build();
+
+ // less than lower bound
+ assertResidualValue(spec, lessThan("value", 100), 110, Expression.Operation.FALSE);
+ assertResidualValue(spec, lessThan("value", 100), 100, Expression.Operation.FALSE);
+ assertResidualValue(spec, lessThan("value", 100), 90, Expression.Operation.TRUE);
+ // less than upper bound
+ assertResidualValue(spec, lessThan("value", 99), 100, Expression.Operation.FALSE);
+ assertResidualPredicate(spec, lessThan("value", 99), 90);
+ assertResidualValue(spec, lessThan("value", 99), 80, Expression.Operation.TRUE);
+
+ // less than equals lower bound
+ assertResidualValue(spec, lessThanOrEqual("value", 100), 110, Expression.Operation.FALSE);
+ assertResidualPredicate(spec, lessThanOrEqual("value", 100), 100);
+ assertResidualValue(spec, lessThanOrEqual("value", 100), 90, Expression.Operation.TRUE);
+ // less than equals upper bound
+ assertResidualValue(spec, lessThanOrEqual("value", 99), 100, Expression.Operation.FALSE);
+ assertResidualValue(spec, lessThanOrEqual("value", 99), 90, Expression.Operation.TRUE);
+ assertResidualValue(spec, lessThanOrEqual("value", 99), 80, Expression.Operation.TRUE);
+
+ // greater than lower bound
+ assertResidualValue(spec, greaterThan("value", 100), 110, Expression.Operation.TRUE);
+ assertResidualPredicate(spec, greaterThan("value", 100), 100);
+ assertResidualValue(spec, greaterThan("value", 100), 90, Expression.Operation.FALSE);
+ // greater than upper bound
+ assertResidualValue(spec, greaterThan("value", 99), 100, Expression.Operation.TRUE);
+ assertResidualValue(spec, greaterThan("value", 99), 90, Expression.Operation.FALSE);
+ assertResidualValue(spec, greaterThan("value", 99), 80, Expression.Operation.FALSE);
+
+ // greater than equals lower bound
+ assertResidualValue(spec, greaterThanOrEqual("value", 100), 110, Expression.Operation.TRUE);
+ assertResidualValue(spec, greaterThanOrEqual("value", 100), 100, Expression.Operation.TRUE);
+ assertResidualValue(spec, greaterThanOrEqual("value", 100), 90, Expression.Operation.FALSE);
+ // greater than equals upper bound
+ assertResidualValue(spec, greaterThanOrEqual("value", 99), 100, Expression.Operation.TRUE);
+ assertResidualPredicate(spec, greaterThanOrEqual("value", 99), 90);
+ assertResidualValue(spec, greaterThanOrEqual("value", 99), 80, Expression.Operation.FALSE);
+
+ // equal lower bound
+ assertResidualValue(spec, equal("value", 100), 110, Expression.Operation.FALSE);
+ assertResidualPredicate(spec, equal("value", 100), 100);
+ assertResidualValue(spec, equal("value", 100), 90, Expression.Operation.FALSE);
+ // equal upper bound
+ assertResidualValue(spec, equal("value", 99), 100, Expression.Operation.FALSE);
+ assertResidualPredicate(spec, equal("value", 99), 90);
+ assertResidualValue(spec, equal("value", 99), 80, Expression.Operation.FALSE);
+
+ // not equal lower bound
+ assertResidualValue(spec, notEqual("value", 100), 110, Expression.Operation.TRUE);
+ assertResidualPredicate(spec, notEqual("value", 100), 100);
+ assertResidualValue(spec, notEqual("value", 100), 90, Expression.Operation.TRUE);
+ // not equal upper bound
+ assertResidualValue(spec, notEqual("value", 99), 100, Expression.Operation.TRUE);
+ assertResidualPredicate(spec, notEqual("value", 99), 90);
+ assertResidualValue(spec, notEqual("value", 99), 80, Expression.Operation.TRUE);
+ }
+
+ @Test
+ public void testStringTruncateTransformResiduals() {
+ Schema schema = new Schema(Types.NestedField.optional(50, "value", Types.StringType.get()));
+ // valid partitions would be two letter strings for eg: ab, bc etc
+ PartitionSpec spec = PartitionSpec.builderFor(schema).truncate("value", 2).build();
+
+ // less than
+ assertResidualValue(spec, lessThan("value", "bcd"), "ab", Expression.Operation.TRUE);
+ assertResidualPredicate(spec, lessThan("value", "bcd"), "bc");
+ assertResidualValue(spec, lessThan("value", "bcd"), "cd", Expression.Operation.FALSE);
+
+ // less than equals
+ assertResidualValue(spec, lessThanOrEqual("value", "bcd"), "ab", Expression.Operation.TRUE);
+ assertResidualPredicate(spec, lessThanOrEqual("value", "bcd"), "bc");
+ assertResidualValue(spec, lessThanOrEqual("value", "bcd"), "cd", Expression.Operation.FALSE);
+
+ // greater than
+ assertResidualValue(spec, greaterThan("value", "bcd"), "ab", Expression.Operation.FALSE);
+ assertResidualPredicate(spec, greaterThan("value", "bcd"), "bc");
+ assertResidualValue(spec, greaterThan("value", "bcd"), "cd", Expression.Operation.TRUE);
+
+ // greater than
+ assertResidualValue(spec, greaterThanOrEqual("value", "bcd"), "ab", Expression.Operation.FALSE);
+ assertResidualPredicate(spec, greaterThanOrEqual("value", "bcd"), "bc");
+ assertResidualValue(spec, greaterThanOrEqual("value", "bcd"), "cd", Expression.Operation.TRUE);
+
+ // equal
+ assertResidualValue(spec, equal("value", "bcd"), "ab", Expression.Operation.FALSE);
+ assertResidualPredicate(spec, equal("value", "bcd"), "bc");
+ assertResidualValue(spec, equal("value", "bcd"), "cd", Expression.Operation.FALSE);
+
+ // not equal
+ assertResidualValue(spec, notEqual("value", "bcd"), "ab", Expression.Operation.TRUE);
+ assertResidualPredicate(spec, notEqual("value", "bcd"), "bc");
+ assertResidualValue(spec, notEqual("value", "bcd"), "cd", Expression.Operation.TRUE);
+ }
+}