You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2022/09/02 21:00:21 UTC
[iceberg] branch master updated: API: Remove source type from Transform (#5601)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 223177faf9 API: Remove source type from Transform (#5601)
223177faf9 is described below
commit 223177faf955bd8f11864477da16013cf5d7cc75
Author: Ryan Blue <bl...@apache.org>
AuthorDate: Fri Sep 2 14:00:15 2022 -0700
API: Remove source type from Transform (#5601)
---
.palantir/revapi.yml | 4 +
.../main/java/org/apache/iceberg/PartitionKey.java | 11 +-
.../java/org/apache/iceberg/PartitionSpec.java | 73 ++---
.../main/java/org/apache/iceberg/SortOrder.java | 11 +-
.../org/apache/iceberg/UnboundPartitionSpec.java | 16 +-
.../java/org/apache/iceberg/UnboundSortOrder.java | 14 +-
.../apache/iceberg/expressions/BoundTransform.java | 5 +-
.../apache/iceberg/expressions/ExpressionUtil.java | 6 +-
.../apache/iceberg/expressions/Expressions.java | 18 +-
.../iceberg/expressions/UnboundTransform.java | 10 +-
.../java/org/apache/iceberg/transforms/Bucket.java | 173 +++++------
.../java/org/apache/iceberg/transforms/Dates.java | 66 +++--
.../java/org/apache/iceberg/transforms/Days.java | 81 ++++++
.../transforms/{VoidTransform.java => Hours.java} | 54 ++--
.../org/apache/iceberg/transforms/Identity.java | 82 ++----
.../java/org/apache/iceberg/transforms/Months.java | 81 ++++++
.../iceberg/transforms/PartitionSpecVisitor.java | 12 +-
.../apache/iceberg/transforms/ProjectionUtil.java | 19 +-
.../iceberg/transforms/SerializationProxies.java | 75 +++++
.../iceberg/transforms/SortOrderVisitor.java | 14 +-
.../{VoidTransform.java => TimeTransform.java} | 51 ++--
.../org/apache/iceberg/transforms/Timestamps.java | 74 +++--
.../org/apache/iceberg/transforms/Transform.java | 65 ++++-
.../org/apache/iceberg/transforms/Transforms.java | 127 ++++++++-
.../org/apache/iceberg/transforms/Truncate.java | 316 +++++++++------------
.../iceberg/transforms/UnknownTransform.java | 21 +-
.../apache/iceberg/transforms/VoidTransform.java | 21 ++
.../transforms/{VoidTransform.java => Years.java} | 59 ++--
.../SerializableFunction.java} | 28 +-
.../org/apache/iceberg/PartitionSpecTestBase.java | 9 +-
.../org/apache/iceberg/TestPartitionPaths.java | 10 +-
.../iceberg/TestPartitionSpecValidation.java | 13 +-
.../iceberg/expressions/TestExpressionHelpers.java | 2 +-
.../apache/iceberg/transforms/TestBucketing.java | 142 +++------
.../transforms/TestBucketingProjection.java | 14 +-
.../org/apache/iceberg/transforms/TestDates.java | 104 ++++---
.../iceberg/transforms/TestDatesProjection.java | 31 +-
.../apache/iceberg/transforms/TestIdentity.java | 46 +--
.../iceberg/transforms/TestNotStartsWith.java | 9 +-
.../apache/iceberg/transforms/TestResiduals.java | 14 +-
.../apache/iceberg/transforms/TestStartsWith.java | 8 +-
.../apache/iceberg/transforms/TestTimestamps.java | 135 +++++----
.../transforms/TestTimestampsProjection.java | 31 +-
.../transforms/TestTransformSerialization.java | 71 +++++
.../apache/iceberg/transforms/TestTruncate.java | 39 ++-
.../transforms/TestTruncatesProjection.java | 23 +-
.../java/org/apache/iceberg/BaseMetadataTable.java | 5 +-
.../apache/iceberg/BaseUpdatePartitionSpec.java | 2 +-
.../java/org/apache/iceberg/LocationProviders.java | 6 +-
.../java/org/apache/iceberg/ManifestsTable.java | 2 +
.../java/org/apache/iceberg/TableMetadata.java | 3 +-
.../iceberg/expressions/ExpressionParser.java | 10 +-
.../org/apache/iceberg/TestReplaceTransaction.java | 2 +-
.../java/org/apache/iceberg/TestTableMetadata.java | 21 +-
.../iceberg/TestTableUpdatePartitionSpec.java | 9 +-
.../apache/iceberg/TestUpdatePartitionSpec.java | 27 +-
.../apache/iceberg/hadoop/TestHadoopCatalog.java | 3 +-
.../apache/iceberg/hadoop/TestHadoopTables.java | 2 +-
.../org/apache/iceberg/jdbc/TestJdbcCatalog.java | 2 +-
.../iceberg/spark/source/WritersBenchmark.java | 6 +-
.../org/apache/iceberg/spark/IcebergSpark.java | 6 +-
.../expressions/TransformExpressions.scala | 25 +-
.../iceberg/spark/source/TestFilteredScan.java | 12 +-
.../iceberg/spark/source/TestIcebergSpark.java | 37 ++-
.../iceberg/spark/source/TestPartitionPruning.java | 26 +-
65 files changed, 1492 insertions(+), 1002 deletions(-)
diff --git a/.palantir/revapi.yml b/.palantir/revapi.yml
index 4c45b8f80d..9d8b5c5077 100644
--- a/.palantir/revapi.yml
+++ b/.palantir/revapi.yml
@@ -18,6 +18,10 @@ acceptedBreaks:
- code: "java.method.removed"
old: "method org.apache.iceberg.RowDelta org.apache.iceberg.RowDelta::validateNoConflictingAppends(org.apache.iceberg.expressions.Expression)"
justification: "Deprecations for 1.0 release"
+ - code: "java.class.defaultSerializationChanged"
+ old: "class org.apache.iceberg.PartitionKey"
+ new: "class org.apache.iceberg.PartitionKey"
+ justification: "Serialization across versions is not supported"
release-base-0.13.0:
org.apache.iceberg:iceberg-api:
- code: "java.class.defaultSerializationChanged"
diff --git a/api/src/main/java/org/apache/iceberg/PartitionKey.java b/api/src/main/java/org/apache/iceberg/PartitionKey.java
index 0f696b59c4..fc56d1a453 100644
--- a/api/src/main/java/org/apache/iceberg/PartitionKey.java
+++ b/api/src/main/java/org/apache/iceberg/PartitionKey.java
@@ -22,8 +22,9 @@ import java.io.Serializable;
import java.lang.reflect.Array;
import java.util.Arrays;
import java.util.List;
+import java.util.function.Function;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.transforms.Transform;
+import org.apache.iceberg.util.SerializableFunction;
/**
* A struct of partition values.
@@ -36,7 +37,7 @@ public class PartitionKey implements StructLike, Serializable {
private final PartitionSpec spec;
private final int size;
private final Object[] partitionTuple;
- private final Transform[] transforms;
+ private final SerializableFunction[] transforms;
private final Accessor<StructLike>[] accessors;
@SuppressWarnings("unchecked")
@@ -46,7 +47,7 @@ public class PartitionKey implements StructLike, Serializable {
List<PartitionField> fields = spec.fields();
this.size = fields.size();
this.partitionTuple = new Object[size];
- this.transforms = new Transform[size];
+ this.transforms = new SerializableFunction[size];
this.accessors = (Accessor<StructLike>[]) Array.newInstance(Accessor.class, size);
Schema schema = spec.schema();
@@ -57,7 +58,7 @@ public class PartitionKey implements StructLike, Serializable {
accessor != null,
"Cannot build accessor for field: " + schema.findField(field.sourceId()));
this.accessors[i] = accessor;
- this.transforms[i] = field.transform();
+ this.transforms[i] = field.transform().bind(accessor.type());
}
}
@@ -101,7 +102,7 @@ public class PartitionKey implements StructLike, Serializable {
@SuppressWarnings("unchecked")
public void partition(StructLike row) {
for (int i = 0; i < partitionTuple.length; i += 1) {
- Transform<Object, Object> transform = transforms[i];
+ Function<Object, Object> transform = transforms[i];
partitionTuple[i] = transform.apply(accessors[i].get(row));
}
}
diff --git a/api/src/main/java/org/apache/iceberg/PartitionSpec.java b/api/src/main/java/org/apache/iceberg/PartitionSpec.java
index e984fc69d8..20f5795f68 100644
--- a/api/src/main/java/org/apache/iceberg/PartitionSpec.java
+++ b/api/src/main/java/org/apache/iceberg/PartitionSpec.java
@@ -59,6 +59,7 @@ public class PartitionSpec implements Serializable {
private final PartitionField[] fields;
private transient volatile ListMultimap<Integer, PartitionField> fieldsBySourceId = null;
private transient volatile Class<?>[] lazyJavaClasses = null;
+ private transient volatile StructType lazyPartitionType = null;
private transient volatile List<PartitionField> fieldList = null;
private final int lastAssignedFieldId;
@@ -123,16 +124,23 @@ public class PartitionSpec implements Serializable {
/** Returns a {@link StructType} for partition data defined by this spec. */
public StructType partitionType() {
- List<Types.NestedField> structFields = Lists.newArrayListWithExpectedSize(fields.length);
+ if (lazyPartitionType == null) {
+ synchronized (this) {
+ if (lazyPartitionType == null) {
+ List<Types.NestedField> structFields = Lists.newArrayListWithExpectedSize(fields.length);
- for (int i = 0; i < fields.length; i += 1) {
- PartitionField field = fields[i];
- Type sourceType = schema.findType(field.sourceId());
- Type resultType = field.transform().getResultType(sourceType);
- structFields.add(Types.NestedField.optional(field.fieldId(), field.name(), resultType));
+ for (PartitionField field : fields) {
+ Type sourceType = schema.findType(field.sourceId());
+ Type resultType = field.transform().getResultType(sourceType);
+ structFields.add(Types.NestedField.optional(field.fieldId(), field.name(), resultType));
+ }
+
+ this.lazyPartitionType = Types.StructType.of(structFields);
+ }
+ }
}
- return Types.StructType.of(structFields);
+ return lazyPartitionType;
}
public Class<?>[] javaClasses() {
@@ -175,9 +183,11 @@ public class PartitionSpec implements Serializable {
public String partitionToPath(StructLike data) {
StringBuilder sb = new StringBuilder();
Class<?>[] javaClasses = javaClasses();
+ List<Types.NestedField> outputFields = partitionType().fields();
for (int i = 0; i < javaClasses.length; i += 1) {
PartitionField field = fields[i];
- String valueString = field.transform().toHumanString(get(data, i, javaClasses[i]));
+ Type type = outputFields.get(i).type();
+ String valueString = field.transform().toHumanString(type, get(data, i, javaClasses[i]));
if (i > 0) {
sb.append("/");
@@ -412,10 +422,7 @@ public class PartitionSpec implements Serializable {
checkAndAddPartitionName(targetName, sourceColumn.fieldId());
PartitionField field =
new PartitionField(
- sourceColumn.fieldId(),
- nextFieldId(),
- targetName,
- Transforms.identity(sourceColumn.type()));
+ sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.identity());
checkForRedundantPartitions(field);
fields.add(field);
return this;
@@ -429,11 +436,7 @@ public class PartitionSpec implements Serializable {
checkAndAddPartitionName(targetName);
Types.NestedField sourceColumn = findSourceColumn(sourceName);
PartitionField field =
- new PartitionField(
- sourceColumn.fieldId(),
- nextFieldId(),
- targetName,
- Transforms.year(sourceColumn.type()));
+ new PartitionField(sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.year());
checkForRedundantPartitions(field);
fields.add(field);
return this;
@@ -447,11 +450,7 @@ public class PartitionSpec implements Serializable {
checkAndAddPartitionName(targetName);
Types.NestedField sourceColumn = findSourceColumn(sourceName);
PartitionField field =
- new PartitionField(
- sourceColumn.fieldId(),
- nextFieldId(),
- targetName,
- Transforms.month(sourceColumn.type()));
+ new PartitionField(sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.month());
checkForRedundantPartitions(field);
fields.add(field);
return this;
@@ -465,11 +464,7 @@ public class PartitionSpec implements Serializable {
checkAndAddPartitionName(targetName);
Types.NestedField sourceColumn = findSourceColumn(sourceName);
PartitionField field =
- new PartitionField(
- sourceColumn.fieldId(),
- nextFieldId(),
- targetName,
- Transforms.day(sourceColumn.type()));
+ new PartitionField(sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.day());
checkForRedundantPartitions(field);
fields.add(field);
return this;
@@ -483,11 +478,7 @@ public class PartitionSpec implements Serializable {
checkAndAddPartitionName(targetName);
Types.NestedField sourceColumn = findSourceColumn(sourceName);
PartitionField field =
- new PartitionField(
- sourceColumn.fieldId(),
- nextFieldId(),
- targetName,
- Transforms.hour(sourceColumn.type()));
+ new PartitionField(sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.hour());
checkForRedundantPartitions(field);
fields.add(field);
return this;
@@ -502,10 +493,7 @@ public class PartitionSpec implements Serializable {
Types.NestedField sourceColumn = findSourceColumn(sourceName);
fields.add(
new PartitionField(
- sourceColumn.fieldId(),
- nextFieldId(),
- targetName,
- Transforms.bucket(sourceColumn.type(), numBuckets)));
+ sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.bucket(numBuckets)));
return this;
}
@@ -518,10 +506,7 @@ public class PartitionSpec implements Serializable {
Types.NestedField sourceColumn = findSourceColumn(sourceName);
fields.add(
new PartitionField(
- sourceColumn.fieldId(),
- nextFieldId(),
- targetName,
- Transforms.truncate(sourceColumn.type(), width)));
+ sourceColumn.fieldId(), nextFieldId(), targetName, Transforms.truncate(width)));
return this;
}
@@ -545,16 +530,10 @@ public class PartitionSpec implements Serializable {
// add a partition field with an auto-increment partition field id starting from
// PARTITION_DATA_ID_START
- Builder add(int sourceId, String name, String transform) {
+ Builder add(int sourceId, String name, Transform<?, ?> transform) {
return add(sourceId, nextFieldId(), name, transform);
}
- Builder add(int sourceId, int fieldId, String name, String transform) {
- Types.NestedField column = schema.findField(sourceId);
- Preconditions.checkNotNull(column, "Cannot find source column: %s", sourceId);
- return add(sourceId, fieldId, name, Transforms.fromString(column.type(), transform));
- }
-
Builder add(int sourceId, int fieldId, String name, Transform<?, ?> transform) {
checkAndAddPartitionName(name, sourceId);
fields.add(new PartitionField(sourceId, fieldId, name, transform));
diff --git a/api/src/main/java/org/apache/iceberg/SortOrder.java b/api/src/main/java/org/apache/iceberg/SortOrder.java
index ee02318de6..43da6057b5 100644
--- a/api/src/main/java/org/apache/iceberg/SortOrder.java
+++ b/api/src/main/java/org/apache/iceberg/SortOrder.java
@@ -36,7 +36,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.Types;
/** A sort order that defines how data and delete files should be ordered in a table. */
public class SortOrder implements Serializable {
@@ -253,14 +252,6 @@ public class SortOrder implements Serializable {
return this;
}
- Builder addSortField(
- String transformAsString, int sourceId, SortDirection direction, NullOrder nullOrder) {
- Types.NestedField column = schema.findField(sourceId);
- ValidationException.check(column != null, "Cannot find source column: %s", sourceId);
- Transform<?, ?> transform = Transforms.fromString(column.type(), transformAsString);
- return addSortField(transform, sourceId, direction, nullOrder);
- }
-
Builder addSortField(
Transform<?, ?> transform, int sourceId, SortDirection direction, NullOrder nullOrder) {
SortField sortField = new SortField(transform, sourceId, direction, nullOrder);
@@ -293,7 +284,7 @@ public class SortOrder implements Serializable {
private Transform<?, ?> toTransform(BoundTerm<?> term) {
if (term instanceof BoundReference) {
- return Transforms.identity(term.type());
+ return Transforms.identity();
} else if (term instanceof BoundTransform) {
return ((BoundTransform<?, ?>) term).transform();
} else {
diff --git a/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java b/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java
index 530d3d442c..27ed738882 100644
--- a/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java
+++ b/api/src/main/java/org/apache/iceberg/UnboundPartitionSpec.java
@@ -20,6 +20,8 @@ package org.apache.iceberg;
import java.util.List;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.transforms.Transform;
+import org.apache.iceberg.transforms.Transforms;
public class UnboundPartitionSpec {
@@ -52,9 +54,9 @@ public class UnboundPartitionSpec {
for (UnboundPartitionField field : fields) {
if (field.partitionId != null) {
- builder.add(field.sourceId, field.partitionId, field.name, field.transformAsString);
+ builder.add(field.sourceId, field.partitionId, field.name, field.transform);
} else {
- builder.add(field.sourceId, field.name, field.transformAsString);
+ builder.add(field.sourceId, field.name, field.transform);
}
}
@@ -94,13 +96,17 @@ public class UnboundPartitionSpec {
}
static class UnboundPartitionField {
- private final String transformAsString;
+ private final Transform<?, ?> transform;
private final int sourceId;
private final Integer partitionId;
private final String name;
+ public Transform<?, ?> transform() {
+ return transform;
+ }
+
public String transformAsString() {
- return transformAsString;
+ return transform.toString();
}
public int sourceId() {
@@ -117,7 +123,7 @@ public class UnboundPartitionSpec {
private UnboundPartitionField(
String transformAsString, int sourceId, Integer partitionId, String name) {
- this.transformAsString = transformAsString;
+ this.transform = Transforms.fromString(transformAsString);
this.sourceId = sourceId;
this.partitionId = partitionId;
this.name = name;
diff --git a/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java b/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java
index 1181b665f8..c9d7d0b2e8 100644
--- a/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java
+++ b/api/src/main/java/org/apache/iceberg/UnboundSortOrder.java
@@ -21,6 +21,8 @@ package org.apache.iceberg;
import java.util.Collections;
import java.util.List;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.transforms.Transform;
+import org.apache.iceberg.transforms.Transforms;
public class UnboundSortOrder {
private static final UnboundSortOrder UNSORTED_ORDER =
@@ -38,8 +40,7 @@ public class UnboundSortOrder {
SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(orderId);
for (UnboundSortField field : fields) {
- builder.addSortField(
- field.transformAsString, field.sourceId, field.direction, field.nullOrder);
+ builder.addSortField(field.transform, field.sourceId, field.direction, field.nullOrder);
}
return builder.build();
@@ -49,8 +50,7 @@ public class UnboundSortOrder {
SortOrder.Builder builder = SortOrder.builderFor(schema).withOrderId(orderId);
for (UnboundSortField field : fields) {
- builder.addSortField(
- field.transformAsString, field.sourceId, field.direction, field.nullOrder);
+ builder.addSortField(field.transform, field.sourceId, field.direction, field.nullOrder);
}
return builder.buildUnchecked();
@@ -114,21 +114,21 @@ public class UnboundSortOrder {
}
static class UnboundSortField {
- private final String transformAsString;
+ private final Transform<?, ?> transform;
private final int sourceId;
private final SortDirection direction;
private final NullOrder nullOrder;
private UnboundSortField(
String transformAsString, int sourceId, SortDirection direction, NullOrder nullOrder) {
- this.transformAsString = transformAsString;
+ this.transform = Transforms.fromString(transformAsString);
this.sourceId = sourceId;
this.direction = direction;
this.nullOrder = nullOrder;
}
public String transformAsString() {
- return transformAsString;
+ return transform.toString();
}
public int sourceId() {
diff --git a/api/src/main/java/org/apache/iceberg/expressions/BoundTransform.java b/api/src/main/java/org/apache/iceberg/expressions/BoundTransform.java
index 32f91018c6..22271aaed9 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/BoundTransform.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/BoundTransform.java
@@ -21,6 +21,7 @@ package org.apache.iceberg.expressions;
import org.apache.iceberg.StructLike;
import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.types.Type;
+import org.apache.iceberg.util.SerializableFunction;
/**
* A transform expression.
@@ -31,15 +32,17 @@ import org.apache.iceberg.types.Type;
public class BoundTransform<S, T> implements BoundTerm<T> {
private final BoundReference<S> ref;
private final Transform<S, T> transform;
+ private final SerializableFunction<S, T> func;
BoundTransform(BoundReference<S> ref, Transform<S, T> transform) {
this.ref = ref;
this.transform = transform;
+ this.func = transform.bind(ref.type());
}
@Override
public T eval(StructLike struct) {
- return transform.apply(ref.eval(struct));
+ return func.apply(ref.eval(struct));
}
@Override
diff --git a/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java b/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java
index fe180a4f05..9c3bec48b6 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/ExpressionUtil.java
@@ -18,17 +18,17 @@
*/
package org.apache.iceberg.expressions;
+import java.util.function.Function;
import java.util.regex.Pattern;
import java.util.stream.Collectors;
import org.apache.iceberg.PartitionSpec;
-import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Types;
/** Expression utility methods. */
public class ExpressionUtil {
- private static final Transform<CharSequence, Integer> HASH_FUNC =
- Transforms.bucket(Types.StringType.get(), Integer.MAX_VALUE);
+ private static final Function<Object, Integer> HASH_FUNC =
+ Transforms.bucket(Integer.MAX_VALUE).bind(Types.StringType.get());
private static final Pattern DATE = Pattern.compile("\\d\\d\\d\\d-\\d\\d-\\d\\d");
private static final Pattern TIME = Pattern.compile("\\d\\d:\\d\\d(:\\d\\d(.\\d{1,6})?)?");
private static final Pattern TIMESTAMP =
diff --git a/api/src/main/java/org/apache/iceberg/expressions/Expressions.java b/api/src/main/java/org/apache/iceberg/expressions/Expressions.java
index 63212fa842..7fad8324c4 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/Expressions.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/Expressions.java
@@ -24,7 +24,6 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
-import org.apache.iceberg.types.Types;
/** Factory methods for creating {@link Expression expressions}. */
public class Expressions {
@@ -74,37 +73,32 @@ public class Expressions {
@SuppressWarnings("unchecked")
public static <T> UnboundTerm<T> bucket(String name, int numBuckets) {
- Transform<?, T> transform =
- (Transform<?, T>) Transforms.bucket(Types.StringType.get(), numBuckets);
+ Transform<?, T> transform = (Transform<?, T>) Transforms.bucket(numBuckets);
return new UnboundTransform<>(ref(name), transform);
}
@SuppressWarnings("unchecked")
public static <T> UnboundTerm<T> year(String name) {
- return new UnboundTransform<>(
- ref(name), (Transform<?, T>) Transforms.year(Types.TimestampType.withZone()));
+ return new UnboundTransform<>(ref(name), (Transform<?, T>) Transforms.year());
}
@SuppressWarnings("unchecked")
public static <T> UnboundTerm<T> month(String name) {
- return new UnboundTransform<>(
- ref(name), (Transform<?, T>) Transforms.month(Types.TimestampType.withZone()));
+ return new UnboundTransform<>(ref(name), (Transform<?, T>) Transforms.month());
}
@SuppressWarnings("unchecked")
public static <T> UnboundTerm<T> day(String name) {
- return new UnboundTransform<>(
- ref(name), (Transform<?, T>) Transforms.day(Types.TimestampType.withZone()));
+ return new UnboundTransform<>(ref(name), (Transform<?, T>) Transforms.day());
}
@SuppressWarnings("unchecked")
public static <T> UnboundTerm<T> hour(String name) {
- return new UnboundTransform<>(
- ref(name), (Transform<?, T>) Transforms.hour(Types.TimestampType.withZone()));
+ return new UnboundTransform<>(ref(name), (Transform<?, T>) Transforms.hour());
}
public static <T> UnboundTerm<T> truncate(String name, int width) {
- return new UnboundTransform<>(ref(name), Transforms.truncate(Types.LongType.get(), width));
+ return new UnboundTransform<>(ref(name), Transforms.truncate(width));
}
public static <T> UnboundPredicate<T> isNull(String name) {
diff --git a/api/src/main/java/org/apache/iceberg/expressions/UnboundTransform.java b/api/src/main/java/org/apache/iceberg/expressions/UnboundTransform.java
index cd92fce391..cae84733c8 100644
--- a/api/src/main/java/org/apache/iceberg/expressions/UnboundTransform.java
+++ b/api/src/main/java/org/apache/iceberg/expressions/UnboundTransform.java
@@ -20,7 +20,6 @@ package org.apache.iceberg.expressions;
import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.transforms.Transform;
-import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Types;
public class UnboundTransform<S, T> implements UnboundTerm<T>, Term {
@@ -41,18 +40,13 @@ public class UnboundTransform<S, T> implements UnboundTerm<T>, Term {
return transform;
}
- @SuppressWarnings("unchecked")
@Override
public BoundTransform<S, T> bind(Types.StructType struct, boolean caseSensitive) {
BoundReference<S> boundRef = ref.bind(struct, caseSensitive);
- Transform<S, T> typeTransform;
try {
- // TODO: Avoid using toString/fromString
- typeTransform =
- (Transform<S, T>) Transforms.fromString(boundRef.type(), transform.toString());
ValidationException.check(
- typeTransform.canTransform(boundRef.type()),
+ transform.canTransform(boundRef.type()),
"Cannot bind: %s cannot transform %s values from '%s'",
transform,
boundRef.type(),
@@ -63,7 +57,7 @@ public class UnboundTransform<S, T> implements UnboundTerm<T>, Term {
transform, boundRef.type(), ref.name());
}
- return new BoundTransform<>(boundRef, typeTransform);
+ return new BoundTransform<>(boundRef, transform);
}
@Override
diff --git a/api/src/main/java/org/apache/iceberg/transforms/Bucket.java b/api/src/main/java/org/apache/iceberg/transforms/Bucket.java
index 32540bb923..912bcd2717 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/Bucket.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/Bucket.java
@@ -18,52 +18,53 @@
*/
package org.apache.iceberg.transforms;
-import static org.apache.iceberg.types.Type.TypeID;
-
+import java.io.Serializable;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
-import java.util.Set;
import java.util.UUID;
+import java.util.function.Function;
import org.apache.iceberg.expressions.BoundPredicate;
import org.apache.iceberg.expressions.BoundTransform;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.UnboundPredicate;
-import org.apache.iceberg.relocated.com.google.common.annotations.VisibleForTesting;
import org.apache.iceberg.relocated.com.google.common.base.Objects;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.relocated.com.google.common.collect.Sets;
-import org.apache.iceberg.relocated.com.google.common.hash.HashFunction;
-import org.apache.iceberg.relocated.com.google.common.hash.Hashing;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.BucketUtil;
+import org.apache.iceberg.util.SerializableFunction;
-abstract class Bucket<T> implements Transform<T, Integer> {
- private static final HashFunction MURMUR3 = Hashing.murmur3_32_fixed();
+class Bucket<T> implements Transform<T, Integer>, Serializable {
+ static <T> Bucket<T> get(int numBuckets) {
+ Preconditions.checkArgument(
+ numBuckets > 0, "Invalid number of buckets: %s (must be > 0)", numBuckets);
+ return new Bucket<>(numBuckets);
+ }
@SuppressWarnings("unchecked")
- static <T> Bucket<T> get(Type type, int numBuckets) {
+ static <T, B extends Bucket<T> & SerializableFunction<T, Integer>> B get(
+ Type type, int numBuckets) {
Preconditions.checkArgument(
numBuckets > 0, "Invalid number of buckets: %s (must be > 0)", numBuckets);
switch (type.typeId()) {
case DATE:
case INTEGER:
- return (Bucket<T>) new BucketInteger(numBuckets);
+ return (B) new BucketInteger(numBuckets);
case TIME:
case TIMESTAMP:
case LONG:
- return (Bucket<T>) new BucketLong(numBuckets);
+ return (B) new BucketLong(numBuckets);
case DECIMAL:
- return (Bucket<T>) new BucketDecimal(numBuckets);
+ return (B) new BucketDecimal(numBuckets);
case STRING:
- return (Bucket<T>) new BucketString(numBuckets);
+ return (B) new BucketString(numBuckets);
case FIXED:
case BINARY:
- return (Bucket<T>) new BucketByteBuffer(numBuckets);
+ return (B) new BucketByteBuffer(numBuckets);
case UUID:
- return (Bucket<T>) new BucketUUID(numBuckets);
+ return (B) new BucketUUID(numBuckets);
default:
throw new IllegalArgumentException("Cannot bucket by type: " + type);
}
@@ -79,8 +80,16 @@ abstract class Bucket<T> implements Transform<T, Integer> {
return numBuckets;
}
- @VisibleForTesting
- abstract int hash(T value);
+ @Override
+ public SerializableFunction<T, Integer> bind(Type type) {
+ Preconditions.checkArgument(canTransform(type), "Cannot bucket by type: %s", type);
+ return get(type, numBuckets);
+ }
+
+ protected int hash(T value) {
+ throw new UnsupportedOperationException(
+ "hash(value) is not supported on the base Bucket class");
+ }
@Override
public Integer apply(T value) {
@@ -90,6 +99,24 @@ abstract class Bucket<T> implements Transform<T, Integer> {
return (hash(value) & Integer.MAX_VALUE) % numBuckets;
}
+ @Override
+ public boolean canTransform(Type type) {
+ switch (type.typeId()) {
+ case INTEGER:
+ case LONG:
+ case DATE:
+ case TIME:
+ case TIMESTAMP:
+ case STRING:
+ case BINARY:
+ case FIXED:
+ case DECIMAL:
+ case UUID:
+ return true;
+ }
+ return false;
+ }
+
@Override
public boolean equals(Object o) {
if (this == o) {
@@ -114,6 +141,7 @@ abstract class Bucket<T> implements Transform<T, Integer> {
@Override
public UnboundPredicate<Integer> project(String name, BoundPredicate<T> predicate) {
+ Function<T, Integer> function = this.bind(predicate.term().type());
if (predicate.term() instanceof BoundTransform) {
return ProjectionUtil.projectTransformPredicate(this, name, predicate);
}
@@ -122,10 +150,10 @@ abstract class Bucket<T> implements Transform<T, Integer> {
return Expressions.predicate(predicate.op(), name);
} else if (predicate.isLiteralPredicate() && predicate.op() == Expression.Operation.EQ) {
return Expressions.predicate(
- predicate.op(), name, apply(predicate.asLiteralPredicate().literal().value()));
+ predicate.op(), name, function.apply(predicate.asLiteralPredicate().literal().value()));
} else if (predicate.isSetPredicate()
&& predicate.op() == Expression.Operation.IN) { // notIn can't be projected
- return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), this);
+ return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), function);
}
// comparison predicates can't be projected, notEq can't be projected
@@ -136,6 +164,7 @@ abstract class Bucket<T> implements Transform<T, Integer> {
@Override
public UnboundPredicate<Integer> projectStrict(String name, BoundPredicate<T> predicate) {
+ Function<T, Integer> function = this.bind(predicate.term().type());
if (predicate.term() instanceof BoundTransform) {
return ProjectionUtil.projectTransformPredicate(this, name, predicate);
}
@@ -145,9 +174,9 @@ abstract class Bucket<T> implements Transform<T, Integer> {
} else if (predicate.isLiteralPredicate() && predicate.op() == Expression.Operation.NOT_EQ) {
// TODO: need to translate not(eq(...)) into notEq in expressions
return Expressions.predicate(
- predicate.op(), name, apply(predicate.asLiteralPredicate().literal().value()));
+ predicate.op(), name, function.apply(predicate.asLiteralPredicate().literal().value()));
} else if (predicate.isSetPredicate() && predicate.op() == Expression.Operation.NOT_IN) {
- return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), this);
+ return ProjectionUtil.transformSet(name, predicate.asSetPredicate(), function);
}
// no strict projection for comparison or equality
@@ -159,111 +188,61 @@ abstract class Bucket<T> implements Transform<T, Integer> {
return Types.IntegerType.get();
}
- private static class BucketInteger extends Bucket<Integer> {
+ private static class BucketInteger extends Bucket<Integer>
+ implements SerializableFunction<Integer, Integer> {
+
private BucketInteger(int numBuckets) {
super(numBuckets);
}
@Override
- public int hash(Integer value) {
+ protected int hash(Integer value) {
return BucketUtil.hash(value);
}
-
- @Override
- public boolean canTransform(Type type) {
- return type.typeId() == TypeID.INTEGER || type.typeId() == TypeID.DATE;
- }
}
- private static class BucketLong extends Bucket<Long> {
- private BucketLong(int numBuckets) {
- super(numBuckets);
- }
-
- @Override
- public int hash(Long value) {
- return BucketUtil.hash(value);
- }
-
- @Override
- public boolean canTransform(Type type) {
- return type.typeId() == TypeID.LONG
- || type.typeId() == TypeID.TIME
- || type.typeId() == TypeID.TIMESTAMP;
- }
- }
+ private static class BucketLong extends Bucket<Long>
+ implements SerializableFunction<Long, Integer> {
- // bucketing by Double is not allowed by the spec, but this has the float hash implementation
- static class BucketFloat extends Bucket<Float> {
- // used by tests because the factory method will not instantiate a bucket function for floats
- BucketFloat(int numBuckets) {
+ private BucketLong(int numBuckets) {
super(numBuckets);
}
@Override
- public int hash(Float value) {
+ protected int hash(Long value) {
return BucketUtil.hash(value);
}
-
- @Override
- public boolean canTransform(Type type) {
- return type.typeId() == TypeID.FLOAT;
- }
}
- // bucketing by Double is not allowed by the spec, but this has the double hash implementation
- static class BucketDouble extends Bucket<Double> {
- // used by tests because the factory method will not instantiate a bucket function for doubles
- BucketDouble(int numBuckets) {
- super(numBuckets);
- }
-
- @Override
- public int hash(Double value) {
- return BucketUtil.hash(value);
- }
+ private static class BucketString extends Bucket<CharSequence>
+ implements SerializableFunction<CharSequence, Integer> {
- @Override
- public boolean canTransform(Type type) {
- return type.typeId() == TypeID.DOUBLE;
- }
- }
-
- private static class BucketString extends Bucket<CharSequence> {
private BucketString(int numBuckets) {
super(numBuckets);
}
@Override
- public int hash(CharSequence value) {
+ protected int hash(CharSequence value) {
return BucketUtil.hash(value);
}
-
- @Override
- public boolean canTransform(Type type) {
- return type.typeId() == TypeID.STRING;
- }
}
- private static class BucketByteBuffer extends Bucket<ByteBuffer> {
- private static final Set<TypeID> SUPPORTED_TYPES = Sets.newHashSet(TypeID.BINARY, TypeID.FIXED);
+ private static class BucketByteBuffer extends Bucket<ByteBuffer>
+ implements SerializableFunction<ByteBuffer, Integer> {
private BucketByteBuffer(int numBuckets) {
super(numBuckets);
}
@Override
- public int hash(ByteBuffer value) {
+ protected int hash(ByteBuffer value) {
return BucketUtil.hash(value);
}
-
- @Override
- public boolean canTransform(Type type) {
- return SUPPORTED_TYPES.contains(type.typeId());
- }
}
- private static class BucketUUID extends Bucket<UUID> {
+ private static class BucketUUID extends Bucket<UUID>
+ implements SerializableFunction<UUID, Integer> {
+
private BucketUUID(int numBuckets) {
super(numBuckets);
}
@@ -272,26 +251,18 @@ abstract class Bucket<T> implements Transform<T, Integer> {
public int hash(UUID value) {
return BucketUtil.hash(value);
}
-
- @Override
- public boolean canTransform(Type type) {
- return type.typeId() == TypeID.UUID;
- }
}
- private static class BucketDecimal extends Bucket<BigDecimal> {
+ private static class BucketDecimal extends Bucket<BigDecimal>
+ implements SerializableFunction<BigDecimal, Integer> {
+
private BucketDecimal(int numBuckets) {
super(numBuckets);
}
@Override
- public int hash(BigDecimal value) {
+ protected int hash(BigDecimal value) {
return BucketUtil.hash(value);
}
-
- @Override
- public boolean canTransform(Type type) {
- return type.typeId() == TypeID.DECIMAL;
- }
}
}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/Dates.java b/api/src/main/java/org/apache/iceberg/transforms/Dates.java
index b313a2f154..0543a0c891 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/Dates.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/Dates.java
@@ -27,44 +27,66 @@ import org.apache.iceberg.expressions.BoundTransform;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.UnboundPredicate;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.SerializableFunction;
enum Dates implements Transform<Integer, Integer> {
YEAR(ChronoUnit.YEARS, "year"),
MONTH(ChronoUnit.MONTHS, "month"),
DAY(ChronoUnit.DAYS, "day");
+ static class Apply implements SerializableFunction<Integer, Integer> {
+ private final ChronoUnit granularity;
+
+ Apply(ChronoUnit granularity) {
+ this.granularity = granularity;
+ }
+
+ @Override
+ public Integer apply(Integer days) {
+ if (days == null) {
+ return null;
+ }
+
+ if (granularity == ChronoUnit.DAYS) {
+ return days;
+ }
+
+ if (days >= 0) {
+ LocalDate date = EPOCH.plusDays(days);
+ return (int) granularity.between(EPOCH, date);
+ } else {
+ // add 1 day to the value to account for the case where there is exactly 1 unit between the
+ // date and epoch because the result will always be decremented.
+ LocalDate date = EPOCH.plusDays(days + 1);
+ return (int) granularity.between(EPOCH, date) - 1;
+ }
+ }
+ }
+
private static final LocalDate EPOCH =
Instant.ofEpochSecond(0).atOffset(ZoneOffset.UTC).toLocalDate();
private final ChronoUnit granularity;
private final String name;
+ private final Apply apply;
Dates(ChronoUnit granularity, String name) {
this.granularity = granularity;
this.name = name;
+ this.apply = new Apply(granularity);
}
@Override
public Integer apply(Integer days) {
- if (days == null) {
- return null;
- }
-
- if (granularity == ChronoUnit.DAYS) {
- return days;
- }
+ return apply.apply(days);
+ }
- if (days >= 0) {
- LocalDate date = EPOCH.plusDays(days);
- return (int) granularity.between(EPOCH, date);
- } else {
- // add 1 day to the value to account for the case where there is exactly 1 unit between the
- // date and epoch
- // because the result will always be decremented.
- LocalDate date = EPOCH.plusDays(days + 1);
- return (int) granularity.between(EPOCH, date) - 1;
- }
+ @Override
+ public SerializableFunction<Integer, Integer> bind(Type type) {
+ Preconditions.checkArgument(canTransform(type), "Cannot bind to unsupported type: %s", type);
+ return apply;
}
@Override
@@ -113,7 +135,7 @@ enum Dates implements Transform<Integer, Integer> {
} else if (pred.isLiteralPredicate()) {
UnboundPredicate<Integer> projected =
- ProjectionUtil.truncateInteger(fieldName, pred.asLiteralPredicate(), this);
+ ProjectionUtil.truncateInteger(fieldName, pred.asLiteralPredicate(), apply);
if (this != DAY) {
return ProjectionUtil.fixInclusiveTimeProjection(projected);
}
@@ -122,7 +144,7 @@ enum Dates implements Transform<Integer, Integer> {
} else if (pred.isSetPredicate() && pred.op() == Expression.Operation.IN) {
UnboundPredicate<Integer> projected =
- ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this);
+ ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), apply);
if (this != DAY) {
return ProjectionUtil.fixInclusiveTimeProjection(projected);
}
@@ -144,7 +166,7 @@ enum Dates implements Transform<Integer, Integer> {
} else if (pred.isLiteralPredicate()) {
UnboundPredicate<Integer> projected =
- ProjectionUtil.truncateIntegerStrict(fieldName, pred.asLiteralPredicate(), this);
+ ProjectionUtil.truncateIntegerStrict(fieldName, pred.asLiteralPredicate(), apply);
if (this != DAY) {
return ProjectionUtil.fixStrictTimeProjection(projected);
}
@@ -153,7 +175,7 @@ enum Dates implements Transform<Integer, Integer> {
} else if (pred.isSetPredicate() && pred.op() == Expression.Operation.NOT_IN) {
UnboundPredicate<Integer> projected =
- ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this);
+ ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), apply);
if (this != DAY) {
return ProjectionUtil.fixStrictTimeProjection(projected);
}
@@ -165,7 +187,7 @@ enum Dates implements Transform<Integer, Integer> {
}
@Override
- public String toHumanString(Integer value) {
+ public String toHumanString(Type outputType, Integer value) {
if (value == null) {
return "null";
}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/Days.java b/api/src/main/java/org/apache/iceberg/transforms/Days.java
new file mode 100644
index 0000000000..f69d5d6110
--- /dev/null
+++ b/api/src/main/java/org/apache/iceberg/transforms/Days.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.transforms;
+
+import java.io.ObjectStreamException;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+
+public class Days<T> extends TimeTransform<T> {
+ private static final Days<?> INSTANCE = new Days<>();
+
+ @SuppressWarnings("unchecked")
+ static <T> Days<T> get() {
+ return (Days<T>) INSTANCE;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ protected Transform<T, Integer> toEnum(Type type) {
+ switch (type.typeId()) {
+ case DATE:
+ return (Transform<T, Integer>) Dates.DAY;
+ case TIMESTAMP:
+ return (Transform<T, Integer>) Timestamps.DAY;
+ default:
+ throw new IllegalArgumentException("Unsupported type: " + type);
+ }
+ }
+
+ @Override
+ public Type getResultType(Type sourceType) {
+ return Types.DateType.get();
+ }
+
+ @Override
+ public boolean satisfiesOrderOf(Transform<?, ?> other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (other instanceof Timestamps) {
+ return Timestamps.DAY.satisfiesOrderOf(other);
+ } else if (other instanceof Dates) {
+ return Dates.DAY.satisfiesOrderOf(other);
+ } else if (other instanceof Days || other instanceof Months || other instanceof Years) {
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public String toHumanString(Type alwaysDate, Integer value) {
+ return value != null ? TransformUtil.humanDay(value) : "null";
+ }
+
+ @Override
+ public String toString() {
+ return "day";
+ }
+
+ Object writeReplace() throws ObjectStreamException {
+ return SerializationProxies.DaysTransformProxy.get();
+ }
+}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java b/api/src/main/java/org/apache/iceberg/transforms/Hours.java
similarity index 52%
copy from api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java
copy to api/src/main/java/org/apache/iceberg/transforms/Hours.java
index 83f7f76bf0..afc14516f3 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/Hours.java
@@ -19,56 +19,66 @@
package org.apache.iceberg.transforms;
import java.io.ObjectStreamException;
-import org.apache.iceberg.expressions.BoundPredicate;
-import org.apache.iceberg.expressions.UnboundPredicate;
import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
-class VoidTransform<S> implements Transform<S, Void> {
- private static final VoidTransform<Object> INSTANCE = new VoidTransform<>();
+public class Hours<T> extends TimeTransform<T> {
+ private static final Hours<?> INSTANCE = new Hours<>();
@SuppressWarnings("unchecked")
- static <T> VoidTransform<T> get() {
- return (VoidTransform<T>) INSTANCE;
+ static <T> Hours<T> get() {
+ return (Hours<T>) INSTANCE;
}
- private VoidTransform() {}
-
@Override
- public Void apply(Object value) {
- return null;
+ @SuppressWarnings("unchecked")
+ protected Transform<T, Integer> toEnum(Type type) {
+ if (type.typeId() == Type.TypeID.TIMESTAMP) {
+ return (Transform<T, Integer>) Timestamps.HOUR;
+ }
+
+ throw new IllegalArgumentException("Unsupported type: " + type);
}
@Override
public boolean canTransform(Type type) {
- return true;
+ return type.typeId() == Type.TypeID.TIMESTAMP;
}
@Override
public Type getResultType(Type sourceType) {
- return sourceType;
+ return Types.IntegerType.get();
}
@Override
- public UnboundPredicate<Void> projectStrict(String name, BoundPredicate<S> predicate) {
- return null;
- }
+ public boolean satisfiesOrderOf(Transform<?, ?> other) {
+ if (this == other) {
+ return true;
+ }
- @Override
- public UnboundPredicate<Void> project(String name, BoundPredicate<S> predicate) {
- return null;
+ if (other instanceof Timestamps) {
+ return other == Timestamps.HOUR;
+ } else if (other instanceof Hours
+ || other instanceof Days
+ || other instanceof Months
+ || other instanceof Years) {
+ return true;
+ }
+
+ return false;
}
@Override
- public String toHumanString(Void value) {
- return "null";
+ public String toHumanString(Type alwaysInt, Integer value) {
+ return value != null ? TransformUtil.humanHour(value) : "null";
}
@Override
public String toString() {
- return "void";
+ return "hour";
}
Object writeReplace() throws ObjectStreamException {
- return SerializationProxies.VoidTransformProxy.get();
+ return SerializationProxies.HoursTransformProxy.get();
}
}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/Identity.java b/api/src/main/java/org/apache/iceberg/transforms/Identity.java
index 78586543a2..365abdf0b9 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/Identity.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/Identity.java
@@ -18,31 +18,49 @@
*/
package org.apache.iceberg.transforms;
-import java.nio.ByteBuffer;
+import java.io.ObjectStreamException;
import org.apache.iceberg.expressions.BoundPredicate;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.UnboundPredicate;
-import org.apache.iceberg.relocated.com.google.common.base.Objects;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.types.Type;
-import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.SerializableFunction;
class Identity<T> implements Transform<T, T> {
+ private static final Identity<?> INSTANCE = new Identity<>();
+
@SuppressWarnings("unchecked")
- public static <I> Identity<I> get(Type type) {
- return new Identity<>(type);
+ public static <I> Identity<I> get() {
+ return (Identity<I>) INSTANCE;
}
- private final Type type;
+ private static class Apply<T> implements SerializableFunction<T, T> {
+ private static final Apply<?> APPLY_INSTANCE = new Apply<>();
+
+ @SuppressWarnings("unchecked")
+ private static <T> Apply<T> get() {
+ return (Apply<T>) APPLY_INSTANCE;
+ }
- private Identity(Type type) {
- this.type = type;
+ @Override
+ public T apply(T t) {
+ return t;
+ }
}
+ private Identity() {}
+
@Override
public T apply(T value) {
return value;
}
+ @Override
+ public SerializableFunction<T, T> bind(Type type) {
+ Preconditions.checkArgument(canTransform(type), "Cannot bind to unsupported type: %s", type);
+ return Apply.get();
+ }
+
@Override
public boolean canTransform(Type maybePrimitive) {
return maybePrimitive.isPrimitiveType();
@@ -87,56 +105,12 @@ class Identity<T> implements Transform<T, T> {
return true;
}
- @Override
- public String toHumanString(T value) {
- if (value == null) {
- return "null";
- }
-
- switch (type.typeId()) {
- case DATE:
- return TransformUtil.humanDay((Integer) value);
- case TIME:
- return TransformUtil.humanTime((Long) value);
- case TIMESTAMP:
- if (((Types.TimestampType) type).shouldAdjustToUTC()) {
- return TransformUtil.humanTimestampWithZone((Long) value);
- } else {
- return TransformUtil.humanTimestampWithoutZone((Long) value);
- }
- case FIXED:
- case BINARY:
- if (value instanceof ByteBuffer) {
- return TransformUtil.base64encode(((ByteBuffer) value).duplicate());
- } else if (value instanceof byte[]) {
- return TransformUtil.base64encode(ByteBuffer.wrap((byte[]) value));
- } else {
- throw new UnsupportedOperationException("Unsupported binary type: " + value.getClass());
- }
- default:
- return value.toString();
- }
- }
-
@Override
public String toString() {
return "identity";
}
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- } else if (!(o instanceof Identity)) {
- return false;
- }
-
- Identity<?> that = (Identity<?>) o;
- return type.equals(that.type);
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(type);
+ Object writeReplace() throws ObjectStreamException {
+ return SerializationProxies.IdentityTransformProxy.get();
}
}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/Months.java b/api/src/main/java/org/apache/iceberg/transforms/Months.java
new file mode 100644
index 0000000000..8fa4d42385
--- /dev/null
+++ b/api/src/main/java/org/apache/iceberg/transforms/Months.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.transforms;
+
+import java.io.ObjectStreamException;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+
+public class Months<T> extends TimeTransform<T> {
+ private static final Months<?> INSTANCE = new Months<>();
+
+ @SuppressWarnings("unchecked")
+ static <T> Months<T> get() {
+ return (Months<T>) INSTANCE;
+ }
+
+ @Override
+ @SuppressWarnings("unchecked")
+ protected Transform<T, Integer> toEnum(Type type) {
+ switch (type.typeId()) {
+ case DATE:
+ return (Transform<T, Integer>) Dates.MONTH;
+ case TIMESTAMP:
+ return (Transform<T, Integer>) Timestamps.MONTH;
+ default:
+ throw new IllegalArgumentException("Unsupported type: " + type);
+ }
+ }
+
+ @Override
+ public Type getResultType(Type sourceType) {
+ return Types.IntegerType.get();
+ }
+
+ @Override
+ public boolean satisfiesOrderOf(Transform<?, ?> other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (other instanceof Timestamps) {
+ return Timestamps.MONTH.satisfiesOrderOf(other);
+ } else if (other instanceof Dates) {
+ return Dates.MONTH.satisfiesOrderOf(other);
+ } else if (other instanceof Months || other instanceof Years) {
+ return true;
+ }
+
+ return false;
+ }
+
+ @Override
+ public String toHumanString(Type alwaysInt, Integer value) {
+ return value != null ? TransformUtil.humanMonth(value) : "null";
+ }
+
+ @Override
+ public String toString() {
+ return "month";
+ }
+
+ Object writeReplace() throws ObjectStreamException {
+ return SerializationProxies.MonthsTransformProxy.get();
+ }
+}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/PartitionSpecVisitor.java b/api/src/main/java/org/apache/iceberg/transforms/PartitionSpecVisitor.java
index eee174bc2d..e4796478bf 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/PartitionSpecVisitor.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/PartitionSpecVisitor.java
@@ -121,13 +121,17 @@ public interface PartitionSpecVisitor<T> {
} else if (transform instanceof Truncate) {
int width = ((Truncate<?>) transform).width();
return visitor.truncate(field.fieldId(), sourceName, field.sourceId(), width);
- } else if (transform == Dates.YEAR || transform == Timestamps.YEAR) {
+ } else if (transform == Dates.YEAR
+ || transform == Timestamps.YEAR
+ || transform instanceof Years) {
return visitor.year(field.fieldId(), sourceName, field.sourceId());
- } else if (transform == Dates.MONTH || transform == Timestamps.MONTH) {
+ } else if (transform == Dates.MONTH
+ || transform == Timestamps.MONTH
+ || transform instanceof Months) {
return visitor.month(field.fieldId(), sourceName, field.sourceId());
- } else if (transform == Dates.DAY || transform == Timestamps.DAY) {
+ } else if (transform == Dates.DAY || transform == Timestamps.DAY || transform instanceof Days) {
return visitor.day(field.fieldId(), sourceName, field.sourceId());
- } else if (transform == Timestamps.HOUR) {
+ } else if (transform == Timestamps.HOUR || transform instanceof Hours) {
return visitor.hour(field.fieldId(), sourceName, field.sourceId());
} else if (transform instanceof VoidTransform) {
return visitor.alwaysNull(field.fieldId(), sourceName, field.sourceId());
diff --git a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java
index a359c9b884..732336d510 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/ProjectionUtil.java
@@ -23,6 +23,7 @@ import static org.apache.iceberg.expressions.Expressions.predicate;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.util.Set;
+import java.util.function.Function;
import org.apache.iceberg.expressions.BoundLiteralPredicate;
import org.apache.iceberg.expressions.BoundPredicate;
import org.apache.iceberg.expressions.BoundSetPredicate;
@@ -39,7 +40,7 @@ class ProjectionUtil {
private ProjectionUtil() {}
static <T> UnboundPredicate<T> truncateInteger(
- String name, BoundLiteralPredicate<Integer> pred, Transform<Integer, T> transform) {
+ String name, BoundLiteralPredicate<Integer> pred, Function<Integer, T> transform) {
int boundary = pred.literal().value();
switch (pred.op()) {
case LT:
@@ -60,7 +61,7 @@ class ProjectionUtil {
}
static <T> UnboundPredicate<T> truncateIntegerStrict(
- String name, BoundLiteralPredicate<Integer> pred, Transform<Integer, T> transform) {
+ String name, BoundLiteralPredicate<Integer> pred, Function<Integer, T> transform) {
int boundary = pred.literal().value();
switch (pred.op()) {
case LT:
@@ -83,7 +84,7 @@ class ProjectionUtil {
}
static <T> UnboundPredicate<T> truncateLongStrict(
- String name, BoundLiteralPredicate<Long> pred, Transform<Long, T> transform) {
+ String name, BoundLiteralPredicate<Long> pred, Function<Long, T> transform) {
long boundary = pred.literal().value();
switch (pred.op()) {
case LT:
@@ -106,7 +107,7 @@ class ProjectionUtil {
}
static <T> UnboundPredicate<T> truncateLong(
- String name, BoundLiteralPredicate<Long> pred, Transform<Long, T> transform) {
+ String name, BoundLiteralPredicate<Long> pred, Function<Long, T> transform) {
long boundary = pred.literal().value();
switch (pred.op()) {
case LT:
@@ -127,7 +128,7 @@ class ProjectionUtil {
}
static <T> UnboundPredicate<T> truncateDecimal(
- String name, BoundLiteralPredicate<BigDecimal> pred, Transform<BigDecimal, T> transform) {
+ String name, BoundLiteralPredicate<BigDecimal> pred, Function<BigDecimal, T> transform) {
BigDecimal boundary = pred.literal().value();
switch (pred.op()) {
case LT:
@@ -152,7 +153,7 @@ class ProjectionUtil {
}
static <T> UnboundPredicate<T> truncateDecimalStrict(
- String name, BoundLiteralPredicate<BigDecimal> pred, Transform<BigDecimal, T> transform) {
+ String name, BoundLiteralPredicate<BigDecimal> pred, Function<BigDecimal, T> transform) {
BigDecimal boundary = pred.literal().value();
BigDecimal minusOne =
@@ -182,7 +183,7 @@ class ProjectionUtil {
}
static <S, T> UnboundPredicate<T> truncateArray(
- String name, BoundLiteralPredicate<S> pred, Transform<S, T> transform) {
+ String name, BoundLiteralPredicate<S> pred, Function<S, T> transform) {
S boundary = pred.literal().value();
switch (pred.op()) {
case LT:
@@ -203,7 +204,7 @@ class ProjectionUtil {
}
static <S, T> UnboundPredicate<T> truncateArrayStrict(
- String name, BoundLiteralPredicate<S> pred, Transform<S, T> transform) {
+ String name, BoundLiteralPredicate<S> pred, Function<S, T> transform) {
S boundary = pred.literal().value();
switch (pred.op()) {
case LT:
@@ -251,7 +252,7 @@ class ProjectionUtil {
}
static <S, T> UnboundPredicate<T> transformSet(
- String fieldName, BoundSetPredicate<S> predicate, Transform<S, T> transform) {
+ String fieldName, BoundSetPredicate<S> predicate, Function<S, T> transform) {
return predicate(
predicate.op(),
fieldName,
diff --git a/api/src/main/java/org/apache/iceberg/transforms/SerializationProxies.java b/api/src/main/java/org/apache/iceberg/transforms/SerializationProxies.java
index 5ae7235f60..90f59622a7 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/SerializationProxies.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/SerializationProxies.java
@@ -43,4 +43,79 @@ class SerializationProxies {
return VoidTransform.get();
}
}
+
+ static class IdentityTransformProxy implements Serializable {
+ private static final IdentityTransformProxy INSTANCE = new IdentityTransformProxy();
+
+ static IdentityTransformProxy get() {
+ return INSTANCE;
+ }
+
+ /** Constructor for Java serialization. */
+ IdentityTransformProxy() {}
+
+ Object readResolve() throws ObjectStreamException {
+ return Identity.get();
+ }
+ }
+
+ static class YearsTransformProxy implements Serializable {
+ private static final YearsTransformProxy INSTANCE = new YearsTransformProxy();
+
+ static YearsTransformProxy get() {
+ return INSTANCE;
+ }
+
+ /** Constructor for Java serialization. */
+ YearsTransformProxy() {}
+
+ Object readResolve() throws ObjectStreamException {
+ return Years.get();
+ }
+ }
+
+ static class MonthsTransformProxy implements Serializable {
+ private static final MonthsTransformProxy INSTANCE = new MonthsTransformProxy();
+
+ static MonthsTransformProxy get() {
+ return INSTANCE;
+ }
+
+ /** Constructor for Java serialization. */
+ MonthsTransformProxy() {}
+
+ Object readResolve() throws ObjectStreamException {
+ return Months.get();
+ }
+ }
+
+ static class DaysTransformProxy implements Serializable {
+ private static final DaysTransformProxy INSTANCE = new DaysTransformProxy();
+
+ static DaysTransformProxy get() {
+ return INSTANCE;
+ }
+
+ /** Constructor for Java serialization. */
+ DaysTransformProxy() {}
+
+ Object readResolve() throws ObjectStreamException {
+ return Days.get();
+ }
+ }
+
+ static class HoursTransformProxy implements Serializable {
+ private static final HoursTransformProxy INSTANCE = new HoursTransformProxy();
+
+ static HoursTransformProxy get() {
+ return INSTANCE;
+ }
+
+ /** Constructor for Java serialization. */
+ HoursTransformProxy() {}
+
+ Object readResolve() throws ObjectStreamException {
+ return Hours.get();
+ }
+ }
}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/SortOrderVisitor.java b/api/src/main/java/org/apache/iceberg/transforms/SortOrderVisitor.java
index ed3327571f..680e095270 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/SortOrderVisitor.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/SortOrderVisitor.java
@@ -84,16 +84,22 @@ public interface SortOrderVisitor<T> {
results.add(
visitor.truncate(
sourceName, field.sourceId(), width, field.direction(), field.nullOrder()));
- } else if (transform == Dates.YEAR || transform == Timestamps.YEAR) {
+ } else if (transform == Dates.YEAR
+ || transform == Timestamps.YEAR
+ || transform instanceof Years) {
results.add(
visitor.year(sourceName, field.sourceId(), field.direction(), field.nullOrder()));
- } else if (transform == Dates.MONTH || transform == Timestamps.MONTH) {
+ } else if (transform == Dates.MONTH
+ || transform == Timestamps.MONTH
+ || transform instanceof Months) {
results.add(
visitor.month(sourceName, field.sourceId(), field.direction(), field.nullOrder()));
- } else if (transform == Dates.DAY || transform == Timestamps.DAY) {
+ } else if (transform == Dates.DAY
+ || transform == Timestamps.DAY
+ || transform instanceof Days) {
results.add(
visitor.day(sourceName, field.sourceId(), field.direction(), field.nullOrder()));
- } else if (transform == Timestamps.HOUR) {
+ } else if (transform == Timestamps.HOUR || transform instanceof Hours) {
results.add(
visitor.hour(sourceName, field.sourceId(), field.direction(), field.nullOrder()));
} else if (transform instanceof UnknownTransform) {
diff --git a/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java b/api/src/main/java/org/apache/iceberg/transforms/TimeTransform.java
similarity index 51%
copy from api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java
copy to api/src/main/java/org/apache/iceberg/transforms/TimeTransform.java
index 83f7f76bf0..01ea8130aa 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/TimeTransform.java
@@ -18,57 +18,50 @@
*/
package org.apache.iceberg.transforms;
-import java.io.ObjectStreamException;
import org.apache.iceberg.expressions.BoundPredicate;
+import org.apache.iceberg.expressions.BoundTransform;
import org.apache.iceberg.expressions.UnboundPredicate;
import org.apache.iceberg.types.Type;
+import org.apache.iceberg.util.SerializableFunction;
-class VoidTransform<S> implements Transform<S, Void> {
- private static final VoidTransform<Object> INSTANCE = new VoidTransform<>();
-
- @SuppressWarnings("unchecked")
- static <T> VoidTransform<T> get() {
- return (VoidTransform<T>) INSTANCE;
- }
-
- private VoidTransform() {}
+abstract class TimeTransform<S> implements Transform<S, Integer> {
+ protected abstract Transform<S, Integer> toEnum(Type type);
@Override
- public Void apply(Object value) {
- return null;
+ public SerializableFunction<S, Integer> bind(Type type) {
+ return toEnum(type).bind(type);
}
@Override
- public boolean canTransform(Type type) {
+ public boolean preservesOrder() {
return true;
}
@Override
- public Type getResultType(Type sourceType) {
- return sourceType;
+ public boolean canTransform(Type type) {
+ return type.typeId() == Type.TypeID.DATE || type.typeId() == Type.TypeID.TIMESTAMP;
}
@Override
- public UnboundPredicate<Void> projectStrict(String name, BoundPredicate<S> predicate) {
- return null;
- }
+ public UnboundPredicate<Integer> project(String name, BoundPredicate<S> predicate) {
+ if (predicate.term() instanceof BoundTransform) {
+ return ProjectionUtil.projectTransformPredicate(this, name, predicate);
+ }
- @Override
- public UnboundPredicate<Void> project(String name, BoundPredicate<S> predicate) {
- return null;
+ return toEnum(predicate.term().type()).project(name, predicate);
}
@Override
- public String toHumanString(Void value) {
- return "null";
- }
+ public UnboundPredicate<Integer> projectStrict(String name, BoundPredicate<S> predicate) {
+ if (predicate.term() instanceof BoundTransform) {
+ return ProjectionUtil.projectTransformPredicate(this, name, predicate);
+ }
- @Override
- public String toString() {
- return "void";
+ return toEnum(predicate.term().type()).projectStrict(name, predicate);
}
- Object writeReplace() throws ObjectStreamException {
- return SerializationProxies.VoidTransformProxy.get();
+ @Override
+ public String dedupName() {
+ return "time";
}
}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java
index 4760337072..e63ee39ac1 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/Timestamps.java
@@ -27,8 +27,10 @@ import org.apache.iceberg.expressions.BoundTransform;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.UnboundPredicate;
+import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.SerializableFunction;
enum Timestamps implements Transform<Long, Integer> {
YEAR(ChronoUnit.YEARS, "year"),
@@ -36,39 +38,59 @@ enum Timestamps implements Transform<Long, Integer> {
DAY(ChronoUnit.DAYS, "day"),
HOUR(ChronoUnit.HOURS, "hour");
+ static class Apply implements SerializableFunction<Long, Integer> {
+ private final ChronoUnit granularity;
+
+ Apply(ChronoUnit granularity) {
+ this.granularity = granularity;
+ }
+
+ @Override
+ public Integer apply(Long timestampMicros) {
+ if (timestampMicros == null) {
+ return null;
+ }
+
+ if (timestampMicros >= 0) {
+ OffsetDateTime timestamp =
+ Instant.ofEpochSecond(
+ Math.floorDiv(timestampMicros, 1_000_000),
+ Math.floorMod(timestampMicros, 1_000_000) * 1000)
+ .atOffset(ZoneOffset.UTC);
+ return (int) granularity.between(EPOCH, timestamp);
+ } else {
+ // add 1 micro to the value to account for the case where there is exactly 1 unit between
+ // the timestamp and epoch because the result will always be decremented.
+ OffsetDateTime timestamp =
+ Instant.ofEpochSecond(
+ Math.floorDiv(timestampMicros, 1_000_000),
+ Math.floorMod(timestampMicros + 1, 1_000_000) * 1000)
+ .atOffset(ZoneOffset.UTC);
+ return (int) granularity.between(EPOCH, timestamp) - 1;
+ }
+ }
+ }
+
private static final OffsetDateTime EPOCH = Instant.ofEpochSecond(0).atOffset(ZoneOffset.UTC);
private final ChronoUnit granularity;
private final String name;
+ private final SerializableFunction<Long, Integer> apply;
Timestamps(ChronoUnit granularity, String name) {
this.granularity = granularity;
this.name = name;
+ this.apply = new Apply(granularity);
}
@Override
public Integer apply(Long timestampMicros) {
- if (timestampMicros == null) {
- return null;
- }
+ return apply.apply(timestampMicros);
+ }
- if (timestampMicros >= 0) {
- OffsetDateTime timestamp =
- Instant.ofEpochSecond(
- Math.floorDiv(timestampMicros, 1_000_000),
- Math.floorMod(timestampMicros, 1_000_000) * 1000)
- .atOffset(ZoneOffset.UTC);
- return (int) granularity.between(EPOCH, timestamp);
- } else {
- // add 1 micro to the value to account for the case where there is exactly 1 unit between the
- // timestamp and epoch
- // because the result will always be decremented.
- OffsetDateTime timestamp =
- Instant.ofEpochSecond(
- Math.floorDiv(timestampMicros, 1_000_000),
- Math.floorMod(timestampMicros + 1, 1_000_000) * 1000)
- .atOffset(ZoneOffset.UTC);
- return (int) granularity.between(EPOCH, timestamp) - 1;
- }
+ @Override
+ public SerializableFunction<Long, Integer> bind(Type type) {
+ Preconditions.checkArgument(canTransform(type), "Cannot bind to unsupported type: %s", type);
+ return apply;
}
@Override
@@ -117,12 +139,12 @@ enum Timestamps implements Transform<Long, Integer> {
} else if (pred.isLiteralPredicate()) {
UnboundPredicate<Integer> projected =
- ProjectionUtil.truncateLong(fieldName, pred.asLiteralPredicate(), this);
+ ProjectionUtil.truncateLong(fieldName, pred.asLiteralPredicate(), apply);
return ProjectionUtil.fixInclusiveTimeProjection(projected);
} else if (pred.isSetPredicate() && pred.op() == Expression.Operation.IN) {
UnboundPredicate<Integer> projected =
- ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this);
+ ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), apply);
return ProjectionUtil.fixInclusiveTimeProjection(projected);
}
@@ -140,12 +162,12 @@ enum Timestamps implements Transform<Long, Integer> {
} else if (pred.isLiteralPredicate()) {
UnboundPredicate<Integer> projected =
- ProjectionUtil.truncateLongStrict(fieldName, pred.asLiteralPredicate(), this);
+ ProjectionUtil.truncateLongStrict(fieldName, pred.asLiteralPredicate(), apply);
return ProjectionUtil.fixStrictTimeProjection(projected);
} else if (pred.isSetPredicate() && pred.op() == Expression.Operation.NOT_IN) {
UnboundPredicate<Integer> projected =
- ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), this);
+ ProjectionUtil.transformSet(fieldName, pred.asSetPredicate(), apply);
return ProjectionUtil.fixStrictTimeProjection(projected);
}
@@ -153,7 +175,7 @@ enum Timestamps implements Transform<Long, Integer> {
}
@Override
- public String toHumanString(Integer value) {
+ public String toHumanString(Type outputType, Integer value) {
if (value == null) {
return "null";
}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/Transform.java b/api/src/main/java/org/apache/iceberg/transforms/Transform.java
index 9e61bf377e..6905eddc65 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/Transform.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/Transform.java
@@ -19,9 +19,13 @@
package org.apache.iceberg.transforms;
import java.io.Serializable;
+import java.nio.ByteBuffer;
+import java.util.function.Function;
import org.apache.iceberg.expressions.BoundPredicate;
import org.apache.iceberg.expressions.UnboundPredicate;
import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.SerializableFunction;
/**
* A transform function used for partitioning.
@@ -38,8 +42,23 @@ public interface Transform<S, T> extends Serializable {
*
* @param value a source value
* @return a transformed partition value
+ * @deprecated use {@link #bind(Type)} instead; will be removed in 2.0.0
*/
- T apply(S value);
+ @Deprecated
+ default T apply(S value) {
+ throw new UnsupportedOperationException(
+ "apply(value) is deprecated, use bind(Type).apply(value)");
+ }
+
+ /**
+ * Returns a function that applies this transform to values of the given {@link Type type}.
+ *
+ * @param type an Iceberg {@link Type}
+ * @return a {@link Function} that applies this transform to values of the given type.
+ */
+ default SerializableFunction<S, T> bind(Type type) {
+ throw new UnsupportedOperationException("bind is not implemented");
+ }
/**
* Checks whether this function can be applied to the given {@link Type}.
@@ -84,7 +103,7 @@ public interface Transform<S, T> extends Serializable {
/**
* Transforms a {@link BoundPredicate predicate} to an inclusive predicate on the partition values
- * produced by {@link #apply(Object)}.
+ * produced by the transform.
*
* <p>This inclusive transform guarantees that if pred(v) is true, then projected(apply(v)) is
* true.
@@ -97,7 +116,7 @@ public interface Transform<S, T> extends Serializable {
/**
* Transforms a {@link BoundPredicate predicate} to a strict predicate on the partition values
- * produced by {@link #apply(Object)}.
+ * produced by the transform.
*
* <p>This strict transform guarantees that if strict(apply(v)) is true, then pred(v) is also
* true.
@@ -124,9 +143,47 @@ public interface Transform<S, T> extends Serializable {
*
* @param value a transformed value
* @return a human-readable String representation of the value
+ * @deprecated use {@link #toHumanString(Type, Object)} instead; will be removed in 2.0.0
*/
+ @Deprecated
default String toHumanString(T value) {
- return String.valueOf(value);
+ if (value instanceof ByteBuffer) {
+ return TransformUtil.base64encode(((ByteBuffer) value).duplicate());
+ } else if (value instanceof byte[]) {
+ return TransformUtil.base64encode(ByteBuffer.wrap((byte[]) value));
+ } else {
+ return String.valueOf(value);
+ }
+ }
+
+ default String toHumanString(Type type, T value) {
+ if (value == null) {
+ return "null";
+ }
+
+ switch (type.typeId()) {
+ case DATE:
+ return TransformUtil.humanDay((Integer) value);
+ case TIME:
+ return TransformUtil.humanTime((Long) value);
+ case TIMESTAMP:
+ if (((Types.TimestampType) type).shouldAdjustToUTC()) {
+ return TransformUtil.humanTimestampWithZone((Long) value);
+ } else {
+ return TransformUtil.humanTimestampWithoutZone((Long) value);
+ }
+ case FIXED:
+ case BINARY:
+ if (value instanceof ByteBuffer) {
+ return TransformUtil.base64encode(((ByteBuffer) value).duplicate());
+ } else if (value instanceof byte[]) {
+ return TransformUtil.base64encode(ByteBuffer.wrap((byte[]) value));
+ } else {
+ throw new UnsupportedOperationException("Unsupported binary type: " + value.getClass());
+ }
+ default:
+ return value.toString();
+ }
}
/**
diff --git a/api/src/main/java/org/apache/iceberg/transforms/Transforms.java b/api/src/main/java/org/apache/iceberg/transforms/Transforms.java
index 35ace7a800..0a0c70b746 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/Transforms.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/Transforms.java
@@ -39,20 +39,49 @@ public class Transforms {
private static final Pattern HAS_WIDTH = Pattern.compile("(\\w+)\\[(\\d+)\\]");
+ public static Transform<?, ?> fromString(String transform) {
+ Matcher widthMatcher = HAS_WIDTH.matcher(transform);
+ if (widthMatcher.matches()) {
+ String name = widthMatcher.group(1);
+ int parsedWidth = Integer.parseInt(widthMatcher.group(2));
+ if (name.equalsIgnoreCase("truncate")) {
+ return Truncate.get(parsedWidth);
+ } else if (name.equals("bucket")) {
+ return Bucket.get(parsedWidth);
+ }
+ }
+
+ if (transform.equalsIgnoreCase("identity")) {
+ return Identity.get();
+ } else if (transform.equalsIgnoreCase("year")) {
+ return Years.get();
+ } else if (transform.equalsIgnoreCase("month")) {
+ return Months.get();
+ } else if (transform.equalsIgnoreCase("day")) {
+ return Days.get();
+ } else if (transform.equalsIgnoreCase("hour")) {
+ return Hours.get();
+ } else if (transform.equalsIgnoreCase("void")) {
+ return VoidTransform.get();
+ }
+
+ return new UnknownTransform<>(transform);
+ }
+
public static Transform<?, ?> fromString(Type type, String transform) {
Matcher widthMatcher = HAS_WIDTH.matcher(transform);
if (widthMatcher.matches()) {
String name = widthMatcher.group(1);
int parsedWidth = Integer.parseInt(widthMatcher.group(2));
if (name.equalsIgnoreCase("truncate")) {
- return Truncate.get(type, parsedWidth);
+ return (Transform<?, ?>) Truncate.get(type, parsedWidth);
} else if (name.equals("bucket")) {
- return Bucket.get(type, parsedWidth);
+ return (Transform<?, ?>) Bucket.get(type, parsedWidth);
}
}
if (transform.equalsIgnoreCase("identity")) {
- return Identity.get(type);
+ return Identity.get();
}
try {
@@ -69,7 +98,7 @@ public class Transforms {
return VoidTransform.get();
}
- return new UnknownTransform<>(type, transform);
+ return new UnknownTransform<>(transform);
}
/**
@@ -78,9 +107,11 @@ public class Transforms {
* @param type the {@link Type source type} for the transform
* @param <T> Java type passed to this transform
* @return an identity transform
+ * @deprecated use {@link #identity()} instead; will be removed in 2.0.0
*/
+ @Deprecated
public static <T> Transform<T, T> identity(Type type) {
- return Identity.get(type);
+ return Identity.get();
}
/**
@@ -89,7 +120,9 @@ public class Transforms {
* @param type the {@link Type source type} for the transform
* @param <T> Java type passed to this transform
* @return a year transform
+ * @deprecated use {@link #year()} instead; will be removed in 2.0.0
*/
+ @Deprecated
@SuppressWarnings("unchecked")
public static <T> Transform<T, Integer> year(Type type) {
switch (type.typeId()) {
@@ -108,7 +141,9 @@ public class Transforms {
* @param type the {@link Type source type} for the transform
* @param <T> Java type passed to this transform
* @return a month transform
+ * @deprecated use {@link #month()} instead; will be removed in 2.0.0
*/
+ @Deprecated
@SuppressWarnings("unchecked")
public static <T> Transform<T, Integer> month(Type type) {
switch (type.typeId()) {
@@ -127,7 +162,9 @@ public class Transforms {
* @param type the {@link Type source type} for the transform
* @param <T> Java type passed to this transform
* @return a day transform
+ * @deprecated use {@link #day()} instead; will be removed in 2.0.0
*/
+ @Deprecated
@SuppressWarnings("unchecked")
public static <T> Transform<T, Integer> day(Type type) {
switch (type.typeId()) {
@@ -146,7 +183,9 @@ public class Transforms {
* @param type the {@link Type source type} for the transform
* @param <T> Java type passed to this transform
* @return a hour transform
+ * @deprecated use {@link #hour()} instead; will be removed in 2.0.0
*/
+ @Deprecated
@SuppressWarnings("unchecked")
public static <T> Transform<T, Integer> hour(Type type) {
Preconditions.checkArgument(
@@ -161,7 +200,9 @@ public class Transforms {
* @param numBuckets the number of buckets for the transform to produce
* @param <T> Java type passed to this transform
* @return a transform that buckets values into numBuckets
+ * @deprecated use {@link #bucket(int)} instead; will be removed in 2.0.0
*/
+ @Deprecated
public static <T> Transform<T, Integer> bucket(Type type, int numBuckets) {
return Bucket.get(type, numBuckets);
}
@@ -173,9 +214,83 @@ public class Transforms {
* @param width the width to truncate data values
* @param <T> Java type passed to this transform
* @return a transform that truncates the given type to width
+ * @deprecated use {@link #truncate(int)} instead; will be removed in 2.0.0
*/
+ @Deprecated
public static <T> Transform<T, T> truncate(Type type, int width) {
- return Truncate.get(type, width);
+ return (Transform<T, T>) Truncate.get(type, width);
+ }
+
+ /**
+ * Returns an identity {@link Transform} that can be used for any type.
+ *
+ * @param <T> Java type passed to this transform
+ * @return an identity transform
+ */
+ public static <T> Transform<T, T> identity() {
+ return Identity.get();
+ }
+
+ /**
+ * Returns a year {@link Transform} for date or timestamp types.
+ *
+ * @param <T> Java type passed to this transform
+ * @return a year transform
+ */
+ public static <T> Transform<T, Integer> year() {
+ return Years.get();
+ }
+
+ /**
+ * Returns a year {@link Transform} for date or timestamp types.
+ *
+ * @param <T> Java type passed to this transform
+ * @return a year transform
+ */
+ public static <T> Transform<T, Integer> month() {
+ return Months.get();
+ }
+
+ /**
+ * Returns a year {@link Transform} for date or timestamp types.
+ *
+ * @param <T> Java type passed to this transform
+ * @return a year transform
+ */
+ public static <T> Transform<T, Integer> day() {
+ return Days.get();
+ }
+
+ /**
+ * Returns a year {@link Transform} for date or timestamp types.
+ *
+ * @param <T> Java type passed to this transform
+ * @return a year transform
+ */
+ public static <T> Transform<T, Integer> hour() {
+ return Hours.get();
+ }
+
+ /**
+ * Returns a bucket {@link Transform} for the given number of buckets.
+ *
+ * @param numBuckets the number of buckets for the transform to produce
+ * @param <T> Java type passed to this transform
+ * @return a transform that buckets values into numBuckets
+ */
+ public static <T> Transform<T, Integer> bucket(int numBuckets) {
+ return Bucket.get(numBuckets);
+ }
+
+ /**
+ * Returns a truncate {@link Transform} for the given width.
+ *
+ * @param width the width to truncate data values
+ * @param <T> Java type passed to this transform
+ * @return a transform that truncates the given type to width
+ */
+ public static <T> Transform<T, T> truncate(int width) {
+ return Truncate.get(width);
}
/**
diff --git a/api/src/main/java/org/apache/iceberg/transforms/Truncate.java b/api/src/main/java/org/apache/iceberg/transforms/Truncate.java
index 22958fe42b..a6c0427d81 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/Truncate.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/Truncate.java
@@ -21,6 +21,7 @@ package org.apache.iceberg.transforms;
import java.math.BigDecimal;
import java.math.BigInteger;
import java.nio.ByteBuffer;
+import java.util.function.Function;
import org.apache.iceberg.expressions.BoundLiteralPredicate;
import org.apache.iceberg.expressions.BoundPredicate;
import org.apache.iceberg.expressions.BoundTransform;
@@ -32,34 +33,84 @@ import org.apache.iceberg.relocated.com.google.common.base.Objects;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.util.BinaryUtil;
+import org.apache.iceberg.util.SerializableFunction;
import org.apache.iceberg.util.TruncateUtil;
import org.apache.iceberg.util.UnicodeUtil;
-abstract class Truncate<T> implements Transform<T, T> {
+class Truncate<T> implements Transform<T, T>, Function<T, T> {
+ static <T> Truncate<T> get(int width) {
+ Preconditions.checkArgument(width > 0, "Invalid truncate width: %s (must be > 0)", width);
+ return new Truncate<>(width);
+ }
+
+ @Deprecated
@SuppressWarnings("unchecked")
- static <T> Truncate<T> get(Type type, int width) {
+ static <T, R extends Truncate<T> & SerializableFunction<T, T>> R get(Type type, int width) {
Preconditions.checkArgument(width > 0, "Invalid truncate width: %s (must be > 0)", width);
switch (type.typeId()) {
case INTEGER:
- return (Truncate<T>) new TruncateInteger(width);
+ return (R) new TruncateInteger(width);
case LONG:
- return (Truncate<T>) new TruncateLong(width);
+ return (R) new TruncateLong(width);
case DECIMAL:
- return (Truncate<T>) new TruncateDecimal(width);
+ return (R) new TruncateDecimal(width);
case STRING:
- return (Truncate<T>) new TruncateString(width);
+ return (R) new TruncateString(width);
case BINARY:
- return (Truncate<T>) new TruncateByteBuffer(width);
+ return (R) new TruncateByteBuffer(width);
default:
throw new UnsupportedOperationException("Cannot truncate type: " + type);
}
}
- public abstract Integer width();
+ @SuppressWarnings("checkstyle:VisibilityModifier")
+ protected final int width;
+
+ Truncate(int width) {
+ this.width = width;
+ }
+
+ public Integer width() {
+ return width;
+ }
+
+ @Override
+ public T apply(T value) {
+ throw new UnsupportedOperationException(
+ "apply(value) is deprecated, use bind(Type).apply(value)");
+ }
+
+ @Override
+ public SerializableFunction<T, T> bind(Type type) {
+ Preconditions.checkArgument(canTransform(type), "Cannot bind to unsupported type: %s", type);
+ return (SerializableFunction<T, T>) get(type, width);
+ }
+
+ @Override
+ public boolean canTransform(Type type) {
+ switch (type.typeId()) {
+ case INTEGER:
+ case LONG:
+ case STRING:
+ case BINARY:
+ case DECIMAL:
+ return true;
+ }
+ return false;
+ }
+
+ @Override
+ public UnboundPredicate<T> project(String name, BoundPredicate<T> predicate) {
+ Truncate<T> bound = (Truncate<T>) get(predicate.term().type(), width);
+ return bound.project(name, predicate);
+ }
@Override
- public abstract T apply(T value);
+ public UnboundPredicate<T> projectStrict(String name, BoundPredicate<T> predicate) {
+ Truncate<T> bound = (Truncate<T>) get(predicate.term().type(), width);
+ return bound.projectStrict(name, predicate);
+ }
@Override
public Type getResultType(Type sourceType) {
@@ -71,16 +122,56 @@ abstract class Truncate<T> implements Transform<T, T> {
return true;
}
- private static class TruncateInteger extends Truncate<Integer> {
- private final int width;
+ @Override
+ public boolean satisfiesOrderOf(Transform<?, ?> other) {
+ if (this == other) {
+ return true;
+ }
+
+ if (!(other instanceof Truncate)) {
+ return false;
+ }
+
+ Truncate<?> otherTrunc = (Truncate<?>) other;
+ return otherTrunc.width <= width;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) {
+ return true;
+ } else if (!(o instanceof Truncate)) {
+ return false;
+ }
+
+ Truncate<?> that = (Truncate<?>) o;
+ return width == that.width;
+ }
+
+ @Override
+ public int hashCode() {
+ return Objects.hashCode(width);
+ }
+
+ @Override
+ public String toString() {
+ return "truncate[" + width + "]";
+ }
+
+ private static class TruncateInteger extends Truncate<Integer>
+ implements SerializableFunction<Integer, Integer> {
private TruncateInteger(int width) {
- this.width = width;
+ super(width);
}
@Override
- public Integer width() {
- return width;
+ public SerializableFunction<Integer, Integer> bind(Type type) {
+ Preconditions.checkArgument(
+ type.typeId() == Type.TypeID.INTEGER,
+ "Cannot bind truncate to a different type: %s",
+ type);
+ return this;
}
@Override
@@ -92,11 +183,6 @@ abstract class Truncate<T> implements Transform<T, T> {
return TruncateUtil.truncateInt(width, value);
}
- @Override
- public boolean canTransform(Type type) {
- return type.typeId() == Type.TypeID.INTEGER;
- }
-
@Override
public UnboundPredicate<Integer> project(String name, BoundPredicate<Integer> pred) {
if (pred.term() instanceof BoundTransform) {
@@ -131,40 +217,20 @@ abstract class Truncate<T> implements Transform<T, T> {
}
return null;
}
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- } else if (!(o instanceof TruncateInteger)) {
- return false;
- }
-
- TruncateInteger that = (TruncateInteger) o;
- return width == that.width;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(width);
- }
-
- @Override
- public String toString() {
- return "truncate[" + width + "]";
- }
}
- private static class TruncateLong extends Truncate<Long> {
- private final int width;
+ private static class TruncateLong extends Truncate<Long>
+ implements SerializableFunction<Long, Long> {
private TruncateLong(int width) {
- this.width = width;
+ super(width);
}
@Override
- public Integer width() {
- return width;
+ public SerializableFunction<Long, Long> bind(Type type) {
+ Preconditions.checkArgument(
+ type.typeId() == Type.TypeID.LONG, "Cannot bind truncate to a different type: %s", type);
+ return this;
}
@Override
@@ -176,11 +242,6 @@ abstract class Truncate<T> implements Transform<T, T> {
return TruncateUtil.truncateLong(width, value);
}
- @Override
- public boolean canTransform(Type type) {
- return type.typeId() == Type.TypeID.LONG;
- }
-
@Override
public UnboundPredicate<Long> project(String name, BoundPredicate<Long> pred) {
if (pred.term() instanceof BoundTransform) {
@@ -212,40 +273,22 @@ abstract class Truncate<T> implements Transform<T, T> {
}
return null;
}
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- } else if (!(o instanceof TruncateLong)) {
- return false;
- }
-
- TruncateLong that = (TruncateLong) o;
- return width == that.width;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(width);
- }
-
- @Override
- public String toString() {
- return "truncate[" + width + "]";
- }
}
- private static class TruncateString extends Truncate<CharSequence> {
- private final int length;
+ private static class TruncateString extends Truncate<CharSequence>
+ implements SerializableFunction<CharSequence, CharSequence> {
private TruncateString(int length) {
- this.length = length;
+ super(length);
}
@Override
- public Integer width() {
- return length;
+ public SerializableFunction<CharSequence, CharSequence> bind(Type type) {
+ Preconditions.checkArgument(
+ type.typeId() == Type.TypeID.STRING,
+ "Cannot bind truncate to a different type: %s",
+ type);
+ return this;
}
@Override
@@ -254,12 +297,7 @@ abstract class Truncate<T> implements Transform<T, T> {
return null;
}
- return UnicodeUtil.truncateString(value, length);
- }
-
- @Override
- public boolean canTransform(Type type) {
- return type.typeId() == Type.TypeID.STRING;
+ return UnicodeUtil.truncateString(value, width);
}
@Override
@@ -351,40 +389,22 @@ abstract class Truncate<T> implements Transform<T, T> {
}
return null;
}
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- } else if (!(o instanceof TruncateString)) {
- return false;
- }
-
- TruncateString that = (TruncateString) o;
- return length == that.length;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(length);
- }
-
- @Override
- public String toString() {
- return "truncate[" + length + "]";
- }
}
- private static class TruncateByteBuffer extends Truncate<ByteBuffer> {
- private final int length;
+ private static class TruncateByteBuffer extends Truncate<ByteBuffer>
+ implements SerializableFunction<ByteBuffer, ByteBuffer> {
private TruncateByteBuffer(int length) {
- this.length = length;
+ super(length);
}
@Override
- public Integer width() {
- return length;
+ public SerializableFunction<ByteBuffer, ByteBuffer> bind(Type type) {
+ Preconditions.checkArgument(
+ type.typeId() == Type.TypeID.BINARY || type.typeId() == Type.TypeID.FIXED,
+ "Cannot bind truncate to a different type: %s",
+ type);
+ return this;
}
@Override
@@ -393,12 +413,7 @@ abstract class Truncate<T> implements Transform<T, T> {
return null;
}
- return BinaryUtil.truncateBinaryUnsafe(value, length);
- }
-
- @Override
- public boolean canTransform(Type type) {
- return type.typeId() == Type.TypeID.BINARY;
+ return BinaryUtil.truncateBinaryUnsafe(value, width);
}
@Override
@@ -433,45 +448,25 @@ abstract class Truncate<T> implements Transform<T, T> {
}
return null;
}
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- } else if (!(o instanceof TruncateByteBuffer)) {
- return false;
- }
-
- TruncateByteBuffer that = (TruncateByteBuffer) o;
- return length == that.length;
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(length);
- }
-
- @Override
- public String toHumanString(ByteBuffer value) {
- return value == null ? "null" : TransformUtil.base64encode(value);
- }
-
- @Override
- public String toString() {
- return "truncate[" + length + "]";
- }
}
- private static class TruncateDecimal extends Truncate<BigDecimal> {
+ private static class TruncateDecimal extends Truncate<BigDecimal>
+ implements SerializableFunction<BigDecimal, BigDecimal> {
+
private final BigInteger unscaledWidth;
private TruncateDecimal(int unscaledWidth) {
+ super(unscaledWidth);
this.unscaledWidth = BigInteger.valueOf(unscaledWidth);
}
@Override
- public Integer width() {
- return unscaledWidth.intValue();
+ public SerializableFunction<BigDecimal, BigDecimal> bind(Type type) {
+ Preconditions.checkArgument(
+ type.typeId() == Type.TypeID.DECIMAL,
+ "Cannot bind truncate to a different type: %s",
+ type);
+ return this;
}
@Override
@@ -483,11 +478,6 @@ abstract class Truncate<T> implements Transform<T, T> {
return TruncateUtil.truncateDecimal(unscaledWidth, value);
}
- @Override
- public boolean canTransform(Type type) {
- return type.typeId() == Type.TypeID.DECIMAL;
- }
-
@Override
public UnboundPredicate<BigDecimal> project(String name, BoundPredicate<BigDecimal> pred) {
if (pred.term() instanceof BoundTransform) {
@@ -520,27 +510,5 @@ abstract class Truncate<T> implements Transform<T, T> {
}
return null;
}
-
- @Override
- public boolean equals(Object o) {
- if (this == o) {
- return true;
- } else if (!(o instanceof TruncateDecimal)) {
- return false;
- }
-
- TruncateDecimal that = (TruncateDecimal) o;
- return unscaledWidth.equals(that.unscaledWidth);
- }
-
- @Override
- public int hashCode() {
- return Objects.hashCode(unscaledWidth);
- }
-
- @Override
- public String toString() {
- return "truncate[" + unscaledWidth + "]";
- }
}
}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/UnknownTransform.java b/api/src/main/java/org/apache/iceberg/transforms/UnknownTransform.java
index 95a30beac2..457bd06304 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/UnknownTransform.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/UnknownTransform.java
@@ -23,14 +23,13 @@ import org.apache.iceberg.expressions.BoundPredicate;
import org.apache.iceberg.expressions.UnboundPredicate;
import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.SerializableFunction;
public class UnknownTransform<S, T> implements Transform<S, T> {
- private final Type sourceType;
private final String transform;
- UnknownTransform(Type sourceType, String transform) {
- this.sourceType = sourceType;
+ UnknownTransform(String transform) {
this.transform = transform;
}
@@ -40,12 +39,16 @@ public class UnknownTransform<S, T> implements Transform<S, T> {
String.format("Cannot apply unsupported transform: %s", transform));
}
+ @Override
+ public SerializableFunction<S, T> bind(Type type) {
+ throw new UnsupportedOperationException(
+ String.format("Cannot bind unsupported transform: %s", transform));
+ }
+
@Override
public boolean canTransform(Type type) {
- // assume the transform function can be applied for this type because unknown transform is only
- // used when parsing
- // a transform in an existing table. a different Iceberg version must have already validated it.
- return this.sourceType.equals(type);
+ // assume the transform function can be applied for any type
+ return true;
}
@Override
@@ -78,11 +81,11 @@ public class UnknownTransform<S, T> implements Transform<S, T> {
}
UnknownTransform<?, ?> that = (UnknownTransform<?, ?>) other;
- return sourceType.equals(that.sourceType) && transform.equals(that.transform);
+ return transform.equals(that.transform);
}
@Override
public int hashCode() {
- return Objects.hash(sourceType, transform);
+ return Objects.hash(transform);
}
}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java b/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java
index 83f7f76bf0..cbab2879ff 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java
@@ -19,9 +19,11 @@
package org.apache.iceberg.transforms;
import java.io.ObjectStreamException;
+import java.io.Serializable;
import org.apache.iceberg.expressions.BoundPredicate;
import org.apache.iceberg.expressions.UnboundPredicate;
import org.apache.iceberg.types.Type;
+import org.apache.iceberg.util.SerializableFunction;
class VoidTransform<S> implements Transform<S, Void> {
private static final VoidTransform<Object> INSTANCE = new VoidTransform<>();
@@ -31,6 +33,20 @@ class VoidTransform<S> implements Transform<S, Void> {
return (VoidTransform<T>) INSTANCE;
}
+ private static class Apply<S> implements SerializableFunction<S, Void>, Serializable {
+ private static final Apply<?> APPLY_INSTANCE = new Apply<>();
+
+ @SuppressWarnings("unchecked")
+ private static <S> Apply<S> get() {
+ return (Apply<S>) APPLY_INSTANCE;
+ }
+
+ @Override
+ public Void apply(S t) {
+ return null;
+ }
+ }
+
private VoidTransform() {}
@Override
@@ -38,6 +54,11 @@ class VoidTransform<S> implements Transform<S, Void> {
return null;
}
+ @Override
+ public SerializableFunction<S, Void> bind(Type type) {
+ return Apply.get();
+ }
+
@Override
public boolean canTransform(Type type) {
return true;
diff --git a/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java b/api/src/main/java/org/apache/iceberg/transforms/Years.java
similarity index 50%
copy from api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java
copy to api/src/main/java/org/apache/iceberg/transforms/Years.java
index 83f7f76bf0..6c1eee5785 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/VoidTransform.java
+++ b/api/src/main/java/org/apache/iceberg/transforms/Years.java
@@ -19,56 +19,63 @@
package org.apache.iceberg.transforms;
import java.io.ObjectStreamException;
-import org.apache.iceberg.expressions.BoundPredicate;
-import org.apache.iceberg.expressions.UnboundPredicate;
import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
-class VoidTransform<S> implements Transform<S, Void> {
- private static final VoidTransform<Object> INSTANCE = new VoidTransform<>();
+class Years<T> extends TimeTransform<T> {
+ private static final Years<?> INSTANCE = new Years<>();
@SuppressWarnings("unchecked")
- static <T> VoidTransform<T> get() {
- return (VoidTransform<T>) INSTANCE;
- }
-
- private VoidTransform() {}
-
- @Override
- public Void apply(Object value) {
- return null;
+ static <T> Years<T> get() {
+ return (Years<T>) INSTANCE;
}
@Override
- public boolean canTransform(Type type) {
- return true;
+ @SuppressWarnings("unchecked")
+ protected Transform<T, Integer> toEnum(Type type) {
+ switch (type.typeId()) {
+ case DATE:
+ return (Transform<T, Integer>) Dates.YEAR;
+ case TIMESTAMP:
+ return (Transform<T, Integer>) Timestamps.YEAR;
+ default:
+ throw new IllegalArgumentException("Unsupported type: " + type);
+ }
}
@Override
public Type getResultType(Type sourceType) {
- return sourceType;
+ return Types.IntegerType.get();
}
@Override
- public UnboundPredicate<Void> projectStrict(String name, BoundPredicate<S> predicate) {
- return null;
- }
+ public boolean satisfiesOrderOf(Transform<?, ?> other) {
+ if (this == other) {
+ return true;
+ }
- @Override
- public UnboundPredicate<Void> project(String name, BoundPredicate<S> predicate) {
- return null;
+ if (other instanceof Timestamps) {
+ return Timestamps.YEAR.satisfiesOrderOf(other);
+ } else if (other instanceof Dates) {
+ return Dates.YEAR.satisfiesOrderOf(other);
+ } else if (other instanceof Years) {
+ return true;
+ }
+
+ return false;
}
@Override
- public String toHumanString(Void value) {
- return "null";
+ public String toHumanString(Type alwaysInt, Integer value) {
+ return value != null ? TransformUtil.humanYear(value) : "null";
}
@Override
public String toString() {
- return "void";
+ return "year";
}
Object writeReplace() throws ObjectStreamException {
- return SerializationProxies.VoidTransformProxy.get();
+ return SerializationProxies.YearsTransformProxy.get();
}
}
diff --git a/api/src/main/java/org/apache/iceberg/transforms/SerializationProxies.java b/api/src/main/java/org/apache/iceberg/util/SerializableFunction.java
similarity index 55%
copy from api/src/main/java/org/apache/iceberg/transforms/SerializationProxies.java
copy to api/src/main/java/org/apache/iceberg/util/SerializableFunction.java
index 5ae7235f60..0c191d905d 100644
--- a/api/src/main/java/org/apache/iceberg/transforms/SerializationProxies.java
+++ b/api/src/main/java/org/apache/iceberg/util/SerializableFunction.java
@@ -16,31 +16,15 @@
* specific language governing permissions and limitations
* under the License.
*/
-package org.apache.iceberg.transforms;
+package org.apache.iceberg.util;
-import java.io.ObjectStreamException;
import java.io.Serializable;
+import java.util.function.Function;
/**
- * Stand-in classes for expression classes in Java Serialization.
+ * A concrete transform function that applies a transform to values of a certain type.
*
- * <p>These are used so that transform classes can be singletons and use identical equality.
+ * @param <S> Java class of source values
+ * @param <T> Java class of transformed values
*/
-class SerializationProxies {
- private SerializationProxies() {}
-
- static class VoidTransformProxy implements Serializable {
- private static final VoidTransformProxy INSTANCE = new VoidTransformProxy();
-
- static VoidTransformProxy get() {
- return INSTANCE;
- }
-
- /** Constructor for Java serialization. */
- VoidTransformProxy() {}
-
- Object readResolve() throws ObjectStreamException {
- return VoidTransform.get();
- }
- }
-}
+public interface SerializableFunction<S, T> extends Function<S, T>, Serializable {}
diff --git a/api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java b/api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java
index f6d076a446..5e4ca1fb11 100644
--- a/api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java
+++ b/api/src/test/java/org/apache/iceberg/PartitionSpecTestBase.java
@@ -18,6 +18,7 @@
*/
package org.apache.iceberg;
+import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Types;
@SuppressWarnings("checkstyle:HideUtilityClassConstructor")
@@ -69,8 +70,12 @@ public class PartitionSpecTestBase {
PartitionSpec.builderFor(SCHEMA).truncate("l", 10).build(),
PartitionSpec.builderFor(SCHEMA).truncate("dec", 10).build(),
PartitionSpec.builderFor(SCHEMA).truncate("s", 10).build(),
- PartitionSpec.builderFor(SCHEMA).add(6, "dec_unsupported", "unsupported").build(),
- PartitionSpec.builderFor(SCHEMA).add(6, 1111, "dec_unsupported", "unsupported").build(),
+ PartitionSpec.builderFor(SCHEMA)
+ .add(6, "dec_unsupported", Transforms.fromString("unsupported"))
+ .build(),
+ PartitionSpec.builderFor(SCHEMA)
+ .add(6, 1111, "dec_unsupported", Transforms.fromString("unsupported"))
+ .build(),
PartitionSpec.builderFor(SCHEMA).alwaysNull("ts").build(),
};
}
diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java b/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java
index d5a8767b26..886a03415c 100644
--- a/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java
+++ b/api/src/test/java/org/apache/iceberg/TestPartitionPaths.java
@@ -21,6 +21,7 @@ package org.apache.iceberg;
import org.apache.iceberg.TestHelpers.Row;
import org.apache.iceberg.expressions.Literal;
import org.apache.iceberg.transforms.Transform;
+import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Types;
import org.junit.Assert;
import org.junit.Test;
@@ -33,17 +34,16 @@ public class TestPartitionPaths {
Types.NestedField.optional(3, "ts", Types.TimestampType.withoutZone()));
@Test
- @SuppressWarnings("unchecked")
public void testPartitionPath() {
PartitionSpec spec = PartitionSpec.builderFor(SCHEMA).hour("ts").bucket("id", 10).build();
- Transform hour = spec.getFieldsBySourceId(3).get(0).transform();
- Transform bucket = spec.getFieldsBySourceId(1).get(0).transform();
+ Transform<Long, Integer> hour = Transforms.hour();
+ Transform<Integer, Integer> bucket = Transforms.bucket(10);
Literal<Long> ts =
Literal.of("2017-12-01T10:12:55.038194").to(Types.TimestampType.withoutZone());
- Object tsHour = hour.apply(ts.value());
- Object idBucket = bucket.apply(1);
+ Object tsHour = hour.bind(Types.TimestampType.withoutZone()).apply(ts.value());
+ Object idBucket = bucket.bind(Types.IntegerType.get()).apply(1);
Row partition = Row.of(tsHour, idBucket);
diff --git a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java
index 14c2a9ab1b..ed1d035cdc 100644
--- a/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java
+++ b/api/src/test/java/org/apache/iceberg/TestPartitionSpecValidation.java
@@ -18,6 +18,7 @@
*/
package org.apache.iceberg;
+import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.types.Types.NestedField;
import org.junit.Assert;
@@ -301,7 +302,7 @@ public class TestPartitionSpecValidation {
PartitionSpec.builderFor(SCHEMA)
.year("ts", "custom_year")
.bucket("ts", 4, "custom_bucket")
- .add(1, "id_partition2", "bucket[4]")
+ .add(1, "id_partition2", Transforms.bucket(4))
.truncate("s", 1, "custom_truncate")
.build();
@@ -316,9 +317,9 @@ public class TestPartitionSpecValidation {
public void testAddPartitionFieldsWithFieldIds() {
PartitionSpec spec =
PartitionSpec.builderFor(SCHEMA)
- .add(1, 1005, "id_partition1", "bucket[4]")
- .add(1, 1006, "id_partition2", "bucket[5]")
- .add(1, 1002, "id_partition3", "bucket[6]")
+ .add(1, 1005, "id_partition1", Transforms.bucket(4))
+ .add(1, 1006, "id_partition2", Transforms.bucket(5))
+ .add(1, 1002, "id_partition3", Transforms.bucket(6))
.build();
Assert.assertEquals(1005, spec.fields().get(0).fieldId());
@@ -331,8 +332,8 @@ public class TestPartitionSpecValidation {
public void testAddPartitionFieldsWithAndWithoutFieldIds() {
PartitionSpec spec =
PartitionSpec.builderFor(SCHEMA)
- .add(1, "id_partition2", "bucket[5]")
- .add(1, 1005, "id_partition1", "bucket[4]")
+ .add(1, "id_partition2", Transforms.bucket(5))
+ .add(1, 1005, "id_partition1", Transforms.bucket(4))
.truncate("s", 1, "custom_truncate")
.build();
diff --git a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java
index 62f6ff9b21..54e275080b 100644
--- a/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java
+++ b/api/src/test/java/org/apache/iceberg/expressions/TestExpressionHelpers.java
@@ -237,6 +237,6 @@ public class TestExpressionHelpers {
}
private <T> UnboundTerm<T> self(String name) {
- return new UnboundTransform<>(ref(name), Transforms.identity(Types.DoubleType.get()));
+ return new UnboundTransform<>(ref(name), Transforms.identity());
}
}
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestBucketing.java b/api/src/test/java/org/apache/iceberg/transforms/TestBucketing.java
index fecf8ca97e..04d8207be8 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestBucketing.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestBucketing.java
@@ -32,6 +32,7 @@ import org.apache.iceberg.expressions.Literal;
import org.apache.iceberg.relocated.com.google.common.hash.HashFunction;
import org.apache.iceberg.relocated.com.google.common.hash.Hashing;
import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.BucketUtil;
import org.junit.Assert;
import org.junit.Before;
import org.junit.BeforeClass;
@@ -61,86 +62,65 @@ public class TestBucketing {
@Test
public void testSpecValues() {
+ Assert.assertEquals("Spec example: hash(true) = 1392991556", 1392991556, BucketUtil.hash(1));
+ Assert.assertEquals("Spec example: hash(34) = 2017239379", 2017239379, BucketUtil.hash(34));
+ Assert.assertEquals("Spec example: hash(34L) = 2017239379", 2017239379, BucketUtil.hash(34L));
Assert.assertEquals(
- "Spec example: hash(true) = 1392991556",
- 1392991556,
- Bucket.<Integer>get(Types.IntegerType.get(), 100).hash(1));
+ "Spec example: hash(17.11F) = -142385009", -142385009, BucketUtil.hash(1.0F));
Assert.assertEquals(
- "Spec example: hash(34) = 2017239379",
- 2017239379,
- Bucket.<Integer>get(Types.IntegerType.get(), 100).hash(34));
- Assert.assertEquals(
- "Spec example: hash(34L) = 2017239379",
- 2017239379,
- Bucket.<Long>get(Types.LongType.get(), 100).hash(34L));
- Assert.assertEquals(
- "Spec example: hash(17.11F) = -142385009",
- -142385009,
- new Bucket.BucketFloat(100).hash(1.0F));
- Assert.assertEquals(
- "Spec example: hash(17.11D) = -142385009",
- -142385009,
- new Bucket.BucketDouble(100).hash(1.0D));
+ "Spec example: hash(17.11D) = -142385009", -142385009, BucketUtil.hash(1.0D));
Assert.assertEquals(
"Spec example: hash(decimal2(14.20)) = -500754589",
-500754589,
- Bucket.<BigDecimal>get(Types.DecimalType.of(9, 2), 100).hash(new BigDecimal("14.20")));
+ BucketUtil.hash(new BigDecimal("14.20")));
Assert.assertEquals(
"Spec example: hash(decimal2(14.20)) = -500754589",
-500754589,
- Bucket.<BigDecimal>get(Types.DecimalType.of(9, 2), 100).hash(new BigDecimal("14.20")));
+ BucketUtil.hash(new BigDecimal("14.20")));
Literal<Integer> date = Literal.of("2017-11-16").to(Types.DateType.get());
Assert.assertEquals(
- "Spec example: hash(2017-11-16) = -653330422",
- -653330422,
- Bucket.<Integer>get(Types.DateType.get(), 100).hash(date.value()));
+ "Spec example: hash(2017-11-16) = -653330422", -653330422, BucketUtil.hash(date.value()));
Literal<Long> timeValue = Literal.of("22:31:08").to(Types.TimeType.get());
Assert.assertEquals(
"Spec example: hash(22:31:08) = -662762989",
-662762989,
- Bucket.<Long>get(Types.TimeType.get(), 100).hash(timeValue.value()));
+ BucketUtil.hash(timeValue.value()));
Literal<Long> timestampVal =
Literal.of("2017-11-16T22:31:08").to(Types.TimestampType.withoutZone());
Assert.assertEquals(
"Spec example: hash(2017-11-16T22:31:08) = -2047944441",
-2047944441,
- Bucket.<Long>get(Types.TimestampType.withoutZone(), 100).hash(timestampVal.value()));
+ BucketUtil.hash(timestampVal.value()));
Literal<Long> timestamptzVal =
Literal.of("2017-11-16T14:31:08-08:00").to(Types.TimestampType.withZone());
Assert.assertEquals(
"Spec example: hash(2017-11-16T14:31:08-08:00) = -2047944441",
-2047944441,
- Bucket.<Long>get(Types.TimestampType.withZone(), 100).hash(timestamptzVal.value()));
+ BucketUtil.hash(timestamptzVal.value()));
Assert.assertEquals(
- "Spec example: hash(\"iceberg\") = 1210000089",
- 1210000089,
- Bucket.<String>get(Types.StringType.get(), 100).hash("iceberg"));
+ "Spec example: hash(\"iceberg\") = 1210000089", 1210000089, BucketUtil.hash("iceberg"));
Assert.assertEquals(
"Spec example: hash(\"iceberg\") = 1210000089",
1210000089,
- Bucket.<Utf8>get(Types.StringType.get(), 100).hash(new Utf8("iceberg")));
+ BucketUtil.hash(new Utf8("iceberg")));
Literal<UUID> uuid =
Literal.of("f79c3e09-677c-4bbd-a479-3f349cb785e7").to(Types.UUIDType.get());
Assert.assertEquals(
"Spec example: hash(f79c3e09-677c-4bbd-a479-3f349cb785e7) = 1488055340",
1488055340,
- Bucket.<UUID>get(Types.UUIDType.get(), 100).hash(uuid.value()));
+ BucketUtil.hash(uuid.value()));
ByteBuffer bytes = ByteBuffer.wrap(new byte[] {0, 1, 2, 3});
Assert.assertEquals(
- "Spec example: hash([00 01 02 03]) = -188683207",
- -188683207,
- Bucket.<ByteBuffer>get(Types.BinaryType.get(), 100).hash(bytes));
+ "Spec example: hash([00 01 02 03]) = -188683207", -188683207, BucketUtil.hash(bytes));
Assert.assertEquals(
- "Spec example: hash([00 01 02 03]) = -188683207",
- -188683207,
- Bucket.<ByteBuffer>get(Types.BinaryType.get(), 100).hash(bytes));
+ "Spec example: hash([00 01 02 03]) = -188683207", -188683207, BucketUtil.hash(bytes));
}
@Test
@@ -148,14 +128,12 @@ public class TestBucketing {
int num = testRandom.nextInt();
ByteBuffer buffer = ByteBuffer.allocate(8);
buffer.order(ByteOrder.LITTLE_ENDIAN);
- buffer.putLong((long) num);
-
- Bucket<Integer> bucketFunc = Bucket.get(Types.IntegerType.get(), 100);
+ buffer.putLong(num);
Assert.assertEquals(
"Integer hash should match hash of little-endian bytes",
hashBytes(buffer.array()),
- bucketFunc.hash(num));
+ BucketUtil.hash(num));
}
@Test
@@ -165,68 +143,30 @@ public class TestBucketing {
buffer.order(ByteOrder.LITTLE_ENDIAN);
buffer.putLong(num);
- Bucket<Long> bucketFunc = Bucket.get(Types.LongType.get(), 100);
-
Assert.assertEquals(
"Long hash should match hash of little-endian bytes",
hashBytes(buffer.array()),
- bucketFunc.hash(num));
+ BucketUtil.hash(num));
}
@Test
public void testIntegerTypePromotion() {
- Bucket<Integer> bucketInts = Bucket.get(Types.IntegerType.get(), 100);
- Bucket<Long> bucketLongs = Bucket.get(Types.LongType.get(), 100);
-
int randomInt = testRandom.nextInt();
Assert.assertEquals(
"Integer and Long bucket results should match",
- bucketInts.apply(randomInt),
- bucketLongs.apply((long) randomInt));
- }
-
- @Test
- public void testFloat() {
- float num = testRandom.nextFloat();
- ByteBuffer buffer = ByteBuffer.allocate(8);
- buffer.order(ByteOrder.LITTLE_ENDIAN);
- buffer.putDouble((double) num);
-
- Bucket<Float> bucketFunc = new Bucket.BucketFloat(100);
-
- Assert.assertEquals(
- "Float hash should match hash of little-endian bytes",
- hashBytes(buffer.array()),
- bucketFunc.hash(num));
- }
-
- @Test
- public void testDouble() {
- double num = testRandom.nextDouble();
- ByteBuffer buffer = ByteBuffer.allocate(8);
- buffer.order(ByteOrder.LITTLE_ENDIAN);
- buffer.putDouble(num);
-
- Bucket<Double> bucketFunc = new Bucket.BucketDouble(100);
-
- Assert.assertEquals(
- "Double hash should match hash of little-endian bytes",
- hashBytes(buffer.array()),
- bucketFunc.hash(num));
+ BucketUtil.hash(randomInt),
+ BucketUtil.hash((long) randomInt));
}
@Test
public void testFloatTypePromotion() {
- Bucket<Float> bucketFloats = new Bucket.BucketFloat(100);
- Bucket<Double> bucketDoubles = new Bucket.BucketDouble(100);
-
float randomFloat = testRandom.nextFloat();
Assert.assertEquals(
"Float and Double bucket results should match",
- bucketFloats.apply(randomFloat),
- bucketDoubles.apply((double) randomFloat));
+ BucketUtil.hash(randomFloat),
+ BucketUtil.hash((double) randomFloat));
}
@Test
@@ -235,12 +175,10 @@ public class TestBucketing {
BigDecimal decimal = BigDecimal.valueOf(num);
byte[] unscaledBytes = decimal.unscaledValue().toByteArray();
- Bucket<BigDecimal> bucketFunc = Bucket.get(Types.DecimalType.of(9, 2), 100);
-
Assert.assertEquals(
"Decimal hash should match hash of backing bytes",
hashBytes(unscaledBytes),
- bucketFunc.hash(decimal));
+ BucketUtil.hash(decimal));
}
@Test
@@ -248,12 +186,10 @@ public class TestBucketing {
String string = "string to test murmur3 hash";
byte[] asBytes = string.getBytes(StandardCharsets.UTF_8);
- Bucket<CharSequence> bucketFunc = Bucket.get(Types.StringType.get(), 100);
-
Assert.assertEquals(
"String hash should match hash of UTF-8 bytes",
hashBytes(asBytes),
- bucketFunc.hash(string));
+ BucketUtil.hash(string));
}
@Test
@@ -263,12 +199,10 @@ public class TestBucketing {
"string has no surrogate pairs", string.length(), string.codePoints().count());
byte[] asBytes = string.getBytes(StandardCharsets.UTF_8);
- Bucket<CharSequence> bucketFunc = Bucket.get(Types.StringType.get(), 100);
-
Assert.assertEquals(
"String hash should match hash of UTF-8 bytes",
hashBytes(asBytes),
- bucketFunc.hash(string));
+ BucketUtil.hash(string));
}
@Test
@@ -276,10 +210,8 @@ public class TestBucketing {
Utf8 utf8 = new Utf8("string to test murmur3 hash");
byte[] asBytes = utf8.toString().getBytes(StandardCharsets.UTF_8);
- Bucket<CharSequence> bucketFunc = Bucket.get(Types.StringType.get(), 100);
-
Assert.assertEquals(
- "String hash should match hash of UTF-8 bytes", hashBytes(asBytes), bucketFunc.hash(utf8));
+ "String hash should match hash of UTF-8 bytes", hashBytes(asBytes), BucketUtil.hash(utf8));
}
@Test
@@ -287,12 +219,10 @@ public class TestBucketing {
byte[] bytes = randomBytes(128);
ByteBuffer buffer = ByteBuffer.wrap(bytes, 5, 100);
- Bucket<ByteBuffer> bucketFunc = Bucket.get(Types.BinaryType.get(), 100);
-
Assert.assertEquals(
"HeapByteBuffer hash should match hash for correct slice",
hashBytes(bytes, 5, 100),
- bucketFunc.hash(buffer));
+ BucketUtil.hash(buffer));
// verify that the buffer was not modified
Assert.assertEquals("Buffer position should not change", 5, buffer.position());
@@ -306,12 +236,10 @@ public class TestBucketing {
ByteBuffer buffer = raw.slice();
Assert.assertEquals("Buffer arrayOffset should be 5", 5, buffer.arrayOffset());
- Bucket<ByteBuffer> bucketFunc = Bucket.get(Types.BinaryType.get(), 100);
-
Assert.assertEquals(
"HeapByteBuffer hash should match hash for correct slice",
hashBytes(bytes, 5, 100),
- bucketFunc.hash(buffer));
+ BucketUtil.hash(buffer));
// verify that the buffer was not modified
Assert.assertEquals("Buffer position should be 0", 0, buffer.position());
@@ -330,12 +258,10 @@ public class TestBucketing {
buffer.put(bytes, 5, 100);
buffer.reset();
- Bucket<ByteBuffer> bucketFunc = Bucket.get(Types.BinaryType.get(), 100);
-
Assert.assertEquals(
"DirectByteBuffer hash should match hash for correct slice",
hashBytes(bytes, 5, 100),
- bucketFunc.hash(buffer));
+ BucketUtil.hash(buffer));
// verify that the buffer was not modified
Assert.assertEquals("Buffer position should not change", 5, buffer.position());
@@ -347,12 +273,10 @@ public class TestBucketing {
byte[] uuidBytes = randomBytes(16);
UUID uuid = newUUID(uuidBytes);
- Bucket<UUID> bucketFunc = Bucket.get(Types.UUIDType.get(), 100);
-
Assert.assertEquals(
"UUID hash should match hash of backing bytes",
hashBytes(uuidBytes),
- bucketFunc.hash(uuid));
+ BucketUtil.hash(uuid));
}
@Test
@@ -361,7 +285,7 @@ public class TestBucketing {
"Should fail if numBucket is less than or equal to zero",
IllegalArgumentException.class,
"Invalid number of buckets: 0 (must be > 0)",
- () -> Bucket.get(Types.IntegerType.get(), 0));
+ () -> Bucket.get(0));
}
private byte[] randomBytes(int length) {
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestBucketingProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestBucketingProjection.java
index 25f10f06ec..34b5a19dfd 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestBucketingProjection.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestBucketingProjection.java
@@ -61,19 +61,18 @@ public class TestBucketingProjection {
Assert.assertNotEquals(
"Strict projection never runs for IN", Expression.Operation.IN, predicate.op());
- Bucket transform = (Bucket) spec.getFieldsBySourceId(1).get(0).transform();
if (predicate.op() == Expression.Operation.NOT_IN) {
Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
String actual =
Lists.newArrayList(values).stream()
.sorted()
- .map(v -> transform.toHumanString(v))
+ .map(String::valueOf)
.collect(Collectors.toList())
.toString();
Assert.assertEquals(expectedLiteral, actual);
} else {
- Literal literal = predicate.literal();
- String output = transform.toHumanString(literal.value());
+ Literal<?> literal = predicate.literal();
+ String output = String.valueOf(literal.value());
Assert.assertEquals(expectedLiteral, output);
}
}
@@ -103,19 +102,18 @@ public class TestBucketingProjection {
Assert.assertNotEquals(
"Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op());
- Bucket transform = (Bucket) spec.getFieldsBySourceId(1).get(0).transform();
if (predicate.op() == Expression.Operation.IN) {
Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
String actual =
Lists.newArrayList(values).stream()
.sorted()
- .map(v -> transform.toHumanString(v))
+ .map(String::valueOf)
.collect(Collectors.toList())
.toString();
Assert.assertEquals(expectedLiteral, actual);
} else {
- Literal literal = predicate.literal();
- String output = transform.toHumanString(literal.value());
+ Literal<?> literal = predicate.literal();
+ String output = String.valueOf(literal.value());
Assert.assertEquals(expectedLiteral, output);
}
}
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDates.java b/api/src/test/java/org/apache/iceberg/transforms/TestDates.java
index 39829221d6..1df5beeaa7 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestDates.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestDates.java
@@ -26,7 +26,8 @@ import org.junit.Test;
public class TestDates {
@Test
- public void testDateTransform() {
+ @SuppressWarnings("deprecation")
+ public void testDeprecatedDateTransform() {
Types.DateType type = Types.DateType.get();
Literal<Integer> date = Literal.of("2017-12-01").to(type);
Literal<Integer> pd = Literal.of("1970-01-01").to(type);
@@ -48,28 +49,57 @@ public class TestDates {
Assert.assertEquals("Should produce -1", -1, (int) days.apply(nd.value()));
}
+ @Test
+ public void testDateTransform() {
+ Types.DateType type = Types.DateType.get();
+ Literal<Integer> date = Literal.of("2017-12-01").to(type);
+ Literal<Integer> pd = Literal.of("1970-01-01").to(type);
+ Literal<Integer> nd = Literal.of("1969-12-31").to(type);
+
+ Transform<Integer, Integer> years = Transforms.year();
+ Assert.assertEquals(
+ "Should produce 2017 - 1970 = 47", 47, (int) years.bind(type).apply(date.value()));
+ Assert.assertEquals(
+ "Should produce 1970 - 1970 = 0", 0, (int) years.bind(type).apply(pd.value()));
+ Assert.assertEquals(
+ "Should produce 1969 - 1970 = -1", -1, (int) years.bind(type).apply(nd.value()));
+
+ Transform<Integer, Integer> months = Transforms.month();
+ Assert.assertEquals(
+ "Should produce 47 * 12 + 11 = 575", 575, (int) months.bind(type).apply(date.value()));
+ Assert.assertEquals(
+ "Should produce 0 * 12 + 0 = 0", 0, (int) months.bind(type).apply(pd.value()));
+ Assert.assertEquals("Should produce -1", -1, (int) months.bind(type).apply(nd.value()));
+
+ Transform<Integer, Integer> days = Transforms.day();
+ Assert.assertEquals("Should produce 17501", 17501, (int) days.bind(type).apply(date.value()));
+ Assert.assertEquals(
+ "Should produce 0 * 365 + 0 = 0", 0, (int) days.bind(type).apply(pd.value()));
+ Assert.assertEquals("Should produce -1", -1, (int) days.bind(type).apply(nd.value()));
+ }
+
@Test
public void testDateToHumanString() {
Types.DateType type = Types.DateType.get();
Literal<Integer> date = Literal.of("2017-12-01").to(type);
- Transform<Integer, Integer> year = Transforms.year(type);
+ Transform<Integer, Integer> year = Transforms.year();
Assert.assertEquals(
"Should produce the correct Human string",
"2017",
- year.toHumanString(year.apply(date.value())));
+ year.toHumanString(type, year.bind(type).apply(date.value())));
- Transform<Integer, Integer> month = Transforms.month(type);
+ Transform<Integer, Integer> month = Transforms.month();
Assert.assertEquals(
"Should produce the correct Human string",
"2017-12",
- month.toHumanString(month.apply(date.value())));
+ month.toHumanString(type, month.bind(type).apply(date.value())));
- Transform<Integer, Integer> day = Transforms.day(type);
+ Transform<Integer, Integer> day = Transforms.day();
Assert.assertEquals(
"Should produce the correct Human string",
"2017-12-01",
- day.toHumanString(day.apply(date.value())));
+ day.toHumanString(type, day.bind(type).apply(date.value())));
}
@Test
@@ -77,23 +107,23 @@ public class TestDates {
Types.DateType type = Types.DateType.get();
Literal<Integer> date = Literal.of("1969-12-30").to(type);
- Transform<Integer, Integer> year = Transforms.year(type);
+ Transform<Integer, Integer> year = Transforms.year();
Assert.assertEquals(
"Should produce the correct Human string",
"1969",
- year.toHumanString(year.apply(date.value())));
+ year.toHumanString(type, year.bind(type).apply(date.value())));
- Transform<Integer, Integer> month = Transforms.month(type);
+ Transform<Integer, Integer> month = Transforms.month();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-12",
- month.toHumanString(month.apply(date.value())));
+ month.toHumanString(type, month.bind(type).apply(date.value())));
- Transform<Integer, Integer> day = Transforms.day(type);
+ Transform<Integer, Integer> day = Transforms.day();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-12-30",
- day.toHumanString(day.apply(date.value())));
+ day.toHumanString(type, day.bind(type).apply(date.value())));
}
@Test
@@ -101,23 +131,23 @@ public class TestDates {
Types.DateType type = Types.DateType.get();
Literal<Integer> date = Literal.of("1970-01-01").to(type);
- Transform<Integer, Integer> year = Transforms.year(type);
+ Transform<Integer, Integer> year = Transforms.year();
Assert.assertEquals(
"Should produce the correct Human string",
"1970",
- year.toHumanString(year.apply(date.value())));
+ year.toHumanString(type, year.bind(type).apply(date.value())));
- Transform<Integer, Integer> month = Transforms.month(type);
+ Transform<Integer, Integer> month = Transforms.month();
Assert.assertEquals(
"Should produce the correct Human string",
"1970-01",
- month.toHumanString(month.apply(date.value())));
+ month.toHumanString(type, month.bind(type).apply(date.value())));
- Transform<Integer, Integer> day = Transforms.day(type);
+ Transform<Integer, Integer> day = Transforms.day();
Assert.assertEquals(
"Should produce the correct Human string",
"1970-01-01",
- day.toHumanString(day.apply(date.value())));
+ day.toHumanString(type, day.bind(type).apply(date.value())));
}
@Test
@@ -125,23 +155,23 @@ public class TestDates {
Types.DateType type = Types.DateType.get();
Literal<Integer> date = Literal.of("1969-01-01").to(type);
- Transform<Integer, Integer> year = Transforms.year(type);
+ Transform<Integer, Integer> year = Transforms.year();
Assert.assertEquals(
"Should produce the correct Human string",
"1969",
- year.toHumanString(year.apply(date.value())));
+ year.toHumanString(type, year.bind(type).apply(date.value())));
- Transform<Integer, Integer> month = Transforms.month(type);
+ Transform<Integer, Integer> month = Transforms.month();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-01",
- month.toHumanString(month.apply(date.value())));
+ month.toHumanString(type, month.bind(type).apply(date.value())));
- Transform<Integer, Integer> day = Transforms.day(type);
+ Transform<Integer, Integer> day = Transforms.day();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-01-01",
- day.toHumanString(day.apply(date.value())));
+ day.toHumanString(type, day.bind(type).apply(date.value())));
}
@Test
@@ -149,49 +179,49 @@ public class TestDates {
Types.DateType type = Types.DateType.get();
Literal<Integer> date = Literal.of("1969-12-31").to(type);
- Transform<Integer, Integer> year = Transforms.year(type);
+ Transform<Integer, Integer> year = Transforms.year();
Assert.assertEquals(
"Should produce the correct Human string",
"1969",
- year.toHumanString(year.apply(date.value())));
+ year.toHumanString(type, year.bind(type).apply(date.value())));
- Transform<Integer, Integer> month = Transforms.month(type);
+ Transform<Integer, Integer> month = Transforms.month();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-12",
- month.toHumanString(month.apply(date.value())));
+ month.toHumanString(type, month.bind(type).apply(date.value())));
- Transform<Integer, Integer> day = Transforms.day(type);
+ Transform<Integer, Integer> day = Transforms.day();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-12-31",
- day.toHumanString(day.apply(date.value())));
+ day.toHumanString(type, day.bind(type).apply(date.value())));
}
@Test
public void testNullHumanString() {
Types.DateType type = Types.DateType.get();
Assert.assertEquals(
- "Should produce \"null\" for null", "null", Transforms.year(type).toHumanString(null));
+ "Should produce \"null\" for null", "null", Transforms.year().toHumanString(type, null));
Assert.assertEquals(
- "Should produce \"null\" for null", "null", Transforms.month(type).toHumanString(null));
+ "Should produce \"null\" for null", "null", Transforms.month().toHumanString(type, null));
Assert.assertEquals(
- "Should produce \"null\" for null", "null", Transforms.day(type).toHumanString(null));
+ "Should produce \"null\" for null", "null", Transforms.day().toHumanString(type, null));
}
@Test
public void testDatesReturnType() {
Types.DateType type = Types.DateType.get();
- Transform<Integer, Integer> year = Transforms.year(type);
+ Transform<Integer, Integer> year = Transforms.year();
Type yearResultType = year.getResultType(type);
Assert.assertEquals(Types.IntegerType.get(), yearResultType);
- Transform<Integer, Integer> month = Transforms.month(type);
+ Transform<Integer, Integer> month = Transforms.month();
Type monthResultType = month.getResultType(type);
Assert.assertEquals(Types.IntegerType.get(), monthResultType);
- Transform<Integer, Integer> day = Transforms.day(type);
+ Transform<Integer, Integer> day = Transforms.day();
Type dayResultType = day.getResultType(type);
Assert.assertEquals(Types.DateType.get(), dayResultType);
}
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java
index a6abdc8f93..5106be0ca6 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestDatesProjection.java
@@ -38,6 +38,7 @@ import org.apache.iceberg.expressions.Projections;
import org.apache.iceberg.expressions.UnboundPredicate;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.junit.Assert;
import org.junit.Test;
@@ -46,6 +47,7 @@ public class TestDatesProjection {
private static final Types.DateType TYPE = Types.DateType.get();
private static final Schema SCHEMA = new Schema(optional(1, "date", TYPE));
+ @SuppressWarnings("unchecked")
public void assertProjectionStrict(
PartitionSpec spec,
UnboundPredicate<?> filter,
@@ -53,26 +55,28 @@ public class TestDatesProjection {
String expectedLiteral) {
Expression projection = Projections.strict(spec).project(filter);
- UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);
+ UnboundPredicate<Integer> predicate = assertAndUnwrapUnbound(projection);
Assert.assertEquals(expectedOp, predicate.op());
Assert.assertNotEquals(
"Strict projection never runs for IN", Expression.Operation.IN, predicate.op());
- Dates transform = (Dates) spec.getFieldsBySourceId(1).get(0).transform();
+ Transform<?, Integer> transform =
+ (Transform<?, Integer>) spec.getFieldsBySourceId(1).get(0).transform();
+ Type type = spec.partitionType().field(spec.getFieldsBySourceId(1).get(0).fieldId()).type();
if (predicate.op() == Expression.Operation.NOT_IN) {
- Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
+ Iterable<Integer> values = Iterables.transform(predicate.literals(), Literal::value);
String actual =
Lists.newArrayList(values).stream()
.sorted()
- .map(v -> transform.toHumanString((Integer) v))
+ .map(v -> transform.toHumanString(type, v))
.collect(Collectors.toList())
.toString();
Assert.assertEquals(expectedLiteral, actual);
} else {
- Literal literal = predicate.literal();
- String output = transform.toHumanString((int) literal.value());
+ Literal<Integer> literal = predicate.literal();
+ String output = transform.toHumanString(type, literal.value());
Assert.assertEquals(expectedLiteral, output);
}
}
@@ -91,32 +95,35 @@ public class TestDatesProjection {
Assert.assertEquals(expectedOp, projection.op());
}
+ @SuppressWarnings("unchecked")
public void assertProjectionInclusive(
PartitionSpec spec,
UnboundPredicate<?> filter,
Expression.Operation expectedOp,
String expectedLiteral) {
Expression projection = Projections.inclusive(spec).project(filter);
- UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);
+ UnboundPredicate<Integer> predicate = assertAndUnwrapUnbound(projection);
Assert.assertEquals(expectedOp, predicate.op());
Assert.assertNotEquals(
"Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op());
- Dates transform = (Dates) spec.getFieldsBySourceId(1).get(0).transform();
+ Transform<?, Integer> transform =
+ (Transform<?, Integer>) spec.getFieldsBySourceId(1).get(0).transform();
+ Type type = spec.partitionType().field(spec.getFieldsBySourceId(1).get(0).fieldId()).type();
if (predicate.op() == Expression.Operation.IN) {
- Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
+ Iterable<Integer> values = Iterables.transform(predicate.literals(), Literal::value);
String actual =
Lists.newArrayList(values).stream()
.sorted()
- .map(v -> transform.toHumanString((Integer) v))
+ .map(v -> transform.toHumanString(type, v))
.collect(Collectors.toList())
.toString();
Assert.assertEquals(expectedLiteral, actual);
} else {
- Literal literal = predicate.literal();
- String output = transform.toHumanString((int) literal.value());
+ Literal<Integer> literal = predicate.literal();
+ String output = transform.toHumanString(type, literal.value());
Assert.assertEquals(expectedLiteral, output);
}
}
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestIdentity.java b/api/src/test/java/org/apache/iceberg/transforms/TestIdentity.java
index e2e3680c7b..a14b7c8618 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestIdentity.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestIdentity.java
@@ -29,59 +29,62 @@ public class TestIdentity {
@Test
public void testNullHumanString() {
Types.LongType longType = Types.LongType.get();
- Transform<Long, Long> identity = Transforms.identity(longType);
+ Transform<Long, Long> identity = Transforms.identity();
- Assert.assertEquals("Should produce \"null\" for null", "null", identity.toHumanString(null));
+ Assert.assertEquals(
+ "Should produce \"null\" for null", "null", identity.toHumanString(longType, null));
}
@Test
public void testBinaryHumanString() {
Types.BinaryType binary = Types.BinaryType.get();
- Transform<ByteBuffer, ByteBuffer> identity = Transforms.identity(binary);
+ Transform<ByteBuffer, ByteBuffer> identity = Transforms.identity();
Assert.assertEquals(
"Should base64-encode binary",
"AQID",
- identity.toHumanString(ByteBuffer.wrap(new byte[] {1, 2, 3})));
+ identity.toHumanString(binary, ByteBuffer.wrap(new byte[] {1, 2, 3})));
}
@Test
public void testFixedHumanString() {
Types.FixedType fixed3 = Types.FixedType.ofLength(3);
- Transform<byte[], byte[]> identity = Transforms.identity(fixed3);
+ Transform<byte[], byte[]> identity = Transforms.identity();
Assert.assertEquals(
- "Should base64-encode binary", "AQID", identity.toHumanString(new byte[] {1, 2, 3}));
+ "Should base64-encode binary",
+ "AQID",
+ identity.toHumanString(fixed3, new byte[] {1, 2, 3}));
}
@Test
public void testDateHumanString() {
Types.DateType date = Types.DateType.get();
- Transform<Integer, Integer> identity = Transforms.identity(date);
+ Transform<Integer, Integer> identity = Transforms.identity();
String dateString = "2017-12-01";
Literal<Integer> dateLit = Literal.of(dateString).to(date);
Assert.assertEquals(
- "Should produce identical date", dateString, identity.toHumanString(dateLit.value()));
+ "Should produce identical date", dateString, identity.toHumanString(date, dateLit.value()));
}
@Test
public void testTimeHumanString() {
Types.TimeType time = Types.TimeType.get();
- Transform<Long, Long> identity = Transforms.identity(time);
+ Transform<Long, Long> identity = Transforms.identity();
String timeString = "10:12:55.038194";
Literal<Long> timeLit = Literal.of(timeString).to(time);
Assert.assertEquals(
- "Should produce identical time", timeString, identity.toHumanString(timeLit.value()));
+ "Should produce identical time", timeString, identity.toHumanString(time, timeLit.value()));
}
@Test
public void testTimestampWithZoneHumanString() {
Types.TimestampType timestamptz = Types.TimestampType.withZone();
- Transform<Long, Long> identity = Transforms.identity(timestamptz);
+ Transform<Long, Long> identity = Transforms.identity();
Literal<Long> ts = Literal.of("2017-12-01T10:12:55.038194-08:00").to(timestamptz);
@@ -89,13 +92,13 @@ public class TestIdentity {
Assert.assertEquals(
"Should produce timestamp with time zone adjusted to UTC",
"2017-12-01T18:12:55.038194Z",
- identity.toHumanString(ts.value()));
+ identity.toHumanString(timestamptz, ts.value()));
}
@Test
public void testTimestampWithoutZoneHumanString() {
Types.TimestampType timestamp = Types.TimestampType.withoutZone();
- Transform<Long, Long> identity = Transforms.identity(timestamp);
+ Transform<Long, Long> identity = Transforms.identity();
String tsString = "2017-12-01T10:12:55.038194";
Literal<Long> ts = Literal.of(tsString).to(timestamp);
@@ -104,35 +107,38 @@ public class TestIdentity {
Assert.assertEquals(
"Should produce identical timestamp without time zone",
tsString,
- identity.toHumanString(ts.value()));
+ identity.toHumanString(timestamp, ts.value()));
}
@Test
public void testLongToHumanString() {
Types.LongType longType = Types.LongType.get();
- Transform<Long, Long> identity = Transforms.identity(longType);
+ Transform<Long, Long> identity = Transforms.identity();
Assert.assertEquals(
- "Should use Long toString", "-1234567890000", identity.toHumanString(-1234567890000L));
+ "Should use Long toString",
+ "-1234567890000",
+ identity.toHumanString(longType, -1234567890000L));
}
@Test
public void testStringToHumanString() {
Types.StringType string = Types.StringType.get();
- Transform<String, String> identity = Transforms.identity(string);
+ Transform<String, String> identity = Transforms.identity();
String withSlash = "a/b/c=d";
- Assert.assertEquals("Should not modify Strings", withSlash, identity.toHumanString(withSlash));
+ Assert.assertEquals(
+ "Should not modify Strings", withSlash, identity.toHumanString(string, withSlash));
}
@Test
public void testBigDecimalToHumanString() {
Types.DecimalType decimal = Types.DecimalType.of(9, 2);
- Transform<BigDecimal, BigDecimal> identity = Transforms.identity(decimal);
+ Transform<BigDecimal, BigDecimal> identity = Transforms.identity();
String decimalString = "-1.50";
BigDecimal bigDecimal = new BigDecimal(decimalString);
Assert.assertEquals(
- "Should not modify Strings", decimalString, identity.toHumanString(bigDecimal));
+ "Should not modify Strings", decimalString, identity.toHumanString(decimal, bigDecimal));
}
}
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestNotStartsWith.java b/api/src/test/java/org/apache/iceberg/transforms/TestNotStartsWith.java
index 54a362a9d3..88c762797b 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestNotStartsWith.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestNotStartsWith.java
@@ -93,7 +93,7 @@ public class TestNotStartsWith {
@Test
public void testTruncateStringWhenProjectedPredicateTermIsLongerThanWidth() {
- Truncate<String> trunc = Truncate.get(Types.StringType.get(), 2);
+ Truncate<String> trunc = Truncate.get(2);
UnboundPredicate<String> expr = notStartsWith(COLUMN, "abcde");
BoundPredicate<String> boundExpr =
(BoundPredicate<String>) Binder.bind(SCHEMA.asStruct(), expr, false);
@@ -127,7 +127,7 @@ public class TestNotStartsWith {
@Test
public void testTruncateStringWhenProjectedPredicateTermIsShorterThanWidth() {
- Truncate<String> trunc = Truncate.get(Types.StringType.get(), 16);
+ Truncate<String> trunc = Truncate.get(16);
UnboundPredicate<String> expr = notStartsWith(COLUMN, "ab");
BoundPredicate<String> boundExpr =
(BoundPredicate<String>) Binder.bind(SCHEMA.asStruct(), expr, false);
@@ -153,7 +153,7 @@ public class TestNotStartsWith {
@Test
public void testTruncateStringWhenProjectedPredicateTermIsEqualToWidth() {
- Truncate<String> trunc = Truncate.get(Types.StringType.get(), 7);
+ Truncate<String> trunc = Truncate.get(7);
UnboundPredicate<String> expr = notStartsWith(COLUMN, "abcdefg");
BoundPredicate<String> boundExpr =
(BoundPredicate<String>) Binder.bind(SCHEMA.asStruct(), expr, false);
@@ -223,6 +223,7 @@ public class TestNotStartsWith {
assertProjection(spec, expectedLiteral, projection, expectedOp);
}
+ @SuppressWarnings("unchecked")
private void assertProjection(
PartitionSpec spec,
String expectedLiteral,
@@ -232,7 +233,7 @@ public class TestNotStartsWith {
Literal<?> literal = predicate.literal();
Truncate<CharSequence> transform =
(Truncate<CharSequence>) spec.getFieldsBySourceId(1).get(0).transform();
- String output = transform.toHumanString((String) literal.value());
+ String output = transform.toHumanString(Types.StringType.get(), (String) literal.value());
Assert.assertEquals(expectedOp, predicate.op());
Assert.assertEquals(expectedLiteral, output);
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestResiduals.java b/api/src/test/java/org/apache/iceberg/transforms/TestResiduals.java
index dc46964c46..c2bb855cfd 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestResiduals.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestResiduals.java
@@ -33,6 +33,7 @@ import static org.apache.iceberg.expressions.Expressions.notIn;
import static org.apache.iceberg.expressions.Expressions.notNaN;
import static org.apache.iceberg.expressions.Expressions.or;
+import java.util.function.Function;
import org.apache.iceberg.PartitionSpec;
import org.apache.iceberg.Schema;
import org.apache.iceberg.TestHelpers.Row;
@@ -40,7 +41,6 @@ import org.apache.iceberg.exceptions.ValidationException;
import org.apache.iceberg.expressions.Expression;
import org.apache.iceberg.expressions.Expressions;
import org.apache.iceberg.expressions.Literal;
-import org.apache.iceberg.expressions.Predicate;
import org.apache.iceberg.expressions.ResidualEvaluator;
import org.apache.iceberg.expressions.UnboundPredicate;
import org.apache.iceberg.types.Types;
@@ -210,10 +210,10 @@ public class TestResiduals {
PartitionSpec spec = PartitionSpec.builderFor(schema).day("ts").build();
- Transform day = spec.getFieldsBySourceId(50).get(0).transform();
- Integer tsDay = (Integer) day.apply(date20191201);
+ Function<Object, Integer> day = Transforms.day().bind(Types.TimestampType.withoutZone());
+ Integer tsDay = day.apply(date20191201);
- Predicate pred = in("ts", date20191201, date20191202);
+ Expression pred = in("ts", date20191201, date20191202);
ResidualEvaluator resEval = ResidualEvaluator.of(spec, pred, true);
Expression residual = resEval.residualFor(Row.of(tsDay));
@@ -318,10 +318,10 @@ public class TestResiduals {
PartitionSpec spec = PartitionSpec.builderFor(schema).day("ts").build();
- Transform day = spec.getFieldsBySourceId(50).get(0).transform();
- Integer tsDay = (Integer) day.apply(date20191201);
+ Function<Object, Integer> day = Transforms.day().bind(Types.TimestampType.withoutZone());
+ Integer tsDay = day.apply(date20191201);
- Predicate pred = notIn("ts", date20191201, date20191202);
+ Expression pred = notIn("ts", date20191201, date20191202);
ResidualEvaluator resEval = ResidualEvaluator.of(spec, pred, true);
Expression residual = resEval.residualFor(Row.of(tsDay));
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestStartsWith.java b/api/src/test/java/org/apache/iceberg/transforms/TestStartsWith.java
index 90143aa7e8..8ceeb91952 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestStartsWith.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestStartsWith.java
@@ -61,8 +61,9 @@ public class TestStartsWith {
}
@Test
+ @SuppressWarnings("unchecked")
public void testTruncateString() {
- Truncate<String> trunc = Truncate.get(Types.StringType.get(), 2);
+ Truncate<String> trunc = Truncate.get(2);
Expression expr = startsWith(COLUMN, "abcde");
BoundPredicate<String> boundExpr =
(BoundPredicate<String>) Binder.bind(SCHEMA.asStruct(), expr, false);
@@ -93,16 +94,17 @@ public class TestStartsWith {
assertProjection(spec, expectedLiteral, projection, expectedOp);
}
+ @SuppressWarnings("unchecked")
private void assertProjection(
PartitionSpec spec,
String expectedLiteral,
Expression projection,
Expression.Operation expectedOp) {
UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);
- Literal literal = predicate.literal();
+ Literal<?> literal = predicate.literal();
Truncate<CharSequence> transform =
(Truncate<CharSequence>) spec.getFieldsBySourceId(1).get(0).transform();
- String output = transform.toHumanString((String) literal.value());
+ String output = transform.toHumanString(Types.StringType.get(), (String) literal.value());
Assert.assertEquals(expectedOp, predicate.op());
Assert.assertEquals(expectedLiteral, output);
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java
index c5ff6788db..aefa98ed26 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestamps.java
@@ -26,7 +26,8 @@ import org.junit.Test;
public class TestTimestamps {
@Test
- public void testTimestampTransform() {
+ @SuppressWarnings("deprecation")
+ public void testDeprecatedTimestampTransform() {
Types.TimestampType type = Types.TimestampType.withoutZone();
Literal<Long> ts = Literal.of("2017-12-01T10:12:55.038194").to(type);
Literal<Long> pts = Literal.of("1970-01-01T00:00:01.000001").to(type);
@@ -53,34 +54,70 @@ public class TestTimestamps {
Assert.assertEquals("Should produce -1", -1, (int) hours.apply(nts.value()));
}
+ @Test
+ public void testTimestampTransform() {
+ Types.TimestampType type = Types.TimestampType.withoutZone();
+ Literal<Long> ts = Literal.of("2017-12-01T10:12:55.038194").to(type);
+ Literal<Long> pts = Literal.of("1970-01-01T00:00:01.000001").to(type);
+ Literal<Long> nts = Literal.of("1969-12-31T23:59:58.999999").to(type);
+
+ Transform<Long, Integer> years = Transforms.year();
+ Assert.assertEquals(
+ "Should produce 2017 - 1970 = 47", 47, (int) years.bind(type).apply(ts.value()));
+ Assert.assertEquals(
+ "Should produce 1970 - 1970 = 0", 0, (int) years.bind(type).apply(pts.value()));
+ Assert.assertEquals(
+ "Should produce 1969 - 1970 = -1", -1, (int) years.bind(type).apply(nts.value()));
+
+ Transform<Long, Integer> months = Transforms.month();
+ Assert.assertEquals(
+ "Should produce 47 * 12 + 11 = 575", 575, (int) months.bind(type).apply(ts.value()));
+ Assert.assertEquals(
+ "Should produce 0 * 12 + 0 = 0", 0, (int) months.bind(type).apply(pts.value()));
+ Assert.assertEquals("Should produce -1", -1, (int) months.bind(type).apply(nts.value()));
+
+ Transform<Long, Integer> days = Transforms.day();
+ Assert.assertEquals("Should produce 17501", 17501, (int) days.bind(type).apply(ts.value()));
+ Assert.assertEquals(
+ "Should produce 0 * 365 + 0 = 0", 0, (int) days.bind(type).apply(pts.value()));
+ Assert.assertEquals("Should produce -1", -1, (int) days.bind(type).apply(nts.value()));
+
+ Transform<Long, Integer> hours = Transforms.hour();
+ Assert.assertEquals(
+ "Should produce 17501 * 24 + 10", 420034, (int) hours.bind(type).apply(ts.value()));
+ Assert.assertEquals(
+ "Should produce 0 * 24 + 0 = 0", 0, (int) hours.bind(type).apply(pts.value()));
+ Assert.assertEquals("Should produce -1", -1, (int) hours.bind(type).apply(nts.value()));
+ }
+
@Test
public void testTimestampWithoutZoneToHumanString() {
Types.TimestampType type = Types.TimestampType.withoutZone();
Literal<Long> date = Literal.of("2017-12-01T10:12:55.038194").to(type);
- Transform<Long, Integer> year = Transforms.year(type);
+ Transform<Long, Integer> year = Transforms.year();
Assert.assertEquals(
"Should produce the correct Human string",
"2017",
- year.toHumanString(year.apply(date.value())));
+ year.toHumanString(type, year.bind(type).apply(date.value())));
- Transform<Long, Integer> month = Transforms.month(type);
+ Transform<Long, Integer> month = Transforms.month();
Assert.assertEquals(
"Should produce the correct Human string",
"2017-12",
- month.toHumanString(month.apply(date.value())));
+ month.toHumanString(type, month.bind(type).apply(date.value())));
- Transform<Long, Integer> day = Transforms.day(type);
+ Transform<Long, Integer> day = Transforms.day();
Assert.assertEquals(
"Should produce the correct Human string",
"2017-12-01",
- day.toHumanString(day.apply(date.value())));
+ day.toHumanString(type, day.bind(type).apply(date.value())));
- Transform<Long, Integer> hour = Transforms.hour(type);
+ Transform<Long, Integer> hour = Transforms.hour();
Assert.assertEquals(
"Should produce the correct Human string",
"2017-12-01-10",
- hour.toHumanString(hour.apply(date.value())));
+ hour.toHumanString(type, hour.bind(type).apply(date.value())));
}
@Test
@@ -88,29 +125,29 @@ public class TestTimestamps {
Types.TimestampType type = Types.TimestampType.withoutZone();
Literal<Long> date = Literal.of("1969-12-30T10:12:55.038194").to(type);
- Transform<Long, Integer> year = Transforms.year(type);
+ Transform<Long, Integer> year = Transforms.year();
Assert.assertEquals(
"Should produce the correct Human string",
"1969",
- year.toHumanString(year.apply(date.value())));
+ year.toHumanString(type, year.bind(type).apply(date.value())));
- Transform<Long, Integer> month = Transforms.month(type);
+ Transform<Long, Integer> month = Transforms.month();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-12",
- month.toHumanString(month.apply(date.value())));
+ month.toHumanString(type, month.bind(type).apply(date.value())));
- Transform<Long, Integer> day = Transforms.day(type);
+ Transform<Long, Integer> day = Transforms.day();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-12-30",
- day.toHumanString(day.apply(date.value())));
+ day.toHumanString(type, day.bind(type).apply(date.value())));
- Transform<Long, Integer> hour = Transforms.hour(type);
+ Transform<Long, Integer> hour = Transforms.hour();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-12-30-10",
- hour.toHumanString(hour.apply(date.value())));
+ hour.toHumanString(type, hour.bind(type).apply(date.value())));
}
@Test
@@ -118,29 +155,29 @@ public class TestTimestamps {
Types.TimestampType type = Types.TimestampType.withoutZone();
Literal<Long> date = Literal.of("1969-12-30T00:00:00.000000").to(type);
- Transform<Long, Integer> year = Transforms.year(type);
+ Transform<Long, Integer> year = Transforms.year();
Assert.assertEquals(
"Should produce the correct Human string",
"1969",
- year.toHumanString(year.apply(date.value())));
+ year.toHumanString(type, year.bind(type).apply(date.value())));
- Transform<Long, Integer> month = Transforms.month(type);
+ Transform<Long, Integer> month = Transforms.month();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-12",
- month.toHumanString(month.apply(date.value())));
+ month.toHumanString(type, month.bind(type).apply(date.value())));
- Transform<Long, Integer> day = Transforms.day(type);
+ Transform<Long, Integer> day = Transforms.day();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-12-30",
- day.toHumanString(day.apply(date.value())));
+ day.toHumanString(type, day.bind(type).apply(date.value())));
- Transform<Long, Integer> hour = Transforms.hour(type);
+ Transform<Long, Integer> hour = Transforms.hour();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-12-30-00",
- hour.toHumanString(hour.apply(date.value())));
+ hour.toHumanString(type, hour.bind(type).apply(date.value())));
}
@Test
@@ -148,29 +185,29 @@ public class TestTimestamps {
Types.TimestampType type = Types.TimestampType.withoutZone();
Literal<Long> date = Literal.of("1969-12-31T23:59:59.999999").to(type);
- Transform<Long, Integer> year = Transforms.year(type);
+ Transform<Long, Integer> year = Transforms.year();
Assert.assertEquals(
"Should produce the correct Human string",
"1969",
- year.toHumanString(year.apply(date.value())));
+ year.toHumanString(type, year.bind(type).apply(date.value())));
- Transform<Long, Integer> month = Transforms.month(type);
+ Transform<Long, Integer> month = Transforms.month();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-12",
- month.toHumanString(month.apply(date.value())));
+ month.toHumanString(type, month.bind(type).apply(date.value())));
- Transform<Long, Integer> day = Transforms.day(type);
+ Transform<Long, Integer> day = Transforms.day();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-12-31",
- day.toHumanString(day.apply(date.value())));
+ day.toHumanString(type, day.bind(type).apply(date.value())));
- Transform<Long, Integer> hour = Transforms.hour(type);
+ Transform<Long, Integer> hour = Transforms.hour();
Assert.assertEquals(
"Should produce the correct Human string",
"1969-12-31-23",
- hour.toHumanString(hour.apply(date.value())));
+ hour.toHumanString(type, hour.bind(type).apply(date.value())));
}
@Test
@@ -178,62 +215,62 @@ public class TestTimestamps {
Types.TimestampType type = Types.TimestampType.withZone();
Literal<Long> date = Literal.of("2017-12-01T10:12:55.038194-08:00").to(type);
- Transform<Long, Integer> year = Transforms.year(type);
+ Transform<Long, Integer> year = Transforms.year();
Assert.assertEquals(
"Should produce the correct Human string",
"2017",
- year.toHumanString(year.apply(date.value())));
+ year.toHumanString(type, year.bind(type).apply(date.value())));
- Transform<Long, Integer> month = Transforms.month(type);
+ Transform<Long, Integer> month = Transforms.month();
Assert.assertEquals(
"Should produce the correct Human string",
"2017-12",
- month.toHumanString(month.apply(date.value())));
+ month.toHumanString(type, month.bind(type).apply(date.value())));
- Transform<Long, Integer> day = Transforms.day(type);
+ Transform<Long, Integer> day = Transforms.day();
Assert.assertEquals(
"Should produce the correct Human string",
"2017-12-01",
- day.toHumanString(day.apply(date.value())));
+ day.toHumanString(type, day.bind(type).apply(date.value())));
// the hour is 18 because the value is always UTC
- Transform<Long, Integer> hour = Transforms.hour(type);
+ Transform<Long, Integer> hour = Transforms.hour();
Assert.assertEquals(
"Should produce the correct Human string",
"2017-12-01-18",
- hour.toHumanString(hour.apply(date.value())));
+ hour.toHumanString(type, hour.bind(type).apply(date.value())));
}
@Test
public void testNullHumanString() {
Types.TimestampType type = Types.TimestampType.withZone();
Assert.assertEquals(
- "Should produce \"null\" for null", "null", Transforms.year(type).toHumanString(null));
+ "Should produce \"null\" for null", "null", Transforms.year().toHumanString(type, null));
Assert.assertEquals(
- "Should produce \"null\" for null", "null", Transforms.month(type).toHumanString(null));
+ "Should produce \"null\" for null", "null", Transforms.month().toHumanString(type, null));
Assert.assertEquals(
- "Should produce \"null\" for null", "null", Transforms.day(type).toHumanString(null));
+ "Should produce \"null\" for null", "null", Transforms.day().toHumanString(type, null));
Assert.assertEquals(
- "Should produce \"null\" for null", "null", Transforms.hour(type).toHumanString(null));
+ "Should produce \"null\" for null", "null", Transforms.hour().toHumanString(type, null));
}
@Test
public void testTimestampsReturnType() {
Types.TimestampType type = Types.TimestampType.withZone();
- Transform<Integer, Integer> year = Transforms.year(type);
+ Transform<Integer, Integer> year = Transforms.year();
Type yearResultType = year.getResultType(type);
Assert.assertEquals(Types.IntegerType.get(), yearResultType);
- Transform<Integer, Integer> month = Transforms.month(type);
+ Transform<Integer, Integer> month = Transforms.month();
Type monthResultType = month.getResultType(type);
Assert.assertEquals(Types.IntegerType.get(), monthResultType);
- Transform<Integer, Integer> day = Transforms.day(type);
+ Transform<Integer, Integer> day = Transforms.day();
Type dayResultType = day.getResultType(type);
Assert.assertEquals(Types.DateType.get(), dayResultType);
- Transform<Integer, Integer> hour = Transforms.hour(type);
+ Transform<Integer, Integer> hour = Transforms.hour();
Type hourResultType = hour.getResultType(type);
Assert.assertEquals(Types.IntegerType.get(), hourResultType);
}
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java
index 3ed5f9bac8..3da034d501 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestTimestampsProjection.java
@@ -38,6 +38,7 @@ import org.apache.iceberg.expressions.Projections;
import org.apache.iceberg.expressions.UnboundPredicate;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.junit.Assert;
import org.junit.Test;
@@ -46,6 +47,7 @@ public class TestTimestampsProjection {
private static final Types.TimestampType TYPE = Types.TimestampType.withoutZone();
private static final Schema SCHEMA = new Schema(optional(1, "timestamp", TYPE));
+ @SuppressWarnings("unchecked")
public void assertProjectionStrict(
PartitionSpec spec,
UnboundPredicate<?> filter,
@@ -53,26 +55,28 @@ public class TestTimestampsProjection {
String expectedLiteral) {
Expression projection = Projections.strict(spec).project(filter);
- UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);
+ UnboundPredicate<Integer> predicate = assertAndUnwrapUnbound(projection);
Assert.assertEquals(expectedOp, predicate.op());
Assert.assertNotEquals(
"Strict projection never runs for IN", Expression.Operation.IN, predicate.op());
- Timestamps transform = (Timestamps) spec.getFieldsBySourceId(1).get(0).transform();
+ Transform<?, Integer> transform =
+ (Transform<?, Integer>) spec.getFieldsBySourceId(1).get(0).transform();
+ Type type = spec.partitionType().field(spec.getFieldsBySourceId(1).get(0).fieldId()).type();
if (predicate.op() == Expression.Operation.NOT_IN) {
- Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
+ Iterable<Integer> values = Iterables.transform(predicate.literals(), Literal::value);
String actual =
Lists.newArrayList(values).stream()
.sorted()
- .map(v -> transform.toHumanString((Integer) v))
+ .map(v -> transform.toHumanString(type, v))
.collect(Collectors.toList())
.toString();
Assert.assertEquals(expectedLiteral, actual);
} else {
- Literal literal = predicate.literal();
- String output = transform.toHumanString((int) literal.value());
+ Literal<Integer> literal = predicate.literal();
+ String output = transform.toHumanString(type, literal.value());
Assert.assertEquals(expectedLiteral, output);
}
}
@@ -91,32 +95,35 @@ public class TestTimestampsProjection {
Assert.assertEquals(expectedOp, projection.op());
}
+ @SuppressWarnings("unchecked")
public void assertProjectionInclusive(
PartitionSpec spec,
UnboundPredicate<?> filter,
Expression.Operation expectedOp,
String expectedLiteral) {
Expression projection = Projections.inclusive(spec).project(filter);
- UnboundPredicate<?> predicate = assertAndUnwrapUnbound(projection);
+ UnboundPredicate<Integer> predicate = assertAndUnwrapUnbound(projection);
Assert.assertEquals(expectedOp, predicate.op());
Assert.assertNotEquals(
"Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op());
- Timestamps transform = (Timestamps) spec.getFieldsBySourceId(1).get(0).transform();
+ Transform<?, Integer> transform =
+ (Transform<?, Integer>) spec.getFieldsBySourceId(1).get(0).transform();
+ Type type = spec.partitionType().field(spec.getFieldsBySourceId(1).get(0).fieldId()).type();
if (predicate.op() == Expression.Operation.IN) {
- Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
+ Iterable<Integer> values = Iterables.transform(predicate.literals(), Literal::value);
String actual =
Lists.newArrayList(values).stream()
.sorted()
- .map(v -> transform.toHumanString((Integer) v))
+ .map(v -> transform.toHumanString(type, v))
.collect(Collectors.toList())
.toString();
Assert.assertEquals(expectedLiteral, actual);
} else {
- Literal literal = predicate.literal();
- String output = transform.toHumanString((int) literal.value());
+ Literal<Integer> literal = predicate.literal();
+ String output = transform.toHumanString(type, literal.value());
Assert.assertEquals(expectedLiteral, output);
}
}
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTransformSerialization.java b/api/src/test/java/org/apache/iceberg/transforms/TestTransformSerialization.java
new file mode 100644
index 0000000000..2c04779c67
--- /dev/null
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestTransformSerialization.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.iceberg.transforms;
+
+import org.apache.iceberg.TestHelpers;
+import org.apache.iceberg.types.Type;
+import org.apache.iceberg.types.Types;
+import org.apache.iceberg.util.SerializableFunction;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class TestTransformSerialization {
+ @Test
+ public void testFunctionSerialization() throws Exception {
+ Type[] types =
+ new Type[] {
+ Types.BooleanType.get(),
+ Types.IntegerType.get(),
+ Types.LongType.get(),
+ Types.FloatType.get(),
+ Types.DoubleType.get(),
+ Types.StringType.get(),
+ Types.DateType.get(),
+ Types.TimeType.get(),
+ Types.TimestampType.withoutZone(),
+ Types.TimestampType.withoutZone(),
+ Types.BinaryType.get(),
+ Types.FixedType.ofLength(4),
+ Types.DecimalType.of(9, 4),
+ Types.UUIDType.get(),
+ };
+
+ Transform<?, ?>[] transforms =
+ new Transform<?, ?>[] {
+ Transforms.identity(),
+ Transforms.bucket(1024),
+ Transforms.year(),
+ Transforms.month(),
+ Transforms.day(),
+ Transforms.hour(),
+ Transforms.truncate(16)
+ };
+
+ for (Type type : types) {
+ for (Transform<?, ?> transform : transforms) {
+ Assert.assertEquals(transform, TestHelpers.roundTripSerialize(transform));
+
+ if (transform.canTransform(type)) {
+ SerializableFunction<?, ?> func = transform.bind(type);
+ Assert.assertTrue(func.getClass().isInstance(TestHelpers.roundTripSerialize(func)));
+ }
+ }
+ }
+ }
+}
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTruncate.java b/api/src/test/java/org/apache/iceberg/transforms/TestTruncate.java
index 6682086a6e..4324998bd4 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestTruncate.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestTruncate.java
@@ -20,15 +20,32 @@ package org.apache.iceberg.transforms;
import java.math.BigDecimal;
import java.nio.ByteBuffer;
+import java.nio.charset.StandardCharsets;
+import java.util.function.Function;
import org.apache.iceberg.AssertHelpers;
import org.apache.iceberg.types.Types;
import org.junit.Assert;
import org.junit.Test;
public class TestTruncate {
+ @Test
+ public void testDeprecatedTruncateInteger() {
+ Truncate<Object> trunc = Truncate.get(Types.IntegerType.get(), 10);
+ Assert.assertEquals(0, (int) trunc.apply(0));
+ Assert.assertEquals(0, (int) trunc.apply(1));
+ Assert.assertEquals(0, (int) trunc.apply(5));
+ Assert.assertEquals(0, (int) trunc.apply(9));
+ Assert.assertEquals(10, (int) trunc.apply(10));
+ Assert.assertEquals(10, (int) trunc.apply(11));
+ Assert.assertEquals(-10, (int) trunc.apply(-1));
+ Assert.assertEquals(-10, (int) trunc.apply(-5));
+ Assert.assertEquals(-10, (int) trunc.apply(-10));
+ Assert.assertEquals(-20, (int) trunc.apply(-11));
+ }
+
@Test
public void testTruncateInteger() {
- Truncate<Integer> trunc = Truncate.get(Types.IntegerType.get(), 10);
+ Function<Object, Object> trunc = Truncate.get(10).bind(Types.IntegerType.get());
Assert.assertEquals(0, (int) trunc.apply(0));
Assert.assertEquals(0, (int) trunc.apply(1));
Assert.assertEquals(0, (int) trunc.apply(5));
@@ -43,7 +60,7 @@ public class TestTruncate {
@Test
public void testTruncateLong() {
- Truncate<Long> trunc = Truncate.get(Types.LongType.get(), 10);
+ Function<Object, Object> trunc = Truncate.get(10).bind(Types.LongType.get());
Assert.assertEquals(0L, (long) trunc.apply(0L));
Assert.assertEquals(0L, (long) trunc.apply(1L));
Assert.assertEquals(0L, (long) trunc.apply(5L));
@@ -59,7 +76,7 @@ public class TestTruncate {
@Test
public void testTruncateDecimal() {
// decimal truncation works by applying the decimal scale to the width: 10 scale 2 = 0.10
- Truncate<BigDecimal> trunc = Truncate.get(Types.DecimalType.of(9, 2), 10);
+ Function<Object, Object> trunc = Truncate.get(10).bind(Types.DecimalType.of(9, 2));
Assert.assertEquals(new BigDecimal("12.30"), trunc.apply(new BigDecimal("12.34")));
Assert.assertEquals(new BigDecimal("12.30"), trunc.apply(new BigDecimal("12.30")));
Assert.assertEquals(new BigDecimal("12.20"), trunc.apply(new BigDecimal("12.29")));
@@ -69,7 +86,7 @@ public class TestTruncate {
@Test
public void testTruncateString() {
- Truncate<String> trunc = Truncate.get(Types.StringType.get(), 5);
+ Function<Object, Object> trunc = Truncate.get(5).bind(Types.StringType.get());
Assert.assertEquals(
"Should truncate strings longer than length", "abcde", trunc.apply("abcdefg"));
Assert.assertEquals("Should not pad strings shorter than length", "abc", trunc.apply("abc"));
@@ -77,16 +94,16 @@ public class TestTruncate {
}
@Test
- public void testTruncateByteBuffer() throws Exception {
- Truncate<ByteBuffer> trunc = Truncate.get(Types.BinaryType.get(), 4);
+ public void testTruncateByteBuffer() {
+ Function<Object, Object> trunc = Truncate.get(4).bind(Types.BinaryType.get());
Assert.assertEquals(
"Should truncate binary longer than length",
- ByteBuffer.wrap("abcd".getBytes("UTF-8")),
- trunc.apply(ByteBuffer.wrap("abcdefg".getBytes("UTF-8"))));
+ ByteBuffer.wrap("abcd".getBytes(StandardCharsets.UTF_8)),
+ trunc.apply(ByteBuffer.wrap("abcdefg".getBytes(StandardCharsets.UTF_8))));
Assert.assertEquals(
"Should not pad binary shorter than length",
- ByteBuffer.wrap("abc".getBytes("UTF-8")),
- trunc.apply(ByteBuffer.wrap("abc".getBytes("UTF-8"))));
+ ByteBuffer.wrap("abc".getBytes(StandardCharsets.UTF_8)),
+ trunc.apply(ByteBuffer.wrap("abc".getBytes(StandardCharsets.UTF_8))));
}
@Test
@@ -95,6 +112,6 @@ public class TestTruncate {
"Should fail if width is less than or equal to zero",
IllegalArgumentException.class,
"Invalid truncate width: 0 (must be > 0)",
- () -> Truncate.get(Types.IntegerType.get(), 0));
+ () -> Truncate.get(0));
}
}
diff --git a/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesProjection.java b/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesProjection.java
index 4cbabd4213..cc0ae0040a 100644
--- a/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesProjection.java
+++ b/api/src/test/java/org/apache/iceberg/transforms/TestTruncatesProjection.java
@@ -40,12 +40,14 @@ import org.apache.iceberg.expressions.Projections;
import org.apache.iceberg.expressions.UnboundPredicate;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
+import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.junit.Assert;
import org.junit.Test;
public class TestTruncatesProjection {
+ @SuppressWarnings("unchecked")
public void assertProjectionStrict(
PartitionSpec spec,
UnboundPredicate<?> filter,
@@ -60,19 +62,21 @@ public class TestTruncatesProjection {
Assert.assertNotEquals(
"Strict projection never runs for IN", Expression.Operation.IN, predicate.op());
- Truncate transform = (Truncate) spec.getFieldsBySourceId(1).get(0).transform();
+ Transform<Object, Object> transform =
+ (Transform<Object, Object>) spec.getFieldsBySourceId(1).get(0).transform();
+ Type type = spec.partitionType().field(spec.getFieldsBySourceId(1).get(0).fieldId()).type();
if (predicate.op() == Expression.Operation.NOT_IN) {
Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
String actual =
Lists.newArrayList(values).stream()
.sorted()
- .map(v -> transform.toHumanString(v))
+ .map(v -> transform.toHumanString(type, v))
.collect(Collectors.toList())
.toString();
Assert.assertEquals(expectedLiteral, actual);
} else {
- Literal literal = predicate.literal();
- String output = transform.toHumanString(literal.value());
+ Literal<?> literal = predicate.literal();
+ String output = transform.toHumanString(type, literal.value());
Assert.assertEquals(expectedLiteral, output);
}
}
@@ -91,6 +95,7 @@ public class TestTruncatesProjection {
Assert.assertEquals(projection.op(), expectedOp);
}
+ @SuppressWarnings("unchecked")
public void assertProjectionInclusive(
PartitionSpec spec,
UnboundPredicate<?> filter,
@@ -104,19 +109,21 @@ public class TestTruncatesProjection {
Assert.assertNotEquals(
"Inclusive projection never runs for NOT_IN", Expression.Operation.NOT_IN, predicate.op());
- Truncate transform = (Truncate) spec.getFieldsBySourceId(1).get(0).transform();
+ Transform<Object, Object> transform =
+ (Transform<Object, Object>) spec.getFieldsBySourceId(1).get(0).transform();
+ Type type = spec.partitionType().field(spec.getFieldsBySourceId(1).get(0).fieldId()).type();
if (predicate.op() == Expression.Operation.IN) {
Iterable<?> values = Iterables.transform(predicate.literals(), Literal::value);
String actual =
Lists.newArrayList(values).stream()
.sorted()
- .map(v -> transform.toHumanString(v))
+ .map(v -> transform.toHumanString(type, v))
.collect(Collectors.toList())
.toString();
Assert.assertEquals(expectedLiteral, actual);
} else {
- Literal literal = predicate.literal();
- String output = transform.toHumanString(literal.value());
+ Literal<?> literal = predicate.literal();
+ String output = transform.toHumanString(type, literal.value());
Assert.assertEquals(expectedLiteral, output);
}
}
diff --git a/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java b/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java
index 5bf8215dc9..31890843f2 100644
--- a/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java
+++ b/core/src/main/java/org/apache/iceberg/BaseMetadataTable.java
@@ -25,6 +25,7 @@ import org.apache.iceberg.encryption.EncryptionManager;
import org.apache.iceberg.io.FileIO;
import org.apache.iceberg.io.LocationProvider;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
+import org.apache.iceberg.transforms.Transforms;
/**
* Base class for metadata tables.
@@ -63,7 +64,9 @@ public abstract class BaseMetadataTable implements Table, HasTableOperations, Se
static PartitionSpec transformSpec(Schema metadataTableSchema, PartitionSpec spec) {
PartitionSpec.Builder identitySpecBuilder =
PartitionSpec.builderFor(metadataTableSchema).checkConflicts(false);
- spec.fields().forEach(pf -> identitySpecBuilder.add(pf.fieldId(), pf.name(), "identity"));
+ for (PartitionField field : spec.fields()) {
+ identitySpecBuilder.add(field.fieldId(), field.name(), Transforms.identity());
+ }
return identitySpecBuilder.build();
}
diff --git a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java
index 6f688848f7..11c74a045e 100644
--- a/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java
+++ b/core/src/main/java/org/apache/iceberg/BaseUpdatePartitionSpec.java
@@ -343,7 +343,7 @@ class BaseUpdatePartitionSpec implements UpdatePartitionSpec {
private Transform<?, ?> toTransform(BoundTerm<?> term) {
if (term instanceof BoundReference) {
- return Transforms.identity(term.type());
+ return Transforms.identity();
} else if (term instanceof BoundTransform) {
return ((BoundTransform<?, ?>) term).transform();
} else {
diff --git a/core/src/main/java/org/apache/iceberg/LocationProviders.java b/core/src/main/java/org/apache/iceberg/LocationProviders.java
index f22060a52a..61ad1c2a57 100644
--- a/core/src/main/java/org/apache/iceberg/LocationProviders.java
+++ b/core/src/main/java/org/apache/iceberg/LocationProviders.java
@@ -19,11 +19,11 @@
package org.apache.iceberg;
import java.util.Map;
+import java.util.function.Function;
import org.apache.hadoop.fs.Path;
import org.apache.iceberg.common.DynConstructors;
import org.apache.iceberg.io.LocationProvider;
import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
-import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.LocationUtil;
@@ -104,8 +104,8 @@ public class LocationProviders {
}
static class ObjectStoreLocationProvider implements LocationProvider {
- private static final Transform<String, Integer> HASH_FUNC =
- Transforms.bucket(Types.StringType.get(), Integer.MAX_VALUE);
+ private static final Function<Object, Integer> HASH_FUNC =
+ Transforms.bucket(Integer.MAX_VALUE).bind(Types.StringType.get());
private final String storageLocation;
private final String context;
diff --git a/core/src/main/java/org/apache/iceberg/ManifestsTable.java b/core/src/main/java/org/apache/iceberg/ManifestsTable.java
index 88f9943ddf..2168a5b1ff 100644
--- a/core/src/main/java/org/apache/iceberg/ManifestsTable.java
+++ b/core/src/main/java/org/apache/iceberg/ManifestsTable.java
@@ -130,12 +130,14 @@ public class ManifestsTable extends BaseMetadataTable {
.get(i)
.transform()
.toHumanString(
+ spec.partitionType().fields().get(i).type(),
Conversions.fromByteBuffer(
spec.partitionType().fields().get(i).type(), summary.lowerBound())),
spec.fields()
.get(i)
.transform()
.toHumanString(
+ spec.partitionType().fields().get(i).type(),
Conversions.fromByteBuffer(
spec.partitionType().fields().get(i).type(), summary.upperBound()))));
}
diff --git a/core/src/main/java/org/apache/iceberg/TableMetadata.java b/core/src/main/java/org/apache/iceberg/TableMetadata.java
index e449a26e42..0a566581ab 100644
--- a/core/src/main/java/org/apache/iceberg/TableMetadata.java
+++ b/core/src/main/java/org/apache/iceberg/TableMetadata.java
@@ -109,8 +109,7 @@ public class TableMetadata implements Serializable {
// look up the name of the source field in the old schema to get the new schema's id
String sourceName = schema.findColumnName(field.sourceId());
// reassign all partition fields with fresh partition field Ids to ensure consistency
- specBuilder.add(
- freshSchema.findField(sourceName).fieldId(), field.name(), field.transform().toString());
+ specBuilder.add(freshSchema.findField(sourceName).fieldId(), field.name(), field.transform());
}
PartitionSpec freshSpec = specBuilder.build();
diff --git a/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java b/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java
index fa9b8649ce..6b5c3bb9e0 100644
--- a/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java
+++ b/core/src/main/java/org/apache/iceberg/expressions/ExpressionParser.java
@@ -35,7 +35,6 @@ import org.apache.iceberg.relocated.com.google.common.base.Preconditions;
import org.apache.iceberg.relocated.com.google.common.collect.ImmutableList;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.transforms.Transforms;
-import org.apache.iceberg.types.Type;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.JsonUtil;
@@ -316,7 +315,7 @@ public class ExpressionParser {
@SuppressWarnings("unchecked")
private static <T> UnboundPredicate<T> predicateFromJson(
Expression.Operation op, JsonNode node, Schema schema) {
- UnboundTerm<T> term = term(JsonUtil.get(TERM, node), schema);
+ UnboundTerm<T> term = term(JsonUtil.get(TERM, node));
Function<JsonNode, T> convertValue;
if (schema != null) {
@@ -399,7 +398,7 @@ public class ExpressionParser {
}
@SuppressWarnings("unchecked")
- private static <T> UnboundTerm<T> term(JsonNode node, Schema schema) {
+ private static <T> UnboundTerm<T> term(JsonNode node) {
if (node.isTextual()) {
return Expressions.ref(node.asText());
} else if (node.isObject()) {
@@ -408,11 +407,10 @@ public class ExpressionParser {
case REFERENCE:
return Expressions.ref(JsonUtil.getString(TERM, node));
case TRANSFORM:
- UnboundTerm<T> child = term(JsonUtil.get(TERM, node), schema);
- Type termType = schema.findType(child.ref().name());
+ UnboundTerm<T> child = term(JsonUtil.get(TERM, node));
String transform = JsonUtil.getString(TRANSFORM, node);
return (UnboundTerm<T>)
- Expressions.transform(child.ref().name(), Transforms.fromString(termType, transform));
+ Expressions.transform(child.ref().name(), Transforms.fromString(transform));
default:
throw new IllegalArgumentException("Cannot parse type as a reference: " + type);
}
diff --git a/core/src/test/java/org/apache/iceberg/TestReplaceTransaction.java b/core/src/test/java/org/apache/iceberg/TestReplaceTransaction.java
index db13113486..fd01b5f7db 100644
--- a/core/src/test/java/org/apache/iceberg/TestReplaceTransaction.java
+++ b/core/src/test/java/org/apache/iceberg/TestReplaceTransaction.java
@@ -96,7 +96,7 @@ public class TestReplaceTransaction extends TableTestBase {
Assert.assertEquals("Direction must match ", ASC, sortOrder.fields().get(0).direction());
Assert.assertEquals(
"Null order must match ", NULLS_FIRST, sortOrder.fields().get(0).nullOrder());
- Transform<?, ?> transform = Transforms.identity(Types.IntegerType.get());
+ Transform<?, ?> transform = Transforms.identity();
Assert.assertEquals("Transform must match", transform, sortOrder.fields().get(0).transform());
}
diff --git a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java
index e6c7109f6a..ab42ebea4f 100644
--- a/core/src/test/java/org/apache/iceberg/TestTableMetadata.java
+++ b/core/src/test/java/org/apache/iceberg/TestTableMetadata.java
@@ -53,6 +53,7 @@ import org.apache.iceberg.relocated.com.google.common.collect.ImmutableMap;
import org.apache.iceberg.relocated.com.google.common.collect.Iterables;
import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
+import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Types;
import org.apache.iceberg.util.JsonUtil;
import org.assertj.core.api.Assertions;
@@ -1069,8 +1070,8 @@ public class TestTableMetadata {
PartitionSpec spec =
PartitionSpec.builderFor(schema)
.withSpecId(5)
- .add(3, 1005, "x_partition", "bucket[4]")
- .add(5, 1003, "z_partition", "bucket[8]")
+ .add(3, 1005, "x_partition", Transforms.bucket(4))
+ .add(5, 1003, "z_partition", Transforms.bucket(8))
.build();
String location = "file://tmp/db/table";
TableMetadata metadata =
@@ -1080,8 +1081,8 @@ public class TestTableMetadata {
PartitionSpec expected =
PartitionSpec.builderFor(metadata.schema())
.withSpecId(0)
- .add(1, 1000, "x_partition", "bucket[4]")
- .add(3, 1001, "z_partition", "bucket[8]")
+ .add(1, 1000, "x_partition", Transforms.bucket(4))
+ .add(3, 1001, "z_partition", Transforms.bucket(8))
.build();
Assert.assertEquals(expected, metadata.spec());
@@ -1094,7 +1095,7 @@ public class TestTableMetadata {
PartitionSpec spec =
PartitionSpec.builderFor(schema)
.withSpecId(5)
- .add(1, 1005, "x_partition", "bucket[4]")
+ .add(1, 1005, "x_partition", Transforms.bucket(4))
.build();
String location = "file://tmp/db/table";
TableMetadata metadata =
@@ -1135,9 +1136,9 @@ public class TestTableMetadata {
PartitionSpec expected =
PartitionSpec.builderFor(updated.schema())
.withSpecId(1)
- .add(1, 1000, "x", "identity")
- .add(2, 1001, "y", "void")
- .add(3, 1002, "z_bucket", "bucket[8]")
+ .add(1, 1000, "x", Transforms.identity())
+ .add(2, 1001, "y", Transforms.alwaysNull())
+ .add(3, 1002, "z_bucket", Transforms.bucket(8))
.build();
Assert.assertEquals(
"Should reassign the partition field IDs and reuse any existing IDs for equivalent fields",
@@ -1171,8 +1172,8 @@ public class TestTableMetadata {
PartitionSpec expected =
PartitionSpec.builderFor(updated.schema())
.withSpecId(1)
- .add(3, 1002, "z_bucket", "bucket[8]")
- .add(1, 1000, "x", "identity")
+ .add(3, 1002, "z_bucket", Transforms.bucket(8))
+ .add(1, 1000, "x", Transforms.identity())
.build();
Assert.assertEquals(
"Should reassign the partition field IDs and reuse any existing IDs for equivalent fields",
diff --git a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java
index 6c3414748a..f770cd2792 100644
--- a/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java
+++ b/core/src/test/java/org/apache/iceberg/TestTableUpdatePartitionSpec.java
@@ -21,6 +21,7 @@ package org.apache.iceberg;
import static org.apache.iceberg.expressions.Expressions.bucket;
import static org.apache.iceberg.expressions.Expressions.truncate;
+import org.apache.iceberg.transforms.Transforms;
import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
@@ -83,7 +84,7 @@ public class TestTableUpdatePartitionSpec extends TableTestBase {
"Should hard delete id and data buckets",
PartitionSpec.builderFor(table.schema())
.withSpecId(2)
- .add(2, 1002, "data_trunc_8", "truncate[8]")
+ .add(2, 1002, "data_trunc_8", Transforms.truncate(8))
.build(),
table.spec());
@@ -179,7 +180,7 @@ public class TestTableUpdatePartitionSpec extends TableTestBase {
"Should hard delete data bucket",
PartitionSpec.builderFor(table.schema())
.withSpecId(1)
- .add(1, 1001, "id_bucket_8", "bucket[8]")
+ .add(1, 1001, "id_bucket_8", Transforms.bucket(8))
.build(),
table.spec());
@@ -202,7 +203,7 @@ public class TestTableUpdatePartitionSpec extends TableTestBase {
"Should remove and then add a bucket field",
PartitionSpec.builderFor(table.schema())
.withSpecId(1)
- .add(2, 1001, "data_bucket_6", "bucket[6]")
+ .add(2, 1001, "data_bucket_6", Transforms.bucket(6))
.build(),
table.spec());
Assert.assertEquals(1001, table.spec().lastAssignedFieldId());
@@ -243,7 +244,7 @@ public class TestTableUpdatePartitionSpec extends TableTestBase {
"Should add a new id bucket",
PartitionSpec.builderFor(table.schema())
.withSpecId(2)
- .add(1, 1001, "id_bucket_8", "bucket[8]")
+ .add(1, 1001, "id_bucket_8", Transforms.bucket(8))
.build(),
table.spec());
Assert.assertEquals(1001, table.spec().lastAssignedFieldId());
diff --git a/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java b/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java
index a4f6f8518c..97210e51b5 100644
--- a/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java
+++ b/core/src/test/java/org/apache/iceberg/TestUpdatePartitionSpec.java
@@ -213,9 +213,8 @@ public class TestUpdatePartitionSpec extends TableTestBase {
Assert.assertEquals(
"Should have a day and an hour time field",
ImmutableList.of(
- new PartitionField(2, 1000, "ts_day", Transforms.day(Types.TimestampType.withZone())),
- new PartitionField(
- 2, 1001, "ts_hour", Transforms.hour(Types.TimestampType.withZone()))),
+ new PartitionField(2, 1000, "ts_day", Transforms.day()),
+ new PartitionField(2, 1001, "ts_hour", Transforms.hour())),
byHour.fields());
}
@@ -254,8 +253,8 @@ public class TestUpdatePartitionSpec extends TableTestBase {
PartitionSpec v2Expected =
PartitionSpec.builderFor(SCHEMA)
- .add(id("ts"), 1001, "ts_day", Transforms.day(Types.TimestampType.withZone()))
- .add(id("id"), 1002, "shard", Transforms.bucket(Types.LongType.get(), 16))
+ .add(id("ts"), 1001, "ts_day", Transforms.day())
+ .add(id("id"), 1002, "shard", Transforms.bucket(16))
.build();
V2Assert.assertEquals("Should match expected spec", v2Expected, updated);
@@ -277,8 +276,8 @@ public class TestUpdatePartitionSpec extends TableTestBase {
PartitionSpec v2Expected =
PartitionSpec.builderFor(SCHEMA)
- .add(id("category"), 1000, "category", Transforms.identity(Types.StringType.get()))
- .add(id("ts"), 1001, "ts_day", Transforms.day(Types.TimestampType.withZone()))
+ .add(id("category"), 1000, "category", Transforms.identity())
+ .add(id("ts"), 1001, "ts_day", Transforms.day())
.build();
V2Assert.assertEquals("Should match expected spec", v2Expected, updated);
@@ -302,8 +301,8 @@ public class TestUpdatePartitionSpec extends TableTestBase {
PartitionSpec v2Expected =
PartitionSpec.builderFor(SCHEMA)
- .add(id("ts"), 1001, "ts_day", Transforms.day(Types.TimestampType.withZone()))
- .add(id("id"), 1002, "shard", Transforms.bucket(Types.LongType.get(), 16))
+ .add(id("ts"), 1001, "ts_day", Transforms.day())
+ .add(id("id"), 1002, "shard", Transforms.bucket(16))
.build();
V2Assert.assertEquals("Should match expected spec", v2Expected, updated);
@@ -325,8 +324,8 @@ public class TestUpdatePartitionSpec extends TableTestBase {
PartitionSpec v2Expected =
PartitionSpec.builderFor(SCHEMA)
- .add(id("category"), 1000, "category", Transforms.identity(Types.StringType.get()))
- .add(id("id"), 1002, "shard", Transforms.bucket(Types.LongType.get(), 16))
+ .add(id("category"), 1000, "category", Transforms.identity())
+ .add(id("id"), 1002, "shard", Transforms.bucket(16))
.build();
V2Assert.assertEquals("Should match expected spec", v2Expected, updated);
@@ -388,9 +387,9 @@ public class TestUpdatePartitionSpec extends TableTestBase {
PartitionSpec v2Expected =
PartitionSpec.builderFor(SCHEMA)
- .add(id("category"), 1000, "category", Transforms.identity(Types.StringType.get()))
- .add(id("id"), 1002, "id_bucket", Transforms.bucket(Types.LongType.get(), 16))
- .add(id("data"), 1003, "prefix", Transforms.truncate(Types.StringType.get(), 4))
+ .add(id("category"), 1000, "category", Transforms.identity())
+ .add(id("id"), 1002, "id_bucket", Transforms.bucket(16))
+ .add(id("data"), 1003, "prefix", Transforms.truncate(4))
.build();
V2Assert.assertEquals("Should match expected spec", v2Expected, updated);
diff --git a/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java b/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java
index 7d5599919d..2f89f6875d 100644
--- a/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java
+++ b/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopCatalog.java
@@ -51,7 +51,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
-import org.apache.iceberg.types.Types;
import org.assertj.core.api.Assertions;
import org.junit.Assert;
import org.junit.Test;
@@ -181,7 +180,7 @@ public class TestHadoopCatalog extends HadoopTableTestBase {
Assert.assertEquals("Direction must match ", ASC, sortOrder.fields().get(0).direction());
Assert.assertEquals(
"Null order must match ", NULLS_FIRST, sortOrder.fields().get(0).nullOrder());
- Transform<?, ?> transform = Transforms.identity(Types.IntegerType.get());
+ Transform<?, ?> transform = Transforms.identity();
Assert.assertEquals("Transform must match", transform, sortOrder.fields().get(0).transform());
}
diff --git a/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopTables.java b/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopTables.java
index ef7c8252a6..e1cd275674 100644
--- a/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopTables.java
+++ b/core/src/test/java/org/apache/iceberg/hadoop/TestHadoopTables.java
@@ -143,7 +143,7 @@ public class TestHadoopTables {
Assert.assertEquals("Direction must match ", ASC, sortOrder.fields().get(0).direction());
Assert.assertEquals(
"Null order must match ", NULLS_FIRST, sortOrder.fields().get(0).nullOrder());
- Transform<?, ?> transform = Transforms.identity(Types.IntegerType.get());
+ Transform<?, ?> transform = Transforms.identity();
Assert.assertEquals("Transform must match", transform, sortOrder.fields().get(0).transform());
}
diff --git a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java
index 68016af8ed..87e97832a0 100644
--- a/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java
+++ b/core/src/test/java/org/apache/iceberg/jdbc/TestJdbcCatalog.java
@@ -250,7 +250,7 @@ public class TestJdbcCatalog extends CatalogTests<JdbcCatalog> {
Assert.assertEquals("Direction must match ", ASC, sortOrder.fields().get(0).direction());
Assert.assertEquals(
"Null order must match ", NULLS_FIRST, sortOrder.fields().get(0).nullOrder());
- Transform<?, ?> transform = Transforms.identity(Types.IntegerType.get());
+ Transform<?, ?> transform = Transforms.identity();
Assert.assertEquals("Transform must match", transform, sortOrder.fields().get(0).transform());
}
diff --git a/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java b/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java
index 8d0b94262a..13ff034e4b 100644
--- a/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java
+++ b/spark/v3.3/spark/src/jmh/java/org/apache/iceberg/spark/source/WritersBenchmark.java
@@ -87,8 +87,10 @@ public abstract class WritersBenchmark extends IcebergSourceBenchmark {
setupSpark();
List<InternalRow> data = Lists.newArrayList(RandomData.generateSpark(SCHEMA, NUM_ROWS, 0L));
- Transform<Integer, Integer> transform = Transforms.bucket(Types.IntegerType.get(), 32);
- data.sort(Comparator.comparingInt(row -> transform.apply(row.getInt(1))));
+ Transform<Integer, Integer> transform = Transforms.bucket(32);
+ data.sort(
+ Comparator.comparingInt(
+ row -> transform.bind(Types.IntegerType.get()).apply(row.getInt(1))));
this.rows = data;
this.positionDeleteRows =
diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/IcebergSpark.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/IcebergSpark.java
index 094364d229..eb2420c0b2 100644
--- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/IcebergSpark.java
+++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/IcebergSpark.java
@@ -18,7 +18,7 @@
*/
package org.apache.iceberg.spark;
-import org.apache.iceberg.transforms.Transform;
+import java.util.function.Function;
import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Type;
import org.apache.spark.sql.SparkSession;
@@ -32,7 +32,7 @@ public class IcebergSpark {
SparkSession session, String funcName, DataType sourceType, int numBuckets) {
SparkTypeToType typeConverter = new SparkTypeToType();
Type sourceIcebergType = typeConverter.atomic(sourceType);
- Transform<Object, Integer> bucket = Transforms.bucket(sourceIcebergType, numBuckets);
+ Function<Object, Integer> bucket = Transforms.bucket(numBuckets).bind(sourceIcebergType);
session
.udf()
.register(
@@ -45,7 +45,7 @@ public class IcebergSpark {
SparkSession session, String funcName, DataType sourceType, int width) {
SparkTypeToType typeConverter = new SparkTypeToType();
Type sourceIcebergType = typeConverter.atomic(sourceType);
- Transform<Object, Object> truncate = Transforms.truncate(sourceIcebergType, width);
+ Function<Object, Object> truncate = Transforms.truncate(width).bind(sourceIcebergType);
session
.udf()
.register(
diff --git a/spark/v3.3/spark/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala b/spark/v3.3/spark/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala
index 2d2c2fb9aa..dffac82af7 100644
--- a/spark/v3.3/spark/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala
+++ b/spark/v3.3/spark/src/main/scala/org/apache/spark/sql/catalyst/expressions/TransformExpressions.scala
@@ -22,6 +22,7 @@ package org.apache.spark.sql.catalyst.expressions
import java.nio.ByteBuffer
import java.nio.CharBuffer
import java.nio.charset.StandardCharsets
+import java.util.function
import org.apache.iceberg.spark.SparkSchemaUtil
import org.apache.iceberg.transforms.Transform
import org.apache.iceberg.transforms.Transforms
@@ -48,7 +49,7 @@ abstract class IcebergTransformExpression
abstract class IcebergTimeTransform
extends IcebergTransformExpression with ImplicitCastInputTypes {
- def transform: Transform[Any, Integer]
+ def transform: function.Function[Any, Integer]
override protected def nullSafeEval(value: Any): Any = {
transform(value).toInt
@@ -62,7 +63,7 @@ abstract class IcebergTimeTransform
case class IcebergYearTransform(child: Expression)
extends IcebergTimeTransform {
- @transient lazy val transform: Transform[Any, Integer] = Transforms.year[Any](icebergInputType)
+ @transient lazy val transform: function.Function[Any, Integer] = Transforms.year[Any]().bind(icebergInputType)
override protected def withNewChildInternal(newChild: Expression): Expression = {
copy(child = newChild)
@@ -72,7 +73,7 @@ case class IcebergYearTransform(child: Expression)
case class IcebergMonthTransform(child: Expression)
extends IcebergTimeTransform {
- @transient lazy val transform: Transform[Any, Integer] = Transforms.month[Any](icebergInputType)
+ @transient lazy val transform: function.Function[Any, Integer] = Transforms.month[Any]().bind(icebergInputType)
override protected def withNewChildInternal(newChild: Expression): Expression = {
copy(child = newChild)
@@ -82,7 +83,7 @@ case class IcebergMonthTransform(child: Expression)
case class IcebergDayTransform(child: Expression)
extends IcebergTimeTransform {
- @transient lazy val transform: Transform[Any, Integer] = Transforms.day[Any](icebergInputType)
+ @transient lazy val transform: function.Function[Any, Integer] = Transforms.day[Any]().bind(icebergInputType)
override protected def withNewChildInternal(newChild: Expression): Expression = {
copy(child = newChild)
@@ -92,7 +93,7 @@ case class IcebergDayTransform(child: Expression)
case class IcebergHourTransform(child: Expression)
extends IcebergTimeTransform {
- @transient lazy val transform: Transform[Any, Integer] = Transforms.hour[Any](icebergInputType)
+ @transient lazy val transform: function.Function[Any, Integer] = Transforms.hour[Any]().bind(icebergInputType)
override protected def withNewChildInternal(newChild: Expression): Expression = {
copy(child = newChild)
@@ -103,15 +104,15 @@ case class IcebergBucketTransform(numBuckets: Int, child: Expression) extends Ic
@transient lazy val bucketFunc: Any => Int = child.dataType match {
case _: DecimalType =>
- val t = Transforms.bucket[Any](icebergInputType, numBuckets)
+ val t = Transforms.bucket[Any](numBuckets).bind(icebergInputType)
d: Any => t(d.asInstanceOf[Decimal].toJavaBigDecimal).toInt
case _: StringType =>
// the spec requires that the hash of a string is equal to the hash of its UTF-8 encoded bytes
// TODO: pass bytes without the copy out of the InternalRow
- val t = Transforms.bucket[ByteBuffer](Types.BinaryType.get(), numBuckets)
+ val t = Transforms.bucket[ByteBuffer](numBuckets).bind(Types.BinaryType.get())
s: Any => t(ByteBuffer.wrap(s.asInstanceOf[UTF8String].getBytes)).toInt
case _ =>
- val t = Transforms.bucket[Any](icebergInputType, numBuckets)
+ val t = Transforms.bucket[Any](numBuckets).bind(icebergInputType)
a: Any => t(a).toInt
}
@@ -130,20 +131,20 @@ case class IcebergTruncateTransform(child: Expression, width: Int) extends Icebe
@transient lazy val truncateFunc: Any => Any = child.dataType match {
case _: DecimalType =>
- val t = Transforms.truncate[java.math.BigDecimal](icebergInputType, width)
+ val t = Transforms.truncate[java.math.BigDecimal](width).bind(icebergInputType)
d: Any => Decimal.apply(t(d.asInstanceOf[Decimal].toJavaBigDecimal))
case _: StringType =>
- val t = Transforms.truncate[CharSequence](icebergInputType, width)
+ val t = Transforms.truncate[CharSequence](width).bind(icebergInputType)
s: Any => {
val charSequence = t(StandardCharsets.UTF_8.decode(ByteBuffer.wrap(s.asInstanceOf[UTF8String].getBytes)))
val bb = StandardCharsets.UTF_8.encode(CharBuffer.wrap(charSequence));
UTF8String.fromBytes(ByteBuffers.toByteArray(bb))
}
case _: BinaryType =>
- val t = Transforms.truncate[ByteBuffer](icebergInputType, width)
+ val t = Transforms.truncate[ByteBuffer](width).bind(icebergInputType)
s: Any => ByteBuffers.toByteArray(t(ByteBuffer.wrap(s.asInstanceOf[Array[Byte]])))
case _ =>
- val t = Transforms.truncate[Any](icebergInputType, width)
+ val t = Transforms.truncate[Any](width).bind(icebergInputType)
a: Any => t(a)
}
diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java
index b30bbf145f..a2b4837ec2 100644
--- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java
+++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestFilteredScan.java
@@ -30,6 +30,7 @@ import java.time.OffsetDateTime;
import java.util.List;
import java.util.Locale;
import java.util.UUID;
+import java.util.function.Function;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
import org.apache.iceberg.DataFile;
@@ -48,7 +49,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Lists;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
import org.apache.iceberg.spark.SparkReadOptions;
import org.apache.iceberg.spark.data.GenericsHelpers;
-import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Types;
import org.apache.spark.sql.Dataset;
@@ -116,23 +116,23 @@ public class TestFilteredScan {
TestFilteredScan.spark = SparkSession.builder().master("local[2]").getOrCreate();
// define UDFs used by partition tests
- Transform<Long, Integer> bucket4 = Transforms.bucket(Types.LongType.get(), 4);
+ Function<Object, Integer> bucket4 = Transforms.bucket(4).bind(Types.LongType.get());
spark.udf().register("bucket4", (UDF1<Long, Integer>) bucket4::apply, IntegerType$.MODULE$);
- Transform<Long, Integer> day = Transforms.day(Types.TimestampType.withZone());
+ Function<Object, Integer> day = Transforms.day().bind(Types.TimestampType.withZone());
spark
.udf()
.register(
"ts_day",
- (UDF1<Timestamp, Integer>) timestamp -> day.apply((Long) fromJavaTimestamp(timestamp)),
+ (UDF1<Timestamp, Integer>) timestamp -> day.apply(fromJavaTimestamp(timestamp)),
IntegerType$.MODULE$);
- Transform<Long, Integer> hour = Transforms.hour(Types.TimestampType.withZone());
+ Function<Object, Integer> hour = Transforms.hour().bind(Types.TimestampType.withZone());
spark
.udf()
.register(
"ts_hour",
- (UDF1<Timestamp, Integer>) timestamp -> hour.apply((Long) fromJavaTimestamp(timestamp)),
+ (UDF1<Timestamp, Integer>) timestamp -> hour.apply(fromJavaTimestamp(timestamp)),
IntegerType$.MODULE$);
spark.udf().register("data_ident", (UDF1<String, String>) data -> data, StringType$.MODULE$);
diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java
index 0be1e0b1bd..37e329a8b9 100644
--- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java
+++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestIcebergSpark.java
@@ -61,7 +61,8 @@ public class TestIcebergSpark {
List<Row> results = spark.sql("SELECT iceberg_bucket_int_16(1)").collectAsList();
Assert.assertEquals(1, results.size());
Assert.assertEquals(
- (int) Transforms.bucket(Types.IntegerType.get(), 16).apply(1), results.get(0).getInt(0));
+ (int) Transforms.bucket(16).bind(Types.IntegerType.get()).apply(1),
+ results.get(0).getInt(0));
}
@Test
@@ -70,7 +71,8 @@ public class TestIcebergSpark {
List<Row> results = spark.sql("SELECT iceberg_bucket_short_16(1S)").collectAsList();
Assert.assertEquals(1, results.size());
Assert.assertEquals(
- (int) Transforms.bucket(Types.IntegerType.get(), 16).apply(1), results.get(0).getInt(0));
+ (int) Transforms.bucket(16).bind(Types.IntegerType.get()).apply(1),
+ results.get(0).getInt(0));
}
@Test
@@ -79,7 +81,8 @@ public class TestIcebergSpark {
List<Row> results = spark.sql("SELECT iceberg_bucket_byte_16(1Y)").collectAsList();
Assert.assertEquals(1, results.size());
Assert.assertEquals(
- (int) Transforms.bucket(Types.IntegerType.get(), 16).apply(1), results.get(0).getInt(0));
+ (int) Transforms.bucket(16).bind(Types.IntegerType.get()).apply(1),
+ results.get(0).getInt(0));
}
@Test
@@ -88,7 +91,7 @@ public class TestIcebergSpark {
List<Row> results = spark.sql("SELECT iceberg_bucket_long_16(1L)").collectAsList();
Assert.assertEquals(1, results.size());
Assert.assertEquals(
- (int) Transforms.bucket(Types.LongType.get(), 16).apply(1L), results.get(0).getInt(0));
+ (int) Transforms.bucket(16).bind(Types.LongType.get()).apply(1L), results.get(0).getInt(0));
}
@Test
@@ -97,7 +100,7 @@ public class TestIcebergSpark {
List<Row> results = spark.sql("SELECT iceberg_bucket_string_16('hello')").collectAsList();
Assert.assertEquals(1, results.size());
Assert.assertEquals(
- (int) Transforms.bucket(Types.StringType.get(), 16).apply("hello"),
+ (int) Transforms.bucket(16).bind(Types.StringType.get()).apply("hello"),
results.get(0).getInt(0));
}
@@ -107,7 +110,7 @@ public class TestIcebergSpark {
List<Row> results = spark.sql("SELECT iceberg_bucket_char_16('hello')").collectAsList();
Assert.assertEquals(1, results.size());
Assert.assertEquals(
- (int) Transforms.bucket(Types.StringType.get(), 16).apply("hello"),
+ (int) Transforms.bucket(16).bind(Types.StringType.get()).apply("hello"),
results.get(0).getInt(0));
}
@@ -117,7 +120,7 @@ public class TestIcebergSpark {
List<Row> results = spark.sql("SELECT iceberg_bucket_varchar_16('hello')").collectAsList();
Assert.assertEquals(1, results.size());
Assert.assertEquals(
- (int) Transforms.bucket(Types.StringType.get(), 16).apply("hello"),
+ (int) Transforms.bucket(16).bind(Types.StringType.get()).apply("hello"),
results.get(0).getInt(0));
}
@@ -129,7 +132,8 @@ public class TestIcebergSpark {
Assert.assertEquals(1, results.size());
Assert.assertEquals(
(int)
- Transforms.bucket(Types.DateType.get(), 16)
+ Transforms.bucket(16)
+ .bind(Types.DateType.get())
.apply(DateTimeUtils.fromJavaDate(Date.valueOf("2021-06-30"))),
results.get(0).getInt(0));
}
@@ -145,7 +149,8 @@ public class TestIcebergSpark {
Assert.assertEquals(1, results.size());
Assert.assertEquals(
(int)
- Transforms.bucket(Types.TimestampType.withZone(), 16)
+ Transforms.bucket(16)
+ .bind(Types.TimestampType.withZone())
.apply(
DateTimeUtils.fromJavaTimestamp(Timestamp.valueOf("2021-06-30 00:00:00.000"))),
results.get(0).getInt(0));
@@ -158,7 +163,8 @@ public class TestIcebergSpark {
Assert.assertEquals(1, results.size());
Assert.assertEquals(
(int)
- Transforms.bucket(Types.BinaryType.get(), 16)
+ Transforms.bucket(16)
+ .bind(Types.BinaryType.get())
.apply(ByteBuffer.wrap(new byte[] {0x00, 0x20, 0x00, 0x1F})),
results.get(0).getInt(0));
}
@@ -169,7 +175,7 @@ public class TestIcebergSpark {
List<Row> results = spark.sql("SELECT iceberg_bucket_decimal_16(11.11)").collectAsList();
Assert.assertEquals(1, results.size());
Assert.assertEquals(
- (int) Transforms.bucket(Types.DecimalType.of(4, 2), 16).apply(new BigDecimal("11.11")),
+ (int) Transforms.bucket(16).bind(Types.DecimalType.of(4, 2)).apply(new BigDecimal("11.11")),
results.get(0).getInt(0));
}
@@ -209,7 +215,7 @@ public class TestIcebergSpark {
List<Row> results = spark.sql("SELECT iceberg_truncate_int_4(1)").collectAsList();
Assert.assertEquals(1, results.size());
Assert.assertEquals(
- Transforms.truncate(Types.IntegerType.get(), 4).apply(1), results.get(0).getInt(0));
+ Transforms.truncate(4).bind(Types.IntegerType.get()).apply(1), results.get(0).getInt(0));
}
@Test
@@ -218,7 +224,7 @@ public class TestIcebergSpark {
List<Row> results = spark.sql("SELECT iceberg_truncate_long_4(1L)").collectAsList();
Assert.assertEquals(1, results.size());
Assert.assertEquals(
- Transforms.truncate(Types.LongType.get(), 4).apply(1L), results.get(0).getLong(0));
+ Transforms.truncate(4).bind(Types.LongType.get()).apply(1L), results.get(0).getLong(0));
}
@Test
@@ -227,7 +233,7 @@ public class TestIcebergSpark {
List<Row> results = spark.sql("SELECT iceberg_truncate_decimal_4(11.11)").collectAsList();
Assert.assertEquals(1, results.size());
Assert.assertEquals(
- Transforms.truncate(Types.DecimalType.of(4, 2), 4).apply(new BigDecimal("11.11")),
+ Transforms.truncate(4).bind(Types.DecimalType.of(4, 2)).apply(new BigDecimal("11.11")),
results.get(0).getDecimal(0));
}
@@ -237,6 +243,7 @@ public class TestIcebergSpark {
List<Row> results = spark.sql("SELECT iceberg_truncate_string_4('hello')").collectAsList();
Assert.assertEquals(1, results.size());
Assert.assertEquals(
- Transforms.truncate(Types.StringType.get(), 4).apply("hello"), results.get(0).getString(0));
+ Transforms.truncate(4).bind(Types.StringType.get()).apply("hello"),
+ results.get(0).getString(0));
}
}
diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java
index ffe21432f0..4ef022c50c 100644
--- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java
+++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionPruning.java
@@ -29,6 +29,7 @@ import java.util.Map;
import java.util.Random;
import java.util.Set;
import java.util.concurrent.TimeUnit;
+import java.util.function.Function;
import java.util.function.Predicate;
import java.util.stream.Collectors;
import org.apache.hadoop.conf.Configuration;
@@ -47,7 +48,6 @@ import org.apache.iceberg.relocated.com.google.common.collect.Maps;
import org.apache.iceberg.relocated.com.google.common.collect.Sets;
import org.apache.iceberg.spark.SparkReadOptions;
import org.apache.iceberg.spark.SparkSchemaUtil;
-import org.apache.iceberg.transforms.Transform;
import org.apache.iceberg.transforms.Transforms;
import org.apache.iceberg.types.Types;
import org.apache.spark.api.java.JavaRDD;
@@ -96,12 +96,12 @@ public class TestPartitionPruning {
private static SparkSession spark = null;
private static JavaSparkContext sparkContext = null;
- private static Transform<Object, Integer> bucketTransform =
- Transforms.bucket(Types.IntegerType.get(), 3);
- private static Transform<Object, Object> truncateTransform =
- Transforms.truncate(Types.StringType.get(), 5);
- private static Transform<Object, Integer> hourTransform =
- Transforms.hour(Types.TimestampType.withoutZone());
+ private static final Function<Object, Integer> BUCKET_FUNC =
+ Transforms.bucket(3).bind(Types.IntegerType.get());
+ private static final Function<Object, Object> TRUNCATE_FUNC =
+ Transforms.truncate(5).bind(Types.StringType.get());
+ private static final Function<Object, Integer> HOUR_FUNC =
+ Transforms.hour().bind(Types.TimestampType.withoutZone());
@BeforeClass
public static void startSpark() {
@@ -112,19 +112,17 @@ public class TestPartitionPruning {
CONF.set(optionKey, CountOpenLocalFileSystem.class.getName());
spark.conf().set(optionKey, CountOpenLocalFileSystem.class.getName());
spark.conf().set("spark.sql.session.timeZone", "UTC");
+ spark.udf().register("bucket3", (Integer num) -> BUCKET_FUNC.apply(num), DataTypes.IntegerType);
spark
.udf()
- .register("bucket3", (Integer num) -> bucketTransform.apply(num), DataTypes.IntegerType);
- spark
- .udf()
- .register("truncate5", (String str) -> truncateTransform.apply(str), DataTypes.StringType);
+ .register("truncate5", (String str) -> TRUNCATE_FUNC.apply(str), DataTypes.StringType);
// NOTE: date transforms take the type long, not Timestamp
spark
.udf()
.register(
"hour",
(Timestamp ts) ->
- hourTransform.apply(
+ HOUR_FUNC.apply(
org.apache.spark.sql.catalyst.util.DateTimeUtils.fromJavaTimestamp(ts)),
DataTypes.IntegerType);
}
@@ -197,7 +195,7 @@ public class TestPartitionPruning {
(Row r) -> {
int bucketId = r.getInt(2);
Set<Integer> buckets =
- Arrays.stream(ids).map(bucketTransform::apply).boxed().collect(Collectors.toSet());
+ Arrays.stream(ids).map(BUCKET_FUNC::apply).boxed().collect(Collectors.toSet());
return buckets.contains(bucketId);
};
@@ -243,7 +241,7 @@ public class TestPartitionPruning {
int hourValue = r.getInt(4);
Instant instant = getInstant("2020-02-03T01:00:00");
Integer hourValueToFilter =
- hourTransform.apply(TimeUnit.MILLISECONDS.toMicros(instant.toEpochMilli()));
+ HOUR_FUNC.apply(TimeUnit.MILLISECONDS.toMicros(instant.toEpochMilli()));
return hourValue >= hourValueToFilter;
};