You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2020/05/25 17:38:55 UTC
[incubator-iceberg] branch master updated: Fix conversion accessors
for nested fields (#1013)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 50461d8 Fix conversion accessors for nested fields (#1013)
50461d8 is described below
commit 50461d86731e17473de47a2678a283fde9ca7e98
Author: Xiang Li <wa...@gmail.com>
AuthorDate: Tue May 26 01:38:48 2020 +0800
Fix conversion accessors for nested fields (#1013)
Accessors that convert values were not correctly wrapped when nested because they extend PositionAccessor, which is replaced when nested by Position2Accessor or similar classes. The short-term fix is to check that the accessor is PositionAccessor, not a subclass. Long term, conversion should not be done in accessors.
---
.../main/java/org/apache/iceberg/Accessors.java | 2 +-
.../apache/iceberg/spark/source/PartitionKey.java | 2 +-
.../iceberg/spark/source/TestPartitionValues.java | 58 ++++++++++++++++++++++
3 files changed, 60 insertions(+), 2 deletions(-)
diff --git a/api/src/main/java/org/apache/iceberg/Accessors.java b/api/src/main/java/org/apache/iceberg/Accessors.java
index 0e0d275..aef8e1d 100644
--- a/api/src/main/java/org/apache/iceberg/Accessors.java
+++ b/api/src/main/java/org/apache/iceberg/Accessors.java
@@ -178,7 +178,7 @@ public class Accessors {
if (isOptional) {
// the wrapped position handles null layers
return new WrappedPositionAccessor(pos, accessor);
- } else if (accessor instanceof PositionAccessor) {
+ } else if (accessor.getClass() == PositionAccessor.class) {
return new Position2Accessor(pos, (PositionAccessor) accessor);
} else if (accessor instanceof Position2Accessor) {
return new Position3Accessor(pos, (Position2Accessor) accessor);
diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/PartitionKey.java b/spark/src/main/java/org/apache/iceberg/spark/source/PartitionKey.java
index 0984237..54903d2 100644
--- a/spark/src/main/java/org/apache/iceberg/spark/source/PartitionKey.java
+++ b/spark/src/main/java/org/apache/iceberg/spark/source/PartitionKey.java
@@ -182,7 +182,7 @@ class PartitionKey implements StructLike {
if (isOptional) {
// the wrapped position handles null layers
return new WrappedPositionAccessor(position, size, accessor);
- } else if (accessor instanceof PositionAccessor) {
+ } else if (accessor.getClass() == PositionAccessor.class) {
return new Position2Accessor(position, size, (PositionAccessor) accessor);
} else if (accessor instanceof Position2Accessor) {
return new Position3Accessor(position, size, (Position2Accessor) accessor);
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java
index 44686d6..c8417fb 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java
@@ -21,6 +21,7 @@ package org.apache.iceberg.spark.source;
import com.google.common.collect.Lists;
import java.io.File;
+import java.util.ArrayList;
import java.util.List;
import org.apache.avro.generic.GenericData;
import org.apache.iceberg.DataFiles;
@@ -38,7 +39,12 @@ import org.apache.iceberg.types.Types;
import org.apache.spark.sql.Dataset;
import org.apache.spark.sql.Encoders;
import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
@@ -373,4 +379,56 @@ public class TestPartitionValues {
TestTables.clearTables();
}
}
+
+ /**
+ * To verify if WrappedPositionAccessor is generated against a string field within a nested field,
+ * rather than a Position2Accessor.
+ * Or when building the partition path, a ClassCastException is thrown with the message like:
+ * Cannot cast org.apache.spark.unsafe.types.UTF8String to java.lang.CharSequence
+ */
+ @Test
+ public void testPartitionedByNestedString() throws Exception {
+ // schema and partition spec
+ Schema nestedSchema = new Schema(
+ Types.NestedField.required(1, "struct",
+ Types.StructType.of(Types.NestedField.required(2, "string", Types.StringType.get()))
+ )
+ );
+ PartitionSpec spec = PartitionSpec.builderFor(nestedSchema).identity("struct.string").build();
+
+ // create table
+ HadoopTables tables = new HadoopTables(spark.sessionState().newHadoopConf());
+ String baseLocation = temp.newFolder("partition_by_nested_string").toString();
+ tables.create(nestedSchema, spec, baseLocation);
+
+ // input data frame
+ StructField[] structFields = {
+ new StructField("struct",
+ DataTypes.createStructType(
+ new StructField[] {
+ new StructField("string", DataTypes.StringType, false, Metadata.empty())
+ }
+ ),
+ false, Metadata.empty()
+ )
+ };
+
+ List<Row> rows = new ArrayList<>();
+ rows.add(RowFactory.create(RowFactory.create("nested_string_value")));
+ Dataset<Row> sourceDF = spark.createDataFrame(rows, new StructType(structFields));
+
+ // write into iceberg
+ sourceDF.write()
+ .format("iceberg")
+ .mode("append")
+ .save(baseLocation);
+
+ // verify
+ List<Row> actual = spark.read()
+ .format("iceberg")
+ .load(baseLocation)
+ .collectAsList();
+
+ Assert.assertEquals("Number of rows should match", rows.size(), actual.size());
+ }
}