You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2020/05/25 17:38:55 UTC

[incubator-iceberg] branch master updated: Fix conversion accessors for nested fields (#1013)

This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-iceberg.git


The following commit(s) were added to refs/heads/master by this push:
     new 50461d8  Fix conversion accessors for nested fields (#1013)
50461d8 is described below

commit 50461d86731e17473de47a2678a283fde9ca7e98
Author: Xiang Li <wa...@gmail.com>
AuthorDate: Tue May 26 01:38:48 2020 +0800

    Fix conversion accessors for nested fields (#1013)
    
    Accessors that convert values were not correctly wrapped when nested because they extend PositionAccessor, which is replaced when nested by Position2Accessor or similar classes. The short-term fix is to check that the accessor is PositionAccessor, not a subclass. Long term, conversion should not be done in accessors.
---
 .../main/java/org/apache/iceberg/Accessors.java    |  2 +-
 .../apache/iceberg/spark/source/PartitionKey.java  |  2 +-
 .../iceberg/spark/source/TestPartitionValues.java  | 58 ++++++++++++++++++++++
 3 files changed, 60 insertions(+), 2 deletions(-)

diff --git a/api/src/main/java/org/apache/iceberg/Accessors.java b/api/src/main/java/org/apache/iceberg/Accessors.java
index 0e0d275..aef8e1d 100644
--- a/api/src/main/java/org/apache/iceberg/Accessors.java
+++ b/api/src/main/java/org/apache/iceberg/Accessors.java
@@ -178,7 +178,7 @@ public class Accessors {
     if (isOptional) {
       // the wrapped position handles null layers
       return new WrappedPositionAccessor(pos, accessor);
-    } else if (accessor instanceof PositionAccessor) {
+    } else if (accessor.getClass() == PositionAccessor.class) {
       return new Position2Accessor(pos, (PositionAccessor) accessor);
     } else if (accessor instanceof Position2Accessor) {
       return new Position3Accessor(pos, (Position2Accessor) accessor);
diff --git a/spark/src/main/java/org/apache/iceberg/spark/source/PartitionKey.java b/spark/src/main/java/org/apache/iceberg/spark/source/PartitionKey.java
index 0984237..54903d2 100644
--- a/spark/src/main/java/org/apache/iceberg/spark/source/PartitionKey.java
+++ b/spark/src/main/java/org/apache/iceberg/spark/source/PartitionKey.java
@@ -182,7 +182,7 @@ class PartitionKey implements StructLike {
     if (isOptional) {
       // the wrapped position handles null layers
       return new WrappedPositionAccessor(position, size, accessor);
-    } else if (accessor instanceof PositionAccessor) {
+    } else if (accessor.getClass() == PositionAccessor.class) {
       return new Position2Accessor(position, size, (PositionAccessor) accessor);
     } else if (accessor instanceof Position2Accessor) {
       return new Position3Accessor(position, size, (Position2Accessor) accessor);
diff --git a/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java b/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java
index 44686d6..c8417fb 100644
--- a/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java
+++ b/spark/src/test/java/org/apache/iceberg/spark/source/TestPartitionValues.java
@@ -21,6 +21,7 @@ package org.apache.iceberg.spark.source;
 
 import com.google.common.collect.Lists;
 import java.io.File;
+import java.util.ArrayList;
 import java.util.List;
 import org.apache.avro.generic.GenericData;
 import org.apache.iceberg.DataFiles;
@@ -38,7 +39,12 @@ import org.apache.iceberg.types.Types;
 import org.apache.spark.sql.Dataset;
 import org.apache.spark.sql.Encoders;
 import org.apache.spark.sql.Row;
+import org.apache.spark.sql.RowFactory;
 import org.apache.spark.sql.SparkSession;
+import org.apache.spark.sql.types.DataTypes;
+import org.apache.spark.sql.types.Metadata;
+import org.apache.spark.sql.types.StructField;
+import org.apache.spark.sql.types.StructType;
 import org.junit.AfterClass;
 import org.junit.Assert;
 import org.junit.BeforeClass;
@@ -373,4 +379,56 @@ public class TestPartitionValues {
       TestTables.clearTables();
     }
   }
+
+  /**
+   * To verify if WrappedPositionAccessor is generated against a string field within a nested field,
+   * rather than a Position2Accessor.
+   * Or when building the partition path, a ClassCastException is thrown with the message like:
+   * Cannot cast org.apache.spark.unsafe.types.UTF8String to java.lang.CharSequence
+   */
+  @Test
+  public void testPartitionedByNestedString() throws Exception {
+    // schema and partition spec
+    Schema nestedSchema = new Schema(
+        Types.NestedField.required(1, "struct",
+            Types.StructType.of(Types.NestedField.required(2, "string", Types.StringType.get()))
+        )
+    );
+    PartitionSpec spec = PartitionSpec.builderFor(nestedSchema).identity("struct.string").build();
+
+    // create table
+    HadoopTables tables = new HadoopTables(spark.sessionState().newHadoopConf());
+    String baseLocation = temp.newFolder("partition_by_nested_string").toString();
+    tables.create(nestedSchema, spec, baseLocation);
+
+    // input data frame
+    StructField[] structFields = {
+        new StructField("struct",
+            DataTypes.createStructType(
+                new StructField[] {
+                    new StructField("string", DataTypes.StringType, false, Metadata.empty())
+                }
+            ),
+            false, Metadata.empty()
+        )
+    };
+
+    List<Row> rows = new ArrayList<>();
+    rows.add(RowFactory.create(RowFactory.create("nested_string_value")));
+    Dataset<Row> sourceDF = spark.createDataFrame(rows, new StructType(structFields));
+
+    // write into iceberg
+    sourceDF.write()
+        .format("iceberg")
+        .mode("append")
+        .save(baseLocation);
+
+    // verify
+    List<Row> actual = spark.read()
+        .format("iceberg")
+        .load(baseLocation)
+        .collectAsList();
+
+    Assert.assertEquals("Number of rows should match", rows.size(), actual.size());
+  }
 }