You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2022/04/03 20:31:49 UTC

[iceberg] branch master updated: Spark: Add identifier fields to Spark table properties (#4475)

This is an automated email from the ASF dual-hosted git repository.

blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git


The following commit(s) were added to refs/heads/master by this push:
     new 8ac7bf133 Spark: Add identifier fields to Spark table properties (#4475)
8ac7bf133 is described below

commit 8ac7bf13346fd9d78249c1735a30bd74d3e158fd
Author: Mingliang Liu <li...@apache.org>
AuthorDate: Sun Apr 3 13:31:45 2022 -0700

    Spark: Add identifier fields to Spark table properties (#4475)
---
 .../src/main/java/org/apache/iceberg/spark/source/SparkTable.java  | 7 ++++++-
 .../java/org/apache/iceberg/spark/actions/TestCreateActions.java   | 4 ++++
 2 files changed, 10 insertions(+), 1 deletion(-)

diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
index cfa390221..0f43dd068 100644
--- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
+++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
@@ -77,7 +77,7 @@ public class SparkTable implements org.apache.spark.sql.connector.catalog.Table,
   private static final Logger LOG = LoggerFactory.getLogger(SparkTable.class);
 
   private static final Set<String> RESERVED_PROPERTIES =
-      ImmutableSet.of("provider", "format", "current-snapshot-id", "location", "sort-order");
+      ImmutableSet.of("provider", "format", "current-snapshot-id", "location", "sort-order", "identifier-fields");
   private static final Set<TableCapability> CAPABILITIES = ImmutableSet.of(
       TableCapability.BATCH_READ,
       TableCapability.BATCH_WRITE,
@@ -164,6 +164,11 @@ public class SparkTable implements org.apache.spark.sql.connector.catalog.Table,
       propsBuilder.put("sort-order", Spark3Util.describe(icebergTable.sortOrder()));
     }
 
+    Set<String> identifierFields = icebergTable.schema().identifierFieldNames();
+    if (!identifierFields.isEmpty()) {
+      propsBuilder.put("identifier-fields", "[" + String.join(",", identifierFields) + "]");
+    }
+
     icebergTable.properties().entrySet().stream()
         .filter(entry -> !RESERVED_PROPERTIES.contains(entry.getKey()))
         .forEach(propsBuilder::put);
diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestCreateActions.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestCreateActions.java
index 8dd363422..8c475bf97 100644
--- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestCreateActions.java
+++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestCreateActions.java
@@ -492,6 +492,10 @@ public class TestCreateActions extends SparkCatalogTestBase {
     Assert.assertTrue("Location isn't correct", table.properties().get("location").endsWith(destTableName));
     Assert.assertEquals("Sort-order isn't correct", "id ASC NULLS FIRST, data DESC NULLS LAST",
         table.properties().get("sort-order"));
+    Assert.assertNull("Identifier fields should be null", table.properties().get("identifier-fields"));
+
+    table.table().updateSchema().allowIncompatibleChanges().requireColumn("id").setIdentifierFields("id").commit();
+    Assert.assertEquals("Identifier fields aren't correct", "[id]", table.properties().get("identifier-fields"));
   }
 
   @Test