You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2022/04/03 20:31:49 UTC
[iceberg] branch master updated: Spark: Add identifier fields to Spark table properties (#4475)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new 8ac7bf133 Spark: Add identifier fields to Spark table properties (#4475)
8ac7bf133 is described below
commit 8ac7bf13346fd9d78249c1735a30bd74d3e158fd
Author: Mingliang Liu <li...@apache.org>
AuthorDate: Sun Apr 3 13:31:45 2022 -0700
Spark: Add identifier fields to Spark table properties (#4475)
---
.../src/main/java/org/apache/iceberg/spark/source/SparkTable.java | 7 ++++++-
.../java/org/apache/iceberg/spark/actions/TestCreateActions.java | 4 ++++
2 files changed, 10 insertions(+), 1 deletion(-)
diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
index cfa390221..0f43dd068 100644
--- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
+++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/source/SparkTable.java
@@ -77,7 +77,7 @@ public class SparkTable implements org.apache.spark.sql.connector.catalog.Table,
private static final Logger LOG = LoggerFactory.getLogger(SparkTable.class);
private static final Set<String> RESERVED_PROPERTIES =
- ImmutableSet.of("provider", "format", "current-snapshot-id", "location", "sort-order");
+ ImmutableSet.of("provider", "format", "current-snapshot-id", "location", "sort-order", "identifier-fields");
private static final Set<TableCapability> CAPABILITIES = ImmutableSet.of(
TableCapability.BATCH_READ,
TableCapability.BATCH_WRITE,
@@ -164,6 +164,11 @@ public class SparkTable implements org.apache.spark.sql.connector.catalog.Table,
propsBuilder.put("sort-order", Spark3Util.describe(icebergTable.sortOrder()));
}
+ Set<String> identifierFields = icebergTable.schema().identifierFieldNames();
+ if (!identifierFields.isEmpty()) {
+ propsBuilder.put("identifier-fields", "[" + String.join(",", identifierFields) + "]");
+ }
+
icebergTable.properties().entrySet().stream()
.filter(entry -> !RESERVED_PROPERTIES.contains(entry.getKey()))
.forEach(propsBuilder::put);
diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestCreateActions.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestCreateActions.java
index 8dd363422..8c475bf97 100644
--- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestCreateActions.java
+++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/actions/TestCreateActions.java
@@ -492,6 +492,10 @@ public class TestCreateActions extends SparkCatalogTestBase {
Assert.assertTrue("Location isn't correct", table.properties().get("location").endsWith(destTableName));
Assert.assertEquals("Sort-order isn't correct", "id ASC NULLS FIRST, data DESC NULLS LAST",
table.properties().get("sort-order"));
+ Assert.assertNull("Identifier fields should be null", table.properties().get("identifier-fields"));
+
+ table.table().updateSchema().allowIncompatibleChanges().requireColumn("id").setIdentifierFields("id").commit();
+ Assert.assertEquals("Identifier fields aren't correct", "[id]", table.properties().get("identifier-fields"));
}
@Test