You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@iceberg.apache.org by bl...@apache.org on 2022/07/22 17:21:33 UTC
[iceberg] branch master updated: Spark: Support partition transforms with using any case (#5335)
This is an automated email from the ASF dual-hosted git repository.
blue pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/iceberg.git
The following commit(s) were added to refs/heads/master by this push:
new be9c491b7a Spark: Support partition transforms with using any case (#5335)
be9c491b7a is described below
commit be9c491b7a5a589523d6342a54f73a3613567e35
Author: southernriver <so...@users.noreply.github.com>
AuthorDate: Sat Jul 23 01:21:28 2022 +0800
Spark: Support partition transforms with using any case (#5335)
---
.../src/main/java/org/apache/iceberg/spark/Spark3Util.java | 5 +++--
.../java/org/apache/iceberg/spark/sql/TestCreateTable.java | 11 +++++++++++
.../src/main/java/org/apache/iceberg/spark/Spark3Util.java | 5 +++--
.../java/org/apache/iceberg/spark/sql/TestCreateTable.java | 11 +++++++++++
.../src/main/java/org/apache/iceberg/spark/Spark3Util.java | 5 +++--
.../java/org/apache/iceberg/spark/sql/TestCreateTable.java | 11 +++++++++++
.../src/main/java/org/apache/iceberg/spark/Spark3Util.java | 5 +++--
.../java/org/apache/iceberg/spark/sql/TestCreateTable.java | 11 +++++++++++
8 files changed, 56 insertions(+), 8 deletions(-)
diff --git a/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
index 7ba3bbbac2..c2c3c69929 100644
--- a/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
+++ b/spark/v3.0/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -351,7 +352,7 @@ public class Spark3Util {
Preconditions.checkArgument(transform.references().length == 1,
"Cannot convert transform with more than one column reference: %s", transform);
String colName = DOT.join(transform.references()[0].fieldNames());
- switch (transform.name()) {
+ switch (transform.name().toLowerCase(Locale.ROOT)) {
case "identity":
return org.apache.iceberg.expressions.Expressions.ref(colName);
case "bucket":
@@ -390,7 +391,7 @@ public class Spark3Util {
Preconditions.checkArgument(transform.references().length == 1,
"Cannot convert transform with more than one column reference: %s", transform);
String colName = DOT.join(transform.references()[0].fieldNames());
- switch (transform.name()) {
+ switch (transform.name().toLowerCase(Locale.ROOT)) {
case "identity":
builder.identity(colName);
break;
diff --git a/spark/v3.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
index 0a4c9368cb..986098543d 100644
--- a/spark/v3.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
+++ b/spark/v3.0/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
@@ -49,6 +49,17 @@ public class TestCreateTable extends SparkCatalogTestBase {
sql("DROP TABLE IF EXISTS %s", tableName);
}
+ @Test
+ public void testTransformIgnoreCase() {
+ Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent));
+ sql("CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " +
+ "USING iceberg partitioned by (HOURS(ts))", tableName);
+ Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent));
+ sql("CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " +
+ "USING iceberg partitioned by (hours(ts))", tableName);
+ Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent));
+ }
+
@Test
public void testCreateTable() {
Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent));
diff --git a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
index 7ba3bbbac2..c2c3c69929 100644
--- a/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
+++ b/spark/v3.1/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
@@ -23,6 +23,7 @@ import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.Collections;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -351,7 +352,7 @@ public class Spark3Util {
Preconditions.checkArgument(transform.references().length == 1,
"Cannot convert transform with more than one column reference: %s", transform);
String colName = DOT.join(transform.references()[0].fieldNames());
- switch (transform.name()) {
+ switch (transform.name().toLowerCase(Locale.ROOT)) {
case "identity":
return org.apache.iceberg.expressions.Expressions.ref(colName);
case "bucket":
@@ -390,7 +391,7 @@ public class Spark3Util {
Preconditions.checkArgument(transform.references().length == 1,
"Cannot convert transform with more than one column reference: %s", transform);
String colName = DOT.join(transform.references()[0].fieldNames());
- switch (transform.name()) {
+ switch (transform.name().toLowerCase(Locale.ROOT)) {
case "identity":
builder.identity(colName);
break;
diff --git a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
index 0a4c9368cb..986098543d 100644
--- a/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
+++ b/spark/v3.1/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
@@ -49,6 +49,17 @@ public class TestCreateTable extends SparkCatalogTestBase {
sql("DROP TABLE IF EXISTS %s", tableName);
}
+ @Test
+ public void testTransformIgnoreCase() {
+ Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent));
+ sql("CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " +
+ "USING iceberg partitioned by (HOURS(ts))", tableName);
+ Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent));
+ sql("CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " +
+ "USING iceberg partitioned by (hours(ts))", tableName);
+ Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent));
+ }
+
@Test
public void testCreateTable() {
Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent));
diff --git a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
index 215f4df55a..d472987957 100644
--- a/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
+++ b/spark/v3.2/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
@@ -22,6 +22,7 @@ package org.apache.iceberg.spark;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -317,7 +318,7 @@ public class Spark3Util {
Preconditions.checkArgument("zorder".equals(transform.name()) || transform.references().length == 1,
"Cannot convert transform with more than one column reference: %s", transform);
String colName = DOT.join(transform.references()[0].fieldNames());
- switch (transform.name()) {
+ switch (transform.name().toLowerCase(Locale.ROOT)) {
case "identity":
return org.apache.iceberg.expressions.Expressions.ref(colName);
case "bucket":
@@ -369,7 +370,7 @@ public class Spark3Util {
Preconditions.checkArgument(transform.references().length == 1,
"Cannot convert transform with more than one column reference: %s", transform);
String colName = DOT.join(transform.references()[0].fieldNames());
- switch (transform.name()) {
+ switch (transform.name().toLowerCase(Locale.ROOT)) {
case "identity":
builder.identity(colName);
break;
diff --git a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
index 0a4c9368cb..986098543d 100644
--- a/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
+++ b/spark/v3.2/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
@@ -49,6 +49,17 @@ public class TestCreateTable extends SparkCatalogTestBase {
sql("DROP TABLE IF EXISTS %s", tableName);
}
+ @Test
+ public void testTransformIgnoreCase() {
+ Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent));
+ sql("CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " +
+ "USING iceberg partitioned by (HOURS(ts))", tableName);
+ Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent));
+ sql("CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " +
+ "USING iceberg partitioned by (hours(ts))", tableName);
+ Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent));
+ }
+
@Test
public void testCreateTable() {
Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent));
diff --git a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
index 215f4df55a..d472987957 100644
--- a/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
+++ b/spark/v3.3/spark/src/main/java/org/apache/iceberg/spark/Spark3Util.java
@@ -22,6 +22,7 @@ package org.apache.iceberg.spark;
import java.nio.ByteBuffer;
import java.util.Arrays;
import java.util.List;
+import java.util.Locale;
import java.util.Map;
import java.util.Objects;
import java.util.Set;
@@ -317,7 +318,7 @@ public class Spark3Util {
Preconditions.checkArgument("zorder".equals(transform.name()) || transform.references().length == 1,
"Cannot convert transform with more than one column reference: %s", transform);
String colName = DOT.join(transform.references()[0].fieldNames());
- switch (transform.name()) {
+ switch (transform.name().toLowerCase(Locale.ROOT)) {
case "identity":
return org.apache.iceberg.expressions.Expressions.ref(colName);
case "bucket":
@@ -369,7 +370,7 @@ public class Spark3Util {
Preconditions.checkArgument(transform.references().length == 1,
"Cannot convert transform with more than one column reference: %s", transform);
String colName = DOT.join(transform.references()[0].fieldNames());
- switch (transform.name()) {
+ switch (transform.name().toLowerCase(Locale.ROOT)) {
case "identity":
builder.identity(colName);
break;
diff --git a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
index 0a4c9368cb..986098543d 100644
--- a/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
+++ b/spark/v3.3/spark/src/test/java/org/apache/iceberg/spark/sql/TestCreateTable.java
@@ -49,6 +49,17 @@ public class TestCreateTable extends SparkCatalogTestBase {
sql("DROP TABLE IF EXISTS %s", tableName);
}
+ @Test
+ public void testTransformIgnoreCase() {
+ Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent));
+ sql("CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " +
+ "USING iceberg partitioned by (HOURS(ts))", tableName);
+ Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent));
+ sql("CREATE TABLE IF NOT EXISTS %s (id BIGINT NOT NULL, ts timestamp) " +
+ "USING iceberg partitioned by (hours(ts))", tableName);
+ Assert.assertTrue("Table should already exist", validationCatalog.tableExists(tableIdent));
+ }
+
@Test
public void testCreateTable() {
Assert.assertFalse("Table should not already exist", validationCatalog.tableExists(tableIdent));