You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kyuubi.apache.org by ch...@apache.org on 2022/07/18 06:15:13 UTC
[incubator-kyuubi] branch master updated: [KYUUBI #3050] Bump Apache Iceberg 0.14.0
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-kyuubi.git
The following commit(s) were added to refs/heads/master by this push:
new 699962242 [KYUUBI #3050] Bump Apache Iceberg 0.14.0
699962242 is described below
commit 699962242048a24d1426e57f5ade201ff4729c91
Author: Cheng Pan <ch...@apache.org>
AuthorDate: Mon Jul 18 14:14:58 2022 +0800
[KYUUBI #3050] Bump Apache Iceberg 0.14.0
### _Why are the changes needed?_
https://github.com/apache/iceberg/releases/tag/apache-iceberg-0.14.0
Notable changes:
- Support Spark 3.3
- Unified package name
### _How was this patch tested?_
- [ ] Add some test cases that check the changes thoroughly including negative and positive cases if possible
- [ ] Add screenshots for manual tests if appropriate
- [x] [Run test](https://kyuubi.apache.org/docs/latest/develop_tools/testing.html#running-tests) locally before make a pull request
Closes #3050 from pan3793/iceberg.
Closes #3050
3e700a11 [Cheng Pan] Fix ut caused by SPARK-37931
039c4e2d [Cheng Pan] Enable Iceberg test w/ Spark 3.3
53cac396 [Cheng Pan] Bump Apache Iceberg 0.14.0
Authored-by: Cheng Pan <ch...@apache.org>
Signed-off-by: Cheng Pan <ch...@apache.org>
---
externals/kyuubi-spark-sql-engine/pom.xml | 2 +-
.../kyuubi/operation/IcebergMetadataTests.scala | 20 +++++++++++++++++---
kyuubi-server/pom.xml | 2 +-
pom.xml | 15 ++++++++-------
4 files changed, 27 insertions(+), 12 deletions(-)
diff --git a/externals/kyuubi-spark-sql-engine/pom.xml b/externals/kyuubi-spark-sql-engine/pom.xml
index 56d8ceca9..6c40d36b8 100644
--- a/externals/kyuubi-spark-sql-engine/pom.xml
+++ b/externals/kyuubi-spark-sql-engine/pom.xml
@@ -137,7 +137,7 @@
<dependency>
<groupId>org.apache.iceberg</groupId>
- <artifactId>${iceberg.name}</artifactId>
+ <artifactId>iceberg-spark-runtime-${spark.binary.version}_${scala.binary.version}</artifactId>
<scope>test</scope>
</dependency>
diff --git a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/IcebergMetadataTests.scala b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/IcebergMetadataTests.scala
index 04a524b44..a9f5ece3f 100644
--- a/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/IcebergMetadataTests.scala
+++ b/kyuubi-common/src/test/scala/org/apache/kyuubi/operation/IcebergMetadataTests.scala
@@ -17,11 +17,22 @@
package org.apache.kyuubi.operation
-import org.apache.kyuubi.IcebergSuiteMixin
+import org.apache.kyuubi.{IcebergSuiteMixin, SPARK_COMPILE_VERSION}
+import org.apache.kyuubi.Utils._
import org.apache.kyuubi.operation.meta.ResultSetSchemaConstant._
trait IcebergMetadataTests extends HiveJDBCTestHelper with IcebergSuiteMixin {
+ def isSparkVersionAtLeast(ver: String): Boolean = {
+ val runtimeMajor = majorVersion(SPARK_COMPILE_VERSION)
+ val targetMajor = majorVersion(ver)
+ (runtimeMajor > targetMajor) || {
+ val runtimeMinor = minorVersion(SPARK_COMPILE_VERSION)
+ val targetMinor = minorVersion(ver)
+ runtimeMajor == targetMajor && runtimeMinor >= targetMinor
+ }
+ }
+
test("get catalogs") {
withJdbcStatement() { statement =>
val metaData = statement.getConnection.getMetaData
@@ -151,9 +162,12 @@ trait IcebergMetadataTests extends HiveJDBCTestHelper with IcebergSuiteMixin {
"map<int, bigint>",
"date",
"timestamp",
- "struct<`X`: bigint, `Y`: double>",
+ // SPARK-37931
+ if (isSparkVersionAtLeast("3.3")) "struct<X: bigint, Y: double>"
+ else "struct<`X`: bigint, `Y`: double>",
"binary",
- "struct<`X`: string>")
+ // SPARK-37931
+ if (isSparkVersionAtLeast("3.3")) "struct<X: string>" else "struct<`X`: string>")
val cols = dataTypes.zipWithIndex.map { case (dt, idx) => s"c$idx" -> dt }
val (colNames, _) = cols.unzip
diff --git a/kyuubi-server/pom.xml b/kyuubi-server/pom.xml
index c5013db44..6f51a9d40 100644
--- a/kyuubi-server/pom.xml
+++ b/kyuubi-server/pom.xml
@@ -384,7 +384,7 @@
<dependency>
<groupId>org.apache.iceberg</groupId>
- <artifactId>${iceberg.name}</artifactId>
+ <artifactId>iceberg-spark-runtime-${spark.binary.version}_${scala.binary.version}</artifactId>
<scope>test</scope>
</dependency>
diff --git a/pom.xml b/pom.xml
index 13d1cc396..a9a67bb48 100644
--- a/pom.xml
+++ b/pom.xml
@@ -138,8 +138,7 @@
<httpclient.version>4.5.13</httpclient.version>
<httpcore.version>4.4.15</httpcore.version>
<hudi.version>0.11.1</hudi.version>
- <iceberg.name>iceberg-spark-runtime-3.2_${scala.binary.version}</iceberg.name>
- <iceberg.version>0.13.2</iceberg.version>
+ <iceberg.version>0.14.0</iceberg.version>
<jackson.version>2.13.3</jackson.version>
<jakarta.servlet-api.version>4.0.4</jakarta.servlet-api.version>
<jakarta.xml-bind.version>2.3.2</jakarta.xml-bind.version>
@@ -165,9 +164,10 @@
<snakeyaml.version>1.30</snakeyaml.version>
<!--
DO NOT forget to change the following properties when change the minor version of Spark:
- `delta.version`, `iceberg.name`, `maven.plugin.scalatest.exclude.tags`
+ `delta.version`, `maven.plugin.scalatest.exclude.tags`
-->
<spark.version>3.2.2</spark.version>
+ <spark.binary.version>3.2</spark.binary.version>
<spark.archive.name>spark-${spark.version}-bin-hadoop3.2.tgz</spark.archive.name>
<spark.archive.mirror>${apache.archive.dist}/spark/spark-${spark.version}</spark.archive.mirror>
<spark.archive.download.skip>false</spark.archive.download.skip>
@@ -1075,7 +1075,7 @@
<dependency>
<groupId>org.apache.iceberg</groupId>
- <artifactId>${iceberg.name}</artifactId>
+ <artifactId>iceberg-spark-runtime-${spark.binary.version}_${scala.binary.version}</artifactId>
<version>${iceberg.version}</version>
</dependency>
@@ -2039,8 +2039,8 @@
<id>spark-3.1</id>
<properties>
<spark.version>3.1.3</spark.version>
+ <spark.binary.version>3.1</spark.binary.version>
<delta.version>1.0.1</delta.version>
- <iceberg.name>iceberg-spark-runtime-3.1_${scala.binary.version}</iceberg.name>
<maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow</maven.plugin.scalatest.exclude.tags>
</properties>
<modules>
@@ -2054,8 +2054,8 @@
<id>spark-3.2</id>
<properties>
<spark.version>3.2.2</spark.version>
+ <spark.binary.version>3.2</spark.binary.version>
<delta.version>2.0.0rc1</delta.version>
- <iceberg.name>iceberg-spark-runtime-3.2_${scala.binary.version}</iceberg.name>
<!--
See https://github.com/apache/incubator-kyuubi/pull/2576
Hudi 0.11 requires hudi-spark3.2-bundle to execute UT
@@ -2073,8 +2073,9 @@
<id>spark-3.3</id>
<properties>
<spark.version>3.3.0</spark.version>
+ <spark.binary.version>3.3</spark.binary.version>
<spark.archive.name>spark-${spark.version}-bin-hadoop3.tgz</spark.archive.name>
- <maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.IcebergTest,org.apache.kyuubi.tags.HudiTest</maven.plugin.scalatest.exclude.tags>
+ <maven.plugin.scalatest.exclude.tags>org.scalatest.tags.Slow,org.apache.kyuubi.tags.DeltaTest,org.apache.kyuubi.tags.HudiTest</maven.plugin.scalatest.exclude.tags>
</properties>
<modules>
<module>extensions/spark/kyuubi-extension-spark-common</module>