You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@paimon.apache.org by lz...@apache.org on 2023/04/24 05:28:04 UTC
[incubator-paimon] branch master updated: [spark] Support Spark 3.4 (#967)
This is an automated email from the ASF dual-hosted git repository.
lzljs3620320 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-paimon.git
The following commit(s) were added to refs/heads/master by this push:
new ee2df54a4 [spark] Support Spark 3.4 (#967)
ee2df54a4 is described below
commit ee2df54a487d07440cda0df83dde64ca690afa87
Author: Nicholas Jiang <pr...@163.com>
AuthorDate: Mon Apr 24 13:27:59 2023 +0800
[spark] Support Spark 3.4 (#967)
---
docs/content/engines/overview.md | 2 +-
docs/content/engines/spark3.md | 4 ++-
docs/content/project/download.md | 2 ++
.../pom.xml | 39 +++++-----------------
paimon-spark/paimon-spark-common/pom.xml | 7 +++-
.../org/apache/paimon/spark/SparkReadITCase.java | 13 +++++---
.../paimon/spark/SparkSchemaEvolutionITCase.java | 9 ++---
paimon-spark/pom.xml | 1 +
8 files changed, 36 insertions(+), 41 deletions(-)
diff --git a/docs/content/engines/overview.md b/docs/content/engines/overview.md
index ea546d955..172914082 100644
--- a/docs/content/engines/overview.md
+++ b/docs/content/engines/overview.md
@@ -36,7 +36,7 @@ Apache Spark and Apache Hive.
|--------|-----------------------------|--------------------------------------------------------------------------------------|--------------------|
| Flink | 1.17/1.16/1.15/1.14 | batch/streaming read, batch/streaming write, create/drop table, create/drop database | Projection, Filter |
| Hive | 3.1/2.3/2.2/2.1/2.1 CDH 6.3 | batch read | Projection, Filter |
-| Spark | 3.3/3.2/3.1 | batch read, batch write, create/drop table, create/drop database | Projection, Filter |
+| Spark | 3.4/3.3/3.2/3.1 | batch read, batch write, create/drop table, create/drop database | Projection, Filter |
| Spark | 2.4 | batch read | Projection, Filter |
| Trino | 388/358 | batch read, create/drop table, create/drop database | Projection, Filter |
| Presto | 0.236 and above | batch read | Projection, Filter |
diff --git a/docs/content/engines/spark3.md b/docs/content/engines/spark3.md
index c43df56a4..60c43330b 100644
--- a/docs/content/engines/spark3.md
+++ b/docs/content/engines/spark3.md
@@ -30,7 +30,7 @@ This documentation is a guide for using Paimon in Spark3.
## Preparing Paimon Jar File
-Paimon currently supports Spark 3.3, 3.2 and 3.1. We recommend the latest Spark version for a better experience.
+Paimon currently supports Spark 3.4, 3.3, 3.2 and 3.1. We recommend the latest Spark version for a better experience.
Download the jar file with corresponding version.
@@ -38,6 +38,7 @@ Download the jar file with corresponding version.
| Version | Jar |
|-----------|----------------------------------------------------------------------------------------------------------------------------------------------------------------------|
+| Spark 3.4 | [paimon-spark-3.4-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.4/{{< version >}}/paimon-spark-3.4-{{< version >}}.jar) |
| Spark 3.3 | [paimon-spark-3.3-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.3/{{< version >}}/paimon-spark-3.3-{{< version >}}.jar) |
| Spark 3.2 | [paimon-spark-3.2-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.2/{{< version >}}/paimon-spark-3.2-{{< version >}}.jar) |
| Spark 3.1 | [paimon-spark-3.1-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.1/{{< version >}}/paimon-spark-3.1-{{< version >}}.jar) |
@@ -48,6 +49,7 @@ Download the jar file with corresponding version.
| Version | Jar |
|-----------|-------------------------------------------------------------------------------------------------------------------------------------|
+| Spark 3.4 | [paimon-spark-3.4-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.4/{{< version >}}/) |
| Spark 3.3 | [paimon-spark-3.3-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.3/{{< version >}}/) |
| Spark 3.2 | [paimon-spark-3.2-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.2/{{< version >}}/) |
| Spark 3.1 | [paimon-spark-3.1-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.1/{{< version >}}/) |
diff --git a/docs/content/project/download.md b/docs/content/project/download.md
index 83e4ce84e..87cdf95bc 100644
--- a/docs/content/project/download.md
+++ b/docs/content/project/download.md
@@ -38,6 +38,7 @@ This documentation is a guide for downloading Paimon Jars.
| Flink 1.16 | [paimon-flink-1.16-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.16/{{< version >}}/) |
| Flink 1.15 | [paimon-flink-1.15-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.15/{{< version >}}/) |
| Flink 1.14 | [paimon-flink-1.14-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-flink-1.14/{{< version >}}/) |
+| Spark 3.4 | [paimon-spark-3.4-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.4/{{< version >}}/) |
| Spark 3.3 | [paimon-spark-3.3-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.3/{{< version >}}/) |
| Spark 3.2 | [paimon-spark-3.2-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.2/{{< version >}}/) |
| Spark 3.1 | [paimon-spark-3.1-{{< version >}}.jar](https://repository.apache.org/snapshots/org/apache/paimon/paimon-spark-3.1/{{< version >}}/) |
@@ -61,6 +62,7 @@ This documentation is a guide for downloading Paimon Jars.
| Flink 1.16 | [paimon-flink-1.16-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.16/{{< version >}}/paimon-flink-1.16-{{< version >}}.jar) |
| Flink 1.15 | [paimon-flink-1.15-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.15/{{< version >}}/paimon-flink-1.15-{{< version >}}.jar) |
| Flink 1.14 | [paimon-flink-1.14-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-flink-1.14/{{< version >}}/paimon-flink-1.14-{{< version >}}.jar) |
+| Spark 3.4 | [paimon-spark-3.4-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.4/{{< version >}}/paimon-spark-3.4-{{< version >}}.jar) |
| Spark 3.3 | [paimon-spark-3.3-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.3/{{< version >}}/paimon-spark-3.3-{{< version >}}.jar) |
| Spark 3.2 | [paimon-spark-3.2-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.2/{{< version >}}/paimon-spark-3.2-{{< version >}}.jar) |
| Spark 3.1 | [paimon-spark-3.1-{{< version >}}.jar](https://repo.maven.apache.org/maven2/org/apache/paimon/paimon-spark-3.1/{{< version >}}/paimon-spark-3.1-{{< version >}}.jar) |
diff --git a/paimon-spark/paimon-spark-common/pom.xml b/paimon-spark/paimon-spark-3.4/pom.xml
similarity index 72%
copy from paimon-spark/paimon-spark-common/pom.xml
copy to paimon-spark/paimon-spark-3.4/pom.xml
index d4e2038a2..d23051626 100644
--- a/paimon-spark/paimon-spark-common/pom.xml
+++ b/paimon-spark/paimon-spark-3.4/pom.xml
@@ -23,46 +23,29 @@ under the License.
<modelVersion>4.0.0</modelVersion>
<parent>
- <artifactId>paimon-spark</artifactId>
<groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-spark</artifactId>
<version>0.4-SNAPSHOT</version>
</parent>
- <packaging>jar</packaging>
-
- <artifactId>paimon-spark-common</artifactId>
- <name>Paimon : Spark : Common</name>
+ <artifactId>paimon-spark-3.4</artifactId>
+ <name>Paimon : Spark : 3.4</name>
<properties>
- <spark.version>3.3.2</spark.version>
+ <spark.version>3.4.0</spark.version>
</properties>
<dependencies>
<dependency>
- <groupId>org.apache.spark</groupId>
- <artifactId>spark-sql_2.12</artifactId>
- <version>${spark.version}</version>
- <exclusions>
- <exclusion>
- <groupId>log4j</groupId>
- <artifactId>log4j</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.slf4j</groupId>
- <artifactId>slf4j-log4j12</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
- </exclusions>
+ <groupId>org.apache.paimon</groupId>
+ <artifactId>paimon-spark-common</artifactId>
+ <version>${project.version}</version>
</dependency>
<dependency>
<groupId>org.apache.spark</groupId>
- <artifactId>spark-hive_2.12</artifactId>
+ <artifactId>spark-sql_2.12</artifactId>
<version>${spark.version}</version>
- <scope>test</scope>
<exclusions>
<exclusion>
<groupId>log4j</groupId>
@@ -72,10 +55,6 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
- <exclusion>
- <groupId>org.apache.orc</groupId>
- <artifactId>orc-core</artifactId>
- </exclusion>
</exclusions>
</dependency>
</dependencies>
@@ -95,7 +74,7 @@ under the License.
<configuration>
<artifactSet>
<includes combine.children="append">
- <include>org.apache.paimon:paimon-bundle</include>
+ <include>org.apache.paimon:paimon-spark-common</include>
</includes>
</artifactSet>
</configuration>
diff --git a/paimon-spark/paimon-spark-common/pom.xml b/paimon-spark/paimon-spark-common/pom.xml
index d4e2038a2..b97617882 100644
--- a/paimon-spark/paimon-spark-common/pom.xml
+++ b/paimon-spark/paimon-spark-common/pom.xml
@@ -34,7 +34,7 @@ under the License.
<name>Paimon : Spark : Common</name>
<properties>
- <spark.version>3.3.2</spark.version>
+ <spark.version>3.4.0</spark.version>
</properties>
<dependencies>
@@ -51,6 +51,11 @@ under the License.
<groupId>org.slf4j</groupId>
<artifactId>slf4j-log4j12</artifactId>
</exclusion>
+ <!-- SPARK-40511 upgrades SLF4J2, which is not compatible w/ SLF4J1 -->
+ <exclusion>
+ <groupId>org.apache.logging.log4j</groupId>
+ <artifactId>log4j-slf4j2-impl</artifactId>
+ </exclusion>
<exclusion>
<groupId>org.apache.orc</groupId>
<artifactId>orc-core</artifactId>
diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadITCase.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadITCase.java
index 08653c8af..bbe5f2a56 100644
--- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadITCase.java
+++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkReadITCase.java
@@ -285,7 +285,8 @@ public class SparkReadITCase extends SparkReadTestBase {
+ "b STRING)\n"
+ "PARTITIONED BY (a)\n");
assertThat(spark.sql("DESCRIBE PartitionedTable").collectAsList().toString())
- .isEqualTo("[[a,bigint,], [b,string,], [,,], [# Partitioning,,], [Part 0,a,]]");
+ .isEqualTo(
+ "[[a,bigint,null], [b,string,null], [# Partition Information,,], [# col_name,data_type,comment], [a,bigint,null]]");
}
@Test
@@ -453,10 +454,14 @@ public class SparkReadITCase extends SparkReadTestBase {
spark.sql(ddl);
assertThatThrownBy(() -> spark.sql(ddl))
.isInstanceOf(TableAlreadyExistsException.class)
- .hasMessageContaining(String.format("Table default.%s already exists", tableName));
+ .hasMessageContaining(
+ String.format(
+ "Cannot create table or view `default`.`%s` because it already "
+ + "exists",
+ tableName));
assertThatThrownBy(() -> spark.sql(ddl.replace("default", "foo")))
.isInstanceOf(NoSuchNamespaceException.class)
- .hasMessageContaining("Namespace 'foo' not found");
+ .hasMessageContaining("The schema `foo` cannot be found");
assertThatThrownBy(
() ->
@@ -551,7 +556,7 @@ public class SparkReadITCase extends SparkReadTestBase {
assertThatThrownBy(() -> spark.sql("CREATE NAMESPACE bar"))
.isInstanceOf(NamespaceAlreadyExistsException.class)
- .hasMessageContaining("Namespace 'bar' already exists");
+ .hasMessageContaining("Cannot create schema `bar` because it already exists");
assertThat(
spark.sql("SHOW NAMESPACES").collectAsList().stream()
diff --git a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java
index de6387991..770dd2195 100644
--- a/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java
+++ b/paimon-spark/paimon-spark-common/src/test/java/org/apache/paimon/spark/SparkSchemaEvolutionITCase.java
@@ -135,11 +135,12 @@ public class SparkSchemaEvolutionITCase extends SparkReadTestBase {
// TODO: add test case for hive catalog table
assertThatThrownBy(() -> spark.sql("ALTER TABLE t3 RENAME TO t4"))
.isInstanceOf(AnalysisException.class)
- .hasMessageContaining("Table or view not found: t3");
+ .hasMessageContaining("The table or view `t3` cannot be found");
assertThatThrownBy(() -> spark.sql("ALTER TABLE t1 RENAME TO t2"))
.isInstanceOf(AnalysisException.class)
- .hasMessageContaining("Table default.t2 already exists");
+ .hasMessageContaining(
+ "Cannot create table or view default.t2 because it already exists");
spark.sql("ALTER TABLE t1 RENAME TO t3");
List<Row> tables = spark.sql("SHOW TABLES").collectAsList();
@@ -176,8 +177,8 @@ public class SparkSchemaEvolutionITCase extends SparkReadTestBase {
assertThatThrownBy(() -> table.select("a", "c"))
.isInstanceOf(AnalysisException.class)
.hasMessageContaining(
- "Column 'a' does not exist. Did you mean one of the following? "
- + "[paimon.default.testRenameColumn.b, paimon.default.testRenameColumn.c, paimon.default.testRenameColumn.aa]");
+ "A column or function parameter with name `a` cannot be resolved. Did you mean one of the following? "
+ + "[`paimon`.`default`.`testRenameColumn`.`b`, `paimon`.`default`.`testRenameColumn`.`c`, `paimon`.`default`.`testRenameColumn`.`aa`]");
}
@Test
diff --git a/paimon-spark/pom.xml b/paimon-spark/pom.xml
index 3300789c2..dddf8babf 100644
--- a/paimon-spark/pom.xml
+++ b/paimon-spark/pom.xml
@@ -40,6 +40,7 @@ under the License.
<modules>
<module>paimon-spark-common</module>
<module>paimon-spark-2</module>
+ <module>paimon-spark-3.4</module>
<module>paimon-spark-3.3</module>
<module>paimon-spark-3.1</module>
<module>paimon-spark-3.2</module>