You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uniffle.apache.org by ck...@apache.org on 2023/01/10 10:58:05 UTC
[incubator-uniffle] branch master updated: [ISSUE-461] Support Spark 3.3 (#463)
This is an automated email from the ASF dual-hosted git repository.
ckj pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
The following commit(s) were added to refs/heads/master by this push:
new 56fa22d2 [ISSUE-461] Support Spark 3.3 (#463)
56fa22d2 is described below
commit 56fa22d286fd1980a757ccce333dccf139749be5
Author: Kaijie Chen <ck...@apache.org>
AuthorDate: Tue Jan 10 18:58:00 2023 +0800
[ISSUE-461] Support Spark 3.3 (#463)
### What changes were proposed in this pull request?
1. Add maven profile `spark3.3`
2. Enable tests for Spark 3.3 in CI
3. Fix broken tests
### Why are the changes needed?
#461
### Does this PR introduce _any_ user-facing change?
No.
### How was this patch tested?
https://github.com/kaijchen/incubator-uniffle/actions/runs/3879507232
---
.github/workflows/parallel.yml | 1 +
README.md | 2 +-
.../org/apache/uniffle/test/GetReaderTest.java | 4 +
pom.xml | 89 ++++++++++++++++++++++
4 files changed, 95 insertions(+), 1 deletion(-)
diff --git a/.github/workflows/parallel.yml b/.github/workflows/parallel.yml
index 0a79020b..821c7ef9 100644
--- a/.github/workflows/parallel.yml
+++ b/.github/workflows/parallel.yml
@@ -62,6 +62,7 @@ jobs:
- spark3
- spark3.2
- spark3.2.0
+ - spark3.3
- mr
- kubernetes
fail-fast: false
diff --git a/README.md b/README.md
index 05da7d10..2454e0a4 100644
--- a/README.md
+++ b/README.md
@@ -60,7 +60,7 @@ The shuffle data is stored with index file and data file. Data file has all bloc
## Supported Spark Version
-Currently supports Spark 2.3.x, Spark 2.4.x, Spark 3.0.x, Spark 3.1.x, Spark 3.2.x
+Currently supports Spark 2.3.x, Spark 2.4.x, Spark 3.0.x, Spark 3.1.x, Spark 3.2.x, Spark 3.3.x
Note: To support dynamic allocation, the patch(which is included in client-spark/patch folder) should be applied to Spark
diff --git a/integration-test/spark3/src/test/java/org/apache/uniffle/test/GetReaderTest.java b/integration-test/spark3/src/test/java/org/apache/uniffle/test/GetReaderTest.java
index 6e6bbea6..7b9590e5 100644
--- a/integration-test/spark3/src/test/java/org/apache/uniffle/test/GetReaderTest.java
+++ b/integration-test/spark3/src/test/java/org/apache/uniffle/test/GetReaderTest.java
@@ -265,6 +265,10 @@ public class GetReaderTest extends IntegrationTestBase {
return null;
}
+ public int cpus() {
+ return Runtime.getRuntime().availableProcessors();
+ }
+
@Override
public java.util.Map<String, ResourceInformation> resourcesJMap() {
return null;
diff --git a/pom.xml b/pom.xml
index 18189d79..d1adaa67 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1322,6 +1322,95 @@
</dependencyManagement>
</profile>
+ <profile>
+ <id>spark3.3</id>
+ <properties>
+ <scala.binary.version>2.12</scala.binary.version>
+ <spark.version>3.3.1</spark.version>
+ <client.type>3</client.type>
+ <jackson.version>2.13.4</jackson.version>
+ </properties>
+ <modules>
+ <module>client-spark/common</module>
+ <module>client-spark/spark3</module>
+ <module>integration-test/spark-common</module>
+ <module>integration-test/spark3</module>
+ </modules>
+ <dependencyManagement>
+ <dependencies>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-client-spark3</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-core_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.spark</groupId>
+ <artifactId>spark-sql_${scala.binary.version}</artifactId>
+ <version>${spark.version}</version>
+ <exclusions>
+ <exclusion>
+ <groupId>org.apache.hadoop</groupId>
+ <artifactId>*</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-databind</artifactId>
+ </exclusion>
+ <exclusion>
+ <groupId>com.fasterxml.jackson.core</groupId>
+ <artifactId>jackson-core</artifactId>
+ </exclusion>
+ </exclusions>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-client-spark-common</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ </dependency>
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-integration-common-test</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+
+ <dependency>
+ <groupId>org.apache.uniffle</groupId>
+ <artifactId>rss-integration-spark-common-test</artifactId>
+ <version>${project.version}</version>
+ <type>test-jar</type>
+ <scope>test</scope>
+ </dependency>
+ </dependencies>
+ </dependencyManagement>
+ </profile>
+
<profile>
<id>spark3.2.0</id>
<properties>