You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by xu...@apache.org on 2022/03/08 09:01:42 UTC
[hudi] branch master updated: [HUDI-3574] Improve maven module configs for different spark profiles (#4970)
This is an automated email from the ASF dual-hosted git repository.
xushiyan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/hudi.git
The following commit(s) were added to refs/heads/master by this push:
new 2538580 [HUDI-3574] Improve maven module configs for different spark profiles (#4970)
2538580 is described below
commit 25385805aab3ff00ca9accc406d0f7b0cf2d9e55
Author: ForwardXu <fo...@gmail.com>
AuthorDate: Tue Mar 8 17:01:05 2022 +0800
[HUDI-3574] Improve maven module configs for different spark profiles (#4970)
---
.github/workflows/bot.yml | 7 +------
hudi-spark-datasource/pom.xml | 5 -----
pom.xml | 21 +++++++++++++++++++++
3 files changed, 22 insertions(+), 11 deletions(-)
diff --git a/.github/workflows/bot.yml b/.github/workflows/bot.yml
index 20515f7..ebf3cac 100644
--- a/.github/workflows/bot.yml
+++ b/.github/workflows/bot.yml
@@ -18,16 +18,12 @@ jobs:
include:
- scala: "scala-2.11"
spark: "spark2"
- skipModules: ""
- scala: "scala-2.11"
spark: "spark2,spark-shade-unbundle-avro"
- skipModules: ""
- scala: "scala-2.12"
spark: "spark3.1.x"
- skipModules: "!hudi-spark-datasource/hudi-spark3"
- scala: "scala-2.12"
spark: "spark3.1.x,spark-shade-unbundle-avro"
- skipModules: "!hudi-spark-datasource/hudi-spark3"
- scala: "scala-2.12"
spark: "spark3"
- scala: "scala-2.12"
@@ -44,5 +40,4 @@ jobs:
env:
SCALA_PROFILE: ${{ matrix.scala }}
SPARK_PROFILE: ${{ matrix.spark }}
- SKIP_MODULES: ${{ matrix.skipModules }}
- run: mvn install -P "$SCALA_PROFILE,$SPARK_PROFILE" -pl "$SKIP_MODULES" -DskipTests=true -Dmaven.javadoc.skip=true -B -V
+ run: mvn install -P "$SCALA_PROFILE,$SPARK_PROFILE" -DskipTests=true -Dmaven.javadoc.skip=true -B -V
diff --git a/hudi-spark-datasource/pom.xml b/hudi-spark-datasource/pom.xml
index 7aaf398..a19a603 100644
--- a/hudi-spark-datasource/pom.xml
+++ b/hudi-spark-datasource/pom.xml
@@ -33,11 +33,6 @@
<modules>
<module>hudi-spark-common</module>
<module>hudi-spark</module>
- <module>hudi-spark2</module>
- <module>hudi-spark3</module>
- <module>hudi-spark3.1.x</module>
- <module>hudi-spark3-common</module>
- <module>hudi-spark2-common</module>
</modules>
</project>
diff --git a/pom.xml b/pom.xml
index 1b28ae1..01fa0e9 100644
--- a/pom.xml
+++ b/pom.xml
@@ -1581,10 +1581,23 @@
<!-- Exists for backwards compatibility; profile doesn't do anything -->
<profile>
<id>spark2</id>
+ <modules>
+ <module>hudi-spark-datasource/hudi-spark2</module>
+ <module>hudi-spark-datasource/hudi-spark2-common</module>
+ </modules>
+ <activation>
+ <activeByDefault>true</activeByDefault>
+ <property>
+ <name>spark2</name>
+ <!-- add spark2 moudule to all profile -->
+ <value>!disabled</value>
+ </property>
+ </activation>
</profile>
<profile>
<id>spark3</id>
<properties>
+ <spark3.version>3.2.1</spark3.version>
<spark.version>${spark3.version}</spark.version>
<sparkbundle.version>${spark3.version}</sparkbundle.version>
<scala.version>${scala12.version}</scala.version>
@@ -1603,6 +1616,10 @@
<skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
<skipITs>true</skipITs>
</properties>
+ <modules>
+ <module>hudi-spark-datasource/hudi-spark3</module>
+ <module>hudi-spark-datasource/hudi-spark3-common</module>
+ </modules>
<activation>
<property>
<name>spark3</name>
@@ -1629,6 +1646,10 @@
<skip.hudi-spark2.unit.tests>true</skip.hudi-spark2.unit.tests>
<skipITs>true</skipITs>
</properties>
+ <modules>
+ <module>hudi-spark-datasource/hudi-spark3.1.x</module>
+ <module>hudi-spark-datasource/hudi-spark3-common</module>
+ </modules>
<activation>
<property>
<name>spark3.1.x</name>