You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@uniffle.apache.org by js...@apache.org on 2022/07/01 06:57:55 UTC
[incubator-uniffle] 11/17: Support build_distribution.sh to specify different mvn build options for Spark2 and Spark3 (#203)
This is an automated email from the ASF dual-hosted git repository.
jshao pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git
commit 392c88129f2706043ebb87cc89e9e2cde5733647
Author: cxzl25 <cx...@users.noreply.github.com>
AuthorDate: Tue Jun 28 10:09:01 2022 +0800
Support build_distribution.sh to specify different mvn build options for Spark2 and Spark3 (#203)
What changes were proposed in this pull request?
Add --spark2-mvn, --spark3-mvn parameters in build_distribution.sh to support compiling different profiles, we can pass in different maven parameters, such as profile, spark version.
Add --help parameters in build_distribution.sh, fix typo.
gitignore ignores the tar package generated by build.
README added how to use build_distribution.sh.
Why are the changes needed?
If we use such a command to build, Spark2 will also use the Spark3 version to compile, so we'd better distinguish the build options of different versions.
./build_distribution.sh -Pspark3.2
Does this PR introduce any user-facing change?
No
How was this patch tested?
local test
---
.gitignore | 1 +
README.md | 16 ++++++++++++++++
build_distribution.sh | 53 +++++++++++++++++++++++++++++++++++++++++++++++----
pom.xml | 4 ++--
4 files changed, 68 insertions(+), 6 deletions(-)
diff --git a/.gitignore b/.gitignore
index 5c39d59..b6164b2 100644
--- a/.gitignore
+++ b/.gitignore
@@ -20,4 +20,5 @@ reports/
metastore_db/
derby.log
dependency-reduced-pom.xml
+rss-*.tgz
diff --git a/README.md b/README.md
index 9ad8299..51a1ed0 100644
--- a/README.md
+++ b/README.md
@@ -50,10 +50,26 @@ To build it, run:
mvn -DskipTests clean package
+Build against profile Spark2(2.4.6)
+
+ mvn -DskipTests clean package -Pspark2
+
+Build against profile Spark3(3.1.2)
+
+ mvn -DskipTests clean package -Pspark3
+
+Build against Spark 3.2.x
+
+ mvn -DskipTests clean package -Pspark3.2
+
To package the Firestorm, run:
./build_distribution.sh
+Package against Spark 3.2.x, run:
+
+ ./build_distribution.sh --spark3-profile 'spark3.2'
+
rss-xxx.tgz will be generated for deployment
## Deploy
diff --git a/build_distribution.sh b/build_distribution.sh
index baf50e4..214a2ed 100755
--- a/build_distribution.sh
+++ b/build_distribution.sh
@@ -32,12 +32,57 @@ RSS_HOME="$(
function exit_with_usage() {
set +x
- echo "$0 - tool for making binary distributions of Rmote Shuffle Service"
+ echo "./build_distribution.sh - Tool for making binary distributions of Remote Shuffle Service"
echo ""
- echo "usage:"
+ echo "Usage:"
+ echo "+------------------------------------------------------------------------------------------------------+"
+ echo "| ./build_distribution.sh [--spark2-profile <spark2 profile id>] [--spark2-mvn <custom maven options>] |"
+ echo "| [--spark3-profile <spark3 profile id>] [--spark3-mvn <custom maven options>] |"
+ echo "| <maven build options> |"
+ echo "+------------------------------------------------------------------------------------------------------+"
exit 1
}
+SPARK2_PROFILE_ID="spark2"
+SPARK2_MVN_OPTS=""
+SPARK3_PROFILE_ID="spark3"
+SPARK3_MVN_OPTS=""
+while (( "$#" )); do
+ case $1 in
+ --spark2-profile)
+ SPARK2_PROFILE_ID="$2"
+ shift
+ ;;
+ --spark2-mvn)
+ SPARK2_MVN_OPTS=$2
+ shift
+ ;;
+ --spark3-profile)
+ SPARK3_PROFILE_ID="$2"
+ shift
+ ;;
+ --spark3-mvn)
+ SPARK3_MVN_OPTS=$2
+ shift
+ ;;
+ --help)
+ exit_with_usage
+ ;;
+ --*)
+ echo "Error: $1 is not supported"
+ exit_with_usage
+ ;;
+ -*)
+ break
+ ;;
+ *)
+ echo "Error: $1 is not supported"
+ exit_with_usage
+ ;;
+ esac
+ shift
+done
+
cd $RSS_HOME
if [ -z "$JAVA_HOME" ]; then
@@ -99,7 +144,7 @@ cp "${RSS_HOME}"/coordinator/target/jars/* ${COORDINATOR_JAR_DIR}
CLIENT_JAR_DIR="${DISTDIR}/jars/client"
mkdir -p $CLIENT_JAR_DIR
-BUILD_COMMAND_SPARK2=("$MVN" clean package -Pspark2 -pl client-spark/spark2 -DskipTests -am $@)
+BUILD_COMMAND_SPARK2=("$MVN" clean package -P$SPARK2_PROFILE_ID -pl client-spark/spark2 -DskipTests -am $@ $SPARK2_MVN_OPTS)
# Actually build the jar
echo -e "\nBuilding with..."
@@ -114,7 +159,7 @@ SPARK_CLIENT2_JAR="${RSS_HOME}/client-spark/spark2/target/shaded/rss-client-spar
echo "copy $SPARK_CLIENT2_JAR to ${SPARK_CLIENT2_JAR_DIR}"
cp $SPARK_CLIENT2_JAR ${SPARK_CLIENT2_JAR_DIR}
-BUILD_COMMAND_SPARK3=("$MVN" clean package -Pspark3 -pl client-spark/spark3 -DskipTests -am $@)
+BUILD_COMMAND_SPARK3=("$MVN" clean package -P$SPARK3_PROFILE_ID -pl client-spark/spark3 -DskipTests -am $@ $SPARK3_MVN_OPTS)
echo -e "\nBuilding with..."
echo -e "\$ ${BUILD_COMMAND_SPARK3[@]}\n"
diff --git a/pom.xml b/pom.xml
index aef4fec..2f412f6 100644
--- a/pom.xml
+++ b/pom.xml
@@ -795,12 +795,12 @@
<plugin>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs-maven-plugin</artifactId>
- <version>${spotbugs-maven-plugin.version}</version>
+ <version>${spotbugs-maven-plugin.version}</version>
<dependencies>
<dependency>
<groupId>com.github.spotbugs</groupId>
<artifactId>spotbugs</artifactId>
- <version>${spotbugs.version}</version>
+ <version>${spotbugs.version}</version>
</dependency>
</dependencies>
<configuration>