You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@celeborn.apache.org by ch...@apache.org on 2023/01/05 02:31:36 UTC
[incubator-celeborn] branch main updated: [CELEBORN-175][BUILD] Include Flink client into the binary release tarball (#1120)
This is an automated email from the ASF dual-hosted git repository.
chengpan pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/incubator-celeborn.git
The following commit(s) were added to refs/heads/main by this push:
new 33c23875 [CELEBORN-175][BUILD] Include Flink client into the binary release tarball (#1120)
33c23875 is described below
commit 33c2387594a0ecf276bfa06cc4605aae018d1898
Author: Shuang <lv...@gmail.com>
AuthorDate: Thu Jan 5 10:31:32 2023 +0800
[CELEBORN-175][BUILD] Include Flink client into the binary release tarball (#1120)
* [CELEBORN-175] with flink plugin when release packages
* improve
* flink use scala 2.11.x as default
* remove unused version align
* add space before end slash
* improve error message
* update default flink version
---
build/make-distribution.sh | 38 +++++++++++++++++++++++++++++++++-
client-flink/common/pom.xml | 2 +-
client-flink/flink-1.14-shaded/pom.xml | 4 ++--
client-flink/flink-1.14/pom.xml | 4 ++--
pom.xml | 15 ++++++--------
5 files changed, 48 insertions(+), 15 deletions(-)
diff --git a/build/make-distribution.sh b/build/make-distribution.sh
index c96ff3d0..998597b0 100755
--- a/build/make-distribution.sh
+++ b/build/make-distribution.sh
@@ -185,13 +185,49 @@ function build_spark_client {
cp "$PROJECT_DIR"/client-spark/spark-$SPARK_MAJOR_VERSION-shaded/target/celeborn-client-spark-${SPARK_MAJOR_VERSION}-shaded_$SCALA_VERSION-$VERSION.jar "$DIST_DIR/spark/"
}
+function build_flink_client {
+ FLINK_VERSION=$("$MVN" help:evaluate -Dexpression=flink.version $@ 2>/dev/null \
+ | grep -v "INFO" \
+ | grep -v "WARNING" \
+ | tail -n 1)
+ SCALA_VERSION=$("$MVN" help:evaluate -Dexpression=scala.binary.version $@ 2>/dev/null \
+ | grep -v "INFO" \
+ | grep -v "WARNING" \
+ | tail -n 1)
+ FLINK_BINARY_VERSION=${FLINK_VERSION%.*}
+
+ # Store the command as an array because $MVN variable might have spaces in it.
+ # Normal quoting tricks don't work.
+ # See: http://mywiki.wooledge.org/BashFAQ/050
+ BUILD_COMMAND=("$MVN" clean package -DskipTests -pl :celeborn-client-flink-${FLINK_BINARY_VERSION}-shaded_$SCALA_VERSION -am $@)
+
+ # Actually build the jar
+ echo -e "\nBuilding with..."
+ echo -e "\$ ${BUILD_COMMAND[@]}\n"
+
+ "${BUILD_COMMAND[@]}"
+
+ ## flink spark client jars
+ mkdir -p "$DIST_DIR/flink"
+ cp "$PROJECT_DIR"/client-flink/flink-$FLINK_BINARY_VERSION-shaded/target/celeborn-client-flink-${FLINK_BINARY_VERSION}-shaded_$SCALA_VERSION-$VERSION.jar "$DIST_DIR/flink/"
+}
+
if [ "$RELEASE" == "true" ]; then
build_service
build_spark_client -Pspark-2.4
build_spark_client -Pspark-3.3
+ build_flink_client -Pflink-1.14
else
+ ## build release package on demand
build_service $@
- build_spark_client $@
+ if [[ $@ == *"spark"* && $@ != *"flink"* ]]; then
+ build_spark_client $@
+ elif [[ $@ == *"flink"* && $@ != *"spark"* ]]; then
+ build_flink_client $@
+ else
+ echo "Error: unsupported command $@, currently we do not support compiling spark and flink at the same time."
+ exit -1
+ fi
fi
# Copy configuration templates
diff --git a/client-flink/common/pom.xml b/client-flink/common/pom.xml
index 5a4b6505..ec585e1d 100644
--- a/client-flink/common/pom.xml
+++ b/client-flink/common/pom.xml
@@ -24,7 +24,7 @@
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>celeborn-client-flink-common-${flink.binary.version}_${scala.binary.version}</artifactId>
+ <artifactId>celeborn-client-flink-common_${scala.binary.version}</artifactId>
<packaging>jar</packaging>
<name>Celeborn Client for Flink Common</name>
diff --git a/client-flink/flink-1.14-shaded/pom.xml b/client-flink/flink-1.14-shaded/pom.xml
index d380ab8d..48922069 100644
--- a/client-flink/flink-1.14-shaded/pom.xml
+++ b/client-flink/flink-1.14-shaded/pom.xml
@@ -24,14 +24,14 @@
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>celeborn-client-flink-${flink.binary.version}-shaded_${scala.binary.version}</artifactId>
+ <artifactId>celeborn-client-flink-1.14-shaded_${scala.binary.version}</artifactId>
<packaging>jar</packaging>
<name>Celeborn Shaded Client for Flink 1.14</name>
<dependencies>
<dependency>
<groupId>org.apache.celeborn</groupId>
- <artifactId>celeborn-client-flink-${flink.binary.version}_${scala.binary.version}</artifactId>
+ <artifactId>celeborn-client-flink-1.14_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
</dependencies>
diff --git a/client-flink/flink-1.14/pom.xml b/client-flink/flink-1.14/pom.xml
index 8645add4..eb9c90be 100644
--- a/client-flink/flink-1.14/pom.xml
+++ b/client-flink/flink-1.14/pom.xml
@@ -24,7 +24,7 @@
<relativePath>../../pom.xml</relativePath>
</parent>
- <artifactId>celeborn-client-flink-${flink.binary.version}_${scala.binary.version}</artifactId>
+ <artifactId>celeborn-client-flink-1.14_${scala.binary.version}</artifactId>
<packaging>jar</packaging>
<name>Celeborn Client for Flink 1.14</name>
@@ -41,7 +41,7 @@
</dependency>
<dependency>
<groupId>org.apache.celeborn</groupId>
- <artifactId>celeborn-client-flink-common-${flink.binary.version}_${scala.binary.version}</artifactId>
+ <artifactId>celeborn-client-flink-common_${scala.binary.version}</artifactId>
<version>${project.version}</version>
</dependency>
<dependency>
diff --git a/pom.xml b/pom.xml
index 0b736c2f..9422e4af 100644
--- a/pom.xml
+++ b/pom.xml
@@ -57,13 +57,15 @@
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>
<java.version>8</java.version>
+ <scala211.version>2.11.12</scala211.version>
+ <scala211.binary.version>2.11</scala211.binary.version>
<scala.version>2.12.15</scala.version>
<scala.binary.version>2.12</scala.binary.version>
<maven.compiler.source>${java.version}</maven.compiler.source>
<maven.compiler.target>${java.version}</maven.compiler.target>
<maven.version>3.6.3</maven.version>
- <flink.version>1.14.0</flink.version>
+ <flink.version>1.14.6</flink.version>
<hadoop.version>3.2.1</hadoop.version>
<spark.version>3.3.1</spark.version>
@@ -1019,14 +1021,9 @@
<module>client-flink/flink-1.14-shaded</module>
</modules>
<properties>
- <flink.version>1.14.0</flink.version>
- <flink.binary.version>1.14</flink.binary.version>
- <jackson.version>2.6.7</jackson.version>
- <jackson.databind.version>2.6.7.3</jackson.databind.version>
- <lz4-java.version>1.4.0</lz4-java.version>
- <scala.version>2.12.15</scala.version>
- <scala.binary.version>2.12</scala.binary.version>
- <zstd-jni.version>1.4.4-3</zstd-jni.version>
+ <flink.version>1.14.6</flink.version>
+ <scala.version>${scala211.version}</scala.version>
+ <scala.binary.version>${scala211.binary.version}</scala.binary.version>
</properties>
</profile>