[GitHub] [incubator-celeborn] cxzl25 commented on a diff in pull request #1921: [CELEBORN-989] Add support for making distribution package via SBT

["cxzl25 (via GitHub)" <gi...@apache.org>]

cxzl25 commented on code in PR #1921:
URL: https://github.com/apache/incubator-celeborn/pull/1921#discussion_r1330368883


##########
build/make-distribution.sh:
##########
@@ -236,42 +241,133 @@ function build_mr_client {
     cp "$PROJECT_DIR"/client-mr/mr-shaded/target/celeborn-client-mr-shaded_${SCALA_VERSION}-$VERSION.jar "$DIST_DIR/mr/"
 }
 
-if [ "$RELEASE" == "true" ]; then
-  build_service
-  build_spark_client -Pspark-2.4
-  build_spark_client -Pspark-3.4
-  build_flink_client -Pflink-1.14
-  build_flink_client -Pflink-1.15
-  build_flink_client -Pflink-1.17
-  build_mr_client mr
+
+#########################
+#     sbt functions     #
+#########################
+
+function sbt_build_service {
+  VERSION=$("$SBT" "Show / version" | awk '/\[info\]/{ver=$2} END{print ver}')
+
+  SCALA_VERSION=$("$SBT" "Show / scalaBinaryVersion" | awk '/\[info\]/{ver=$2} END{print ver}')
+
+  echo "Celeborn version is $VERSION"
+  echo "Making apache-celeborn-$VERSION-$NAME.tgz"
+
+  echo "Celeborn $VERSION$GITREVSTRING" > "$DIST_DIR/RELEASE"
+  echo "Build flags: $@" >> "$DIST_DIR/RELEASE"
+
+  BUILD_COMMAND=("$SBT" clean package $@)
+
+  # Actually build the jar
+  echo -e "\nBuilding with..."
+  echo -e "\$ ${BUILD_COMMAND[@]}\n"
+
+  "${BUILD_COMMAND[@]}"
+
+  $SBT "celeborn-master/copyJars;celeborn-worker/copyJars"
+
+  mkdir -p "$DIST_DIR/jars"
+  mkdir -p "$DIST_DIR/master-jars"
+  mkdir -p "$DIST_DIR/worker-jars"
+
+  ## Copy master jars
+  cp "$PROJECT_DIR"/master/target/scala-$SCALA_VERSION/celeborn-master_$SCALA_VERSION-$VERSION.jar "$DIST_DIR/master-jars/"
+  cp "$PROJECT_DIR"/master/target/scala-$SCALA_VERSION/jars/*.jar "$DIST_DIR/jars/"
+  for jar in $(ls "$PROJECT_DIR/master/target/scala-$SCALA_VERSION/jars"); do
+    (cd $DIST_DIR/master-jars; ln -snf "../jars/$jar" .)
+  done
+  ## Copy worker jars
+  cp "$PROJECT_DIR"/worker/target/scala-$SCALA_VERSION/celeborn-worker_$SCALA_VERSION-$VERSION.jar "$DIST_DIR/worker-jars/"
+  cp "$PROJECT_DIR"/worker/target/scala-$SCALA_VERSION/jars/*.jar "$DIST_DIR/jars/"
+  for jar in $(ls "$PROJECT_DIR/worker/target/scala-$SCALA_VERSION/jars"); do
+    (cd $DIST_DIR/worker-jars; ln -snf "../jars/$jar" .)
+  done
+}
+
+function sbt_build_client {
+  PROFILE="$1"
+  # get the client shaded project
+  CLIENT_PROJECT=$("$SBT" "$PROFILE" projects | grep shaded | awk '{print$2}')
+  # get the shaded jar file path
+  ASSEMBLY_OUTPUT_PATH=$("$SBT" "$PROFILE" "show $CLIENT_PROJECT/assembly/assemblyOutputPath" | awk '/\[info\]/{ver=$2} END{print ver}')
+  # build the shaded jar
+  $SBT $PROFILE "clean;$CLIENT_PROJECT/assembly"
+
+  PROFILE_PREFIX=$(echo "$PROFILE" | cut -d'-' -f2)
+  CLIENT_SUB_DIR="${PROFILE_PREFIX:1}"
+  mkdir -p "$DIST_DIR/$CLIENT_SUB_DIR"
+  cp "$ASSEMBLY_OUTPUT_PATH" "$DIST_DIR/$CLIENT_SUB_DIR/"
+}
+
+
+if [ "$SBT_ENABLED" == "true" ]; then
+  sbt_build_service

Review Comment:
   ```suggestion
     sbt_build_service "$@"
   ```



-- 
This is an automated message from the Apache Git Service.
To respond to the message, please log on to GitHub and use the
URL above to go to the specific comment.

To unsubscribe, e-mail: issues-unsubscribe@celeborn.apache.org

For queries about this service, please contact Infrastructure at:
users@infra.apache.org