[incubator-uniffle] branch master updated: [ISSUE-48][FEATURE] Add Uniffle Dockerfile (#132)

[ro...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

roryqi pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-uniffle.git


The following commit(s) were added to refs/heads/master by this push:
     new 18663edb [ISSUE-48][FEATURE] Add Uniffle Dockerfile (#132)
18663edb is described below

commit 18663edb48089ece34dbd6df2845ee2388b157df
Author: jasonawang <ja...@tencent.com>
AuthorDate: Wed Aug 10 10:03:29 2022 +0800

    [ISSUE-48][FEATURE] Add Uniffle Dockerfile (#132)
    
    ### What changes were proposed in this pull request?
    To solve issue #48 ，I add the Dockerfile for Uniffle.
    
    ### Why are the changes needed?
    Support K8S Operator
    
    ### Does this PR introduce _any_ user-facing change?
    Yes, we will add the document later
    
    ### Does this PR introduce _any_ user-facing change?
    Yes, we will add the document later
---
 .gitignore                                         |   3 +-
 deploy/kubernetes/docker/Dockerfile                |  53 +++++++++++
 deploy/kubernetes/docker/build.sh                  | 106 +++++++++++++++++++++
 .../kubernetes/docker/{ => hadoopconfig}/.gitkeep  |   0
 deploy/kubernetes/docker/start.sh                  |  57 +++++++++++
 5 files changed, 218 insertions(+), 1 deletion(-)

diff --git a/.gitignore b/.gitignore
index b6164b2c..ce609e1c 100644
--- a/.gitignore
+++ b/.gitignore
@@ -21,4 +21,5 @@ metastore_db/
 derby.log
 dependency-reduced-pom.xml
 rss-*.tgz
-
+hadoop-*.tar.gz
+deploy/kubernetes/docker/hadoopconfig/*
diff --git a/deploy/kubernetes/docker/Dockerfile b/deploy/kubernetes/docker/Dockerfile
new file mode 100644
index 00000000..d1ed46bd
--- /dev/null
+++ b/deploy/kubernetes/docker/Dockerfile
@@ -0,0 +1,53 @@
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+FROM openjdk:8-jdk-slim
+
+ARG HADOOP_VERSION
+ARG RSS_VERSION
+ARG AUTHOR=unknown
+ARG GIT_BRANCH=unknown
+ARG GIT_COMMIT=unknown
+
+LABEL author=${AUTHOR}
+LABEL git-branch=${GIT_BRANCH}
+LABEL git-commit=${GIT_COMMIT}
+
+RUN apt-get update && apt-get install -y zlib1g zlib1g-dev lzop lsof netcat dnsutils less procps iputils-ping \
+      && apt-get clean && rm -rf /var/lib/apt/lists/*
+
+RUN useradd -ms /bin/bash rssadmin
+RUN mkdir -p /data/rssadmin/
+RUN chown -R rssadmin:rssadmin /data
+USER rssadmin
+
+COPY rss-${RSS_VERSION}.tgz /data/rssadmin
+RUN tar -xvf /data/rssadmin/rss-${RSS_VERSION}.tgz -C /data/rssadmin
+RUN mv /data/rssadmin/rss-${RSS_VERSION} /data/rssadmin/rss
+RUN rm -rf /data/rssadmin/rss-${RSS_VERSION}.tgz
+
+COPY start.sh /data/rssadmin/rss/bin
+
+COPY hadoop-${HADOOP_VERSION}.tar.gz /data/rssadmin
+RUN tar -zxvf /data/rssadmin/hadoop-${HADOOP_VERSION}.tar.gz -C /data/rssadmin
+RUN mv /data/rssadmin/hadoop-${HADOOP_VERSION} /data/rssadmin/hadoop
+RUN rm -rf /data/rssadmin/hadoop-${HADOOP_VERSION}.tar.gz
+COPY hadoopconfig/ /data/rssadmin/hadoop/etc/hadoop
+
+ENV RSS_VERSION ${RSS_VERSION}
+ENV HADOOP_VERSION ${HADOOP_VERSION}
+CMD bash /data/rssadmin/rss/bin/start.sh
diff --git a/deploy/kubernetes/docker/build.sh b/deploy/kubernetes/docker/build.sh
new file mode 100644
index 00000000..cdebba0d
--- /dev/null
+++ b/deploy/kubernetes/docker/build.sh
@@ -0,0 +1,106 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+function exit_with_usage() {
+  set +x
+  echo "./build.sh - Tool for building docker images of Remote Shuffle Service"
+  echo ""
+  echo "Usage:"
+  echo "+------------------------------------------------------------------------------------------------------+"
+  echo "| ./build.sh [--hadoop-version <hadoop version>] [--registry <registry url>] [--author <author name>]  |"
+  echo "+------------------------------------------------------------------------------------------------------+"
+  exit 1
+}
+
+UNKNOWN_REGISTRY="UNKNOWN_REGISTRY"
+
+REGISTRY=$UNKNOWN_REGISTRY
+HADOOP_VERSION=2.8.5
+AUTHOR=$(whoami)
+
+while (( "$#" )); do
+  case $1 in
+    --registry)
+      REGISTRY="$2"
+      shift
+      ;;
+    --hadoop-version)
+      HADOOP_VERSION="$2"
+      shift
+      ;;
+    --author)
+      AUTHOR="$2"
+      shift
+      ;;
+    --help)
+      exit_with_usage
+      ;;
+    --*)
+      echo "Error: $1 is not supported"
+      exit_with_usage
+      ;;
+    -*)
+      break
+      ;;
+    *)
+      echo "Error: $1 is not supported"
+      exit_with_usage
+      ;;
+  esac
+  shift
+done
+
+if [ "$REGISTRY" == $UNKNOWN_REGISTRY ]; \
+  then echo "please set registry url"; exit; \
+fi
+
+HADOOP_FILE=hadoop-${HADOOP_VERSION}.tar.gz
+HADOOP_URL=https://archive.apache.org/dist/hadoop/core/hadoop-${HADOOP_VERSION}/${HADOOP_FILE}
+echo "HADOOP_URL is $HADOOP_URL"
+if [ ! -e "$HADOOP_FILE" ]; \
+  then wget "$HADOOP_URL"; \
+  else echo "${HADOOP_FILE} has been downloaded"; \
+fi
+
+RSS_DIR=../../..
+cd $RSS_DIR || exit
+RSS_VERSION=$(mvn help:evaluate -Dexpression=project.version 2>/dev/null | grep -v "INFO" | grep -v "WARNING" | tail -n 1)
+RSS_FILE=rss-${RSS_VERSION}.tgz
+if [ ! -e "$RSS_FILE" ]; \
+  then sh build_distribution.sh \
+  else echo "$RSS_FILE has been built"; \
+fi
+cd "$OLDPWD" || exit
+cp "$RSS_DIR/$RSS_FILE" .
+
+IMAGE=$REGISTRY/rss-server:$RSS_VERSION
+
+echo "building image: $IMAGE"
+GIT_BRANCH=$(git rev-parse --abbrev-ref HEAD)
+GIT_COMMIT=$(git describe --dirty --always --tags | sed 's/-/./g')
+docker build -t "$IMAGE" \
+             --build-arg RSS_VERSION="$RSS_VERSION" \
+             --build-arg HADOOP_VERSION="$HADOOP_VERSION" \
+             --build-arg AUTHOR="$AUTHOR" \
+             --build-arg GIT_COMMIT="$GIT_COMMIT" \
+             --build-arg GIT_BRANCH="$GIT_BRANCH" \
+             -f Dockerfile --no-cache .
+
+echo "pushing image: $IMAGE"
+docker push "$IMAGE"
diff --git a/deploy/kubernetes/docker/.gitkeep b/deploy/kubernetes/docker/hadoopconfig/.gitkeep
similarity index 100%
rename from deploy/kubernetes/docker/.gitkeep
rename to deploy/kubernetes/docker/hadoopconfig/.gitkeep
diff --git a/deploy/kubernetes/docker/start.sh b/deploy/kubernetes/docker/start.sh
new file mode 100644
index 00000000..d7ca5fb2
--- /dev/null
+++ b/deploy/kubernetes/docker/start.sh
@@ -0,0 +1,57 @@
+#!/bin/bash
+
+#
+# Licensed to the Apache Software Foundation (ASF) under one or more
+# contributor license agreements.  See the NOTICE file distributed with
+# this work for additional information regarding copyright ownership.
+# The ASF licenses this file to You under the Apache License, Version 2.0
+# (the "License"); you may not use this file except in compliance with
+# the License.  You may obtain a copy of the License at
+#
+#    http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+basedir='/data/rssadmin/rss'
+cd $basedir || exit
+
+coordinator_conf=$basedir'/conf/coordinator.conf'
+echo "coordinator_conf: $coordinator_conf"
+server_conf=$basedir'/conf/server.conf'
+echo "server_conf: $server_conf"
+
+
+if [ "$SERVICE_NAME" == "coordinator" ];then
+
+    bash ${basedir}/bin/start-coordinator.sh &
+    sleep 10
+    while : ; do
+        pid=$(/usr/bin/lsof -i:"${COORDINATOR_RPC_PORT}" -sTCP:LISTEN)
+        if [ "$pid" = "" ]; then
+          break
+        else
+          echo "coordinator pid:$pid is alive"
+          sleep 10
+        fi
+    done
+fi
+
+if [ "$SERVICE_NAME" == "server" ];then
+
+    bash ${basedir}/bin/start-shuffle-server.sh &
+    sleep 10
+    while : ; do
+        pid=$(/usr/bin/lsof -i:"$SERVER_RPC_PORT" -sTCP:LISTEN)
+        if [ "$pid" = "" ]; then
+          break
+        else
+          echo "shuffle server pid:$pid is alive"
+          sleep 10
+        fi
+    done
+fi