You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by wa...@apache.org on 2022/10/20 10:19:05 UTC
[incubator-pegasus] branch master updated: ci: download apache-zookeeper-3.7.0-bin.tar.gz during release building for unit tests once third-parties have change (#1186)
This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 359500ee5 ci: download apache-zookeeper-3.7.0-bin.tar.gz during release building for unit tests once third-parties have change (#1186)
359500ee5 is described below
commit 359500ee51b7be60f1e9883a89f50efbfee6eae1
Author: Dan Wang <wa...@apache.org>
AuthorDate: Thu Oct 20 18:18:58 2022 +0800
ci: download apache-zookeeper-3.7.0-bin.tar.gz during release building for unit tests once third-parties have change (#1186)
---
.github/workflows/lint_and_test_cpp.yaml | 16 ++++++++++++----
run.sh | 7 +++++++
scripts/clear_zk.sh | 17 +++++++++++++----
scripts/download_package.sh | 5 +++++
scripts/start_zk.sh | 30 +++++++-----------------------
scripts/stop_zk.sh | 11 ++++++++---
6 files changed, 52 insertions(+), 34 deletions(-)
diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml
index fd170a3b6..9dd164656 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -94,6 +94,7 @@ jobs:
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake --build build/ -j $(nproc)
../scripts/download_hadoop.sh hadoop-bin
+ ../scripts/download_zk.sh zookeeper-bin
- name: Compilation
run: |
ccache -p
@@ -107,8 +108,9 @@ jobs:
- name: Tar files
run: |
mv thirdparty/hadoop-bin ./
+ mv thirdparty/zookeeper-bin ./
rm -rf thirdparty
- tar -zcvhf release__builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*'
+ tar -zcvhf release__builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin zookeeper-bin --exclude='*CMakeFiles*'
- name: Upload Artifact
uses: actions/upload-artifact@v3
with:
@@ -212,6 +214,7 @@ jobs:
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake --build build/ -j $(nproc)
../scripts/download_hadoop.sh hadoop-bin
+ ../scripts/download_zk.sh zookeeper-bin
- name: Compilation
run: |
ccache -p
@@ -221,8 +224,9 @@ jobs:
- name: Tar files
run: |
mv thirdparty/hadoop-bin ./
+ mv thirdparty/zookeeper-bin ./
rm -rf thirdparty
- tar -zcvhf release_address_builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*'
+ tar -zcvhf release_address_builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin zookeeper-bin --exclude='*CMakeFiles*'
- name: Upload Artifact
uses: actions/upload-artifact@v3
with:
@@ -331,6 +335,7 @@ jobs:
# cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
# cmake --build build/ -j $(nproc)
# ../scripts/download_hadoop.sh hadoop-bin
+# ../scripts/download_zk.sh zookeeper-bin
# - name: Compilation
# run: |
# ccache -p
@@ -340,8 +345,9 @@ jobs:
# - name: Tar files
# run: |
# mv thirdparty/hadoop-bin ./
+# mv thirdparty/zookeeper-bin ./
# rm -rf thirdparty
-# tar -zcvhf release_undefined_builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*'
+# tar -zcvhf release_undefined_builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin zookeeper-bin --exclude='*CMakeFiles*'
# - name: Upload Artifact
# uses: actions/upload-artifact@v3
# with:
@@ -443,6 +449,7 @@ jobs:
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/
cmake --build build/ -j $(nproc)
../scripts/download_hadoop.sh hadoop-bin
+ ../scripts/download_zk.sh zookeeper-bin
- name: Compilation
run: |
ccache -p
@@ -456,8 +463,9 @@ jobs:
- name: Tar files
run: |
mv thirdparty/hadoop-bin ./
+ mv thirdparty/zookeeper-bin ./
rm -rf thirdparty
- tar -zcvhf release_jemalloc_builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*'
+ tar -zcvhf release_jemalloc_builder.tar src/builder/output src/builder/bin src/builder/src/server/test/config.ini hadoop-bin zookeeper-bin --exclude='*CMakeFiles*'
- name: Upload Artifact
uses: actions/upload-artifact@v3
with:
diff --git a/run.sh b/run.sh
index aa73be178..4218d83c7 100755
--- a/run.sh
+++ b/run.sh
@@ -491,6 +491,13 @@ function run_start_zk()
type nc >/dev/null 2>&1 || { echo >&2 "start zk failed, need install netcat command..."; exit 1;}
INSTALL_DIR=`pwd`/.zk_install
+ if [ ! -d "${INSTALL_DIR}/zookeeper-bin" ]; then
+ if [ -d "zookeeper-bin" ]; then
+ # this zookeeper-bin must have been got from github action workflows, thus just
+ # move it to ${INSTALL_DIR} to prevent from downloading
+ mv zookeeper-bin ${INSTALL_DIR}/
+ fi
+ fi
PORT=22181
while [[ $# > 0 ]]; do
key="$1"
diff --git a/scripts/clear_zk.sh b/scripts/clear_zk.sh
index c94cbe03a..311e05a95 100755
--- a/scripts/clear_zk.sh
+++ b/scripts/clear_zk.sh
@@ -34,11 +34,20 @@ then
fi
cd $INSTALL_DIR
+
+# If the old dir for zk bin exists, just use it
ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin
-if [ -d "$ZOOKEEPER_HOME" ]
-then
- $ZOOKEEPER_HOME/bin/zkServer.sh stop
- rm -rf $ZOOKEEPER_HOME/data &>/dev/null
+if [ -d "${ZOOKEEPER_HOME}" ]; then
+ ${ZOOKEEPER_HOME}/bin/zkServer.sh stop
+ rm -rf ${ZOOKEEPER_HOME} &> /dev/null
+ echo "Deleting old zookeeper ... DELETED"
+fi
+
+ZOOKEEPER_HOME=`pwd`/zookeeper-bin
+
+if [ -d "${ZOOKEEPER_HOME}" ]; then
+ ${ZOOKEEPER_HOME}/bin/zkServer.sh stop
+ rm -rf ${ZOOKEEPER_HOME}/data &> /dev/null
echo "Clearing zookeeper ... CLEARED"
fi
diff --git a/scripts/download_package.sh b/scripts/download_package.sh
index bc9202bfa..538393402 100755
--- a/scripts/download_package.sh
+++ b/scripts/download_package.sh
@@ -71,4 +71,9 @@ if [ ! -d ${DIR_NAME} ]; then
exit 1
fi
+if [ -d ${TARGET_PATH} ]; then
+ echo "TARGET_PATH ${TARGET_PATH} has been generated, which means it and ${DIR_NAME} are the same dir thus do not do mv any more"
+ exit 0
+fi
+
mv ${DIR_NAME} ${TARGET_PATH}
diff --git a/scripts/start_zk.sh b/scripts/start_zk.sh
index 0811652ff..2bc37356b 100755
--- a/scripts/start_zk.sh
+++ b/scripts/start_zk.sh
@@ -18,6 +18,8 @@
set -e
+CWD=$(cd $(dirname $0) && pwd)
+
# Options:
# INSTALL_DIR <dir>
# PORT <port>
@@ -42,32 +44,14 @@ fi
cd "$INSTALL_DIR" || exit
-ZOOKEEPER_ROOT=apache-zookeeper-3.7.0-bin
-ZOOKEEPER_TAR_NAME=${ZOOKEEPER_ROOT}.tar.gz
-ZOOKEEPER_TAR_MD5_VALUE="8ffa97e7e6b0b2cf1d022e5156a7561a"
+# If the old dir for zk bin exists, just use it
+ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin
-if [ ! -f $ZOOKEEPER_TAR_NAME ]; then
- echo "Downloading zookeeper..."
- download_url="http://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com/apache-zookeeper-3.7.0-bin.tar.gz"
- if ! wget -T 10 -t 5 $download_url; then
- echo "ERROR: download zookeeper failed"
- exit 1
- fi
- if [ `md5sum $ZOOKEEPER_TAR_NAME | awk '{print$1}'` != $ZOOKEEPER_TAR_MD5_VALUE ]; then
- echo "check file $ZOOKEEPER_TAR_NAME md5sum failed!"
- exit 1
- fi
-fi
-
-if [ ! -d $ZOOKEEPER_ROOT ]; then
- echo "Decompressing zookeeper..."
- if ! tar xf $ZOOKEEPER_TAR_NAME; then
- echo "ERROR: decompress zookeeper failed"
- exit 1
- fi
+if [ ! -d "${ZOOKEEPER_HOME}" ]; then
+ ${CWD}/download_zk.sh zookeeper-bin
+ ZOOKEEPER_HOME=`pwd`/zookeeper-bin
fi
-ZOOKEEPER_HOME=`pwd`/$ZOOKEEPER_ROOT
ZOOKEEPER_PORT=$PORT
cp $ZOOKEEPER_HOME/conf/zoo_sample.cfg $ZOOKEEPER_HOME/conf/zoo.cfg
diff --git a/scripts/stop_zk.sh b/scripts/stop_zk.sh
index 313fd4f54..c30868048 100755
--- a/scripts/stop_zk.sh
+++ b/scripts/stop_zk.sh
@@ -33,9 +33,14 @@ then
fi
cd $INSTALL_DIR
+
+# If the old dir for zk bin exists, just use it
ZOOKEEPER_HOME=`pwd`/apache-zookeeper-3.7.0-bin
-if [ -d "$ZOOKEEPER_HOME" ]
-then
- $ZOOKEEPER_HOME/bin/zkServer.sh stop
+if [ ! -d "${ZOOKEEPER_HOME}" ]; then
+ ZOOKEEPER_HOME=`pwd`/zookeeper-bin
+fi
+
+if [ -d "${ZOOKEEPER_HOME}" ]; then
+ ${ZOOKEEPER_HOME}/bin/zkServer.sh stop
fi
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org