You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by wa...@apache.org on 2022/10/11 06:08:13 UTC
[incubator-pegasus] branch master updated: build(images): download apache-zookeeper-3.7.0-bin.tar.gz during third-parties image building for unit tests (#1178)
This is an automated email from the ASF dual-hosted git repository.
wangdan pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new d5f0e5c3b build(images): download apache-zookeeper-3.7.0-bin.tar.gz during third-parties image building for unit tests (#1178)
d5f0e5c3b is described below
commit d5f0e5c3b7592c4d884c87b99228fbf981324d7c
Author: Dan Wang <wa...@apache.org>
AuthorDate: Tue Oct 11 14:08:07 2022 +0800
build(images): download apache-zookeeper-3.7.0-bin.tar.gz during third-parties image building for unit tests (#1178)
---
.github/workflows/lint_and_test_cpp.yaml | 8 +--
.github/workflows/thirdparty-regular-push.yml | 2 +
docker/thirdparties-bin/Dockerfile | 4 +-
scripts/download_hadoop.sh | 47 ++---------------
scripts/download_package.sh | 74 +++++++++++++++++++++++++++
scripts/download_zk.sh | 31 +++++++++++
6 files changed, 119 insertions(+), 47 deletions(-)
diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml
index a6ebdf0e5..bd90db09b 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -93,7 +93,7 @@ jobs:
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake --build build/ -j $(nproc)
- ./scripts/download_hadoop.sh hadoop-bin
+ ../scripts/download_hadoop.sh hadoop-bin
- name: Compilation
run: |
ccache -p
@@ -209,7 +209,7 @@ jobs:
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake --build build/ -j $(nproc)
- ./scripts/download_hadoop.sh hadoop-bin
+ ../scripts/download_hadoop.sh hadoop-bin
- name: Compilation
run: |
ccache -p
@@ -321,7 +321,7 @@ jobs:
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake --build build/ -j $(nproc)
- ./scripts/download_hadoop.sh hadoop-bin
+ ../scripts/download_hadoop.sh hadoop-bin
- name: Compilation
run: |
ccache -p
@@ -433,7 +433,7 @@ jobs:
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/
cmake --build build/ -j $(nproc)
- ./scripts/download_hadoop.sh hadoop-bin
+ ../scripts/download_hadoop.sh hadoop-bin
- name: Compilation
run: |
ccache -p
diff --git a/.github/workflows/thirdparty-regular-push.yml b/.github/workflows/thirdparty-regular-push.yml
index 64fc35653..3a591b06a 100644
--- a/.github/workflows/thirdparty-regular-push.yml
+++ b/.github/workflows/thirdparty-regular-push.yml
@@ -171,6 +171,7 @@ jobs:
OS_VERSION=${{ matrix.osversion }}
ROCKSDB_PORTABLE=ON
HADOOP_BIN_PATH=hadoop-bin
+ ZOOKEEPER_BIN_PATH=zookeeper-bin
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
@@ -207,5 +208,6 @@ jobs:
ROCKSDB_PORTABLE=ON
USE_JEMALLOC=ON
HADOOP_BIN_PATH=hadoop-bin
+ ZOOKEEPER_BIN_PATH=zookeeper-bin
- name: Image digest
run: echo ${{ steps.docker_build.outputs.digest }}
diff --git a/docker/thirdparties-bin/Dockerfile b/docker/thirdparties-bin/Dockerfile
index 5604621d9..ab9171063 100644
--- a/docker/thirdparties-bin/Dockerfile
+++ b/docker/thirdparties-bin/Dockerfile
@@ -29,12 +29,14 @@ ARG GITHUB_REPOSITORY_URL=https://github.com/apache/incubator-pegasus.git
ARG ROCKSDB_PORTABLE=OFF
ARG USE_JEMALLOC=OFF
ARG HADOOP_BIN_PATH
+ARG ZOOKEEPER_BIN_PATH
RUN git clone --depth=1 --branch=${GITHUB_BRANCH} ${GITHUB_REPOSITORY_URL} \
&& cd incubator-pegasus/thirdparty \
&& unzip /root/thirdparties-src.zip -d . \
&& cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=${ROCKSDB_PORTABLE} -DUSE_JEMALLOC=${USE_JEMALLOC} -B build/ . \
&& cmake --build build/ -j $(($(nproc)/2+1)) \
&& ../scripts/download_hadoop.sh ${HADOOP_BIN_PATH} \
- && zip -r ~/thirdparties-bin.zip output/ build/Source/rocksdb/cmake build/Source/http-parser build/Source/hadoop build/Download/zookeeper ${HADOOP_BIN_PATH} \
+ && ../scripts/download_zk.sh ${ZOOKEEPER_BIN_PATH} \
+ && zip -r ~/thirdparties-bin.zip output/ build/Source/rocksdb/cmake build/Source/http-parser build/Source/hadoop build/Download/zookeeper ${HADOOP_BIN_PATH} ${ZOOKEEPER_BIN_PATH} \
&& cd ~ \
&& rm -rf incubator-pegasus;
diff --git a/scripts/download_hadoop.sh b/scripts/download_hadoop.sh
index 0d4f66677..13fef962c 100755
--- a/scripts/download_hadoop.sh
+++ b/scripts/download_hadoop.sh
@@ -19,50 +19,13 @@
set -e
-if [ $# -lt 1 ]; then
- echo "Target HADOOP_BIN_PATH is not provided, thus do not try to download hadoop"
- exit 0
-fi
+CWD=$(cd $(dirname $0) && pwd)
-HADOOP_BIN_PATH=$1
-if [ -d ${HADOOP_BIN_PATH} ]; then
- echo "Target HADOOP_BIN_PATH ${HADOOP_BIN_PATH} has existed, thus do not try to download hadoop"
- exit 0
+if [ $# -ge 1 ]; then
+ HADOOP_BIN_PATH=$1
fi
HADOOP_VERSION=2.8.4
HADOOP_DIR_NAME=hadoop-${HADOOP_VERSION}
-HADOOP_PACKAGE_NAME=${HADOOP_DIR_NAME}.tar.gz
-if [ ! -f ${HADOOP_PACKAGE_NAME} ]; then
- echo "Downloading hadoop..."
-
- DOWNLOAD_URL="https://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com/${HADOOP_PACKAGE_NAME}"
- if ! wget -T 10 -t 5 ${DOWNLOAD_URL}; then
- echo "ERROR: download hadoop failed"
- exit 1
- fi
-
- HADOOP_PACKAGE_MD5="b30b409bb69185003b3babd1504ba224"
- if [ `md5sum ${HADOOP_PACKAGE_NAME} | awk '{print$1}'` != ${HADOOP_PACKAGE_MD5} ]; then
- echo "Check file ${HADOOP_PACKAGE_NAME} md5sum failed!"
- exit 1
- fi
-fi
-
-rm -rf ${HADOOP_DIR_NAME}
-
-echo "Decompressing hadoop..."
-if ! tar xf ${HADOOP_PACKAGE_NAME}; then
- echo "ERROR: decompress hadoop failed"
- rm -f ${HADOOP_PACKAGE_NAME}
- exit 1
-fi
-
-rm -f ${HADOOP_PACKAGE_NAME}
-
-if [ ! -d ${HADOOP_DIR_NAME} ]; then
- echo "ERROR: ${HADOOP_DIR_NAME} does not exist"
- exit 1
-fi
-
-mv ${HADOOP_DIR_NAME} ${HADOOP_BIN_PATH}
+HADOOP_PACKAGE_MD5="b30b409bb69185003b3babd1504ba224"
+${CWD}/download_package.sh ${HADOOP_DIR_NAME} ${HADOOP_PACKAGE_MD5} ${HADOOP_BIN_PATH}
diff --git a/scripts/download_package.sh b/scripts/download_package.sh
new file mode 100755
index 000000000..bc9202bfa
--- /dev/null
+++ b/scripts/download_package.sh
@@ -0,0 +1,74 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+if [ $# -lt 2 ]; then
+ echo "Invalid arguments !"
+ echo "USAGE: $0 <DIR_NAME> <PACKAGE_MD5> [TARGET_PATH]"
+ exit 1
+fi
+
+DIR_NAME=$1
+PACKAGE_MD5=$2
+
+if [ $# -lt 3 ]; then
+ echo "TARGET_PATH is not provided, thus do not try to download ${DIR_NAME}"
+ exit 0
+fi
+
+TARGET_PATH=$3
+if [ -d ${TARGET_PATH} ]; then
+ echo "TARGET_PATH ${TARGET_PATH} has existed, thus do not try to download ${DIR_NAME}"
+ exit 0
+fi
+
+PACKAGE_NAME=${DIR_NAME}.tar.gz
+if [ ! -f ${PACKAGE_NAME} ]; then
+ echo "Downloading ${DIR_NAME}..."
+
+ DOWNLOAD_URL="https://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com/${PACKAGE_NAME}"
+ if ! wget -T 10 -t 5 ${DOWNLOAD_URL}; then
+ echo "ERROR: download ${DIR_NAME} failed"
+ exit 1
+ fi
+
+ if [ `md5sum ${PACKAGE_NAME} | awk '{print$1}'` != ${PACKAGE_MD5} ]; then
+ echo "Check file ${PACKAGE_NAME} md5sum failed!"
+ exit 1
+ fi
+fi
+
+rm -rf ${DIR_NAME}
+
+echo "Decompressing ${DIR_NAME}..."
+if ! tar xf ${PACKAGE_NAME}; then
+ echo "ERROR: decompress ${DIR_NAME} failed"
+ rm -f ${PACKAGE_NAME}
+ exit 1
+fi
+
+rm -f ${PACKAGE_NAME}
+
+if [ ! -d ${DIR_NAME} ]; then
+ echo "ERROR: ${DIR_NAME} does not exist"
+ exit 1
+fi
+
+mv ${DIR_NAME} ${TARGET_PATH}
diff --git a/scripts/download_zk.sh b/scripts/download_zk.sh
new file mode 100755
index 000000000..41922452a
--- /dev/null
+++ b/scripts/download_zk.sh
@@ -0,0 +1,31 @@
+#!/bin/bash
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+CWD=$(cd $(dirname $0) && pwd)
+
+if [ $# -ge 1 ]; then
+ ZOOKEEPER_BIN_PATH=$1
+fi
+
+ZOOKEEPER_VERSION=3.7.0
+ZOOKEEPER_DIR_NAME=apache-zookeeper-${ZOOKEEPER_VERSION}-bin
+ZOOKEEPER_PACKAGE_MD5="8ffa97e7e6b0b2cf1d022e5156a7561a"
+${CWD}/download_package.sh ${ZOOKEEPER_DIR_NAME} ${ZOOKEEPER_PACKAGE_MD5} ${ZOOKEEPER_BIN_PATH}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org