You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@pegasus.apache.org by yu...@apache.org on 2022/09/27 11:36:26 UTC
[incubator-pegasus] branch master updated: ci: download hadoop-2.8.4.tar.gz during release building for unit tests once third-parties have changed (#1168)
This is an automated email from the ASF dual-hosted git repository.
yuchenhe pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-pegasus.git
The following commit(s) were added to refs/heads/master by this push:
new 760c049ec ci: download hadoop-2.8.4.tar.gz during release building for unit tests once third-parties have changed (#1168)
760c049ec is described below
commit 760c049ec4f7bc28913982feca44c09aa1c6a9e1
Author: Dan Wang <wa...@apache.org>
AuthorDate: Tue Sep 27 19:36:20 2022 +0800
ci: download hadoop-2.8.4.tar.gz during release building for unit tests once third-parties have changed (#1168)
---
.github/workflows/lint_and_test_cpp.yaml | 24 ++++++++++++++++--------
scripts/config_hdfs.sh | 28 +---------------------------
2 files changed, 17 insertions(+), 35 deletions(-)
diff --git a/.github/workflows/lint_and_test_cpp.yaml b/.github/workflows/lint_and_test_cpp.yaml
index beda4f668..a6ebdf0e5 100644
--- a/.github/workflows/lint_and_test_cpp.yaml
+++ b/.github/workflows/lint_and_test_cpp.yaml
@@ -93,6 +93,7 @@ jobs:
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake --build build/ -j $(nproc)
+ ./scripts/download_hadoop.sh hadoop-bin
- name: Compilation
run: |
ccache -p
@@ -105,8 +106,9 @@ jobs:
run: ./run.sh pack_tools
- name: Tar files
run: |
+ mv thirdparty/hadoop-bin ./
rm -rf thirdparty
- tar -zcvhf release__builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini --exclude='*CMakeFiles*'
+ tar -zcvhf release__builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*'
- name: Upload Artifact
uses: actions/upload-artifact@v3
with:
@@ -171,7 +173,7 @@ jobs:
run: |
export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
ulimit -s unlimited
- ./scripts/config_hdfs.sh
+ . ./scripts/config_hdfs.sh
./run.sh test -m ${{ matrix.test_module }}
build_ASAN:
@@ -207,6 +209,7 @@ jobs:
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake --build build/ -j $(nproc)
+ ./scripts/download_hadoop.sh hadoop-bin
- name: Compilation
run: |
ccache -p
@@ -215,8 +218,9 @@ jobs:
ccache -s
- name: Tar files
run: |
+ mv thirdparty/hadoop-bin ./
rm -rf thirdparty
- tar -zcvhf release_address_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini --exclude='*CMakeFiles*'
+ tar -zcvhf release_address_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*'
- name: Upload Artifact
uses: actions/upload-artifact@v3
with:
@@ -281,7 +285,7 @@ jobs:
run: |
export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
ulimit -s unlimited
- ./scripts/config_hdfs.sh
+ . ./scripts/config_hdfs.sh
./run.sh test -m ${{ matrix.test_module }}
build_UBSAN:
@@ -317,6 +321,7 @@ jobs:
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -B build/
cmake --build build/ -j $(nproc)
+ ./scripts/download_hadoop.sh hadoop-bin
- name: Compilation
run: |
ccache -p
@@ -325,8 +330,9 @@ jobs:
ccache -s
- name: Tar files
run: |
+ mv thirdparty/hadoop-bin ./
rm -rf thirdparty
- tar -zcvhf release_undefined_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini --exclude='*CMakeFiles*'
+ tar -zcvhf release_undefined_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*'
- name: Upload Artifact
uses: actions/upload-artifact@v3
with:
@@ -391,7 +397,7 @@ jobs:
run: |
export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
ulimit -s unlimited
- ./scripts/config_hdfs.sh
+ . ./scripts/config_hdfs.sh
./run.sh test -m ${{ matrix.test_module }}
build_with_jemalloc:
@@ -427,6 +433,7 @@ jobs:
mkdir build
cmake -DCMAKE_BUILD_TYPE=Release -DROCKSDB_PORTABLE=ON -DUSE_JEMALLOC=ON -B build/
cmake --build build/ -j $(nproc)
+ ./scripts/download_hadoop.sh hadoop-bin
- name: Compilation
run: |
ccache -p
@@ -439,8 +446,9 @@ jobs:
run: ./run.sh pack_tools -j
- name: Tar files
run: |
+ mv thirdparty/hadoop-bin ./
rm -rf thirdparty
- tar -zcvhf release_jemalloc_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini --exclude='*CMakeFiles*'
+ tar -zcvhf release_jemalloc_builder.tar DSN_ROOT/ src/builder/bin src/builder/src/server/test/config.ini hadoop-bin --exclude='*CMakeFiles*'
- name: Upload Artifact
uses: actions/upload-artifact@v3
with:
@@ -475,7 +483,7 @@ jobs:
run: |
export LD_LIBRARY_PATH=`pwd`/thirdparty/output/lib:/usr/lib/jvm/java-8-openjdk-amd64/jre/lib/amd64/server
ulimit -s unlimited
- ./scripts/config_hdfs.sh
+ . ./scripts/config_hdfs.sh
./run.sh test -m ${{ matrix.test_module }}
build_pegasus_on_macos:
diff --git a/scripts/config_hdfs.sh b/scripts/config_hdfs.sh
index 5ff58d542..caa1cb342 100755
--- a/scripts/config_hdfs.sh
+++ b/scripts/config_hdfs.sh
@@ -39,35 +39,9 @@ fi
JAVA_JVM_LIBRARY_DIR=$(dirname $(find "${JAVA_HOME}/" -name libjvm.so | head -1))
export LD_LIBRARY_PATH=${JAVA_JVM_LIBRARY_DIR}:$LD_LIBRARY_PATH
-# download hdfs for unit test
-HDFS_ROOT=hadoop-2.8.4
-HDFS_TAR_NAME=${HDFS_ROOT}.tar.gz
-HDFS_TAR_MD5_VALUE="b30b409bb69185003b3babd1504ba224"
-if [ ! -f $HDFS_ROOT ]; then
- echo "Downloading hadoop..."
- download_url="https://pegasus-thirdparty-package.oss-cn-beijing.aliyuncs.com/hadoop-2.8.4.tar.gz"
- if ! wget -T 10 -t 5 $download_url; then
- echo "ERROR: download hadoop failed"
- exit 1
- fi
- if [ `md5sum $HDFS_TAR_NAME | awk '{print$1}'` != $HDFS_TAR_MD5_VALUE ]; then
- echo "check file $HDFS_TAR_NAME md5sum failed!"
- exit 1
- fi
-fi
-
-if [ ! -d $HDFS_ROOT ]; then
- echo "Decompressing HDFS..."
- if ! tar xf $HDFS_TAR_NAME; then
- echo "ERROR: decompress hadoop failed"
- exit 1
- fi
- rm -f $HDFS_TAR_NAME
-fi
-
# Set CLASSPATH to all the Hadoop jars needed to run Hadoop itself as well as
# the right configuration directory containing core-site.xml or hdfs-site.xml.
-PEGASUS_HADOOP_HOME=`pwd`/${HDFS_ROOT}
+PEGASUS_HADOOP_HOME=`pwd`/hadoop-bin
# Prefer the HADOOP_HOME set in the environment, but use the pegasus's hadoop dir otherwise.
export HADOOP_HOME="${HADOOP_HOME:-${PEGASUS_HADOOP_HOME}}"
if [ ! -d "$HADOOP_HOME/etc/hadoop" ] || [ ! -d "$HADOOP_HOME/share/hadoop" ]; then
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@pegasus.apache.org
For additional commands, e-mail: commits-help@pegasus.apache.org