You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/06/17 18:26:28 UTC
[GitHub] marcoabreu closed pull request #11322: [MXNET-554] Add ccache
reporting to CI
marcoabreu closed pull request #11322: [MXNET-554] Add ccache reporting to CI
URL: https://github.com/apache/incubator-mxnet/pull/11322
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/ci/build.py b/ci/build.py
index b6821b328a1..680f4574fd8 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -162,6 +162,7 @@ def container_run(platform: str,
'-e', 'CCACHE_MAXSIZE={}'.format(CCACHE_MAXSIZE),
'-e', 'CCACHE_TEMPDIR=/tmp/ccache', # temp dir should be local and not shared
'-e', "CCACHE_DIR=/work/ccache", # this path is inside the container as /work/ccache is mounted
+ '-e', "CCACHE_LOGFILE=/tmp/ccache.log", # a container-scoped log, useful for ccache verification.
tag]
runlist.extend(command)
cmd = ' '.join(runlist)
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 7170b8b22cc..6cefeea9fbc 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -108,6 +108,23 @@ build_jetson() {
popd
}
+report_ccache_usage() {
+ set -ex
+ pushd .
+
+ # Show global ccache summary at the end of each run.
+ ccache -s
+ if [ -e $CCACHE_LOGFILE ]
+ then
+ # Display local ccache log, excluding some overly verbose output.
+ cat $CCACHE_LOGFILE | grep -v "Config:" | grep -v "stats.lock"
+ else
+ echo "No ccache log found."
+ fi
+
+ popd
+}
+
build_armv6() {
set -ex
pushd .
@@ -136,6 +153,7 @@ build_armv6() {
-G Ninja /work/mxnet
ninja -v
+ report_ccache_usage
build_wheel
popd
}
@@ -167,6 +185,7 @@ build_armv7() {
-G Ninja /work/mxnet
ninja -v
+ report_ccache_usage
build_wheel
popd
}
@@ -186,6 +205,7 @@ build_amzn_linux_cpu() {
-DUSE_DIST_KVSTORE=ON\
-G Ninja /work/mxnet
ninja -v
+ report_ccache_usage
export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
}
@@ -202,6 +222,7 @@ build_arm64() {
-DUSE_MKL_IF_AVAILABLE=OFF\
-G Ninja /work/mxnet
ninja -v
+ report_ccache_usage
export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
cd /work/mxnet/python
python setup.py bdist_wheel --universal
@@ -225,6 +246,7 @@ build_android_armv7() {
-DUSE_MKL_IF_AVAILABLE=OFF\
-G Ninja /work/mxnet
ninja -v
+ report_ccache_usage
export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
cd /work/mxnet/python
python setup.py bdist_wheel --universal
@@ -260,6 +282,7 @@ build_centos7_cpu() {
cd /work/mxnet
export CC="ccache gcc"
export CXX="ccache g++"
+
make \
DEV=1 \
USE_LAPACK=1 \
@@ -267,6 +290,8 @@ build_centos7_cpu() {
USE_BLAS=openblas \
USE_DIST_KVSTORE=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_centos7_mkldnn() {
@@ -274,6 +299,7 @@ build_centos7_mkldnn() {
cd /work/mxnet
export CC="ccache gcc"
export CXX="ccache g++"
+
make \
DEV=1 \
USE_LAPACK=1 \
@@ -281,6 +307,8 @@ build_centos7_mkldnn() {
USE_MKLDNN=1 \
USE_BLAS=openblas \
-j$(nproc)
+
+ report_ccache_usage
}
build_centos7_gpu() {
@@ -314,6 +342,8 @@ build_ubuntu_cpu_openblas() {
USE_BLAS=openblas \
USE_DIST_KVSTORE=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_cpu_clang39() {
@@ -330,6 +360,8 @@ build_ubuntu_cpu_clang39() {
USE_OPENMP=0 \
USE_DIST_KVSTORE=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_cpu_clang50() {
@@ -346,6 +378,8 @@ build_ubuntu_cpu_clang50() {
USE_OPENMP=1 \
USE_DIST_KVSTORE=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_cpu_clang39_mkldnn() {
@@ -362,6 +396,8 @@ build_ubuntu_cpu_clang39_mkldnn() {
USE_MKLDNN=1 \
USE_OPENMP=0 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_cpu_clang50_mkldnn() {
@@ -378,17 +414,23 @@ build_ubuntu_cpu_clang50_mkldnn() {
USE_MKLDNN=1 \
USE_OPENMP=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_cpu_mkldnn() {
set -ex
+
build_ccache_wrappers
+
make \
DEV=1 \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
USE_MKLDNN=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_gpu() {
@@ -397,7 +439,9 @@ build_ubuntu_gpu() {
build_ubuntu_gpu_mkldnn() {
set -ex
+
build_ccache_wrappers
+
make \
DEV=1 \
USE_CPP_PACKAGE=1 \
@@ -407,6 +451,8 @@ build_ubuntu_gpu_mkldnn() {
USE_CUDA_PATH=/usr/local/cuda \
USE_CUDNN=1 \
-j$(nproc)
+
+ report_ccache_usage
}
build_ubuntu_gpu_cuda91_cudnn7() {
@@ -453,6 +499,7 @@ build_ubuntu_gpu_cmake_mkldnn() {
/work/mxnet
ninja -v
+ report_ccache_usage
# libmkldnn.so.0 is a link file. We need an actual binary file named libmkldnn.so.0.
cp 3rdparty/mkldnn/src/libmkldnn.so.0 3rdparty/mkldnn/src/libmkldnn.so.0.tmp
mv 3rdparty/mkldnn/src/libmkldnn.so.0.tmp 3rdparty/mkldnn/src/libmkldnn.so.0
@@ -474,6 +521,7 @@ build_ubuntu_gpu_cmake() {
/work/mxnet
ninja -v
+ report_ccache_usage
}
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services