You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/11/29 17:21:58 UTC
[GitHub] marcoabreu closed pull request #11520: [WIP] Improve Ccache
marcoabreu closed pull request #11520: [WIP] Improve Ccache
URL: https://github.com/apache/incubator-mxnet/pull/11520
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.gitignore b/.gitignore
index c8a813649bb..7b8ae55e5dd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -167,10 +167,6 @@ python/.eggs
tests/Makefile
tests/mxnet_unit_tests
-# generated wrappers for ccache
-cc
-cxx
-
# Code coverage related
.coverage
*.gcov
diff --git a/Makefile b/Makefile
index 16ea59f3d58..11f2c7953bc 100644
--- a/Makefile
+++ b/Makefile
@@ -463,7 +463,7 @@ build/src/%.o: src/%.cc | mkldnn
build/src/%_gpu.o: src/%.cu | mkldnn
@mkdir -p $(@D)
- $(NVCC) $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS)" -M -MT build/src/$*_gpu.o $< >build/src/$*_gpu.d
+ $(NVCC) $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS)" --generate-dependencies -MT build/src/$*_gpu.o $< >build/src/$*_gpu.d
$(NVCC) -c -o $@ $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS)" $<
# A nvcc bug cause it to generate "generic/xxx.h" dependencies from torch headers.
@@ -479,7 +479,7 @@ build/plugin/%.o: plugin/%.cc
%_gpu.o: %.cu
@mkdir -p $(@D)
- $(NVCC) $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS) -Isrc/operator" -M -MT $*_gpu.o $< >$*_gpu.d
+ $(NVCC) $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS) -Isrc/operator" --generate-dependencies -MT $*_gpu.o $< >$*_gpu.d
$(NVCC) -c -o $@ $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS) -Isrc/operator" $<
%.o: %.cc $(CORE_INC)
@@ -660,7 +660,7 @@ jnilint:
rclean:
$(RM) -r R-package/src/image_recordio.h R-package/NAMESPACE R-package/man R-package/R/mxnet_generated.R \
- R-package/inst R-package/src/*.o R-package/src/*.so mxnet_*.tar.gz
+ R-package/inst R-package/src/*.o R-package/src/*.so mxnet_*.tar.gz deps
ifneq ($(EXTRA_OPERATORS),)
clean: rclean cyclean $(EXTRA_PACKAGES_CLEAN)
diff --git a/ci/docker/install/centos7_adduser.sh b/ci/docker/install/centos7_adduser.sh
index ba72c9b9228..f9d2402c955 100755
--- a/ci/docker/install/centos7_adduser.sh
+++ b/ci/docker/install/centos7_adduser.sh
@@ -34,4 +34,9 @@ then
mkdir /work/mxnet
mkdir /work/build
chown -R jenkins_slave /work/
+
+ # Later on, we have to override the links because underlying build systems ignore our compiler settings. Thus,
+ # we have to give the process the proper permission to these files. This is hacky, but unfortunately
+ # there's no better way to do this without patching all our submodules.
+ chown -R jenkins_slave /usr/local/bin
fi
diff --git a/ci/docker/install/ubuntu_adduser.sh b/ci/docker/install/ubuntu_adduser.sh
index 515a80f63b0..a7668bac2ab 100755
--- a/ci/docker/install/ubuntu_adduser.sh
+++ b/ci/docker/install/ubuntu_adduser.sh
@@ -40,4 +40,9 @@ then
mkdir /work/mxnet
mkdir /work/build
chown -R jenkins_slave /work/
+
+ # Later on, we have to override the links because underlying build systems ignore our compiler settings. Thus,
+ # we have to give the process the proper permission to these files. This is hacky, but unfortunately
+ # there's no better way to do this without patching all our submodules.
+ chown -R jenkins_slave /usr/local/bin
fi
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 1fc10bf0e08..fec141e9720 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -36,35 +36,37 @@ clean_repo() {
git submodule update --init --recursive
}
-build_ccache_wrappers() {
- set -ex
-
- rm -f cc
- rm -f cxx
-
- touch cc
- touch cxx
-
- if [ -z ${CC+x} ]; then
- echo "No \$CC set, defaulting to gcc";
- export CC=gcc
- fi
-
- if [ -z ${CXX+x} ]; then
- echo "No \$CXX set, defaulting to g++";
- export CXX=g++
- fi
-
- # this function is nessesary for cuda enabled make based builds, since nvcc needs just an executable for -ccbin
-
- echo -e "#!/bin/sh\n/usr/local/bin/ccache ${CC} \"\$@\"\n" >> cc
- echo -e "#!/bin/sh\n/usr/local/bin/ccache ${CXX} \"\$@\"\n" >> cxx
-
- chmod +x cc
- chmod +x cxx
-
- export CC=`pwd`/cc
- export CXX=`pwd`/cxx
+install_ccache_wrappers() {
+ set -ex
+
+ # Recommended by CCache: https://ccache.samba.org/manual.html#_run_modes
+ # Add to the beginning of path to ensure this redirection is picked up instead
+ # of the original ones. Especially CUDA/NVCC appends itself to the beginning of the
+ # path and thus this redirect is ignored. This change fixes this problem
+ # This hacky approach with symbolic links is required because underlying build
+ # systems of our submodules ignore our CMake settings. If they use Makefile,
+ # we can't influence them at all in general and NVCC also prefers to hardcode their
+ # compiler instead of respecting the settings. Thus, we take this brutal approach
+ # and just redirect everything of this installer has been called.
+ # In future, we could do these links during image build time of the container.
+ # But in the beginning, we'll make this opt-in. In future, loads of processes like
+ # the scala make step or numpy compilation and other pip package generations
+ # could be heavily sped up by using ccache as well.
+ export PATH=/usr/local/bin:$PATH
+ ln -s ccache /usr/local/bin/gcc
+ ln -s ccache /usr/local/bin/g++
+ ln -s ccache /usr/local/bin/g++-8
+ ln -s ccache /usr/local/bin/gcc-8
+ ln -s ccache /usr/local/bin/nvcc
+ ln -s ccache /usr/local/bin/clang++-3.9
+ ln -s ccache /usr/local/bin/clang-3.9
+ ln -s ccache /usr/local/bin/clang++-5.0
+ ln -s ccache /usr/local/bin/clang-5.0
+ ln -s ccache /usr/local/bin/clang++-6.0
+ ln -s ccache /usr/local/bin/clang-6.0
+
+ # TODO: Remove
+ export CCACHE_LOGFILE=/work/mxnet/ccache-log
}
build_wheel() {
@@ -128,7 +130,6 @@ build_armv6() {
# to be linked additionally.
# We do not need OpenMP, since most armv6 systems have only 1 core
-
cmake \
-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
@@ -227,6 +228,8 @@ build_android_armv8() {
cd /work/build
cmake\
-DANDROID=ON \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DUSE_CUDA=OFF\
-DUSE_SSE=OFF\
-DUSE_LAPACK=OFF\
@@ -242,9 +245,8 @@ build_android_armv8() {
build_centos7_cpu() {
set -ex
cd /work/mxnet
- export CC="ccache gcc"
- export CXX="ccache g++"
+ install_ccache_wrappers
make \
DEV=1 \
USE_LAPACK=1 \
@@ -257,6 +259,7 @@ build_centos7_cpu() {
build_amzn_linux_cpu() {
cd /work/build
+ install_ccache_wrappers
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
@@ -277,9 +280,8 @@ build_amzn_linux_cpu() {
build_centos7_mkldnn() {
set -ex
cd /work/mxnet
- export CC="ccache gcc"
- export CXX="ccache g++"
-
+
+ install_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
@@ -293,8 +295,7 @@ build_centos7_mkldnn() {
build_centos7_gpu() {
set -ex
cd /work/mxnet
- # unfortunately this build has problems in 3rdparty dependencies with ccache and make
- # build_ccache_wrappers
+ install_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
@@ -315,8 +316,7 @@ build_ubuntu_cpu() {
build_ubuntu_cpu_openblas() {
set -ex
- export CC="ccache gcc"
- export CXX="ccache g++"
+ install_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
@@ -330,6 +330,7 @@ build_ubuntu_cpu_cmake_debug() {
set -ex
pushd .
cd /work/build
+ install_ccache_wrappers
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
@@ -351,6 +352,8 @@ build_ubuntu_cpu_cmake_asan() {
pushd .
cd /work/build
+ install_ccache_wrappers
+ # TODO: Check if we can remove the g++-8 override
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
@@ -377,10 +380,10 @@ build_ubuntu_cpu_cmake_asan() {
build_ubuntu_cpu_clang39() {
set -ex
- export CXX=clang++-3.9
+ export CXX=clang++-3.9
export CC=clang-3.9
- build_ccache_wrappers
- make \
+ install_ccache_wrappers
+ make \
ENABLE_TESTCOVERAGE=1 \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
@@ -395,7 +398,7 @@ build_ubuntu_cpu_clang60() {
export CXX=clang++-6.0
export CC=clang-6.0
- build_ccache_wrappers
+ install_ccache_wrappers
make \
ENABLE_TESTCOVERAGE=1 \
@@ -415,6 +418,7 @@ build_ubuntu_cpu_clang_tidy() {
pushd .
cd /work/build
+ install_ccache_wrappers
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
@@ -438,8 +442,7 @@ build_ubuntu_cpu_clang39_mkldnn() {
export CXX=clang++-3.9
export CC=clang-3.9
- build_ccache_wrappers
-
+ install_ccache_wrappers
make \
ENABLE_TESTCOVERAGE=1 \
USE_CPP_PACKAGE=1 \
@@ -455,8 +458,7 @@ build_ubuntu_cpu_clang60_mkldnn() {
export CXX=clang++-6.0
export CC=clang-6.0
- build_ccache_wrappers
-
+ install_ccache_wrappers
make \
ENABLE_TESTCOVERAGE=1 \
USE_CPP_PACKAGE=1 \
@@ -469,8 +471,7 @@ build_ubuntu_cpu_clang60_mkldnn() {
build_ubuntu_cpu_mkldnn() {
set -ex
- build_ccache_wrappers
-
+ install_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
@@ -488,7 +489,10 @@ build_ubuntu_gpu_tensorrt() {
set -ex
- build_ccache_wrappers
+ install_ccache_wrappers
+
+ # https://github.com/nicehash/nheqminer/issues/228#issuecomment-283240760
+ export CUDA_CUDART_LIBRARY="/usr/local/cuda/lib64/libcudart.so"
# Build ONNX
pushd .
@@ -498,6 +502,8 @@ build_ubuntu_gpu_tensorrt() {
mkdir -p build
cd build
cmake \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_FLAGS=-I/usr/include/python${PYVER}\
-DBUILD_SHARED_LIBS=ON ..\
-G Ninja
@@ -512,7 +518,10 @@ build_ubuntu_gpu_tensorrt() {
cd 3rdparty/onnx-tensorrt/
mkdir -p build
cd build
- cmake ..
+ cmake \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+ ..
make -j$(nproc)
export LIBRARY_PATH=`pwd`:$LIBRARY_PATH
popd
@@ -543,8 +552,7 @@ build_ubuntu_gpu_tensorrt() {
build_ubuntu_gpu_mkldnn() {
set -ex
- build_ccache_wrappers
-
+ install_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
@@ -561,8 +569,7 @@ build_ubuntu_gpu_mkldnn() {
build_ubuntu_gpu_mkldnn_nocudnn() {
set -ex
- build_ccache_wrappers
-
+ install_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
@@ -577,8 +584,7 @@ build_ubuntu_gpu_mkldnn_nocudnn() {
build_ubuntu_gpu_cuda91_cudnn7() {
set -ex
- # unfortunately this build has problems in 3rdparty dependencies with ccache and make
- # build_ccache_wrappers
+ install_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
@@ -595,6 +601,7 @@ build_ubuntu_gpu_cuda91_cudnn7() {
build_ubuntu_amalgamation() {
set -ex
# Amalgamation can not be run with -j nproc
+ install_ccache_wrappers
make -C amalgamation/ clean
make -C amalgamation/ \
USE_BLAS=openblas \
@@ -604,6 +611,7 @@ build_ubuntu_amalgamation() {
build_ubuntu_amalgamation_min() {
set -ex
# Amalgamation can not be run with -j nproc
+ install_ccache_wrappers
make -C amalgamation/ clean
make -C amalgamation/ \
USE_BLAS=openblas \
@@ -614,9 +622,10 @@ build_ubuntu_amalgamation_min() {
build_ubuntu_gpu_cmake_mkldnn() {
set -ex
cd /work/build
+ install_ccache_wrappers
cmake \
- -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
- -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DENABLE_TESTCOVERAGE=ON \
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
@@ -637,9 +646,10 @@ build_ubuntu_gpu_cmake_mkldnn() {
build_ubuntu_gpu_cmake() {
set -ex
cd /work/build
+ install_ccache_wrappers
cmake \
- -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
- -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DENABLE_TESTCOVERAGE=ON \
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services