You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/12/14 19:54:53 UTC
[GitHub] szha closed pull request #13456: Improve CCache handling
szha closed pull request #13456: Improve CCache handling
URL: https://github.com/apache/incubator-mxnet/pull/13456
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/.gitignore b/.gitignore
index c8a813649bb..7b8ae55e5dd 100644
--- a/.gitignore
+++ b/.gitignore
@@ -167,10 +167,6 @@ python/.eggs
tests/Makefile
tests/mxnet_unit_tests
-# generated wrappers for ccache
-cc
-cxx
-
# Code coverage related
.coverage
*.gcov
diff --git a/Makefile b/Makefile
index 16ea59f3d58..deabb639a78 100644
--- a/Makefile
+++ b/Makefile
@@ -463,7 +463,7 @@ build/src/%.o: src/%.cc | mkldnn
build/src/%_gpu.o: src/%.cu | mkldnn
@mkdir -p $(@D)
- $(NVCC) $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS)" -M -MT build/src/$*_gpu.o $< >build/src/$*_gpu.d
+ $(NVCC) $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS)" --generate-dependencies -MT build/src/$*_gpu.o $< >build/src/$*_gpu.d
$(NVCC) -c -o $@ $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS)" $<
# A nvcc bug cause it to generate "generic/xxx.h" dependencies from torch headers.
@@ -479,7 +479,7 @@ build/plugin/%.o: plugin/%.cc
%_gpu.o: %.cu
@mkdir -p $(@D)
- $(NVCC) $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS) -Isrc/operator" -M -MT $*_gpu.o $< >$*_gpu.d
+ $(NVCC) $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS) -Isrc/operator" --generate-dependencies -MT $*_gpu.o $< >$*_gpu.d
$(NVCC) -c -o $@ $(NVCCFLAGS) $(CUDA_ARCH) -Xcompiler "$(CFLAGS) -Isrc/operator" $<
%.o: %.cc $(CORE_INC)
@@ -664,7 +664,7 @@ rclean:
ifneq ($(EXTRA_OPERATORS),)
clean: rclean cyclean $(EXTRA_PACKAGES_CLEAN)
- $(RM) -r build lib bin *~ */*~ */*/*~ */*/*/*~
+ $(RM) -r build lib bin deps *~ */*~ */*/*~ */*/*/*~
cd $(DMLC_CORE); $(MAKE) clean; cd -
cd $(PS_PATH); $(MAKE) clean; cd -
cd $(NNVM_PATH); $(MAKE) clean; cd -
diff --git a/ci/docker/Dockerfile.build.android_armv7 b/ci/docker/Dockerfile.build.android_armv7
index c601fc5e5ff..a2e98cd2efe 100644
--- a/ci/docker/Dockerfile.build.android_armv7
+++ b/ci/docker/Dockerfile.build.android_armv7
@@ -75,6 +75,11 @@ ENV OpenBLAS_DIR=${CROSS_ROOT}
WORKDIR /work
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.android_armv8 b/ci/docker/Dockerfile.build.android_armv8
index 60376b8efda..f7de8676345 100644
--- a/ci/docker/Dockerfile.build.android_armv8
+++ b/ci/docker/Dockerfile.build.android_armv8
@@ -74,6 +74,12 @@ ENV CXX=${CROSS_ROOT}/bin/${CROSS_TRIPLE}-clang++
COPY install/android_arm64_openblas.sh /work/
RUN /work/android_arm64_openblas.sh
ENV CPLUS_INCLUDE_PATH /work/deps/OpenBLAS
-WORKDIR /work/build
+
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
COPY runtime_functions.sh /work/
+
+WORKDIR /work/build
\ No newline at end of file
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
index 6f16d8c77a0..60e223b7a60 100644
--- a/ci/docker/Dockerfile.build.armv6
+++ b/ci/docker/Dockerfile.build.armv6
@@ -38,5 +38,10 @@ ENV OpenBLAS_DIR=${CROSS_ROOT}
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.armv7 b/ci/docker/Dockerfile.build.armv7
index 5f0223448f1..0b557d5839e 100644
--- a/ci/docker/Dockerfile.build.armv7
+++ b/ci/docker/Dockerfile.build.armv7
@@ -38,5 +38,10 @@ ENV OpenBLAS_DIR=${CROSS_ROOT}
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
diff --git a/ci/docker/Dockerfile.build.armv8 b/ci/docker/Dockerfile.build.armv8
index 27bd425ae9b..ef9c9586559 100644
--- a/ci/docker/Dockerfile.build.armv8
+++ b/ci/docker/Dockerfile.build.armv8
@@ -42,5 +42,10 @@ ENV OpenBLAS_DIR=${CROSS_ROOT}
COPY install/deb_ubuntu_ccache.sh /work/
RUN /work/deb_ubuntu_ccache.sh
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
COPY runtime_functions.sh /work/
WORKDIR /work/build
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
index d128ebc7e2a..30b9b7e3750 100644
--- a/ci/docker/Dockerfile.build.jetson
+++ b/ci/docker/Dockerfile.build.jetson
@@ -82,5 +82,10 @@ ENV NVCCFLAGS "-m64"
ENV CUDA_ARCH "-gencode arch=compute_53,code=sm_53 -gencode arch=compute_62,code=sm_62"
ENV NVCC /usr/local/cuda/bin/nvcc
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
+
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
diff --git a/ci/docker/install/centos7_adduser.sh b/ci/docker/install/centos7_adduser.sh
index ba72c9b9228..f9d2402c955 100755
--- a/ci/docker/install/centos7_adduser.sh
+++ b/ci/docker/install/centos7_adduser.sh
@@ -34,4 +34,9 @@ then
mkdir /work/mxnet
mkdir /work/build
chown -R jenkins_slave /work/
+
+ # Later on, we have to override the links because underlying build systems ignore our compiler settings. Thus,
+ # we have to give the process the proper permission to these files. This is hacky, but unfortunately
+ # there's no better way to do this without patching all our submodules.
+ chown -R jenkins_slave /usr/local/bin
fi
diff --git a/ci/docker/install/ubuntu_adduser.sh b/ci/docker/install/ubuntu_adduser.sh
index 515a80f63b0..a7668bac2ab 100755
--- a/ci/docker/install/ubuntu_adduser.sh
+++ b/ci/docker/install/ubuntu_adduser.sh
@@ -40,4 +40,9 @@ then
mkdir /work/mxnet
mkdir /work/build
chown -R jenkins_slave /work/
+
+ # Later on, we have to override the links because underlying build systems ignore our compiler settings. Thus,
+ # we have to give the process the proper permission to these files. This is hacky, but unfortunately
+ # there's no better way to do this without patching all our submodules.
+ chown -R jenkins_slave /usr/local/bin
fi
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 1fc10bf0e08..c10df3194a8 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -39,32 +39,59 @@ clean_repo() {
build_ccache_wrappers() {
set -ex
- rm -f cc
- rm -f cxx
-
- touch cc
- touch cxx
-
if [ -z ${CC+x} ]; then
echo "No \$CC set, defaulting to gcc";
export CC=gcc
fi
-
- if [ -z ${CXX+x} ]; then
+ if [ -z ${CXX+x} ]; then
echo "No \$CXX set, defaulting to g++";
export CXX=g++
fi
- # this function is nessesary for cuda enabled make based builds, since nvcc needs just an executable for -ccbin
-
- echo -e "#!/bin/sh\n/usr/local/bin/ccache ${CC} \"\$@\"\n" >> cc
- echo -e "#!/bin/sh\n/usr/local/bin/ccache ${CXX} \"\$@\"\n" >> cxx
-
- chmod +x cc
- chmod +x cxx
-
- export CC=`pwd`/cc
- export CXX=`pwd`/cxx
+ # Recommended by CCache: https://ccache.samba.org/manual.html#_run_modes
+ # Add to the beginning of path to ensure this redirection is picked up instead
+ # of the original ones. Especially CUDA/NVCC appends itself to the beginning of the
+ # path and thus this redirect is ignored. This change fixes this problem
+ # This hacky approach with symbolic links is required because underlying build
+ # systems of our submodules ignore our CMake settings. If they use Makefile,
+ # we can't influence them at all in general and NVCC also prefers to hardcode their
+ # compiler instead of respecting the settings. Thus, we take this brutal approach
+ # and just redirect everything of this installer has been called.
+ # In future, we could do these links during image build time of the container.
+ # But in the beginning, we'll make this opt-in. In future, loads of processes like
+ # the scala make step or numpy compilation and other pip package generations
+ # could be heavily sped up by using ccache as well.
+ mkdir /tmp/ccache-redirects
+ export PATH=/tmp/ccache-redirects:$PATH
+ ln -s ccache /tmp/ccache-redirects/gcc
+ ln -s ccache /tmp/ccache-redirects/gcc-8
+ ln -s ccache /tmp/ccache-redirects/g++
+ ln -s ccache /tmp/ccache-redirects/g++-8
+ ln -s ccache /tmp/ccache-redirects/nvcc
+ ln -s ccache /tmp/ccache-redirects/clang++-3.9
+ ln -s ccache /tmp/ccache-redirects/clang-3.9
+ ln -s ccache /tmp/ccache-redirects/clang++-5.0
+ ln -s ccache /tmp/ccache-redirects/clang-5.0
+ ln -s ccache /tmp/ccache-redirects/clang++-6.0
+ ln -s ccache /tmp/ccache-redirects/clang-6.0
+ ln -s ccache /usr/local/bin/gcc
+ ln -s ccache /usr/local/bin/gcc-8
+ ln -s ccache /usr/local/bin/g++
+ ln -s ccache /usr/local/bin/g++-8
+ ln -s ccache /usr/local/bin/nvcc
+ ln -s ccache /usr/local/bin/clang++-3.9
+ ln -s ccache /usr/local/bin/clang-3.9
+ ln -s ccache /usr/local/bin/clang++-5.0
+ ln -s ccache /usr/local/bin/clang-5.0
+ ln -s ccache /usr/local/bin/clang++-6.0
+ ln -s ccache /usr/local/bin/clang-6.0
+
+ export NVCC=ccache
+
+ # Uncomment if you would like to debug CCache hit rates.
+ # You can monitor using tail -f ccache-log
+ # export CCACHE_LOGFILE=/work/mxnet/ccache-log
+ # export CCACHE_DEBUG=1
}
build_wheel() {
@@ -106,6 +133,8 @@ build_jetson() {
set -ex
pushd .
+ build_ccache_wrappers
+
cp make/crosscompile.jetson.mk ./config.mk
make -j$(nproc)
@@ -129,6 +158,7 @@ build_armv6() {
# We do not need OpenMP, since most armv6 systems have only 1 core
+ build_ccache_wrappers
cmake \
-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
@@ -159,6 +189,7 @@ build_armv7() {
# file tries to add -llapack. Lapack functionality though, requires -lgfortran
# to be linked additionally.
+ build_ccache_wrappers
cmake \
-DCMAKE_TOOLCHAIN_FILE=${CMAKE_TOOLCHAIN_FILE} \
-DCMAKE_CROSSCOMPILING=ON \
@@ -181,6 +212,7 @@ build_armv7() {
}
build_armv8() {
+ build_ccache_wrappers
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
@@ -205,6 +237,7 @@ build_armv8() {
build_android_armv7() {
set -ex
cd /work/build
+ build_ccache_wrappers
cmake \
-DANDROID=ON\
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
@@ -225,6 +258,7 @@ build_android_armv7() {
build_android_armv8() {
set -ex
cd /work/build
+ build_ccache_wrappers
cmake\
-DANDROID=ON \
-DUSE_CUDA=OFF\
@@ -244,7 +278,7 @@ build_centos7_cpu() {
cd /work/mxnet
export CC="ccache gcc"
export CXX="ccache g++"
-
+ build_ccache_wrappers
make \
DEV=1 \
USE_LAPACK=1 \
@@ -257,6 +291,7 @@ build_centos7_cpu() {
build_amzn_linux_cpu() {
cd /work/build
+ build_ccache_wrappers
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
@@ -279,7 +314,7 @@ build_centos7_mkldnn() {
cd /work/mxnet
export CC="ccache gcc"
export CXX="ccache g++"
-
+ build_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
@@ -294,7 +329,7 @@ build_centos7_gpu() {
set -ex
cd /work/mxnet
# unfortunately this build has problems in 3rdparty dependencies with ccache and make
- # build_ccache_wrappers
+ build_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
@@ -315,8 +350,9 @@ build_ubuntu_cpu() {
build_ubuntu_cpu_openblas() {
set -ex
- export CC="ccache gcc"
- export CXX="ccache g++"
+ export CC="gcc"
+ export CXX="g++"
+ build_ccache_wrappers
make \
DEV=1 \
ENABLE_TESTCOVERAGE=1 \
@@ -330,6 +366,7 @@ build_ubuntu_cpu_cmake_debug() {
set -ex
pushd .
cd /work/build
+ build_ccache_wrappers
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
@@ -351,11 +388,12 @@ build_ubuntu_cpu_cmake_asan() {
pushd .
cd /work/build
+ export CXX=g++-8
+ export CC=gcc-8
+ build_ccache_wrappers
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
- -DCMAKE_CXX_COMPILER=/usr/bin/g++-8 \
- -DCMAKE_C_COMPILER=/usr/bin/gcc-8 \
-DUSE_CUDA=OFF \
-DUSE_MKL_IF_AVAILABLE=OFF \
-DUSE_OPENMP=OFF \
@@ -377,10 +415,10 @@ build_ubuntu_cpu_cmake_asan() {
build_ubuntu_cpu_clang39() {
set -ex
- export CXX=clang++-3.9
+ export CXX=clang++-3.9
export CC=clang-3.9
- build_ccache_wrappers
- make \
+ build_ccache_wrappers
+ make \
ENABLE_TESTCOVERAGE=1 \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
@@ -415,6 +453,7 @@ build_ubuntu_cpu_clang_tidy() {
pushd .
cd /work/build
+ build_ccache_wrappers
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
@@ -498,6 +537,8 @@ build_ubuntu_gpu_tensorrt() {
mkdir -p build
cd build
cmake \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_C_COMPILER_LAUNCHER=ccache \
-DCMAKE_CXX_FLAGS=-I/usr/include/python${PYVER}\
-DBUILD_SHARED_LIBS=ON ..\
-G Ninja
@@ -512,7 +553,10 @@ build_ubuntu_gpu_tensorrt() {
cd 3rdparty/onnx-tensorrt/
mkdir -p build
cd build
- cmake ..
+ cmake \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_C_COMPILER_LAUNCHER=ccache \
+ ..
make -j$(nproc)
export LIBRARY_PATH=`pwd`:$LIBRARY_PATH
popd
@@ -595,6 +639,7 @@ build_ubuntu_gpu_cuda91_cudnn7() {
build_ubuntu_amalgamation() {
set -ex
# Amalgamation can not be run with -j nproc
+ build_ccache_wrappers
make -C amalgamation/ clean
make -C amalgamation/ \
USE_BLAS=openblas \
@@ -604,6 +649,7 @@ build_ubuntu_amalgamation() {
build_ubuntu_amalgamation_min() {
set -ex
# Amalgamation can not be run with -j nproc
+ build_ccache_wrappers
make -C amalgamation/ clean
make -C amalgamation/ \
USE_BLAS=openblas \
@@ -614,9 +660,11 @@ build_ubuntu_amalgamation_min() {
build_ubuntu_gpu_cmake_mkldnn() {
set -ex
cd /work/build
+ build_ccache_wrappers
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
-DENABLE_TESTCOVERAGE=ON \
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
@@ -637,9 +685,11 @@ build_ubuntu_gpu_cmake_mkldnn() {
build_ubuntu_gpu_cmake() {
set -ex
cd /work/build
+ build_ccache_wrappers
cmake \
-DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DCMAKE_C_COMPILER_LAUNCHER=ccache \
+ -DCMAKE_CUDA_COMPILER_LAUNCHER=ccache \
-DENABLE_TESTCOVERAGE=ON \
-DUSE_CUDA=1 \
-DUSE_CUDNN=1 \
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services