You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/06/06 09:13:13 UTC
[GitHub] marcoabreu closed pull request #11036: [MXNET-472] Add ccache
support to CI
marcoabreu closed pull request #11036: [MXNET-472] Add ccache support to CI
URL: https://github.com/apache/incubator-mxnet/pull/11036
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/Makefile b/Makefile
index 951b29b41cf..0e26858a601 100644
--- a/Makefile
+++ b/Makefile
@@ -477,7 +477,7 @@ endif
$(PS_PATH)/build/libps.a: PSLITE
PSLITE:
- $(MAKE) CXX=$(CXX) DEPS_PATH=$(DEPS_PATH) -C $(PS_PATH) ps
+ $(MAKE) CXX="$(CXX)" DEPS_PATH="$(DEPS_PATH)" -C $(PS_PATH) ps
$(DMLC_CORE)/libdmlc.a: DMLCCORE
diff --git a/ci/build.py b/ci/build.py
index deae1d733a8..a037b244b3d 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -33,6 +33,7 @@
import shutil
import subprocess
import sys
+import tempfile
from copy import deepcopy
from itertools import chain
from subprocess import call, check_call
@@ -115,10 +116,19 @@ def is_mxnet_root(path: str) -> bool:
def buildir() -> str:
return os.path.join(get_mxnet_root(), "build")
+def default_ccache_dir() -> str:
+ if 'CCACHE_DIR' in os.environ:
+ ccache_dir = os.path.realpath(os.environ['CCACHE_DIR'])
+ os.makedirs(ccache_dir, exist_ok=True)
+ return ccache_dir
+ #return os.path.join(buildir(), "ccache")
+ # Share ccache across containers (should we have a separate dir per platform?)
+ return os.path.join(tempfile.gettempdir(), "ci_ccache")
def container_run(platform: str,
docker_binary: str,
shared_memory_size: str,
+ local_ccache_dir: str,
command: List[str],
dry_run: bool = False,
into_container: bool = False) -> str:
@@ -127,11 +137,15 @@ def container_run(platform: str,
local_build_folder = buildir()
# We need to create it first, otherwise it will be created by the docker daemon with root only permissions
os.makedirs(local_build_folder, exist_ok=True)
+ os.makedirs(local_ccache_dir, exist_ok=True)
+ logging.info("Using ccache directory: %s", local_ccache_dir)
runlist = [docker_binary, 'run', '--rm', '-t',
'--shm-size={}'.format(shared_memory_size),
'-v', "{}:/work/mxnet".format(mx_root), # mount mxnet root
'-v', "{}:/work/build".format(local_build_folder), # mount mxnet/build for storing build artifacts
+ '-v', "{}:/work/ccache".format(local_ccache_dir),
'-u', '{}:{}'.format(os.getuid(), os.getgid()),
+ '-e', "CCACHE_DIR=/work/ccache", # this path is inside the container as /work/ccache is mounted
tag]
runlist.extend(command)
cmd = ' '.join(runlist)
@@ -166,12 +180,11 @@ def main() -> int:
os.chdir(base)
logging.getLogger().setLevel(logging.INFO)
-
def script_name() -> str:
return os.path.split(sys.argv[0])[1]
-
logging.basicConfig(format='{}: %(asctime)-15s %(message)s'.format(script_name()))
+
parser = argparse.ArgumentParser(description="""Utility for building and testing MXNet on docker
containers""",epilog="")
parser.add_argument("-p", "--platform",
@@ -219,6 +232,11 @@ def script_name() -> str:
help="command to run in the container",
nargs='*', action='append', type=str)
+ parser.add_argument("--ccache-dir",
+ default=default_ccache_dir(),
+ help="Ccache directory",
+ type=str)
+
args = parser.parse_args()
command = list(chain(*args.command))
docker_binary = get_docker_binary(args.nvidiadocker)
@@ -240,15 +258,15 @@ def script_name() -> str:
return 0
if command:
- container_run(platform, docker_binary, shared_memory_size, command)
+ container_run(platform, docker_binary, shared_memory_size, args.ccache_dir, command)
elif args.print_docker_run:
- print(container_run(platform, docker_binary, shared_memory_size, [], True))
+ print(container_run(platform, docker_binary, shared_memory_size, args.ccache_dir, [], True))
elif args.into_container:
- container_run(platform, docker_binary, shared_memory_size, [], False, True)
+ container_run(platform, docker_binary, shared_memory_size, args.ccache_dir, [], False, True)
else:
cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", "build_{}".format(platform)]
logging.info("No command specified, trying default build: %s", ' '.join(cmd))
- container_run(platform, docker_binary, shared_memory_size, cmd)
+ container_run(platform, docker_binary, shared_memory_size, args.ccache_dir, cmd)
elif args.all:
platforms = get_platforms()
@@ -266,7 +284,7 @@ def script_name() -> str:
build_platform = "build_{}".format(platform)
cmd = ["/work/mxnet/ci/docker/runtime_functions.sh", build_platform]
shutil.rmtree(buildir(), ignore_errors=True)
- container_run(platform, docker_binary, shared_memory_size, cmd)
+ container_run(platform, docker_binary, shared_memory_size, args.ccache_dir, cmd)
plat_buildir = os.path.join(get_mxnet_root(), build_platform)
shutil.move(buildir(), plat_buildir)
logging.info("Built files left in: %s", plat_buildir)
diff --git a/ci/docker/Dockerfile.build.armv6 b/ci/docker/Dockerfile.build.armv6
index 20739dabe2e..21221238d46 100755
--- a/ci/docker/Dockerfile.build.armv6
+++ b/ci/docker/Dockerfile.build.armv6
@@ -20,6 +20,8 @@
FROM dockcross/linux-armv6
+RUN apt update && apt install -y ccache
+
ENV ARCH armv6l
ENV FC=/usr/bin/${CROSS_TRIPLE}-gfortran
ENV HOSTCC gcc
diff --git a/ci/docker/Dockerfile.build.jetson b/ci/docker/Dockerfile.build.jetson
index c358edb1fb0..dd8da321472 100755
--- a/ci/docker/Dockerfile.build.jetson
+++ b/ci/docker/Dockerfile.build.jetson
@@ -63,5 +63,9 @@ ENV NVCCFLAGS "-m64"
ENV CUDA_ARCH "-gencode arch=compute_53,code=sm_53 -gencode arch=compute_62,code=sm_62"
ENV NVCC /usr/local/cuda/bin/nvcc
+# we need to install latest ccache that contains fixes for working with nvcc
+
+RUN apt update && apt install -y ccache
+
COPY runtime_functions.sh /work/
WORKDIR /work/mxnet
diff --git a/ci/docker/install/centos7_core.sh b/ci/docker/install/centos7_core.sh
index 1d7e120d6ae..fbdbd3cab5d 100755
--- a/ci/docker/install/centos7_core.sh
+++ b/ci/docker/install/centos7_core.sh
@@ -37,3 +37,4 @@ yum -y install cmake
yum -y install wget
yum -y install unzip
yum -y install ninja-build
+yum -y install ccache
diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh
index 65ad4726597..a71ab15c639 100755
--- a/ci/docker/install/ubuntu_core.sh
+++ b/ci/docker/install/ubuntu_core.sh
@@ -40,4 +40,5 @@ apt-get install -y \
software-properties-common \
sudo \
unzip \
- wget
+ wget \
+ ccache
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 7abe767c869..d99c8a2d27c 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -38,9 +38,12 @@ clean_repo() {
build_jetson() {
set -ex
pushd .
- mv make/crosscompile.jetson.mk make/config.mk
+ mv make/config.mk make/config.tmp
+ cp -f make/crosscompile.jetson.mk make/config.mk
make -j$(nproc)
+ mv make/config.tmp make/config.mk
+
export MXNET_LIBRARY_PATH=`pwd`/libmxnet.so
cd /work/mxnet/python
python setup.py bdist_wheel --universal
@@ -62,6 +65,7 @@ build_jetson() {
build_armv6() {
set -ex
pushd .
+
cd /work/build
# Lapack functionality will be included and statically linked to openblas.
@@ -73,6 +77,7 @@ build_armv6() {
cmake \
-DCMAKE_TOOLCHAIN_FILE=$CROSS_ROOT/Toolchain.cmake \
+ -DCMAKE_CXX_COMPILER_LAUNCHER=ccache \
-DUSE_CUDA=OFF \
-DUSE_OPENCV=OFF \
-DUSE_OPENMP=OFF \
@@ -166,6 +171,8 @@ build_android_arm64() {
build_centos7_cpu() {
set -ex
cd /work/mxnet
+ export CC="ccache gcc"
+ export CXX="ccache g++"
make \
DEV=1 \
USE_LAPACK=1 \
@@ -178,6 +185,8 @@ build_centos7_cpu() {
build_centos7_mkldnn() {
set -ex
cd /work/mxnet
+ export CC="ccache gcc"
+ export CXX="ccache g++"
make \
DEV=1 \
USE_LAPACK=1 \
@@ -202,8 +211,14 @@ build_centos7_gpu() {
-j$(nproc)
}
+build_ubuntu_cpu() {
+ build_ubuntu_cpu_openblas
+}
+
build_ubuntu_cpu_openblas() {
set -ex
+ export CC="ccache gcc"
+ export CXX="ccache g++"
make \
DEV=1 \
USE_CPP_PACKAGE=1 \
@@ -214,37 +229,39 @@ build_ubuntu_cpu_openblas() {
build_ubuntu_cpu_clang39() {
set -ex
+ export CC="ccache clang-3.9"
+ export CXX="ccache clang++-3.9"
make \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
USE_OPENMP=0 \
USE_DIST_KVSTORE=1 \
- CXX=clang++-3.9 \
- CC=clang-3.9 \
-j$(nproc)
}
build_ubuntu_cpu_clang50() {
set -ex
+ export CC="ccache clang-5.0"
+ export CXX="ccache clang++-5.0"
make \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
USE_OPENMP=1 \
USE_DIST_KVSTORE=1 \
- CXX=clang++-5.0 \
- CC=clang-5.0 \
-j$(nproc)
}
build_ubuntu_cpu_clang39_mkldnn() {
+ # mkldnn gets compiled with a script
+ # and compilation fails with clang-3.9
set -ex
make \
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
USE_MKLDNN=1 \
USE_OPENMP=0 \
- CXX=clang++-3.9 \
- CC=clang-3.9 \
+ CC="ccache clang-3.9" \
+ CXX="ccache clang++-3.9" \
-j$(nproc)
}
@@ -255,8 +272,8 @@ build_ubuntu_cpu_clang50_mkldnn() {
USE_BLAS=openblas \
USE_MKLDNN=1 \
USE_OPENMP=1 \
- CXX=clang++-5.0 \
- CC=clang-5.0 \
+ CC="ccache clang-5.0" \
+ CXX="ccache clang++-5.0" \
-j$(nproc)
}
@@ -267,6 +284,8 @@ build_ubuntu_cpu_mkldnn() {
USE_CPP_PACKAGE=1 \
USE_BLAS=openblas \
USE_MKLDNN=1 \
+ CC="ccache gcc" \
+ CXX="ccache g++" \
-j$(nproc)
}
@@ -381,7 +400,7 @@ unittest_ubuntu_python3_cpu() {
unittest_ubuntu_python3_cpu_mkldnn() {
set -ex
- export PYTHONPATH=./python/
+ export PYTHONPATH=./python/
# MXNET_MKLDNN_DEBUG is buggy and produces false positives
# https://github.com/apache/incubator-mxnet/issues/10026
#export MXNET_MKLDNN_DEBUG=1 # Ignored if not present
diff --git a/make/config.mk b/make/config.mk
index dd67c33cc9e..8cc3b2fa230 100644
--- a/make/config.mk
+++ b/make/config.mk
@@ -37,8 +37,8 @@
# choice of compiler
#--------------------
-export CC = gcc
-export CXX = g++
+#export CC = gcc
+#export CXX = g++
export NVCC = nvcc
# whether compile with options for MXNet developer
diff --git a/prepare_mkldnn.sh b/prepare_mkldnn.sh
index b8d5c7e033f..a7d42de659f 100755
--- a/prepare_mkldnn.sh
+++ b/prepare_mkldnn.sh
@@ -17,7 +17,6 @@
# specific language governing permissions and limitations
# under the License.
-# set -ex
#
# All modification made by Intel Corporation: © 2016 Intel Corporation
#
@@ -55,6 +54,8 @@
# OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE.
#
+set -e
+
MXNET_ROOTDIR="$(pwd)"
MKLDNN_ROOTDIR="$MXNET_ROOTDIR/3rdparty/mkldnn/"
MKLDNN_SRCDIR="$MKLDNN_ROOTDIR/src"
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services