[incubator-mxnet] branch master updated: Enable shared ccache and fix test (#11269)

[jx...@apache.org]

This is an automated email from the ASF dual-hosted git repository.

jxie pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git


The following commit(s) were added to refs/heads/master by this push:
     new 2eadfa2  Enable shared ccache and fix test (#11269)
2eadfa2 is described below

commit 2eadfa29c5cdb39bf30af28fb3265b5243984d91
Author: Marco de Abreu <ma...@users.noreply.github.com>
AuthorDate: Thu Jun 14 14:46:02 2018 -0700

    Enable shared ccache and fix test (#11269)
    
    * Enable shared ccache
    
    * Increase cache size
    
    * Don't fail if EFS ccache is not available
---
 ci/build.py             | 12 ++++++++----
 ci/test_docker_cache.py |  3 +++
 2 files changed, 11 insertions(+), 4 deletions(-)

diff --git a/ci/build.py b/ci/build.py
index 35f8b47..b6821b3 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -40,7 +40,7 @@ from itertools import chain
 from subprocess import call, check_call
 from typing import *
 
-CCACHE_MAXSIZE = '10G'
+CCACHE_MAXSIZE = '500G'
 
 def get_platforms(path: Optional[str] = "docker"):
     """Get a list of architectures given our dockerfiles"""
@@ -124,9 +124,12 @@ def buildir() -> str:
 def default_ccache_dir() -> str:
     # Share ccache across containers
     if 'CCACHE_DIR' in os.environ:
-        ccache_dir = os.path.realpath(os.environ['CCACHE_DIR'])
-        os.makedirs(ccache_dir, exist_ok=True)
-        return ccache_dir
+        try:
+            ccache_dir = os.path.realpath(os.environ['CCACHE_DIR'])
+            os.makedirs(ccache_dir, exist_ok=True)
+            return ccache_dir
+        except PermissionError:
+            logging.info('Unable to make dirs at %s, falling back to local temp dir', ccache_dir)
     # In osx tmpdir is not mountable by default
     if platform.system() == 'Darwin':
         ccache_dir = "/tmp/_mxnet_ccache"
@@ -157,6 +160,7 @@ def container_run(platform: str,
                '-v', "{}:/work/ccache".format(local_ccache_dir),
                '-u', '{}:{}'.format(os.getuid(), os.getgid()),
                '-e', 'CCACHE_MAXSIZE={}'.format(CCACHE_MAXSIZE),
+               '-e', 'CCACHE_TEMPDIR=/tmp/ccache',  # temp dir should be local and not shared
                '-e', "CCACHE_DIR=/work/ccache",  # this path is inside the container as /work/ccache is mounted
                tag]
     runlist.extend(command)
diff --git a/ci/test_docker_cache.py b/ci/test_docker_cache.py
index fa8833f..3f471db 100644
--- a/ci/test_docker_cache.py
+++ b/ci/test_docker_cache.py
@@ -28,6 +28,7 @@ import os
 import logging
 import subprocess
 import sys
+from unittest.mock import MagicMock
 
 sys.path.append(os.path.dirname(__file__))
 import docker_cache
@@ -87,6 +88,8 @@ class TestDockerCache(unittest.TestCase):
         base = os.path.split(os.path.realpath(__file__))[0]
         os.chdir(base)
 
+        docker_cache._login_dockerhub = MagicMock()  # Override login
+
         # Stop in case previous execution was dirty
         try:
             self._stop_local_docker_registry()

-- 
To stop receiving notification emails like this one, please contact
jxie@apache.org.