You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by la...@apache.org on 2020/04/03 20:22:06 UTC
[incubator-mxnet] branch master updated: Fix Windows GPU CI (#17962)
This is an automated email from the ASF dual-hosted git repository.
lausen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 66ee118 Fix Windows GPU CI (#17962)
66ee118 is described below
commit 66ee118a455c5e267566f0e3fdc482985f803df0
Author: Leonard Lausen <la...@amazon.com>
AuthorDate: Fri Apr 3 20:20:47 2020 +0000
Fix Windows GPU CI (#17962)
Update Windows CI to use VS 2019 and enable x64 bit toolchain. Previously we are using an older 32 bit toolchain causing OOM errors during linking. Switching to x64 bit toolchain on the older VS version previously used by the CI was attempted in #17912 and did not work. Update to Cuda 10.2 as it is required by VS 2019. Switch to ninja-build on Windows to speed up build as ninja-build is now preinstalled. Remove logic to install cmake 3.16 on every PR as cmake 3.17 is now preinstalled. [...]
Co-authored-by: vexilligera <ve...@gmail.com>
---
3rdparty/dmlc-core | 2 +-
ci/build_windows.py | 68 +++++++++++++++++++++++++++++++++--------------------
2 files changed, 43 insertions(+), 27 deletions(-)
diff --git a/3rdparty/dmlc-core b/3rdparty/dmlc-core
index c8f7f9c..14bf7e8 160000
--- a/3rdparty/dmlc-core
+++ b/3rdparty/dmlc-core
@@ -1 +1 @@
-Subproject commit c8f7f9c81dfcf2489d3bbc6d3b2a9200a20fbcde
+Subproject commit 14bf7e884328eb97bfde160ec6f64c20f5337459
diff --git a/ci/build_windows.py b/ci/build_windows.py
index 2590d21..b9c17a8 100755
--- a/ci/build_windows.py
+++ b/ci/build_windows.py
@@ -33,13 +33,15 @@ import time
import zipfile
from distutils.dir_util import copy_tree
from enum import Enum
-from subprocess import check_call
+from subprocess import check_call, call
from util import *
KNOWN_VCVARS = {
+ # https://gitlab.kitware.com/cmake/cmake/issues/18920
'VS 2015': r'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat',
- 'VS 2017': r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat'
+ 'VS 2017': r'C:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat',
+ 'VS 2019': r'C:\Program Files (x86)\Microsoft Visual Studio\2019\Community\VC\Auxiliary\Build\vcvars64.bat',
}
@@ -54,6 +56,8 @@ class BuildFlavour(Enum):
CMAKE_FLAGS = {
'WIN_CPU': (
+ '-DCMAKE_C_COMPILER=cl '
+ '-DCMAKE_CXX_COMPILER=cl '
'-DUSE_CUDA=OFF '
'-DUSE_CUDNN=OFF '
'-DENABLE_CUDA_RTC=OFF '
@@ -67,6 +71,8 @@ CMAKE_FLAGS = {
'-DCMAKE_BUILD_TYPE=Release')
, 'WIN_CPU_MKLDNN': (
+ '-DCMAKE_C_COMPILER=cl '
+ '-DCMAKE_CXX_COMPILER=cl '
'-DUSE_CUDA=OFF '
'-DUSE_CUDNN=OFF '
'-DENABLE_CUDA_RTC=OFF '
@@ -80,6 +86,8 @@ CMAKE_FLAGS = {
'-DCMAKE_BUILD_TYPE=Release')
, 'WIN_CPU_MKLDNN_MKL': (
+ '-DCMAKE_C_COMPILER=cl '
+ '-DCMAKE_CXX_COMPILER=cl '
'-DUSE_CUDA=OFF '
'-DUSE_CUDNN=OFF '
'-DENABLE_CUDA_RTC=OFF '
@@ -93,6 +101,8 @@ CMAKE_FLAGS = {
'-DCMAKE_BUILD_TYPE=Release')
, 'WIN_CPU_MKL': (
+ '-DCMAKE_C_COMPILER=cl '
+ '-DCMAKE_CXX_COMPILER=cl '
'-DUSE_CUDA=OFF '
'-DUSE_CUDNN=OFF '
'-DENABLE_CUDA_RTC=OFF '
@@ -106,6 +116,8 @@ CMAKE_FLAGS = {
'-DCMAKE_BUILD_TYPE=Release')
, 'WIN_GPU': (
+ '-DCMAKE_C_COMPILER=cl '
+ '-DCMAKE_CXX_COMPILER=cl '
'-DUSE_CUDA=ON '
'-DUSE_CUDNN=ON '
'-DENABLE_CUDA_RTC=ON '
@@ -120,6 +132,8 @@ CMAKE_FLAGS = {
'-DCMAKE_BUILD_TYPE=Release')
, 'WIN_GPU_MKLDNN': (
+ '-DCMAKE_C_COMPILER=cl '
+ '-DCMAKE_CXX_COMPILER=cl '
'-DUSE_CUDA=ON '
'-DUSE_CUDNN=ON '
'-DENABLE_CUDA_RTC=ON '
@@ -140,38 +154,40 @@ def windows_build(args):
logging.info("Using vcvars environment:\n{}".format(args.vcvars))
path = args.output
- os.makedirs(path, exist_ok=True)
- mxnet_root = get_mxnet_root()
- logging.info("Found MXNet root: {}".format(mxnet_root))
+ # cuda thrust + VS 2019 is flaky: try multiple times if fail
+ MAXIMUM_TRY = 5
+ build_try = 0
+
+ while build_try < MAXIMUM_TRY:
+ if os.path.exists(path):
+ shutil.rmtree(path)
+ os.makedirs(path, exist_ok=True)
- url = 'https://github.com/Kitware/CMake/releases/download/v3.16.1/cmake-3.16.1-win64-x64.zip'
- with tempfile.TemporaryDirectory() as tmpdir:
- cmake_file_path = download_file(url, tmpdir)
- with zipfile.ZipFile(cmake_file_path, 'r') as zip_ref:
- # Create $tmpdir\cmake-3.16.1-win64-x64\bin\cmake.exe
- zip_ref.extractall(tmpdir)
+ mxnet_root = get_mxnet_root()
+ logging.info("Found MXNet root: {}".format(mxnet_root))
with remember_cwd():
os.chdir(path)
- cmd = "\"{}\" && {} -G \"NMake Makefiles JOM\" {} {}".format(
- args.vcvars,
- os.path.join(tmpdir, 'cmake-3.16.1-win64-x64', 'bin', 'cmake.exe'),
- CMAKE_FLAGS[args.flavour], mxnet_root)
+ cmd = "\"{}\" && cmake -GNinja {} {}".format(args.vcvars,
+ CMAKE_FLAGS[args.flavour],
+ mxnet_root)
logging.info("Generating project with CMake:\n{}".format(cmd))
check_call(cmd, shell=True)
- cmd = "\"{}\" && jom".format(args.vcvars)
- logging.info("Building with jom:\n{}".format(cmd))
+ cmd = "\"{}\" && ninja".format(args.vcvars)
+ logging.info("Building:\n{}".format(cmd))
t0 = int(time.time())
- check_call(cmd, shell=True)
-
- logging.info(
- "Build flavour: {} complete in directory: \"{}\"".format(
- args.flavour, os.path.abspath(path)))
- logging.info("Build took {}".format(
- datetime.timedelta(seconds=int(time.time() - t0))))
+ ret = call(cmd, shell=True)
+
+ if ret != 0:
+ build_try += 1
+ logging.info("{} build(s) have failed".format(build_try))
+ else:
+ logging.info("Build flavour: {} complete in directory: \"{}\"".format(args.flavour, os.path.abspath(path)))
+ logging.info("Build took {}".format(datetime.timedelta(seconds=int(time.time() - t0))))
+ break
windows_package(args)
@@ -233,7 +249,7 @@ def main():
parser.add_argument("--vcvars",
help="vcvars batch file location, typically inside vs studio install dir",
- default=KNOWN_VCVARS['VS 2015'],
+ default=KNOWN_VCVARS['VS 2019'],
type=str)
parser.add_argument("--arch",
@@ -258,7 +274,7 @@ def main():
if 'OpenCV_DIR' not in os.environ:
os.environ["OpenCV_DIR"] = "C:\\Program Files\\OpenCV-v3.4.1\\build"
if 'CUDA_PATH' not in os.environ:
- os.environ["CUDA_PATH"] = "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v9.2"
+ os.environ["CUDA_PATH"] = "C:\\Program Files\\NVIDIA GPU Computing Toolkit\\CUDA\\v10.2"
if 'MKL_ROOT' not in os.environ:
os.environ["MKL_ROOT"] = "C:\\Program Files (x86)\\IntelSWTools\\compilers_and_libraries\\windows\\mkl"
windows_build(args)