You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/08/03 12:05:43 UTC
[GitHub] marcoabreu closed pull request #11947: [MXNET-769] Usability
improvements to windows builds
marcoabreu closed pull request #11947: [MXNET-769] Usability improvements to windows builds
URL: https://github.com/apache/incubator-mxnet/pull/11947
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/CMakeLists.txt b/CMakeLists.txt
index 483108a6841..000bbbf17ea 100644
--- a/CMakeLists.txt
+++ b/CMakeLists.txt
@@ -372,13 +372,13 @@ endif()
# ---[ LAPack
if(USE_LAPACK)
+ message("USE_LAPACK is ON")
add_definitions(-DMXNET_USE_LAPACK=1)
if (NOT MSVC)
list(APPEND mxnet_LINKER_LIBS lapack)
endif()
endif()
-message("USE LAPACK ${USE_LAPACK}")
# ---[ jemalloc
if(USE_JEMALLOC)
diff --git a/Jenkinsfile b/Jenkinsfile
index 6d21f496426..69934976730 100644
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -378,28 +378,8 @@ try {
ws('workspace/build-cpu') {
withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) {
init_git_win()
- bat """mkdir build_vc14_cpu
- call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat"
- cd build_vc14_cpu
- cmake -G \"Visual Studio 14 2015 Win64\" -DUSE_CUDA=0 -DUSE_CUDNN=0 -DUSE_NVRTC=0 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DUSE_MKL_IF_AVAILABLE=0 ${env.WORKSPACE}"""
- bat 'C:\\mxnet\\build_vc14_cpu.bat'
-
- bat '''rmdir /s/q pkg_vc14_cpu
- mkdir pkg_vc14_cpu\\lib
- mkdir pkg_vc14_cpu\\python
- mkdir pkg_vc14_cpu\\include
- mkdir pkg_vc14_cpu\\build
- copy build_vc14_cpu\\Release\\libmxnet.lib pkg_vc14_cpu\\lib
- copy build_vc14_cpu\\Release\\libmxnet.dll pkg_vc14_cpu\\build
- xcopy python pkg_vc14_cpu\\python /E /I /Y
- xcopy include pkg_vc14_cpu\\include /E /I /Y
- xcopy 3rdparty\\dmlc-core\\include pkg_vc14_cpu\\include /E /I /Y
- xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_cpu\\include\\mshadow /E /I /Y
- xcopy 3rdparty\\nnvm\\include pkg_vc14_cpu\\nnvm\\include /E /I /Y
- del /Q *.7z
- 7z.exe a vc14_cpu.7z pkg_vc14_cpu\\
- '''
- stash includes: 'vc14_cpu.7z', name: 'vc14_cpu'
+ powershell 'python ci/build_windows.py -f WIN_CPU'
+ stash includes: 'windows_package.7z', name: 'windows_package_cpu'
}
}
}
@@ -411,28 +391,9 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
ws('workspace/build-gpu') {
withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0']) {
- init_git_win()
- bat """mkdir build_vc14_gpu
- call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat"
- cd build_vc14_gpu
- cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN=52 -DCUDA_ARCH_PTX=52 -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release -DUSE_MKL_IF_AVAILABLE=0 ${env.WORKSPACE}"""
- bat 'C:\\mxnet\\build_vc14_gpu.bat'
- bat '''rmdir /s/q pkg_vc14_gpu
- mkdir pkg_vc14_gpu\\lib
- mkdir pkg_vc14_gpu\\python
- mkdir pkg_vc14_gpu\\include
- mkdir pkg_vc14_gpu\\build
- copy build_vc14_gpu\\libmxnet.lib pkg_vc14_gpu\\lib
- copy build_vc14_gpu\\libmxnet.dll pkg_vc14_gpu\\build
- xcopy python pkg_vc14_gpu\\python /E /I /Y
- xcopy include pkg_vc14_gpu\\include /E /I /Y
- xcopy 3rdparty\\dmlc-core\\include pkg_vc14_gpu\\include /E /I /Y
- xcopy 3rdparty\\mshadow\\mshadow pkg_vc14_gpu\\include\\mshadow /E /I /Y
- xcopy 3rdparty\\nnvm\\include pkg_vc14_gpu\\nnvm\\include /E /I /Y
- del /Q *.7z
- 7z.exe a vc14_gpu.7z pkg_vc14_gpu\\
- '''
- stash includes: 'vc14_gpu.7z', name: 'vc14_gpu'
+ init_git_win()
+ powershell 'python ci/build_windows.py -f WIN_GPU'
+ stash includes: 'windows_package.7z', name: 'windows_package_gpu'
}
}
}
@@ -443,37 +404,9 @@ try {
timeout(time: max_time, unit: 'MINUTES') {
ws('workspace/build-gpu') {
withEnv(['OpenBLAS_HOME=C:\\mxnet\\openblas', 'OpenCV_DIR=C:\\mxnet\\opencv_vc14', 'CUDA_PATH=C:\\CUDA\\v8.0','BUILD_NAME=vc14_gpu_mkldnn']) {
- init_git_win()
- bat """mkdir build_%BUILD_NAME%
- call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat"
- cd build_%BUILD_NAME%
- copy ${env.WORKSPACE}\\3rdparty\\mkldnn\\config_template.vcxproj.user ${env.WORKSPACE}\\config_template.vcxproj.user /y
- cmake -G \"NMake Makefiles JOM\" -DUSE_CUDA=1 -DUSE_CUDNN=1 -DUSE_NVRTC=1 -DUSE_OPENCV=1 -DUSE_OPENMP=1 -DUSE_PROFILER=1 -DUSE_BLAS=open -DUSE_LAPACK=1 -DUSE_DIST_KVSTORE=0 -DCUDA_ARCH_NAME=Manual -DCUDA_ARCH_BIN=52 -DCUDA_ARCH_PTX=52 -DUSE_MKLDNN=1 -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" -DCMAKE_BUILD_TYPE=Release ${env.WORKSPACE}"""
- bat '''
- call "C:\\Program Files (x86)\\Microsoft Visual Studio 14.0\\VC\\bin\\x86_amd64\\vcvarsx86_amd64.bat"
- cd build_%BUILD_NAME%
- set /a cores=%NUMBER_OF_PROCESSORS% * 2
- jom -j %cores%
- '''
- bat '''rmdir /s/q pkg_%BUILD_NAME%
- mkdir pkg_%BUILD_NAME%\\lib
- mkdir pkg_%BUILD_NAME%\\python
- mkdir pkg_%BUILD_NAME%\\include
- mkdir pkg_%BUILD_NAME%\\build
- copy build_%BUILD_NAME%\\libmxnet.lib pkg_%BUILD_NAME%\\lib
- copy build_%BUILD_NAME%\\libmxnet.dll pkg_%BUILD_NAME%\\build
- copy build_%BUILD_NAME%\\3rdparty\\mkldnn\\src\\mkldnn.dll pkg_%BUILD_NAME%\\build
- copy build_%BUILD_NAME%\\libiomp5md.dll pkg_%BUILD_NAME%\\build
- copy build_%BUILD_NAME%\\mklml.dll pkg_%BUILD_NAME%\\build
- xcopy python pkg_%BUILD_NAME%\\python /E /I /Y
- xcopy include pkg_%BUILD_NAME%\\include /E /I /Y
- xcopy 3rdparty\\dmlc-core\\include pkg_%BUILD_NAME%\\include /E /I /Y
- xcopy 3rdparty\\mshadow\\mshadow pkg_%BUILD_NAME%\\include\\mshadow /E /I /Y
- xcopy 3rdparty\\nnvm\\include pkg_%BUILD_NAME%\\nnvm\\include /E /I /Y
- del /Q *.7z
- 7z.exe a %BUILD_NAME%.7z pkg_%BUILD_NAME%\\
- '''
- stash includes: 'vc14_gpu_mkldnn.7z', name: 'vc14_gpu_mkldnn'
+ init_git_win()
+ powershell 'python ci/build_windows.py -f WIN_GPU_MKLDNN'
+ stash includes: 'windows_package.7z', name: 'windows_package_gpu_mkldnn'
}
}
}
@@ -843,16 +776,8 @@ try {
ws('workspace/ut-python-cpu') {
try {
init_git_win()
- unstash 'vc14_cpu'
- bat '''rmdir /s/q pkg_vc14_cpu
- 7z x -y vc14_cpu.7z'''
- bat """xcopy C:\\mxnet\\data data /E /I /Y
- xcopy C:\\mxnet\\model model /E /I /Y
- call activate py2
- pip install mock
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc
- C:\\mxnet\\test_cpu.bat"""
+ unstash 'windows_package_cpu'
+ powershell 'ci/windows/test_py2_cpu.ps1'
} finally {
collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python2_cpu.xml')
}
@@ -866,15 +791,8 @@ try {
ws('workspace/ut-python-cpu') {
try {
init_git_win()
- unstash 'vc14_cpu'
- bat '''rmdir /s/q pkg_vc14_cpu
- 7z x -y vc14_cpu.7z'''
- bat """xcopy C:\\mxnet\\data data /E /I /Y
- xcopy C:\\mxnet\\model model /E /I /Y
- call activate py3
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_cpu\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_cpu\\python\\*.pyc
- C:\\mxnet\\test_cpu.bat"""
+ unstash 'windows_package_cpu'
+ powershell 'ci/windows/test_py3_cpu.ps1'
} finally {
collect_test_results_windows('nosetests_unittest.xml', 'nosetests_unittest_windows_python3_cpu.xml')
}
@@ -888,19 +806,11 @@ try {
ws('workspace/ut-python-gpu') {
try {
init_git_win()
- unstash 'vc14_gpu'
- bat '''rmdir /s/q pkg_vc14_gpu
- 7z x -y vc14_gpu.7z'''
- bat """xcopy C:\\mxnet\\data data /E /I /Y
- xcopy C:\\mxnet\\model model /E /I /Y
- call activate py2
- pip install mock
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc
- C:\\mxnet\\test_gpu.bat"""
+ unstash 'windows_package_gpu'
+ powershell 'ci/windows/test_py2_gpu.ps1'
} finally {
- collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml')
- collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml')
+ collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python2_gpu.xml')
+ collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python2_gpu.xml')
}
}
}
@@ -912,18 +822,11 @@ try {
ws('workspace/ut-python-gpu') {
try {
init_git_win()
- unstash 'vc14_gpu'
- bat '''rmdir /s/q pkg_vc14_gpu
- 7z x -y vc14_gpu.7z'''
- bat """xcopy C:\\mxnet\\data data /E /I /Y
- xcopy C:\\mxnet\\model model /E /I /Y
- call activate py3
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu\\python\\*.pyc
- C:\\mxnet\\test_gpu.bat"""
+ unstash 'windows_package_gpu'
+ powershell 'ci/windows/test_py3_gpu.ps1'
} finally {
- collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml')
- collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml')
+ collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu.xml')
+ collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu.xml')
}
}
}
@@ -935,18 +838,11 @@ try {
ws('workspace/ut-python-gpu') {
try {
init_git_win()
- unstash 'vc14_gpu_mkldnn'
- bat '''rmdir /s/q pkg_vc14_gpu_mkldnn
- 7z x -y vc14_gpu_mkldnn.7z'''
- bat """xcopy C:\\mxnet\\data data /E /I /Y
- xcopy C:\\mxnet\\model model /E /I /Y
- call activate py3
- set PYTHONPATH=${env.WORKSPACE}\\pkg_vc14_gpu_mkldnn\\python
- del /S /Q ${env.WORKSPACE}\\pkg_vc14_gpu_mkldnn\\python\\*.pyc
- C:\\mxnet\\test_gpu.bat"""
+ unstash 'windows_package_gpu_mkldnn'
+ powershell 'ci/windows/test_py3_gpu.ps1'
} finally {
- collect_test_results_windows('nosetests_gpu_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu_mkldnn.xml')
- collect_test_results_windows('nosetests_gpu_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu_mkldnn.xml')
+ collect_test_results_windows('nosetests_forward.xml', 'nosetests_gpu_forward_windows_python3_gpu_mkldnn.xml')
+ collect_test_results_windows('nosetests_operator.xml', 'nosetests_gpu_operator_windows_python3_gpu_mkldnn.xml')
}
}
}
diff --git a/ci/build.py b/ci/build.py
index 09f2d4709bd..a9d6a63537f 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -39,6 +39,7 @@
from itertools import chain
from subprocess import call, check_call
from typing import *
+from util import *
CCACHE_MAXSIZE = '500G'
@@ -138,24 +139,9 @@ def _get_local_image_id(docker_binary, docker_tag):
return image_id
-def get_mxnet_root() -> str:
- curpath = os.path.abspath(os.path.dirname(__file__))
-
- def is_mxnet_root(path: str) -> bool:
- return os.path.exists(os.path.join(path, ".mxnet_root"))
-
- while not is_mxnet_root(curpath):
- parent = os.path.abspath(os.path.join(curpath, os.pardir))
- if parent == curpath:
- raise RuntimeError("Got to the root and couldn't find a parent folder with .mxnet_root")
- curpath = parent
- return curpath
-
-
def buildir() -> str:
return os.path.join(get_mxnet_root(), "build")
-
def default_ccache_dir() -> str:
# Share ccache across containers
if 'CCACHE_DIR' in os.environ:
diff --git a/ci/build_windows.py b/ci/build_windows.py
new file mode 100755
index 00000000000..5eca58db7b7
--- /dev/null
+++ b/ci/build_windows.py
@@ -0,0 +1,253 @@
+#!/usr/bin/env python
+# -*- coding: utf-8 -*-
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+"""User friendly / multi platform builder script"""
+
+import subprocess
+import logging
+import os
+import tempfile
+import sys
+from distutils import spawn
+import logging
+from subprocess import check_call
+import platform
+import argparse
+from util import *
+import json
+from enum import Enum
+import time
+import datetime
+import shutil
+import glob
+from distutils.dir_util import copy_tree
+
+KNOWN_VCVARS = [
+ # VS 2015
+ r'C:\Program Files (x86)\Microsoft Visual Studio 14.0\VC\bin\x86_amd64\vcvarsx86_amd64.bat'
+ # VS 2017
+ , r'c:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat'
+]
+
+class BuildFlavour(Enum):
+ WIN_CPU = 'WIN_CPU'
+ WIN_CPU_MKLDNN = 'WIN_CPU_MKLDNN'
+ WIN_GPU = 'WIN_GPU'
+ WIN_GPU_MKLDNN = 'WIN_GPU_MKLDNN'
+
+CMAKE_FLAGS = {
+ 'WIN_CPU': '-DUSE_CUDA=0 \
+ -DUSE_CUDNN=0 \
+ -DUSE_NVRTC=0 \
+ -DUSE_OPENCV=1 \
+ -DUSE_OPENMP=1 \
+ -DUSE_PROFILER=1 \
+ -DUSE_BLAS=open \
+ -DUSE_LAPACK=1 \
+ -DUSE_DIST_KVSTORE=0 \
+ -DBUILD_CPP_EXAMPLES=1 \
+ -DUSE_MKL_IF_AVAILABLE=0'
+
+ ,'WIN_CPU_MKLDNN': '-DUSE_CUDA=0 \
+ -DUSE_CUDNN=0 \
+ -DUSE_NVRTC=0 \
+ -DUSE_OPENCV=1 \
+ -DUSE_OPENMP=1 \
+ -DUSE_PROFILER=1 \
+ -DUSE_BLAS=open \
+ -DUSE_LAPACK=1 \
+ -DUSE_DIST_KVSTORE=0 \
+ -DUSE_MKL_IF_AVAILABLE=1'
+
+ ,'WIN_GPU': '-DUSE_CUDA=1 \
+ -DUSE_CUDNN=1 \
+ -DUSE_NVRTC=1 \
+ -DUSE_OPENCV=1 \
+ -DUSE_OPENMP=1 \
+ -DUSE_PROFILER=1 \
+ -DUSE_BLAS=open \
+ -DUSE_LAPACK=1 \
+ -DUSE_DIST_KVSTORE=0 \
+ -DCUDA_ARCH_NAME=Manual \
+ -DCUDA_ARCH_BIN=52 \
+ -DCUDA_ARCH_PTX=52 \
+ -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 /DNDEBUG" \
+ -DUSE_MKL_IF_AVAILABLE=0 \
+ -DCMAKE_BUILD_TYPE=Release'
+
+ ,'WIN_GPU_MKLDNN': '-DUSE_CUDA=1 \
+ -DUSE_CUDNN=1 \
+ -DUSE_NVRTC=1 \
+ -DUSE_OPENCV=1 \
+ -DUSE_OPENMP=1 \
+ -DUSE_PROFILER=1 \
+ -DUSE_BLAS=open \
+ -DUSE_LAPACK=1 \
+ -DUSE_DIST_KVSTORE=0 \
+ -DCUDA_ARCH_NAME=Manual \
+ -DCUDA_ARCH_BIN=52 \
+ -DCUDA_ARCH_PTX=52 \
+ -DUSE_MKLDNN=1 \
+ -DCMAKE_CXX_FLAGS_RELEASE="/FS /MD /O2 /Ob2 \
+ /DNDEBUG" \
+ -DCMAKE_BUILD_TYPE=Release'
+
+}
+
+
+def get_vcvars_environment(architecture, vcvars):
+ """
+ Returns a dictionary containing the environment variables set up by vcvars
+ """
+ result = None
+ python = sys.executable
+
+ vcvars_list = [vcvars]
+ vcvars_list.extend(KNOWN_VCVARS)
+ for vcvars in vcvars_list:
+ if os.path.isfile(vcvars):
+ process = subprocess.Popen('("%s" %s>nul) && "%s" -c "import os; import json; print(json.dumps(dict(os.environ)))"' % (vcvars, architecture, python), stdout=subprocess.PIPE, shell=True)
+ stdout, stderr = process.communicate()
+ exitcode = process.wait()
+ if exitcode == 0:
+ logging.info("Using build environment from: %s", vcvars)
+ return(json.loads(stdout.strip()))
+ else:
+ raise RuntimeError('Failed cloning environment from vcvars file: %s stdout: %s stderr: %s', vcvars, stdout, stderr)
+ raise RuntimeError('Couldn\'t find vcvars batch file: %s', vcvars)
+
+
+def windows_build(args):
+ vcvars_env = get_vcvars_environment(args.arch, args.vcvars)
+ logging.debug("vcvars environment: %s", vcvars_env)
+ os.environ.update(vcvars_env)
+
+ path = args.output
+ os.makedirs(path, exist_ok=True)
+ mxnet_root = get_mxnet_root()
+ logging.info("Found mxnet root: {}".format(mxnet_root))
+ with remember_cwd():
+ os.chdir(path)
+ logging.info("Generating project with CMake")
+ check_call("cmake -G \"Visual Studio 14 2015 Win64\" {} {}".format(CMAKE_FLAGS[args.flavour], mxnet_root), shell=True)
+ logging.info("Building with visual studio")
+ t0 = int(time.time())
+ check_call(["msbuild", "mxnet.sln","/p:configuration=release;platform=x64", "/maxcpucount","/v:minimal"])
+ logging.info("Build flavour: %s complete in directory: \"%s\"", args.flavour, os.path.abspath(path))
+ logging.info("Build took %s" , datetime.timedelta(seconds=int(time.time()-t0)))
+ windows_package(args)
+
+def windows_package(args):
+ pkgfile = 'windows_package.7z'
+ pkgdir = os.path.abspath('windows_package')
+ logging.info("Packaging libraries and headers in package: %s", pkgfile)
+ j = os.path.join
+ pkgdir_lib = os.path.abspath(j(pkgdir, 'lib'))
+ with remember_cwd():
+ os.chdir(args.output)
+ logging.info("Looking for static libraries and dlls in: \"%s", os.getcwd())
+ libs = list(glob.iglob('**/*.lib', recursive=True))
+ dlls = list(glob.iglob('**/*.dll', recursive=True))
+ os.makedirs(pkgdir_lib, exist_ok=True)
+ for lib in libs:
+ logging.info("packing lib: %s", lib)
+ shutil.copy(lib, pkgdir_lib)
+ for dll in dlls:
+ logging.info("packing dll: %s", dll)
+ shutil.copy(dll, pkgdir_lib)
+ os.chdir(get_mxnet_root())
+ logging.info('packing python bindings')
+ copy_tree('python', j(pkgdir, 'python'))
+ logging.info('packing headers')
+ copy_tree('include', j(pkgdir, 'include'))
+ copy_tree(j('3rdparty','dmlc-core','include'), j(pkgdir, 'include'))
+ copy_tree(j('3rdparty','mshadow', 'mshadow'), j(pkgdir, 'include', 'mshadow'))
+ copy_tree(j('3rdparty','tvm','nnvm', 'include'), j(pkgdir,'include', 'nnvm', 'include'))
+ logging.info("Compressing package: %s", pkgfile)
+ check_call(['7z', 'a', pkgfile, pkgdir])
+
+
+def nix_build(args):
+ path = args.output
+ os.makedirs(path, exist_ok=True)
+ with remember_cwd():
+ os.chdir(path)
+ logging.info("Generating project with CMake")
+ check_call("cmake \
+ -DUSE_CUDA=OFF \
+ -DUSE_OPENCV=OFF \
+ -DUSE_OPENMP=OFF \
+ -DCMAKE_BUILD_TYPE=Debug \
+ -GNinja ..", shell=True)
+ check_call("ninja", shell=True)
+
+def main():
+ logging.getLogger().setLevel(logging.INFO)
+ logging.basicConfig(format='%(asctime)-15s %(message)s')
+ logging.info("MXNet Windows build helper")
+
+ parser = argparse.ArgumentParser()
+ parser.add_argument("-o", "--output",
+ help="output directory",
+ default='build',
+ type=str)
+
+ parser.add_argument("--vcvars",
+ help="vcvars batch file location, typically inside vs studio install dir",
+ default=r'c:\Program Files (x86)\Microsoft Visual Studio\2017\Community\VC\Auxiliary\Build\vcvarsx86_amd64.bat',
+ type=str)
+
+ parser.add_argument("--arch",
+ help="architecture",
+ default='x64',
+ type=str)
+
+ parser.add_argument("-f", "--flavour",
+ help="build flavour",
+ default='WIN_CPU',
+ choices=[x.name for x in BuildFlavour],
+ type=str)
+
+ args = parser.parse_args()
+ logging.info("Build flavour: %s", args.flavour)
+
+ system = platform.system()
+ if system == 'Windows':
+ logging.info("Detected Windows platform")
+ if 'OpenBLAS_HOME' not in os.environ:
+ os.environ["OpenBLAS_HOME"] = "C:\\mxnet\\openblas"
+ if 'OpenCV_DIR' not in os.environ:
+ os.environ["OpenCV_DIR"] = "C:\\mxnet\\opencv_vc14"
+ if 'CUDA_PATH' not in os.environ:
+ os.environ["CUDA_PATH"] = "C:\\CUDA\\v8.0"
+ windows_build(args)
+
+ elif system == 'Linux' or system == 'Darwin':
+ nix_build(args)
+
+ else:
+ logging.error("Don't know how to build for {} yet".format(platform.system()))
+
+ return 0
+
+
+if __name__ == '__main__':
+ sys.exit(main())
+
diff --git a/ci/util.py b/ci/util.py
new file mode 100644
index 00000000000..22631f30435
--- /dev/null
+++ b/ci/util.py
@@ -0,0 +1,43 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import os
+import contextlib
+
+def get_mxnet_root() -> str:
+ curpath = os.path.abspath(os.path.dirname(__file__))
+
+ def is_mxnet_root(path: str) -> bool:
+ return os.path.exists(os.path.join(path, ".mxnet_root"))
+
+ while not is_mxnet_root(curpath):
+ parent = os.path.abspath(os.path.join(curpath, os.pardir))
+ if parent == curpath:
+ raise RuntimeError("Got to the root and couldn't find a parent folder with .mxnet_root")
+ curpath = parent
+ return curpath
+
+@contextlib.contextmanager
+def remember_cwd():
+ '''
+ Restore current directory when exiting context
+ '''
+ curdir = os.getcwd()
+ try: yield
+ finally: os.chdir(curdir)
+
+
diff --git a/ci/windows/test_py2_cpu.ps1 b/ci/windows/test_py2_cpu.ps1
new file mode 100644
index 00000000000..1623d295610
--- /dev/null
+++ b/ci/windows/test_py2_cpu.ps1
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+7z x -y windows_package.7z
+$env:PYTHONPATH=join-path $pwd.Path windows_package\python
+$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt
+c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
+if (! $?) { Throw ("Error running unittest") }
+c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_train.xml tests\python\train
+if (! $?) { Throw ("Error running train tests") }
diff --git a/ci/windows/test_py2_gpu.ps1 b/ci/windows/test_py2_gpu.ps1
new file mode 100644
index 00000000000..13cd5366e0d
--- /dev/null
+++ b/ci/windows/test_py2_gpu.ps1
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+7z x -y windows_package.7z
+$env:PYTHONPATH=join-path $pwd.Path windows_package\python
+$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+c:\Anaconda3\envs\py2\Scripts\pip install -r tests\requirements.txt
+c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
+if (! $?) { Throw ("Error running unittest") }
+c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py
+if (! $?) { Throw ("Error running tests") }
+c:\Anaconda3\envs\py2\python.exe -m nose -v --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py
+if (! $?) { Throw ("Error running tests") }
+c:\Anaconda3\envs\py2\python.exe -m nose -v tests\python\train
+if (! $?) { Throw ("Error running tests") }
diff --git a/ci/windows/test_py3_cpu.ps1 b/ci/windows/test_py3_cpu.ps1
new file mode 100644
index 00000000000..98d4e410e8f
--- /dev/null
+++ b/ci/windows/test_py3_cpu.ps1
@@ -0,0 +1,25 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+7z x -y windows_package.7z
+$env:PYTHONPATH=join-path $pwd.Path windows_package\python
+$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt
+c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
+if (! $?) { Throw ("Error running unittest") }
+c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_train.xml tests\python\train
+if (! $?) { Throw ("Error running train tests") }
diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1
new file mode 100644
index 00000000000..b94b4f389be
--- /dev/null
+++ b/ci/windows/test_py3_gpu.ps1
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+7z x -y windows_package.7z
+$env:PYTHONPATH=join-path $pwd.Path windows_package\python
+$env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
+c:\Anaconda3\envs\py3\Scripts\pip install -r tests\requirements.txt
+c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
+if (! $?) { Throw ("Error running unittest") }
+c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py
+if (! $?) { Throw ("Error running tests") }
+c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py
+if (! $?) { Throw ("Error running tests") }
+c:\Anaconda3\envs\py3\python.exe -m nose -v --with-xunit --xunit-file nosetests_train.xml tests\python\train
+if (! $?) { Throw ("Error running tests") }
diff --git a/tests/python/gpu/test_forward.py b/tests/python/gpu/test_forward.py
index 126ccabaa7b..02b0256024d 100644
--- a/tests/python/gpu/test_forward.py
+++ b/tests/python/gpu/test_forward.py
@@ -24,11 +24,13 @@
sys.path.insert(0, os.path.join(curr_path, '../unittest'))
from common import setup_module, with_seed, teardown
from mxnet.gluon import utils
+import tarfile
def _get_model():
if not os.path.exists('model/Inception-7-symbol.json'):
- download('http://data.mxnet.io/models/imagenet/inception-v3.tar.gz', dirname='model')
- os.system("cd model; tar -xf inception-v3.tar.gz --strip-components 1")
+ download('http://data.mxnet.io/models/imagenet/inception-v3.tar.gz')
+ with tarfile.open(name="inception-v3.tar.gz", mode="r:gz") as tf:
+ tf.extractall()
def _dump_images(shape):
import skimage.io
diff --git a/tests/requirements.txt b/tests/requirements.txt
new file mode 100644
index 00000000000..0eca73fbb02
--- /dev/null
+++ b/tests/requirements.txt
@@ -0,0 +1,3 @@
+# Requirements for tests, those are installed before running on the virtualenv
+mock
+nose
diff --git a/tools/license_header.py b/tools/license_header.py
index 0ee4049338b..7aef33b7121 100755
--- a/tools/license_header.py
+++ b/tools/license_header.py
@@ -82,7 +82,7 @@
_LANGS = {'.cc':'*', '.h':'*', '.cu':'*', '.cuh':'*', '.py':'#',
'.pm':'#', '.scala':'*', '.cc':'*', '.sh':'#', '.cmake':'#',
'.java':'*', '.sh':'#', '.cpp':'*', '.hpp':'*', '.c':'*',
- '.bat':'rem', '.pl':'#', '.m':'%', '.R':'#', '.mk':'#', '.cfg':'#', '.t':'#'}
+ '.bat':'rem', '.pl':'#', '.m':'%', '.R':'#', '.mk':'#', '.cfg':'#', '.t':'#', '.ps1': '#'}
# Previous license header, which will be removed
_OLD_LICENSE = re.compile('.*Copyright.*by Contributors')
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services