You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by je...@apache.org on 2021/10/20 01:44:40 UTC
[incubator-mxnet] branch v1.x updated: [v1.x] Port #20648, #20676,
#19236 to v1.x (#20677)
This is an automated email from the ASF dual-hosted git repository.
jevans pushed a commit to branch v1.x
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/v1.x by this push:
new b1ac65d [v1.x] Port #20648, #20676, #19236 to v1.x (#20677)
b1ac65d is described below
commit b1ac65d7ffc2aebd925a126118c8113b26141135
Author: Joe Evans <jo...@gmail.com>
AuthorDate: Tue Oct 19 18:42:59 2021 -0700
[v1.x] Port #20648, #20676, #19236 to v1.x (#20677)
* [v1.9.x] CI fixes (#20648)
* Update openssl package in ubuntu_core.sh (used in ubuntu 16.04 images) to avoid bug triggered by let's encrypt expired ca cert.
* Reduce number of parallel docker builds we run at once to 2, since there are multiple jobs (on different branches) running at the same time.
* Use different mirror to download binutils to avoid expired let's encrypt CA cert.
* [CI] UPgrade windows CI
* fix python 3.8 ctypes dll load with windows (#19236)
Co-authored-by: barry-jin <ba...@gmail.com>
Co-authored-by: Hu Shiwen <ya...@gmail.com>
---
ci/build_windows.py | 25 +++++++++++++++++++++----
ci/docker/install/requirements | 4 ++--
ci/docker/install/ubuntu_binutils.sh | 2 +-
ci/docker_cache.py | 2 +-
ci/jenkins/Jenkins_steps.groovy | 12 ++++++------
ci/windows/test_py3_cpu.ps1 | 8 ++++----
ci/windows/test_py3_gpu.ps1 | 12 ++++++------
python/mxnet/base.py | 7 ++++++-
8 files changed, 47 insertions(+), 25 deletions(-)
diff --git a/ci/build_windows.py b/ci/build_windows.py
index 87f1db7..89d7f83 100755
--- a/ci/build_windows.py
+++ b/ci/build_windows.py
@@ -151,6 +151,8 @@ CMAKE_FLAGS = {
def windows_build(args):
logging.info("Using vcvars environment:\n{}".format(args.vcvars))
+ if args.vcvars_ver:
+ logging.info("Using vcvars version:\n{}".format(args.vcvars_ver))
path = args.output
@@ -171,13 +173,22 @@ def windows_build(args):
env = os.environ.copy()
if 'GPU' in args.flavour:
env["CXXFLAGS"] = '/FS /MD /O2 /Ob2'
- cmd = "\"{}\" && cmake -GNinja {} {}".format(args.vcvars,
- CMAKE_FLAGS[args.flavour],
- mxnet_root)
+ if not args.vcvars_ver:
+ cmd = "\"{}\" && cmake -GNinja {} {}".format(args.vcvars,
+ CMAKE_FLAGS[args.flavour],
+ mxnet_root)
+ else:
+ cmd = "\"{}\" -vcvars_ver={} && cmake -GNinja {} {}".format(args.vcvars,
+ args.vcvars_ver,
+ CMAKE_FLAGS[args.flavour],
+ mxnet_root)
logging.info("Generating project with CMake:\n{}".format(cmd))
check_call(cmd, shell=True, env=env)
- cmd = "\"{}\" && ninja".format(args.vcvars)
+ if not args.vcvars_ver:
+ cmd = "\"{}\" && ninja".format(args.vcvars)
+ else:
+ cmd = "\"{}\" -vcvars_ver={} && ninja".format(args.vcvars, args.vcvars_ver)
logging.info("Building:\n{}".format(cmd))
t0 = int(time.time())
@@ -260,6 +271,12 @@ def main():
default=KNOWN_VCVARS['VS 2019'],
type=str)
+ parser.add_argument("--vcvars_ver",
+ help="Optionally specifies the Visual Studio compiler toolset to use.\
+ By default, the environment is set to use the current Visual Studio compiler toolset.",
+ default=None,
+ type=str)
+
parser.add_argument("--arch",
help="architecture",
default='x64',
diff --git a/ci/docker/install/requirements b/ci/docker/install/requirements
index 210914a..463a7f9 100644
--- a/ci/docker/install/requirements
+++ b/ci/docker/install/requirements
@@ -20,7 +20,7 @@
boto3==1.9.229
cpplint==1.3.0
-Cython==0.29.7
+Cython==0.29.24
decorator==4.4.0
h5py<3
mock==2.0.0
@@ -32,4 +32,4 @@ astroid==2.3.3 # pylint and astroid need to be aligned
requests<2.19.0,>=2.18.4
scipy==1.2.1
setuptools
-coverage
\ No newline at end of file
+coverage
diff --git a/ci/docker/install/ubuntu_binutils.sh b/ci/docker/install/ubuntu_binutils.sh
index 4ef99a8..c688642 100755
--- a/ci/docker/install/ubuntu_binutils.sh
+++ b/ci/docker/install/ubuntu_binutils.sh
@@ -28,7 +28,7 @@ apt-get install -y \
wget
mkdir /opt/binutils_install && mkdir /opt/binutils && cd /opt/binutils
-wget -nv https://ftp.gnu.org/gnu/binutils/binutils-2.27.tar.gz
+wget -nv http://mirrors.kernel.org/gnu/binutils/binutils-2.27.tar.gz
tar -xvf binutils-2.27.tar.gz && cd binutils-2.27
./configure --prefix=/opt/binutils_other --exec-prefix=/opt/binutils_install
make -j$(nproc)
diff --git a/ci/docker_cache.py b/ci/docker_cache.py
index f0a72b5..f9d5b81 100755
--- a/ci/docker_cache.py
+++ b/ci/docker_cache.py
@@ -38,7 +38,7 @@ from util import retry
DOCKER_CACHE_NUM_RETRIES = 3
DOCKER_CACHE_TIMEOUT_MINS = 45
-PARALLEL_BUILDS = 5
+PARALLEL_BUILDS = 2
DOCKER_CACHE_RETRY_SECONDS = 5
DOCKER_BUILD_NUM_RETRIES = 2
diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 9f96c7a..2441d94 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -558,7 +558,7 @@ def compile_windows_cpu() {
ws('workspace/build-cpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
- powershell 'py -3 ci/build_windows.py -f WIN_CPU'
+ powershell 'py -3 ci/build_windows.py -f WIN_CPU --vcvars_ver 14.28'
stash includes: 'windows_package.7z', name: 'windows_package_cpu'
}
}
@@ -572,7 +572,7 @@ def compile_windows_cpu_mkldnn() {
ws('workspace/build-cpu-mkldnn') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
- powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKLDNN'
+ powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKLDNN --vcvars_ver 14.28'
stash includes: 'windows_package.7z', name: 'windows_package_cpu_mkldnn'
}
}
@@ -586,7 +586,7 @@ def compile_windows_cpu_mkldnn_mkl() {
ws('workspace/build-cpu-mkldnn-mkl') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
- powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKLDNN_MKL'
+ powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKLDNN_MKL --vcvars_ver 14.28'
stash includes: 'windows_package.7z', name: 'windows_package_cpu_mkldnn_mkl'
}
}
@@ -600,7 +600,7 @@ def compile_windows_cpu_mkl() {
ws('workspace/build-cpu-mkl') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
- powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKL'
+ powershell 'py -3 ci/build_windows.py -f WIN_CPU_MKL --vcvars_ver 14.28'
stash includes: 'windows_package.7z', name: 'windows_package_cpu_mkl'
}
}
@@ -614,7 +614,7 @@ def compile_windows_gpu() {
ws('workspace/build-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
- powershell 'py -3 ci/build_windows.py -f WIN_GPU'
+ powershell 'py -3 ci/build_windows.py -f WIN_GPU --vcvars_ver 14.28'
stash includes: 'windows_package.7z', name: 'windows_package_gpu'
}
}
@@ -628,7 +628,7 @@ def compile_windows_gpu_mkldnn() {
ws('workspace/build-gpu') {
timeout(time: max_time, unit: 'MINUTES') {
utils.init_git_win()
- powershell 'py -3 ci/build_windows.py -f WIN_GPU_MKLDNN'
+ powershell 'py -3 ci/build_windows.py -f WIN_GPU_MKLDNN --vcvars_ver 14.28'
stash includes: 'windows_package.7z', name: 'windows_package_gpu_mkldnn'
}
}
diff --git a/ci/windows/test_py3_cpu.ps1 b/ci/windows/test_py3_cpu.ps1
index 8f520bb..152e527 100644
--- a/ci/windows/test_py3_cpu.ps1
+++ b/ci/windows/test_py3_cpu.ps1
@@ -23,12 +23,12 @@ $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
$env:MXNET_SUBGRAPH_VERBOSE=0
$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home')
-C:\Python37\Scripts\pip install -r tests\requirements.txt
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
+C:\Python38\Scripts\pip install -r tests\requirements.txt
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
if ($LastExitCode -ne 0) { Throw ("Error running unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train
if ($LastExitCode -ne 0) { Throw ("Error running train tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
# Adding this extra test since it's not possible to set env var on the fly in Windows.
$env:MXNET_SAFE_ACCUMULATION=1
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest\test_operator.py:test_norm
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest\test_operator.py:test_norm
if ($LastExitCode -ne 0) { Throw ("Error running unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
diff --git a/ci/windows/test_py3_gpu.ps1 b/ci/windows/test_py3_gpu.ps1
index 0ce3d95..3632fbc 100644
--- a/ci/windows/test_py3_gpu.ps1
+++ b/ci/windows/test_py3_gpu.ps1
@@ -23,16 +23,16 @@ $env:MXNET_STORAGE_FALLBACK_LOG_VERBOSE=0
$env:MXNET_SUBGRAPH_VERBOSE=0
$env:MXNET_HOME=[io.path]::combine($PSScriptRoot, 'mxnet_home')
-C:\Python37\Scripts\pip install -r tests\requirements.txt
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
+C:\Python38\Scripts\pip install -r tests\requirements.txt
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_unittest.xml tests\python\unittest
if ($LastExitCode -ne 0) { Throw ("Error running unittest, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py
if ($LastExitCode -ne 0) { Throw ("Error running tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_forward.xml tests\python\gpu\test_forward.py
if ($LastExitCode -ne 0) { Throw ("Error running tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_train.xml tests\python\train
if ($LastExitCode -ne 0) { Throw ("Error running tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
# Adding this extra test since it's not possible to set env var on the fly in Windows.
$env:MXNET_SAFE_ACCUMULATION=1
-C:\Python37\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py:test_norm
+C:\Python38\python.exe -m nose -v --with-timer --timer-ok 1 --timer-warning 15 --timer-filter warning,error --with-xunit --xunit-file nosetests_operator.xml tests\python\gpu\test_operator_gpu.py:test_norm
if ($LastExitCode -ne 0) { Throw ("Error running tests, python exited with status code " + ('{0:X}' -f $LastExitCode)) }
diff --git a/python/mxnet/base.py b/python/mxnet/base.py
index ca98116..586aeb9 100644
--- a/python/mxnet/base.py
+++ b/python/mxnet/base.py
@@ -339,7 +339,12 @@ def classproperty(func):
def _load_lib():
"""Load library by searching possible path."""
lib_path = libinfo.find_lib_path()
- lib = ctypes.CDLL(lib_path[0], ctypes.RTLD_LOCAL)
+ if sys.version_info >= (3, 8) and os.name == "nt":
+ # use LOAD_WITH_ALTERED_SEARCH_PATH, For simplicity, let's just fill the numbers.
+ # pylint: disable=E1123
+ lib = ctypes.CDLL(lib_path[0], winmode=0x00000008)
+ else:
+ lib = ctypes.CDLL(lib_path[0], ctypes.RTLD_LOCAL)
# DMatrix functions
lib.MXGetLastError.restype = ctypes.c_char_p
return lib