You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ib...@apache.org on 2019/01/23 15:47:11 UTC
[incubator-mxnet] branch master updated: Julia: add windows-cpu
build (#13937)
This is an automated email from the ASF dual-hosted git repository.
iblis pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new ce8b083 Julia: add windows-cpu build (#13937)
ce8b083 is described below
commit ce8b083f1607025bd352457447038bc7ae186fab
Author: Iblis Lin <ib...@hs.ntnu.edu.tw>
AuthorDate: Wed Jan 23 23:46:51 2019 +0800
Julia: add windows-cpu build (#13937)
- Julia v0.7
- Julia v1.0
---
ci/docker/runtime_functions.sh | 16 ++-----
ci/jenkins/Jenkins_steps.groovy | 28 +++++++++++
ci/jenkins/Jenkinsfile_windows_cpu | 8 ++--
ci/windows/test_jl07_cpu.ps1 | 56 ++++++++++++++++++++++
ci/windows/test_jl10_cpu.ps1 | 56 ++++++++++++++++++++++
julia/.gitignore | 1 +
julia/Project.toml | 26 ++++++++++
julia/REQUIRE | 2 +-
julia/appveyor.yml | 56 ----------------------
julia/deps/build.jl | 2 +-
julia/src/util.jl | 6 +--
.../nightly/apache_rat_license_check/rat-excludes | 1 +
tools/license_header.py | 3 ++
13 files changed, 184 insertions(+), 77 deletions(-)
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 4c70c1a..9f9dd0c 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -922,29 +922,21 @@ unittest_ubuntu_cpu_julia() {
export PATH="$1/bin:$PATH"
export MXNET_HOME='/work/mxnet'
export JULIA_DEPOT_PATH='/work/julia-depot'
- export DEVDIR="$JULIA_DEPOT_PATH/dev"
julia -e 'using InteractiveUtils; versioninfo()'
- # install package
- mkdir -p $DEVDIR
- ln -sf ${MXNET_HOME}/julia ${DEVDIR}/MXNet
-
- # register MXNet.jl and dependencies
- julia -e 'using Pkg; Pkg.develop("MXNet")'
-
# FIXME
export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'
export LD_LIBRARY_PATH=/work/mxnet/lib:$LD_LIBRARY_PATH
# use the prebuilt binary from $MXNET_HOME/lib
- julia -e 'using Pkg; Pkg.build("MXNet")'
+ julia --project=./julia -e 'using Pkg; Pkg.build("MXNet")'
# run the script `julia/test/runtests.jl`
- julia -e 'using Pkg; Pkg.test("MXNet")'
+ julia --project=./julia -e 'using Pkg; Pkg.test("MXNet")'
# See https://github.com/dmlc/MXNet.jl/pull/303#issuecomment-341171774
- julia -e 'using MXNet; mx._sig_checker()'
+ julia --project=./julia -e 'using MXNet; mx._sig_checker()'
}
unittest_ubuntu_cpu_julia07() {
@@ -1280,10 +1272,8 @@ deploy_jl_docs() {
export PATH="/work/julia10/bin:$PATH"
export MXNET_HOME='/work/mxnet'
export JULIA_DEPOT_PATH='/work/julia-depot'
- export DEVDIR="$JULIA_DEPOT_PATH/dev"
julia -e 'using InteractiveUtils; versioninfo()'
- mkdir -p $DEVDIR
# FIXME
export LD_PRELOAD='/usr/lib/x86_64-linux-gnu/libjemalloc.so'
diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 9db3588..984672c 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -1210,6 +1210,34 @@ def test_windows_python3_cpu() {
}]
}
+def test_windows_julia07_cpu() {
+ return ['Julia 0.7: CPU Win': {
+ node(NODE_WINDOWS_CPU) {
+ ws('workspace/ut-julia07-cpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git_win()
+ unstash 'windows_package_cpu'
+ powershell 'ci/windows/test_jl07_cpu.ps1'
+ }
+ }
+ }
+ }]
+}
+
+def test_windows_julia10_cpu() {
+ return ['Julia 1.0: CPU Win': {
+ node(NODE_WINDOWS_CPU) {
+ ws('workspace/ut-julia10-cpu') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git_win()
+ unstash 'windows_package_cpu'
+ powershell 'ci/windows/test_jl10_cpu.ps1'
+ }
+ }
+ }
+ }]
+}
+
def test_qemu_armv7_cpu() {
return ['ARMv7 QEMU': {
node(NODE_LINUX_CPU) {
diff --git a/ci/jenkins/Jenkinsfile_windows_cpu b/ci/jenkins/Jenkinsfile_windows_cpu
index a8746db..5bc40d6 100644
--- a/ci/jenkins/Jenkinsfile_windows_cpu
+++ b/ci/jenkins/Jenkinsfile_windows_cpu
@@ -35,12 +35,14 @@ utils.main_wrapper(
core_logic: {
utils.parallel_stage('Build', [
custom_steps.compile_windows_cpu()
- ])
+ ])
utils.parallel_stage('Tests', [
custom_steps.test_windows_python2_cpu(),
- custom_steps.test_windows_python3_cpu()
- ])
+ custom_steps.test_windows_python3_cpu(),
+ custom_steps.test_windows_julia07_cpu(),
+ custom_steps.test_windows_julia10_cpu()
+ ])
}
,
failure_handler: {
diff --git a/ci/windows/test_jl07_cpu.ps1 b/ci/windows/test_jl07_cpu.ps1
new file mode 100644
index 0000000..6cd34ef
--- /dev/null
+++ b/ci/windows/test_jl07_cpu.ps1
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+7z x -y windows_package.7z
+
+# set default output encoding to utf8
+$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8'
+
+$env:MXNET_HOME = [System.IO.Path]::GetFullPath('.\windows_package')
+$env:JULIA_URL = "https://julialang-s3.julialang.org/bin/winnt/x64/0.7/julia-0.7.0-win64.exe"
+$env:JULIA_DEPOT_PATH = [System.IO.Path]::GetFullPath('.\julia-depot')
+
+$JULIA_DIR = [System.IO.Path]::GetFullPath('.\julia07')
+$JULIA = "$JULIA_DIR\bin\julia"
+
+# Download most recent Julia Windows binary
+[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12
+(New-Object System.Net.WebClient).DownloadFile($env:JULIA_URL, "julia-binary.exe")
+if (! $?) { Throw ("Error on downloading Julia Windows binary") }
+
+# Run installer silently, output to C:\julia07\julia
+Start-Process -Wait "julia-binary.exe" -ArgumentList "/S /D=$JULIA_DIR"
+if (! $?) { Throw ("Error on installing Julia") }
+
+& $JULIA -e "using InteractiveUtils; versioninfo()"
+
+dir
+
+$src='
+ using Pkg
+ Pkg.activate(".\\julia")
+ Pkg.build()
+ Pkg.test()
+'
+
+$src > .\ci-build.jl
+
+# Redirect all stderr output to stdout,
+# since Julia loggers output stuffs to stderr.
+# Then, stderr triggers powershell NativeCommandError.
+& $JULIA .\ci-build.jl 2>&1 | %{ "$_" }
+if ($LastExitCode -eq 1) { Throw ("Error") }
diff --git a/ci/windows/test_jl10_cpu.ps1 b/ci/windows/test_jl10_cpu.ps1
new file mode 100644
index 0000000..96c4190
--- /dev/null
+++ b/ci/windows/test_jl10_cpu.ps1
@@ -0,0 +1,56 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+7z x -y windows_package.7z
+
+# set default output encoding to utf8
+$PSDefaultParameterValues['Out-File:Encoding'] = 'utf8'
+
+$env:MXNET_HOME = [System.IO.Path]::GetFullPath('.\windows_package')
+$env:JULIA_URL = "https://julialang-s3.julialang.org/bin/winnt/x64/1.0/julia-1.0.3-win64.exe"
+$env:JULIA_DEPOT_PATH = [System.IO.Path]::GetFullPath('.\julia-depot')
+
+$JULIA_DIR = [System.IO.Path]::GetFullPath('.\julia10')
+$JULIA = "$JULIA_DIR\bin\julia"
+
+# Download most recent Julia Windows binary
+[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12
+(New-Object System.Net.WebClient).DownloadFile($env:JULIA_URL, "julia-binary.exe")
+if (! $?) { Throw ("Error on downloading Julia Windows binary") }
+
+# Run installer silently, output to C:\julia10\julia
+Start-Process -Wait "julia-binary.exe" -ArgumentList "/S /D=$JULIA_DIR"
+if (! $?) { Throw ("Error on installing Julia") }
+
+& $JULIA -e "using InteractiveUtils; versioninfo()"
+
+dir
+
+$src='
+ using Pkg
+ Pkg.activate(".\\julia")
+ Pkg.build()
+ Pkg.test()
+'
+
+$src > .\ci-build.jl
+
+# Redirect all stderr output to stdout,
+# since Julia loggers output stuffs to stderr.
+# Then, stderr triggers powershell NativeCommandError.
+& $JULIA .\ci-build.jl 2>&1 | %{ "$_" }
+if ($LastExitCode -eq 1) { Throw ("Error") }
diff --git a/julia/.gitignore b/julia/.gitignore
index 3687ed4..e7b35fa 100644
--- a/julia/.gitignore
+++ b/julia/.gitignore
@@ -8,3 +8,4 @@ deps/src
deps/usr
deps/deps.jl
.vscode
+/Manifest.toml
diff --git a/julia/Project.toml b/julia/Project.toml
new file mode 100644
index 0000000..82a94c5
--- /dev/null
+++ b/julia/Project.toml
@@ -0,0 +1,26 @@
+name = "MXNet"
+uuid = "a7949054-b901-59c6-b8e3-7238c29bf7f0"
+authors = ["Chiyuan Zhang <pl...@gmail.com>", "Valentin Churavy <vc...@mit.edu>", "Iblis Lin <ib...@hs.ntnu.edu.tw>"]
+version = "1.5.0"
+
+[deps]
+BinDeps = "9e28174c-4ba2-5203-b857-d8d62c4213ee"
+Formatting = "59287772-0a20-5a39-b81b-1366585eb4c0"
+JSON = "682c06a0-de6a-54ab-a142-c8b1cf79cde6"
+Libdl = "8f399da3-3557-5675-b5ff-fb832c97cbdb"
+LinearAlgebra = "37e2e46d-f89d-539d-b4ee-838fcccc9c8e"
+MacroTools = "1914dd2f-81c6-5fcd-8719-6d5c9610ff09"
+Markdown = "d6f4376e-aef5-505a-96c1-9c027394607a"
+Printf = "de0858da-6303-5e67-8744-51eddeeeb8d7"
+Random = "9a3f8284-a2c9-5f02-9a11-845980a1fd5c"
+Reexport = "189a3867-3050-52da-a836-e630ba90ab69"
+Statistics = "10745b16-79ce-11e8-11f9-7d13ad32a3b2"
+
+[compat]
+julia = "≥0.7"
+
+[extras]
+Test = "8dfed614-e22c-5e08-85e1-65c5234f0b40"
+
+[targets]
+test = ["Test"]
diff --git a/julia/REQUIRE b/julia/REQUIRE
index b53f0c3..8008da3 100644
--- a/julia/REQUIRE
+++ b/julia/REQUIRE
@@ -1,4 +1,4 @@
-julia 0.6
+julia 0.7
Formatting
BinDeps
JSON
diff --git a/julia/appveyor.yml b/julia/appveyor.yml
deleted file mode 100644
index 50e275c..0000000
--- a/julia/appveyor.yml
+++ /dev/null
@@ -1,56 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-environment:
- matrix:
- - JULIA_URL: "https://julialang-s3.julialang.org/bin/winnt/x64/0.6/julia-0.6-latest-win64.exe"
-
-branches:
- only:
- - master
- - stable
-
-notifications:
- - provider: Email
- on_build_success: false
- on_build_failure: false
- on_build_status_changed: false
-
-install:
- - ps: "[System.Net.ServicePointManager]::SecurityProtocol = [System.Net.SecurityProtocolType]::Tls12"
-# If there's a newer build queued for the same PR, cancel this one
- - ps: if ($env:APPVEYOR_PULL_REQUEST_NUMBER -and $env:APPVEYOR_BUILD_NUMBER -ne ((Invoke-RestMethod `
- https://ci.appveyor.com/api/projects/$env:APPVEYOR_ACCOUNT_NAME/$env:APPVEYOR_PROJECT_SLUG/history?recordsNumber=50).builds | `
- Where-Object pullRequestId -eq $env:APPVEYOR_PULL_REQUEST_NUMBER)[0].buildNumber) { `
- throw "There are newer queued builds for this pull request, failing early." }
-
-# Download most recent Julia Windows binary
- - ps: (new-object net.webclient).DownloadFile(
- $env:JULIA_URL,
- "C:\projects\julia-binary.exe")
-# Run installer silently, output to C:\projects\julia
- - C:\projects\julia-binary.exe /S /D=C:\projects\julia
-
-build_script:
-# Need to convert from shallow to complete for Pkg.clone to work
- - IF EXIST .git\shallow (git fetch --unshallow)
- - C:\projects\julia\bin\julia -e "versioninfo();
- Pkg.clone(pwd(), \"MXNet\"); Pkg.build(\"MXNet\")"
-
-test_script:
- - C:\projects\julia\bin\julia --check-bounds=yes -e "Pkg.test(\"MXNet\")"
-
diff --git a/julia/deps/build.jl b/julia/deps/build.jl
index 7a37803..9a719be 100644
--- a/julia/deps/build.jl
+++ b/julia/deps/build.jl
@@ -86,7 +86,7 @@ if HAS_CUDA
if HAS_CUDNN
@info("Found a CuDNN installation.")
end
- @info("CUDA_HOME -> $(get(ENV, "CUDA_HOME", nothing))")
+ @info("CUDA_HOME -> $(get(ENV, "CUDA_HOME", "nothing"))")
else
@info("Did not find a CUDA installation, using CPU-only version of MXNet.")
end
diff --git a/julia/src/util.jl b/julia/src/util.jl
index ae2e50b..ac7f4fc 100644
--- a/julia/src/util.jl
+++ b/julia/src/util.jl
@@ -35,12 +35,12 @@ function get_mnist_ubyte()
filenames = Dict((x[1] => joinpath(mnist_dir, x[2]) for x ∈ pairs(filenames)))
if !all(isfile, values(filenames))
cd(mnist_dir) do
- mnist_dir = download("http://data.mxnet.io/mxnet/data/mnist.zip", "mnist.zip")
+ data = download("http://data.mxnet.io/mxnet/data/mnist.zip", "mnist.zip")
try
- run(`unzip -u $mnist_dir`)
+ run(`unzip -u $data`)
catch
try
- run(pipe(`7z x $mnist_dir`,stdout = devnull))
+ run(pipeline(`7z x $data`,stdout = devnull))
catch
error("Extraction Failed:No extraction program found in path")
end
diff --git a/tests/nightly/apache_rat_license_check/rat-excludes b/tests/nightly/apache_rat_license_check/rat-excludes
index 782ef40..93ac16e 100755
--- a/tests/nightly/apache_rat_license_check/rat-excludes
+++ b/tests/nightly/apache_rat_license_check/rat-excludes
@@ -48,6 +48,7 @@ moderngpu/*
deformable_im2col.cuh
deformable_im2col.h
REQUIRE
+Project.toml
include/*
.*.iml
.*.json.ref
diff --git a/tools/license_header.py b/tools/license_header.py
index 11cc928..b9acbf1 100755
--- a/tools/license_header.py
+++ b/tools/license_header.py
@@ -91,6 +91,9 @@ _WHITE_LIST = [
# Licensed under 2-Clause BSD in header
'example/ssd/dataset/pycocotools/coco.py',
+
+ # Julia package metadata, generated by Pkg3.jl
+ 'julia/Project.toml',
]
# language extensions and the according commment mark