You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by la...@apache.org on 2019/01/10 01:50:14 UTC
[incubator-mxnet] branch master updated: Dockerfiles for Publish
Testing (#13707)
This is an automated email from the ASF dual-hosted git repository.
lanking pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 30b552d Dockerfiles for Publish Testing (#13707)
30b552d is described below
commit 30b552d5da9f32fa14f2b2c56f7ecafbebb4da41
Author: Zach Kimberg <za...@kimberg.com>
AuthorDate: Wed Jan 9 17:49:57 2019 -0800
Dockerfiles for Publish Testing (#13707)
---
ci/build.py | 11 ++++---
...cala.sh => Dockerfile.publish.test.centos7_cpu} | 31 ++++++++++++--------
...cala.sh => Dockerfile.publish.test.centos7_gpu} | 31 ++++++++++++--------
...a.sh => Dockerfile.publish.test.ubuntu1404_cpu} | 32 ++++++++++++--------
...a.sh => Dockerfile.publish.test.ubuntu1404_gpu} | 33 +++++++++++++--------
...a.sh => Dockerfile.publish.test.ubuntu1604_cpu} | 32 ++++++++++++--------
...a.sh => Dockerfile.publish.test.ubuntu1604_gpu} | 32 ++++++++++++--------
...a.sh => Dockerfile.publish.test.ubuntu1804_cpu} | 34 ++++++++++++++--------
...a.sh => Dockerfile.publish.test.ubuntu1804_gpu} | 34 ++++++++++++++--------
..._scala.sh => Dockerfile.publish.ubuntu1404_cpu} | 29 ++++++++++--------
..._scala.sh => Dockerfile.publish.ubuntu1404_gpu} | 29 ++++++++++--------
.../install/{centos7_scala.sh => centos7_base.sh} | 16 +++++-----
ci/docker/install/centos7_scala.sh | 10 ++++++-
.../install/{ubuntu_core.sh => ubuntu_base.sh} | 9 +-----
ci/docker/install/ubuntu_core.sh | 9 +++++-
ci/docker/install/ubuntu_publish.sh | 25 +++++++++++++---
ci/docker/install/ubuntu_scala.sh | 27 +++++++++++++----
ci/docker/runtime_functions.sh | 6 ++--
ci/publish/Jenkinsfile | 22 +++++++-------
{scala-package/dev => ci/publish/scala}/build.sh | 9 ++----
.../dev => ci/publish/scala}/buildkey.py | 0
{scala-package/dev => ci/publish/scala}/deploy.sh | 7 ++---
.../dev/test.sh => ci/publish/scala/fullDeploy.sh | 6 ++--
{scala-package/dev => ci/publish/scala}/test.sh | 7 ++++-
ci/test_docker_cache.py | 4 +--
scala-package/.gitignore | 1 -
scala-package/packageTest/pom.xml | 2 +-
27 files changed, 304 insertions(+), 184 deletions(-)
diff --git a/ci/build.py b/ci/build.py
index e5cf933..1c7a4f8 100755
--- a/ci/build.py
+++ b/ci/build.py
@@ -92,22 +92,24 @@ def get_dockerfiles_path():
def get_platforms(path: str = get_dockerfiles_path()) -> List[str]:
"""Get a list of architectures given our dockerfiles"""
- dockerfiles = glob.glob(os.path.join(path, "Dockerfile.build.*"))
+ dockerfiles = glob.glob(os.path.join(path, "Dockerfile.*"))
dockerfiles = list(filter(lambda x: x[-1] != '~', dockerfiles))
- files = list(map(lambda x: re.sub(r"Dockerfile.build.(.*)", r"\1", x), dockerfiles))
+ files = list(map(lambda x: re.sub(r"Dockerfile.(.*)", r"\1", x), dockerfiles))
platforms = list(map(lambda x: os.path.split(x)[1], sorted(files)))
return platforms
def get_docker_tag(platform: str, registry: str) -> str:
""":return: docker tag to be used for the container"""
+ platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform)
if not registry:
registry = "mxnet_local"
- return "{0}/build.{1}".format(registry, platform)
+ return "{0}/{1}".format(registry, platform)
def get_dockerfile(platform: str, path=get_dockerfiles_path()) -> str:
- return os.path.join(path, "Dockerfile.build.{0}".format(platform))
+ platform = platform if any(x in platform for x in ['build.', 'publish.']) else 'build.{}'.format(platform)
+ return os.path.join(path, "Dockerfile.{0}".format(platform))
def get_docker_binary(use_nvidia_docker: bool) -> str:
@@ -523,6 +525,7 @@ def main() -> int:
elif args.all:
platforms = get_platforms()
+ platforms = [platform for platform in platforms if 'build.' in platform]
logging.info("Building for all architectures: %s", platforms)
logging.info("Artifacts will be produced in the build/ directory.")
for platform in platforms:
diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.centos7_cpu
old mode 100755
new mode 100644
similarity index 61%
copy from ci/docker/install/centos7_scala.sh
copy to ci/docker/Dockerfile.publish.test.centos7_cpu
index ea46de9..7d28445
--- a/ci/docker/install/centos7_scala.sh
+++ b/ci/docker/Dockerfile.publish.test.centos7_cpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +15,24 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to build and run MXNet on CentOS 7 for CPU
+
+FROM centos:7
+
+WORKDIR /work/deps
+
+COPY install/centos7_base.sh /work/
+RUN /work/centos7_base.sh
+
+COPY install/centos7_scala.sh /work/
+RUN /work/centos7_scala.sh
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
+ARG USER_ID=0
+COPY install/centos7_adduser.sh /work/
+RUN /work/centos7_adduser.sh
-set -ex
+ENV PYTHONPATH=./python/
+WORKDIR /work/mxnet
-yum install -y java-1.8.0-openjdk-devel
-# Build from source with Maven
-wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar xzf apache-maven-3.3.9-bin.tar.gz
-mkdir /usr/local/maven
-mv apache-maven-3.3.9/ /usr/local/maven/
-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+COPY runtime_functions.sh /work/
diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.centos7_gpu
old mode 100755
new mode 100644
similarity index 61%
copy from ci/docker/install/centos7_scala.sh
copy to ci/docker/Dockerfile.publish.test.centos7_gpu
index ea46de9..e7f5846
--- a/ci/docker/install/centos7_scala.sh
+++ b/ci/docker/Dockerfile.publish.test.centos7_gpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +15,24 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to build and run MXNet on CentOS 7 for CPU
+
+FROM nvidia/cuda:9.2-cudnn7-devel-centos7
+
+WORKDIR /work/deps
+
+COPY install/centos7_base.sh /work/
+RUN /work/centos7_base.sh
+
+COPY install/centos7_scala.sh /work/
+RUN /work/centos7_scala.sh
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
+ARG USER_ID=0
+COPY install/centos7_adduser.sh /work/
+RUN /work/centos7_adduser.sh
-set -ex
+ENV PYTHONPATH=./python/
+WORKDIR /work/mxnet
-yum install -y java-1.8.0-openjdk-devel
-# Build from source with Maven
-wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar xzf apache-maven-3.3.9-bin.tar.gz
-mkdir /usr/local/maven
-mv apache-maven-3.3.9/ /usr/local/maven/
-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+COPY runtime_functions.sh /work/
diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.ubuntu1404_cpu
old mode 100755
new mode 100644
similarity index 61%
copy from ci/docker/install/centos7_scala.sh
copy to ci/docker/Dockerfile.publish.test.ubuntu1404_cpu
index ea46de9..0358376
--- a/ci/docker/install/centos7_scala.sh
+++ b/ci/docker/Dockerfile.publish.test.ubuntu1404_cpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +15,25 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 14.04 for CPU
+
+FROM ubuntu:14.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
-set -ex
+COPY runtime_functions.sh /work/
-yum install -y java-1.8.0-openjdk-devel
-# Build from source with Maven
-wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar xzf apache-maven-3.3.9-bin.tar.gz
-mkdir /usr/local/maven
-mv apache-maven-3.3.9/ /usr/local/maven/
-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.ubuntu1404_gpu
old mode 100755
new mode 100644
similarity index 61%
copy from ci/docker/install/centos7_scala.sh
copy to ci/docker/Dockerfile.publish.test.ubuntu1404_gpu
index ea46de9..854dd68
--- a/ci/docker/install/centos7_scala.sh
+++ b/ci/docker/Dockerfile.publish.test.ubuntu1404_gpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +15,26 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to run MXNet on Ubuntu 14.04 for GPU
+
+# Use CPU with setup_gpu script
+FROM ubuntu:14.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
-set -ex
+COPY runtime_functions.sh /work/
-yum install -y java-1.8.0-openjdk-devel
-# Build from source with Maven
-wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar xzf apache-maven-3.3.9-bin.tar.gz
-mkdir /usr/local/maven
-mv apache-maven-3.3.9/ /usr/local/maven/
-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.ubuntu1604_cpu
old mode 100755
new mode 100644
similarity index 61%
copy from ci/docker/install/centos7_scala.sh
copy to ci/docker/Dockerfile.publish.test.ubuntu1604_cpu
index ea46de9..bbb7b6a
--- a/ci/docker/install/centos7_scala.sh
+++ b/ci/docker/Dockerfile.publish.test.ubuntu1604_cpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +15,25 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 16.04 for CPU
+
+FROM ubuntu:16.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
-set -ex
+COPY runtime_functions.sh /work/
-yum install -y java-1.8.0-openjdk-devel
-# Build from source with Maven
-wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar xzf apache-maven-3.3.9-bin.tar.gz
-mkdir /usr/local/maven
-mv apache-maven-3.3.9/ /usr/local/maven/
-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.ubuntu1604_gpu
old mode 100755
new mode 100644
similarity index 61%
copy from ci/docker/install/centos7_scala.sh
copy to ci/docker/Dockerfile.publish.test.ubuntu1604_gpu
index ea46de9..660461d
--- a/ci/docker/install/centos7_scala.sh
+++ b/ci/docker/Dockerfile.publish.test.ubuntu1604_gpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +15,25 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to run MXNet on Ubuntu 16.04 for GPU
+
+FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu16.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
-set -ex
+COPY runtime_functions.sh /work/
-yum install -y java-1.8.0-openjdk-devel
-# Build from source with Maven
-wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar xzf apache-maven-3.3.9-bin.tar.gz
-mkdir /usr/local/maven
-mv apache-maven-3.3.9/ /usr/local/maven/
-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu
old mode 100755
new mode 100644
similarity index 61%
copy from ci/docker/install/centos7_scala.sh
copy to ci/docker/Dockerfile.publish.test.ubuntu1804_cpu
index ea46de9..e3a8c19
--- a/ci/docker/install/centos7_scala.sh
+++ b/ci/docker/Dockerfile.publish.test.ubuntu1804_cpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +15,27 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 18.04 for CPU
+
+FROM ubuntu:18.04
+
+WORKDIR /work/deps
+
+ENV DEBIAN_FRONTEND noninteractive
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
-set -ex
+COPY runtime_functions.sh /work/
-yum install -y java-1.8.0-openjdk-devel
-# Build from source with Maven
-wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar xzf apache-maven-3.3.9-bin.tar.gz
-mkdir /usr/local/maven
-mv apache-maven-3.3.9/ /usr/local/maven/
-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu
old mode 100755
new mode 100644
similarity index 60%
copy from ci/docker/install/centos7_scala.sh
copy to ci/docker/Dockerfile.publish.test.ubuntu1804_gpu
index ea46de9..99f7e0d
--- a/ci/docker/install/centos7_scala.sh
+++ b/ci/docker/Dockerfile.publish.test.ubuntu1804_gpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +15,27 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to run MXNet on Ubuntu 18.04 for GPU
+
+FROM nvidia/cuda:9.2-cudnn7-devel-ubuntu18.04
+
+WORKDIR /work/deps
+
+ENV DEBIAN_FRONTEND noninteractive
+
+COPY install/ubuntu_base.sh /work/
+RUN /work/ubuntu_base.sh
+
+COPY install/ubuntu_scala.sh /work/
+RUN /work/ubuntu_scala.sh
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
-set -ex
+COPY runtime_functions.sh /work/
-yum install -y java-1.8.0-openjdk-devel
-# Build from source with Maven
-wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar xzf apache-maven-3.3.9-bin.tar.gz
-mkdir /usr/local/maven
-mv apache-maven-3.3.9/ /usr/local/maven/
-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.ubuntu1404_cpu
old mode 100755
new mode 100644
similarity index 61%
copy from ci/docker/install/centos7_scala.sh
copy to ci/docker/Dockerfile.publish.ubuntu1404_cpu
index ea46de9..04ce94f
--- a/ci/docker/install/centos7_scala.sh
+++ b/ci/docker/Dockerfile.publish.ubuntu1404_cpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +15,22 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to build and run MXNet on Ubuntu 14.04 for CPU
+
+FROM ubuntu:14.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_publish.sh /work/
+RUN /work/ubuntu_publish.sh
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
-set -ex
+COPY runtime_functions.sh /work/
-yum install -y java-1.8.0-openjdk-devel
-# Build from source with Maven
-wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar xzf apache-maven-3.3.9-bin.tar.gz
-mkdir /usr/local/maven
-mv apache-maven-3.3.9/ /usr/local/maven/
-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/Dockerfile.publish.ubuntu1404_gpu
old mode 100755
new mode 100644
similarity index 61%
copy from ci/docker/install/centos7_scala.sh
copy to ci/docker/Dockerfile.publish.ubuntu1404_gpu
index ea46de9..9855986
--- a/ci/docker/install/centos7_scala.sh
+++ b/ci/docker/Dockerfile.publish.ubuntu1404_gpu
@@ -1,5 +1,4 @@
-#!/usr/bin/env bash
-
+# -*- mode: dockerfile -*-
# Licensed to the Apache Software Foundation (ASF) under one
# or more contributor license agreements. See the NOTICE file
# distributed with this work for additional information
@@ -16,16 +15,22 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
+#
+# Dockerfile to run MXNet on Ubuntu 14.04 for GPU
+
+FROM ubuntu:14.04
+
+WORKDIR /work/deps
+
+COPY install/ubuntu_publish.sh /work/
+RUN /work/ubuntu_publish.sh
-# build and install are separated so changes to build don't invalidate
-# the whole docker cache for the image
+ARG USER_ID=0
+ARG GROUP_ID=0
+COPY install/ubuntu_adduser.sh /work/
+RUN /work/ubuntu_adduser.sh
-set -ex
+COPY runtime_functions.sh /work/
-yum install -y java-1.8.0-openjdk-devel
-# Build from source with Maven
-wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar xzf apache-maven-3.3.9-bin.tar.gz
-mkdir /usr/local/maven
-mv apache-maven-3.3.9/ /usr/local/maven/
-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+WORKDIR /work/mxnet
+ENV LD_LIBRARY_PATH=${LD_LIBRARY_PATH}:/usr/local/lib
diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/install/centos7_base.sh
similarity index 72%
copy from ci/docker/install/centos7_scala.sh
copy to ci/docker/install/centos7_base.sh
index ea46de9..3b84aeb 100755
--- a/ci/docker/install/centos7_scala.sh
+++ b/ci/docker/install/centos7_base.sh
@@ -22,10 +22,12 @@
set -ex
-yum install -y java-1.8.0-openjdk-devel
-# Build from source with Maven
-wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar xzf apache-maven-3.3.9-bin.tar.gz
-mkdir /usr/local/maven
-mv apache-maven-3.3.9/ /usr/local/maven/
-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+# Multipackage installation does not fail in yum
+yum -y install epel-release
+yum -y install git
+yum -y install wget
+yum -y install make
+yum -y install cmake
+yum -y install unzip
+yum -y install ninja-build
+yum -y install gcc-gfortran
diff --git a/ci/docker/install/centos7_scala.sh b/ci/docker/install/centos7_scala.sh
index ea46de9..5c43f01 100755
--- a/ci/docker/install/centos7_scala.sh
+++ b/ci/docker/install/centos7_scala.sh
@@ -23,9 +23,17 @@
set -ex
yum install -y java-1.8.0-openjdk-devel
+export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk
+export PATH=$JAVA_HOME/bin:$PATH
# Build from source with Maven
-wget http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
+wget -q http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
tar xzf apache-maven-3.3.9-bin.tar.gz
mkdir /usr/local/maven
mv apache-maven-3.3.9/ /usr/local/maven/
alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+
+echo "export JAVA_HOME=/usr/lib/jvm/jre-1.8.0-openjdk" >> /etc/profile.d/maven.sh
+echo "export M3_HOME=/usr/local/src/apache-maven" >> /etc/profile.d/maven.sh
+echo "export PATH=$M3_HOME/bin:$JAVA_HOME/bin:$PATH" >> /etc/profile.d/maven.sh
+chmod +x /etc/profile.d/maven.sh
+source /etc/profile.d/maven.sh
diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_base.sh
similarity index 86%
copy from ci/docker/install/ubuntu_core.sh
copy to ci/docker/install/ubuntu_base.sh
index 64f8af3..b34c0b3 100755
--- a/ci/docker/install/ubuntu_core.sh
+++ b/ci/docker/install/ubuntu_base.sh
@@ -23,20 +23,13 @@
set -ex
apt-get update || true
apt-get install -y \
- apt-transport-https \
build-essential \
ca-certificates \
cmake \
curl \
git \
- libatlas-base-dev \
- libcurl4-openssl-dev \
- libjemalloc-dev \
- liblapack-dev \
- libopenblas-dev \
- libopencv-dev \
- libzmq3-dev \
ninja-build \
+ libgfortran3 \
software-properties-common \
sudo \
unzip \
diff --git a/ci/docker/install/ubuntu_core.sh b/ci/docker/install/ubuntu_core.sh
index 64f8af3..4382aa6 100755
--- a/ci/docker/install/ubuntu_core.sh
+++ b/ci/docker/install/ubuntu_core.sh
@@ -26,7 +26,6 @@ apt-get install -y \
apt-transport-https \
build-essential \
ca-certificates \
- cmake \
curl \
git \
libatlas-base-dev \
@@ -41,3 +40,11 @@ apt-get install -y \
sudo \
unzip \
wget
+
+
+# Ubuntu 14.04
+if [[ $(lsb_release -r | grep 14.04) ]]; then
+ apt-get install -y cmake3
+else
+ apt-get install -y cmake
+fi
diff --git a/ci/docker/install/ubuntu_publish.sh b/ci/docker/install/ubuntu_publish.sh
index bc3513d..1ad6ab9 100644
--- a/ci/docker/install/ubuntu_publish.sh
+++ b/ci/docker/install/ubuntu_publish.sh
@@ -35,14 +35,31 @@ apt-get install -y git \
nasm \
libtool \
curl \
+ wget \
+ sudo \
+ gnupg \
+ gnupg2 \
+ gnupg-agent \
pandoc \
python3-pip \
automake \
pkg-config \
openjdk-8-jdk
-curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
-tar xzf apache-maven-3.3.9-bin.tar.gz
-mkdir /usr/local/maven
-mv apache-maven-3.3.9/ /usr/local/maven/
+curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
+tar xzf apache-maven-3.3.9-bin.tar.gz
+mkdir /usr/local/maven
+mv apache-maven-3.3.9/ /usr/local/maven/
update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
update-ca-certificates -f
+
+apt-get install -y python python3
+
+# the version of the pip shipped with ubuntu may be too lower, install a recent version here
+wget -nv https://bootstrap.pypa.io/get-pip.py
+python3 get-pip.py
+python2 get-pip.py
+
+apt-get remove -y python3-urllib3
+
+pip2 install nose cpplint==1.3.0 pylint==1.9.3 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
+pip3 install nose cpplint==1.3.0 pylint==2.1.1 'numpy<=1.15.2,>=1.8.2' nose-timer 'requests<2.19.0,>=2.18.4' h5py==2.8.0rc1 scipy==1.0.1 boto3
diff --git a/ci/docker/install/ubuntu_scala.sh b/ci/docker/install/ubuntu_scala.sh
index 22be230..5bade47 100755
--- a/ci/docker/install/ubuntu_scala.sh
+++ b/ci/docker/install/ubuntu_scala.sh
@@ -24,16 +24,31 @@ set -ex
cd "$(dirname "$0")"
# install libraries for mxnet's scala package on ubuntu
echo 'Installing Scala...'
-apt-get update || true
-apt-get install -y software-properties-common
-apt-get update || true
-apt-get install -y openjdk-8-jdk
-apt-get install -y openjdk-8-jre
+# Ubuntu 14.04
+if [[ $(lsb_release -r | grep 14.04) ]]; then
+ add-apt-repository -y ppa:openjdk-r/ppa
+fi
+
+# All Ubuntu
apt-get update || true
apt-get install -y \
- maven \
+ openjdk-8-jdk \
+ openjdk-8-jre \
+ software-properties-common \
gnupg \
gnupg2 \
gnupg-agent \
scala
+
+# Ubuntu 14.04
+if [[ $(lsb_release -r | grep 14.04) ]]; then
+ curl -o apache-maven-3.3.9-bin.tar.gz http://www.eu.apache.org/dist/maven/maven-3/3.3.9/binaries/apache-maven-3.3.9-bin.tar.gz
+ tar xzf apache-maven-3.3.9-bin.tar.gz
+ mkdir /usr/local/maven
+ mv apache-maven-3.3.9/ /usr/local/maven/
+ update-alternatives --install /usr/bin/mvn mvn /usr/local/maven/apache-maven-3.3.9/bin/mvn 1
+ update-ca-certificates -f
+else
+ apt-get install -y maven
+fi
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 4ec6b76..94803ce 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -1274,7 +1274,7 @@ publish_scala_build() {
set -ex
pushd .
scala_prepare
- ./scala-package/dev/build.sh
+ ./ci/publish/scala/build.sh
popd
}
@@ -1282,7 +1282,7 @@ publish_scala_test() {
set -ex
pushd .
scala_prepare
- ./scala-package/dev/test.sh
+ ./ci/publish/scala/test.sh
popd
}
@@ -1290,7 +1290,7 @@ publish_scala_deploy() {
set -ex
pushd .
scala_prepare
- ./scala-package/dev/deploy.sh
+ ./ci/publish/scala/deploy.sh
popd
}
diff --git a/ci/publish/Jenkinsfile b/ci/publish/Jenkinsfile
index 49e7d7a..9a360c6 100644
--- a/ci/publish/Jenkinsfile
+++ b/ci/publish/Jenkinsfile
@@ -21,7 +21,7 @@
// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
//mxnet libraries
-mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/test-classes/**, scala-package/local-snapshot/**'
+mx_scala_pub = 'lib/libmxnet.so, lib/libmxnet.a, 3rdparty/dmlc-core/libdmlc.a, 3rdparty/tvm/nnvm/lib/libnnvm.a, 3rdparty/ps-lite/build/libps.a, deps/lib/libprotobuf-lite.a, deps/lib/libzmq.a, config.mk, scala-package/pom.xml, scala-package/**/pom.xml, scala-package/*/target/**, scala-package/local-snapshot/**'
// timeout in minutes
max_time = 120
@@ -54,22 +54,23 @@ def labels = ['cpu'] // , 'gpu']
for (x in labels) {
def label = x // Required due to language
toBuild["Scala Build ${label}"] = wrapStep(nodeMap[label], "build-scala-${label}") {
- env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label]
- utils.init_git()
- utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE')
- utils.pack_lib("scala_${label}", mx_scala_pub, false)
+ withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) {
+ utils.init_git()
+ utils.docker_run("ubuntu_${label}", 'publish_scala_build', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE')
+ utils.pack_lib("scala_${label}", mx_scala_pub, false)
+ }
}
}
def toTest = [:]
-def systems = ['ubuntu'] // , 'centos7']
+def systems = ['ubuntu1604', 'ubuntu1804', 'centos7']
for (x in labels) {
def label = x // Required due to language
for (y in systems) {
def system = y // Required due to language
toTest["Scala Test ${system} ${label}"] = wrapStep(nodeMap[label], "test-scala-${system}-${label}") {
utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
- utils.docker_run("${system}_${label}", 'publish_scala_test', label == 'gpu')
+ utils.docker_run("publish.test.${system}_${label}", 'publish_scala_test', label == 'gpu')
}
}
}
@@ -78,9 +79,10 @@ def toDeploy = [:]
for (x in labels) {
def label = x // Required due to language
toDeploy["Scala Deploy ${label}"] = wrapStep(nodeMap[label], "deploy-scala-${label}") {
- env.MAVEN_PUBLISH_OS_TYPE = scalaOSMap[label]
- utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
- utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION')
+ withEnv(["MAVEN_PUBLISH_OS_TYPE=${scalaOSMap[label]}"]) {
+ utils.unpack_and_init("scala_${label}", mx_scala_pub, false)
+ utils.docker_run("ubuntu_${label}", 'publish_scala_deploy', label == 'gpu', '500m', 'MAVEN_PUBLISH_OS_TYPE MAVEN_PUBLISH_SECRET_ENDPOINT_URL MAVEN_PUBLISH_SECRET_NAME_CREDENTIALS MAVEN_PUBLISH_SECRET_NAME_GPG DOCKERHUB_SECRET_ENDPOINT_REGION')
+ }
}
}
diff --git a/scala-package/dev/build.sh b/ci/publish/scala/build.sh
similarity index 88%
rename from scala-package/dev/build.sh
rename to ci/publish/scala/build.sh
index c336fd8..17f969a 100755
--- a/scala-package/dev/build.sh
+++ b/ci/publish/scala/build.sh
@@ -24,11 +24,6 @@ set -ex
bash scala-package/dev/compile-mxnet-backend.sh $MAVEN_PUBLISH_OS_TYPE ./
-# Scala steps to deploy
-make scalapkg CI=1
-
# Compile tests for discovery later
-export GPG_TTY=$(tty)
-make scalatestcompile CI=1
-# make scalainstall CI=1
-make scaladeploylocal CI=1
+cd scala-package
+mvn -B deploy
diff --git a/scala-package/dev/buildkey.py b/ci/publish/scala/buildkey.py
similarity index 100%
rename from scala-package/dev/buildkey.py
rename to ci/publish/scala/buildkey.py
diff --git a/scala-package/dev/deploy.sh b/ci/publish/scala/deploy.sh
similarity index 86%
rename from scala-package/dev/deploy.sh
rename to ci/publish/scala/deploy.sh
index 6f845ba..4eb3390 100755
--- a/scala-package/dev/deploy.sh
+++ b/ci/publish/scala/deploy.sh
@@ -23,7 +23,7 @@ set -ex
# export MAVEN_PUBLISH_OS_TYPE=linux-x86_64-cpu
# Run python to configure keys
-python3 $PWD/scala-package/dev/buildkey.py
+python3 ci/publish/scala/buildkey.py
# Updating cache
mkdir -p ~/.gnupg
@@ -34,11 +34,8 @@ echo "pinentry-mode loopback" >> ~/.gnupg/gpg-agent.conf
export GPG_TTY=$(tty)
cd scala-package
-VERSION=$(mvn -q -Dexec.executable="echo" -Dexec.args='${project.version}' --non-recursive exec:exec)
-cd ..
-# echo "\n\n$VERSION\n" | make scalarelease-dryrun
-make scaladeploy CI=1
+mvn -B deploy -Pnightly
# Clear all password .xml files, exp files, and gpg key files
rm -rf ~/.m2/*.xml ~/.m2/key.asc ~/.m2/*.exp
diff --git a/scala-package/dev/test.sh b/ci/publish/scala/fullDeploy.sh
old mode 100755
new mode 100644
similarity index 90%
copy from scala-package/dev/test.sh
copy to ci/publish/scala/fullDeploy.sh
index 03810fb..69d674a
--- a/scala-package/dev/test.sh
+++ b/ci/publish/scala/fullDeploy.sh
@@ -18,6 +18,6 @@
set -ex
-# Test
-cd scala-package/packageTest
-make scalaintegrationtestlocal CI=1
+./ci/publish/scala/build.sh
+./ci/publish/scala/test.sh
+./ci/publish/scala/deploy.sh
diff --git a/scala-package/dev/test.sh b/ci/publish/scala/test.sh
similarity index 88%
rename from scala-package/dev/test.sh
rename to ci/publish/scala/test.sh
index 03810fb..5cef35c 100755
--- a/scala-package/dev/test.sh
+++ b/ci/publish/scala/test.sh
@@ -18,6 +18,11 @@
set -ex
+if [ -z "$JAVA_HOME" ]; then
+ source /etc/profile
+fi
+
# Test
cd scala-package/packageTest
-make scalaintegrationtestlocal CI=1
+# make testlocal CI=1
+make testsnapshot UNIT=1 CI=1
diff --git a/ci/test_docker_cache.py b/ci/test_docker_cache.py
index 358d549..0a3bc46 100644
--- a/ci/test_docker_cache.py
+++ b/ci/test_docker_cache.py
@@ -135,7 +135,7 @@ class TestDockerCache(unittest.TestCase):
"""
platform = 'test_full_cache'
docker_tag = build_util.get_docker_tag(platform=platform, registry=DOCKER_REGISTRY_PATH)
- dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.build.' + platform)
+ dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.' + platform)
try:
with open(dockerfile_path, 'w') as dockerfile_handle:
dockerfile_handle.write(dockerfile_content)
@@ -196,7 +196,7 @@ class TestDockerCache(unittest.TestCase):
"""
platform = 'test_partial_cache'
docker_tag = build_util.get_docker_tag(platform=platform, registry=DOCKER_REGISTRY_PATH)
- dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.build.' + platform)
+ dockerfile_path = os.path.join(DOCKERFILE_DIR, 'Dockerfile.' + platform)
try:
# Write initial Dockerfile
with open(dockerfile_path, 'w') as dockerfile_handle:
diff --git a/scala-package/.gitignore b/scala-package/.gitignore
index bf60e7b..22b12b3 100644
--- a/scala-package/.gitignore
+++ b/scala-package/.gitignore
@@ -8,4 +8,3 @@ core/src/main/scala/org/apache/mxnet/SymbolBase.scala
core/src/main/scala/org/apache/mxnet/SymbolRandomAPIBase.scala
examples/scripts/infer/images/
examples/scripts/infer/models/
-local-snapshot
diff --git a/scala-package/packageTest/pom.xml b/scala-package/packageTest/pom.xml
index 9c5c11c..f7d9e3b 100644
--- a/scala-package/packageTest/pom.xml
+++ b/scala-package/packageTest/pom.xml
@@ -42,7 +42,7 @@
<repositories>
<repository>
<id>local-snapshot</id>
- <url>file://${basedir}/../local-snapshot</url>
+ <url>file://${basedir}/../deploy/target/repo</url>
<snapshots>
<enabled>true</enabled>
</snapshots>