You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by sk...@apache.org on 2018/09/29 04:36:24 UTC
[incubator-mxnet] branch master updated: [MXNET-951] Python
dockerfiles built on pip binaries and build/release script (#12556)
This is an automated email from the ASF dual-hosted git repository.
skm pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 063139d [MXNET-951] Python dockerfiles built on pip binaries and build/release script (#12556)
063139d is described below
commit 063139d6ee12e5b54ebe6f79faddf6cda9ba0897
Author: Meghna Baijal <30...@users.noreply.github.com>
AuthorDate: Fri Sep 28 21:36:08 2018 -0700
[MXNET-951] Python dockerfiles built on pip binaries and build/release script (#12556)
* Initial Commit for docker automation python
* Fixes
* change dir for tests
* Fix more issues
* fix docker tag command
* cosmetic changes
* update README
* update test to fail on version mismatch
* remove debug mode
* Update README.md
* Update README.md
* update README
* Add Licenses
* Some review comments
* Add Cuda80 and cuda92 dockerfiles and build steps
* Add renamed and hence untracked files for cu90
* Update README
* More ways to login
* Update README with login options
* Update README with links to test. test_mxnet link will work only after merge
---
docker/docker-python/Dockerfile.mxnet.python.cpu | 28 +++++
.../docker-python/Dockerfile.mxnet.python.cpu.mkl | 28 +++++
.../docker-python/Dockerfile.mxnet.python.gpu.cu80 | 28 +++++
.../Dockerfile.mxnet.python.gpu.cu80.mkl | 28 +++++
.../docker-python/Dockerfile.mxnet.python.gpu.cu90 | 28 +++++
.../Dockerfile.mxnet.python.gpu.cu90.mkl | 28 +++++
.../docker-python/Dockerfile.mxnet.python.gpu.cu92 | 28 +++++
.../Dockerfile.mxnet.python.gpu.cu92.mkl | 28 +++++
docker/docker-python/README.md | 41 ++++++
docker/docker-python/build_python_dockerfile.sh | 137 +++++++++++++++++++++
docker/docker-python/test_mxnet.py | 26 ++++
11 files changed, 428 insertions(+)
diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu b/docker/docker-python/Dockerfile.mxnet.python.cpu
new file mode 100644
index 0000000..eb437f3
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.cpu
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for CPU
+
+FROM ubuntu:16.04
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet
diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl
new file mode 100644
index 0000000..043932f
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet CPU with MKL
+
+FROM ubuntu:16.04
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-mkl
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80
new file mode 100644
index 0000000..8c83ece
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU
+
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu80
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl
new file mode 100644
index 0000000..a057c1d
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU with MKL
+
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu80mkl
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90
new file mode 100644
index 0000000..1e3d986
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU
+
+FROM nvidia/cuda:9.0-cudnn7-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu90
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl
new file mode 100644
index 0000000..d82abd7
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU with MKL
+
+FROM nvidia/cuda:9.0-cudnn7-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu90mkl
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92
new file mode 100644
index 0000000..ba5c54a
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU
+
+FROM nvidia/cuda:9.2-cudnn7-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu92
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl
new file mode 100644
index 0000000..96a9439
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU with MKL
+
+FROM nvidia/cuda:9.2-cudnn7-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu92mkl
diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md
new file mode 100644
index 0000000..f806d5d
--- /dev/null
+++ b/docker/docker-python/README.md
@@ -0,0 +1,41 @@
+# Release Python Docker Images for MXNet
+
+The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release.
+It uses the appropriate pip binaries to build different docker images as -
+* cpu
+* cpu_mkl
+* latest (same as cpu)
+* gpu_cu90
+* gpu_cu90_mkl
+* gpu (same as gpu_cu90)
+* gpu_cu80
+* gpu_cu80_mkl
+* gpu_cu92
+* gpu_cu92_mkl
+
+
+** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required.
+
+Refer: https://pypi.org/project/mxnet/
+
+### Usage
+`./build_python_dockerfile.sh <mxnet_version> <path_to_cloned_mxnet_repo>`
+
+For example:
+`./build_python_dockerfile.sh 1.3.0 ~/build-docker/incubator-mxnet`
+
+** Note: The build script picks up the latest pip binaries. This means it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly.
+
+### Tests run
+* [test_conv.py](https://github.com/apache/incubator-mxnet/blob/master/tests/python/train/test_conv.py)
+* [train_mnist.py](https://github.com/apache/incubator-mxnet/blob/master/example/image-classification/train_mnist.py)
+* [test_mxnet.py](https://github.com/apache/incubator-mxnet/blob/master/docker/docker-python/test_mxnet.py): This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter.
+
+### Dockerhub Credentials
+Dockerhub credentials will be required to push images at the end of this script.
+Credentials can be provided in the following ways:
+* **Interactive Login:** Run the script as is and it will ask you for credentials interactively.
+* **Be Already Logged in:** Login to the mxnet dockerhub account before you run the build script and the script will complete build, test and push.
+* **Set Environment Variables:** Set the following environment variables which the script will pick up to login to dockerhub at runtime -
+ * $MXNET_DOCKERHUB_PASSWORD
+ * $MXNET_DOCKERHUB_USERNAME
diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh
new file mode 100755
index 0000000..24a44c2
--- /dev/null
+++ b/docker/docker-python/build_python_dockerfile.sh
@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# Check Params
+programname=$0
+
+function usage {
+ echo "usage: $programname [version] [path]"
+ echo " [version] Mxnet Version to build"
+ echo " [path] Path to MXNet repository (to run tests)"
+ echo " "
+ exit 1
+}
+
+if [ $# -le 1 ] || [ $# -ge 3 ]
+then
+ usage
+ exit 1
+fi
+
+# Two params provided
+echo "Building Docker Images for Apache MXNet (Incubating) v$1"
+mxnet_version="${1}"
+test_dir="${2}"
+
+docker_build_image(){
+ echo "Building docker image mxnet/python:${1}"
+ docker build -t mxnet/python:${1} -f ${2} .
+}
+
+docker_tag_image(){
+ docker tag mxnet/python:${1} mxnet/python:${2}
+}
+
+docker_test_image_cpu(){
+ echo "Running tests on mxnet/python:${1}"
+ docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}"
+ docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py"
+ docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py"
+}
+
+docker_test_image_gpu(){
+ echo "Running tests on mxnet/python:${1}"
+ nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}"
+ nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py --gpu"
+ nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py --gpus 2"
+}
+
+# if both $MXNET_DOCKERHUB_PASSWORD and $MXNET_DOCKERHUB_USERNAME environment variables are set, docker will automatically login
+# if env variables are not set, login will be interactive.
+docker_account_login(){
+ if [[ -z $MXNET_DOCKERHUB_PASSWORD ]] || [[ -z $MXNET_DOCKERHUB_USERNAME ]]; then
+ docker login
+ else
+ echo $MXNET_DOCKERHUB_PASSWORD | docker login -u $MXNET_DOCKERHUB_USERNAME --password-stdin
+ fi
+}
+
+docker_account_logout(){
+ docker logout
+}
+
+docker_push_image(){
+ docker push mxnet/python:${1}
+}
+
+
+# Build and Test dockerfiles - CPU
+docker_build_image "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu"
+docker_test_image_cpu "${mxnet_version}_cpu"
+
+docker_build_image "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl"
+docker_test_image_cpu "${mxnet_version}_cpu_mkl"
+
+docker_tag_image "${mxnet_version}_cpu" "latest"
+docker_test_image_cpu "latest"
+
+
+#Build and Test dockerfiles - GPU
+docker_build_image "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90"
+docker_test_image_gpu "${mxnet_version}_gpu_cu90"
+
+docker_build_image "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl"
+docker_test_image_gpu "${mxnet_version}_gpu_cu90_mkl"
+
+docker_tag_image "${mxnet_version}_gpu_cu90" "gpu"
+docker_test_image_gpu "gpu"
+
+docker_build_image "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80"
+docker_test_image_gpu "${mxnet_version}_gpu_cu80"
+
+docker_build_image "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl"
+docker_test_image_gpu "${mxnet_version}_gpu_cu80_mkl"
+
+docker_build_image "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92"
+docker_test_image_gpu "${mxnet_version}_gpu_cu92"
+
+docker_build_image "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl"
+docker_test_image_gpu "${mxnet_version}_gpu_cu92_mkl"
+
+
+# Push dockerfiles
+echo "All images were successfully built. Now login to dockerhub and push images"
+docker_account_login
+
+docker_push_image "${mxnet_version}_cpu"
+docker_push_image "${mxnet_version}_cpu_mkl"
+docker_push_image "latest"
+docker_push_image "${mxnet_version}_gpu_cu90"
+docker_push_image "${mxnet_version}_gpu_cu90_mkl"
+docker_push_image "gpu"
+docker_push_image "${mxnet_version}_gpu_cu80"
+docker_push_image "${mxnet_version}_gpu_cu80_mkl"
+docker_push_image "${mxnet_version}_gpu_cu92"
+docker_push_image "${mxnet_version}_gpu_cu92_mkl"
+
+docker_account_logout
+
+echo "Successfully Built, Tested and Pushed all Images to Dockerhub. Link: https://hub.docker.com/r/mxnet/python/tags/"
diff --git a/docker/docker-python/test_mxnet.py b/docker/docker-python/test_mxnet.py
new file mode 100644
index 0000000..6572058
--- /dev/null
+++ b/docker/docker-python/test_mxnet.py
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This checks that the version of mxnet imported matches the parameter passed to the build script.
+import mxnet as mx
+import sys
+
+pip_version = mx.__version__
+expected_version = sys.argv[1]
+
+if pip_version != expected_version:
+ raise ValueError("ERROR: Incorrect pip version. Please check the parameter passed or pip binary used.")