You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ma...@apache.org on 2019/03/12 13:46:40 UTC
[incubator-mxnet] branch master updated: [MXNET-1093] Add python3
Docker images for each MXNet release (#12791)
This is an automated email from the ASF dual-hosted git repository.
marcoabreu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new ab0ca86 [MXNET-1093] Add python3 Docker images for each MXNet release (#12791)
ab0ca86 is described below
commit ab0ca86d144cff530c416c40044ed2caade623b2
Author: Meghna Baijal <30...@users.noreply.github.com>
AuthorDate: Tue Mar 12 06:46:02 2019 -0700
[MXNET-1093] Add python3 Docker images for each MXNet release (#12791)
* Add dockerfiles for python3
* Fix
* Fix the dockerfile names
* Update the README with python3 images
* Parameterize the mxnet version
* Fix typo
* Reduce the number of docker layers
* Combine build and Test, run builds in parallel. Use variable names instead of numbers
* minor fix
* Update README for build command
* Bug Fix: Script should fail if any background process returns non-zero code
* Changes needed for Benchmarking, 4 new tags, change python to python-dev
* Minor typos
* Minor Rearrangement - sometimes tagging fails
* Move error checking to a method
* Push BAI images to dockerhub
* Create ~/temp dir
* Bug Fix
* Fix merge issues
I don't know if this is tha main merge issue the PR is complaining about on mxnet, but maybe it will help
---
docker/docker-python/Dockerfile.mxnet.python.cpu | 11 +-
.../docker-python/Dockerfile.mxnet.python.cpu.mkl | 11 +-
.../docker-python/Dockerfile.mxnet.python.gpu.cu80 | 11 +-
.../Dockerfile.mxnet.python.gpu.cu80.mkl | 11 +-
.../docker-python/Dockerfile.mxnet.python.gpu.cu90 | 11 +-
.../Dockerfile.mxnet.python.gpu.cu90.mkl | 11 +-
.../docker-python/Dockerfile.mxnet.python.gpu.cu92 | 11 +-
.../Dockerfile.mxnet.python.gpu.cu92.mkl | 11 +-
...net.python.cpu => Dockerfile.mxnet.python3.cpu} | 11 +-
...on.cpu.mkl => Dockerfile.mxnet.python3.cpu.mkl} | 11 +-
....gpu.cu80 => Dockerfile.mxnet.python3.gpu.cu80} | 11 +-
...0.mkl => Dockerfile.mxnet.python3.gpu.cu80.mkl} | 11 +-
....gpu.cu90 => Dockerfile.mxnet.python3.gpu.cu90} | 11 +-
...0.mkl => Dockerfile.mxnet.python3.gpu.cu90.mkl} | 11 +-
....gpu.cu92 => Dockerfile.mxnet.python3.gpu.cu92} | 11 +-
...2.mkl => Dockerfile.mxnet.python3.gpu.cu92.mkl} | 11 +-
docker/docker-python/README.md | 58 ++++---
docker/docker-python/build_python_dockerfile.sh | 178 +++++++++++++++------
18 files changed, 262 insertions(+), 150 deletions(-)
diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu b/docker/docker-python/Dockerfile.mxnet.python.cpu
index eb437f3..0858e99 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.cpu
+++ b/docker/docker-python/Dockerfile.mxnet.python.cpu
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for CPU
FROM ubuntu:16.04
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python get-pip.py
-RUN pip install mxnet
+RUN pip install mxnet==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl
index 043932f..dbb7d29 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl
+++ b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet CPU with MKL
FROM ubuntu:16.04
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python get-pip.py
-RUN pip install mxnet-mkl
+RUN pip install mxnet-mkl==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80
index 8c83ece..cb22721 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for GPU
FROM nvidia/cuda:8.0-cudnn5-devel
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python get-pip.py
-RUN pip install mxnet-cu80
+RUN pip install mxnet-cu80==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl
index a057c1d..eda96c9 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for GPU with MKL
FROM nvidia/cuda:8.0-cudnn5-devel
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python get-pip.py
-RUN pip install mxnet-cu80mkl
+RUN pip install mxnet-cu80mkl==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90
index 1e3d986..cd36b8a 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for GPU
FROM nvidia/cuda:9.0-cudnn7-devel
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python get-pip.py
-RUN pip install mxnet-cu90
+RUN pip install mxnet-cu90==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl
index d82abd7..0b274c4 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for GPU with MKL
FROM nvidia/cuda:9.0-cudnn7-devel
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python get-pip.py
-RUN pip install mxnet-cu90mkl
+RUN pip install mxnet-cu90mkl==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92
index ba5c54a..2c43187 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for GPU
FROM nvidia/cuda:9.2-cudnn7-devel
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python get-pip.py
-RUN pip install mxnet-cu92
+RUN pip install mxnet-cu92==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl
index 96a9439..db20489 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for GPU with MKL
FROM nvidia/cuda:9.2-cudnn7-devel
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python get-pip.py
-RUN pip install mxnet-cu92mkl
+RUN pip install mxnet-cu92mkl==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu b/docker/docker-python/Dockerfile.mxnet.python3.cpu
similarity index 81%
copy from docker/docker-python/Dockerfile.mxnet.python.cpu
copy to docker/docker-python/Dockerfile.mxnet.python3.cpu
index eb437f3..8ad9950 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.cpu
+++ b/docker/docker-python/Dockerfile.mxnet.python3.cpu
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for CPU
FROM ubuntu:16.04
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python3-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python3 get-pip.py
-RUN pip install mxnet
+RUN pip3 install mxnet==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl
similarity index 81%
copy from docker/docker-python/Dockerfile.mxnet.python.cpu.mkl
copy to docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl
index 043932f..c631289 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl
+++ b/docker/docker-python/Dockerfile.mxnet.python3.cpu.mkl
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet CPU with MKL
FROM ubuntu:16.04
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python3-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python3 get-pip.py
-RUN pip install mxnet-mkl
+RUN pip3 install mxnet-mkl==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80
similarity index 81%
copy from docker/docker-python/Dockerfile.mxnet.python.gpu.cu80
copy to docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80
index 8c83ece..58af6bf 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80
+++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for GPU
FROM nvidia/cuda:8.0-cudnn5-devel
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python3-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python3 get-pip.py
-RUN pip install mxnet-cu80
+RUN pip3 install mxnet-cu80==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl
similarity index 81%
copy from docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl
copy to docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl
index a057c1d..059f002 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl
+++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu80.mkl
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for GPU with MKL
FROM nvidia/cuda:8.0-cudnn5-devel
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python3-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python3 get-pip.py
-RUN pip install mxnet-cu80mkl
+RUN pip3 install mxnet-cu80mkl==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90
similarity index 81%
copy from docker/docker-python/Dockerfile.mxnet.python.gpu.cu90
copy to docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90
index 1e3d986..a860de9 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90
+++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for GPU
FROM nvidia/cuda:9.0-cudnn7-devel
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python3-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python3 get-pip.py
-RUN pip install mxnet-cu90
+RUN pip3 install mxnet-cu90==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl
similarity index 81%
copy from docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl
copy to docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl
index d82abd7..c0b6145 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl
+++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu90.mkl
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for GPU with MKL
FROM nvidia/cuda:9.0-cudnn7-devel
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python3-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python3 get-pip.py
-RUN pip install mxnet-cu90mkl
+RUN pip3 install mxnet-cu90mkl==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92
similarity index 81%
copy from docker/docker-python/Dockerfile.mxnet.python.gpu.cu92
copy to docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92
index ba5c54a..6d87796 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92
+++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for GPU
FROM nvidia/cuda:9.2-cudnn7-devel
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python3-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python3 get-pip.py
-RUN pip install mxnet-cu92
+RUN pip3 install mxnet-cu92==$version
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl
similarity index 81%
copy from docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl
copy to docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl
index 96a9439..b73df97 100644
--- a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl
+++ b/docker/docker-python/Dockerfile.mxnet.python3.gpu.cu92.mkl
@@ -19,10 +19,11 @@
# Dockerfile to build MXNet for GPU with MKL
FROM nvidia/cuda:9.2-cudnn7-devel
+ARG version
-RUN apt-get update
-RUN apt-get install -y wget python gcc
-RUN wget https://bootstrap.pypa.io/get-pip.py
-RUN python get-pip.py
+RUN apt-get update && \
+ apt-get install -y wget python3-dev gcc && \
+ wget https://bootstrap.pypa.io/get-pip.py && \
+ python3 get-pip.py
-RUN pip install mxnet-cu92mkl
+RUN pip3 install mxnet-cu92mkl==$version
diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md
index 6d17c50..767be6d 100644
--- a/docker/docker-python/README.md
+++ b/docker/docker-python/README.md
@@ -19,31 +19,40 @@
# Release Python Docker Images for MXNet
-The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release.
-It uses the appropriate pip binaries to build different docker images as -
-* cpu
-* cpu_mkl
-* latest (same as cpu)
-* gpu_cu90
-* gpu_cu90_mkl
-* gpu (same as gpu_cu90)
-* gpu_cu80
-* gpu_cu80_mkl
-* gpu_cu92
-* gpu_cu92_mkl
-
-
-** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required.
+The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release.
+It uses the appropriate pip binaries to build different docker images. Both python2 (default) and python3 images are available as -
+* {version}_cpu
+* {version}_cpu_mkl
+* {version}_gpu_cu90
+* {version}_gpu_cu90_mkl
+* {version}_gpu_cu80
+* {version}_gpu_cu80_mkl
+* {version}_gpu_cu92
+* {version}_gpu_cu92_mkl
+* {version}_cpu_py3
+* {version}_cpu_mkl_py3
+* {version}_gpu_cu90_py3
+* {version}_gpu_cu90_mkl_py3
+* {version}_gpu_cu80_py3
+* {version}_gpu_cu80_mkl_py3
+* {version}_gpu_cu92_py3
+* {version}_gpu_cu92_mkl_py3
-Refer: https://pypi.org/project/mxnet/
+And the following tags will be available without the version string in the image name (for Benchmarking and other use cases):
+* latest (same as {version}_cpu)
+* gpu (same as {version}_gpu_cu90)
+* latest_cpu_mkl_py2 (same as {version}_cpu_mkl)
+* latest_cpu_mkl_py3 (same as {version}_cpu_mkl_py3)
+* latest_gpu_mkl_py2 (same as {version}_gpu_cu90_mkl)
+* latest_gpu_mkl_py3 (same as {version}_gpu_cu90_mkl_py3)
-### Usage
-`./build_python_dockerfile.sh <mxnet_version> <path_to_cloned_mxnet_repo>`
+Refer: https://pypi.org/project/mxnet/
-For example:
-`./build_python_dockerfile.sh 1.3.0 ~/build-docker/incubator-mxnet`
+### Using the Build Script
+`./build_python_dockerfile.sh <mxnet_version> <pip_tag> <path_to_cloned_mxnet_repo>`
-** Note: The build script picks up the latest pip binaries. This means it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly.
+For example:
+`./build_python_dockerfile.sh 1.3.0 1.3.0.post0 ~/build-docker/incubator-mxnet`
### Tests run
* [test_conv.py](https://github.com/apache/incubator-mxnet/blob/master/tests/python/train/test_conv.py)
@@ -58,3 +67,10 @@ Credentials can be provided in the following ways:
* **Set Environment Variables:** Set the following environment variables which the script will pick up to login to dockerhub at runtime -
* $MXNET_DOCKERHUB_PASSWORD
* $MXNET_DOCKERHUB_USERNAME
+
+
+### Using the Docker Images
+* The MXNet Python Docker images can be found here: https://hub.docker.com/r/mxnet/python/
+
+* Docker Pull Command: `docker pull mxnet/python:<image_tag>`
+* Get started: `docker run -it mxnet/python:<image_tag> bash`
diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh
index 24a44c2..d0f24c8 100755
--- a/docker/docker-python/build_python_dockerfile.sh
+++ b/docker/docker-python/build_python_dockerfile.sh
@@ -17,20 +17,19 @@
# specific language governing permissions and limitations
# under the License.
-set -e
-
# Check Params
programname=$0
function usage {
- echo "usage: $programname [version] [path]"
+ echo "usage: $programname [version] [pip_tag] [path]"
echo " [version] Mxnet Version to build"
+ echo " [pip_tag] Pip Tag to use"
echo " [path] Path to MXNet repository (to run tests)"
echo " "
exit 1
}
-if [ $# -le 1 ] || [ $# -ge 3 ]
+if [ $# -le 2 ] || [ $# -ge 4 ]
then
usage
exit 1
@@ -39,29 +38,37 @@ fi
# Two params provided
echo "Building Docker Images for Apache MXNet (Incubating) v$1"
mxnet_version="${1}"
-test_dir="${2}"
+pip_tag="${2}"
+test_dir="${3}"
-docker_build_image(){
- echo "Building docker image mxnet/python:${1}"
- docker build -t mxnet/python:${1} -f ${2} .
-}
+# Remove the logs directory if it already exists else it may fail due to old logs.
+LOGDIR=~/temp/docker_logs
+if [ -d "${LOGDIR}" ]; then
+ rm -rf ${LOGDIR}
+fi
+
+# Create ~/temp if it does not exist
+mkdir -p ~/temp
+mkdir ${LOGDIR}
-docker_tag_image(){
- docker tag mxnet/python:${1} mxnet/python:${2}
-}
+# Functions
docker_test_image_cpu(){
- echo "Running tests on mxnet/python:${1}"
- docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}"
- docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py"
- docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py"
+ image_tag="${1}"
+ python_version="${2}"
+ echo "Running tests on mxnet/python:${image_tag}"
+ docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}"
+ docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/tests/python/train/test_conv.py"
+ docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/example/image-classification/train_mnist.py"
}
docker_test_image_gpu(){
+ image_tag="${1}"
+ python_version="${2}"
echo "Running tests on mxnet/python:${1}"
- nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}"
- nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py --gpu"
- nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py --gpus 2"
+ nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}"
+ nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/tests/python/train/test_conv.py --gpu"
+ nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${image_tag} bash -c "${python_version} /mxnet/example/image-classification/train_mnist.py --gpus 0,1,2,3"
}
# if both $MXNET_DOCKERHUB_PASSWORD and $MXNET_DOCKERHUB_USERNAME environment variables are set, docker will automatically login
@@ -79,59 +86,132 @@ docker_account_logout(){
}
docker_push_image(){
- docker push mxnet/python:${1}
+ image_tag="${1}"
+ docker push mxnet/python:${image_tag}
}
+docker_generate_image_cpu(){
+ image_tag="${1}"
+ dockerfile="${2}"
+ python_version="${3}"
+ echo "Building docker image mxnet/python:${image_tag}"
+ docker build --build-arg version=${pip_tag} -t mxnet/python:${image_tag} -f ${dockerfile} .
+ docker_test_image_cpu ${image_tag} ${python_version}
+}
-# Build and Test dockerfiles - CPU
-docker_build_image "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu"
-docker_test_image_cpu "${mxnet_version}_cpu"
-
-docker_build_image "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl"
-docker_test_image_cpu "${mxnet_version}_cpu_mkl"
-
-docker_tag_image "${mxnet_version}_cpu" "latest"
-docker_test_image_cpu "latest"
-
-
-#Build and Test dockerfiles - GPU
-docker_build_image "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90"
-docker_test_image_gpu "${mxnet_version}_gpu_cu90"
-
-docker_build_image "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl"
-docker_test_image_gpu "${mxnet_version}_gpu_cu90_mkl"
+docker_tag_image_cpu(){
+ original_tag="${1}"
+ image_tag="${2}"
+ python_version="${3}"
+ docker tag mxnet/python:${original_tag} mxnet/python:${image_tag}
+ docker_test_image_cpu ${image_tag} ${python_version}
+}
-docker_tag_image "${mxnet_version}_gpu_cu90" "gpu"
-docker_test_image_gpu "gpu"
+docker_generate_image_gpu(){
+ image_tag="${1}"
+ dockerfile="${2}"
+ python_version="${3}"
+ echo "Building docker image mxnet/python:${1}"
+ docker build --build-arg version=${pip_tag} -t mxnet/python:${image_tag} -f ${dockerfile} .
+ docker_test_image_gpu ${image_tag} ${python_version}
+}
-docker_build_image "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80"
-docker_test_image_gpu "${mxnet_version}_gpu_cu80"
+docker_tag_image_gpu(){
+ original_tag="${1}"
+ image_tag="${2}"
+ python_version="${3}"
+ docker tag mxnet/python:${original_tag} mxnet/python:${image_tag}
+ docker_test_image_gpu ${image_tag} ${python_version}
+}
-docker_build_image "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl"
-docker_test_image_gpu "${mxnet_version}_gpu_cu80_mkl"
+check_errors(){
+ egrep -i "not found|error|returned a non-zero code|fail" ${LOGDIR}/docker*
+ if [ $? -eq 0 ]; then
+ echo "ERROR: One of the build/test commands failed. Refer to the filename above to see which image tag caused it."
+ exit 1
+ else
+ echo "Success: No errors found"
+ fi
+}
-docker_build_image "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92"
-docker_test_image_gpu "${mxnet_version}_gpu_cu92"
+# Build and Test dockerfiles - CPU
+docker_generate_image_cpu "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu" "python" > ${LOGDIR}/docker_cpu.out 2>&1 &
+docker_generate_image_cpu "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl" "python" > ${LOGDIR}/docker_cpu_mkl.out 2>&1 &
-docker_build_image "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl"
-docker_test_image_gpu "${mxnet_version}_gpu_cu92_mkl"
+#Build and Test dockerfiles - GPU
+docker_generate_image_gpu "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90" "python" > ${LOGDIR}/docker_gpu_cu90.out 2>&1 &
+docker_generate_image_gpu "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl" "python" > ${LOGDIR}/docker_gpu_cu90_mkl.out 2>&1 &
+docker_generate_image_gpu "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80" "python" > ${LOGDIR}/docker_gpu_cu80.out 2>&1 &
+docker_generate_image_gpu "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl" "python" > ${LOGDIR}/docker_gpu_cu80_mkl.out 2>&1 &
+docker_generate_image_gpu "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92" "python" > ${LOGDIR}/docker_gpu_cu92.out 2>&1 &
+docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl" "python" > ${LOGDIR}/docker_gpu_cu92_mkl.out 2>&1
+
+echo "Waiting for MXNet Python2 Docker Images to Build"
+wait
+
+# Build and Test Python3 dockerfiles - CPU
+docker_generate_image_cpu "${mxnet_version}_cpu_py3" "Dockerfile.mxnet.python3.cpu" "python3" > ${LOGDIR}/docker_cpu_py3.out 2>&1 &
+docker_generate_image_cpu "${mxnet_version}_cpu_mkl_py3" "Dockerfile.mxnet.python3.cpu.mkl" "python3" > ${LOGDIR}/docker_cpu_mkl_py3.out 2>&1 &
+
+#Build and Test Python3 dockerfiles - GPU
+docker_generate_image_gpu "${mxnet_version}_gpu_cu90_py3" "Dockerfile.mxnet.python3.gpu.cu90" "python3" > ${LOGDIR}/docker_gpu_cu90_py3.out 2>&1 &
+docker_generate_image_gpu "${mxnet_version}_gpu_cu90_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu90.mkl" "python3" > ${LOGDIR}/docker_gpu_cu90_mkl_py3.out 2>&1 &
+docker_generate_image_gpu "${mxnet_version}_gpu_cu80_py3" "Dockerfile.mxnet.python3.gpu.cu80" "python3" > ${LOGDIR}/docker_gpu_cu80_py3.out 2>&1 &
+docker_generate_image_gpu "${mxnet_version}_gpu_cu80_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu80.mkl" "python3" > ${LOGDIR}/docker_gpu_cu80_mkl_py3.out 2>&1 &
+docker_generate_image_gpu "${mxnet_version}_gpu_cu92_py3" "Dockerfile.mxnet.python3.gpu.cu92" "python3" > ${LOGDIR}/docker_gpu_cu92_py3.out 2>&1 &
+docker_generate_image_gpu "${mxnet_version}_gpu_cu92_mkl_py3" "Dockerfile.mxnet.python3.gpu.cu92.mkl" "python3" > ${LOGDIR}/docker_gpu_cu92_mkl_py3.out 2>&1
+
+echo "Waiting for MXNet Python3 Docker Images to Build"
+wait
+
+echo "Re-Tag 6 images with version-free names (for Benchmarking) - only after previous builds complete. "
+docker_tag_image_cpu "${mxnet_version}_cpu" "latest" "python" > ${LOGDIR}/docker_latest.out 2>&1 &
+docker_tag_image_gpu "${mxnet_version}_gpu_cu90" "gpu" "python" > ${LOGDIR}/docker_gpu.out 2>&1 &
+docker_tag_image_cpu "${mxnet_version}_cpu_mkl" "latest_cpu_mkl_py2" "python" > ${LOGDIR}/docker_latest_cpu_mkl_py2.out 2>&1 &
+docker_tag_image_cpu "${mxnet_version}_cpu_mkl_py3" "latest_cpu_mkl_py3" "python3" > ${LOGDIR}/docker_latest_cpu_mkl_py3.out 2>&1 &
+docker_tag_image_gpu "${mxnet_version}_gpu_cu90_mkl" "latest_gpu_mkl_py2" "python" > ${LOGDIR}/docker_latest_gpu_mkl_py2.out 2>&1 &
+docker_tag_image_gpu "${mxnet_version}_gpu_cu90_mkl_py3" "latest_gpu_mkl_py3" "python3" > ${LOGDIR}/docker_latest_gpu_mkl_py3.out 2>&1
+wait
+
+# Parse all the docker logfiles to make sure there is no error. Fail script if error is found.
+check_errors
# Push dockerfiles
echo "All images were successfully built. Now login to dockerhub and push images"
docker_account_login
+# Python2
docker_push_image "${mxnet_version}_cpu"
docker_push_image "${mxnet_version}_cpu_mkl"
-docker_push_image "latest"
docker_push_image "${mxnet_version}_gpu_cu90"
docker_push_image "${mxnet_version}_gpu_cu90_mkl"
-docker_push_image "gpu"
docker_push_image "${mxnet_version}_gpu_cu80"
docker_push_image "${mxnet_version}_gpu_cu80_mkl"
docker_push_image "${mxnet_version}_gpu_cu92"
docker_push_image "${mxnet_version}_gpu_cu92_mkl"
+# Python3
+docker_push_image "${mxnet_version}_cpu_py3"
+docker_push_image "${mxnet_version}_cpu_mkl_py3"
+docker_push_image "${mxnet_version}_gpu_cu90_py3"
+docker_push_image "${mxnet_version}_gpu_cu90_mkl_py3"
+docker_push_image "${mxnet_version}_gpu_cu80_py3"
+docker_push_image "${mxnet_version}_gpu_cu80_mkl_py3"
+docker_push_image "${mxnet_version}_gpu_cu92_py3"
+docker_push_image "${mxnet_version}_gpu_cu92_mkl_py3"
+
+docker_push_image "latest"
+docker_push_image "gpu"
+docker_push_image "latest_cpu_mkl_py2"
+docker_push_image "latest_cpu_mkl_py3"
+docker_push_image "latest_gpu_mkl_py2"
+docker_push_image "latest_gpu_mkl_py3"
+
+
docker_account_logout
echo "Successfully Built, Tested and Pushed all Images to Dockerhub. Link: https://hub.docker.com/r/mxnet/python/tags/"
+
+#Delete the log directory since everything succeeded:
+rm -rf ${LOGDIR}
\ No newline at end of file