You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by GitBox <gi...@apache.org> on 2018/09/29 04:36:09 UTC
[GitHub] sandeep-krishnamurthy closed pull request #12556: [MXNET-951]
Python dockerfiles built on pip binaries and build/release script
sandeep-krishnamurthy closed pull request #12556: [MXNET-951] Python dockerfiles built on pip binaries and build/release script
URL: https://github.com/apache/incubator-mxnet/pull/12556
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu b/docker/docker-python/Dockerfile.mxnet.python.cpu
new file mode 100644
index 00000000000..eb437f3c233
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.cpu
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for CPU
+
+FROM ubuntu:16.04
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet
diff --git a/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl
new file mode 100644
index 00000000000..043932ff7c8
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.cpu.mkl
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet CPU with MKL
+
+FROM ubuntu:16.04
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-mkl
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80
new file mode 100644
index 00000000000..8c83ece434a
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU
+
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu80
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl
new file mode 100644
index 00000000000..a057c1d20cb
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu80.mkl
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU with MKL
+
+FROM nvidia/cuda:8.0-cudnn5-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu80mkl
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90
new file mode 100644
index 00000000000..1e3d9869ac6
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU
+
+FROM nvidia/cuda:9.0-cudnn7-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu90
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl
new file mode 100644
index 00000000000..d82abd7cf52
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu90.mkl
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU with MKL
+
+FROM nvidia/cuda:9.0-cudnn7-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu90mkl
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92 b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92
new file mode 100644
index 00000000000..ba5c54a2a2a
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU
+
+FROM nvidia/cuda:9.2-cudnn7-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu92
diff --git a/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl
new file mode 100644
index 00000000000..96a943980b5
--- /dev/null
+++ b/docker/docker-python/Dockerfile.mxnet.python.gpu.cu92.mkl
@@ -0,0 +1,28 @@
+# -*- mode: dockerfile -*-
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+#
+# Dockerfile to build MXNet for GPU with MKL
+
+FROM nvidia/cuda:9.2-cudnn7-devel
+
+RUN apt-get update
+RUN apt-get install -y wget python gcc
+RUN wget https://bootstrap.pypa.io/get-pip.py
+RUN python get-pip.py
+
+RUN pip install mxnet-cu92mkl
diff --git a/docker/docker-python/README.md b/docker/docker-python/README.md
new file mode 100644
index 00000000000..f806d5d6f45
--- /dev/null
+++ b/docker/docker-python/README.md
@@ -0,0 +1,41 @@
+# Release Python Docker Images for MXNet
+
+The `docker-python` directory can be used to release mxnet python docker images to dockerhub after any mxnet release.
+It uses the appropriate pip binaries to build different docker images as -
+* cpu
+* cpu_mkl
+* latest (same as cpu)
+* gpu_cu90
+* gpu_cu90_mkl
+* gpu (same as gpu_cu90)
+* gpu_cu80
+* gpu_cu80_mkl
+* gpu_cu92
+* gpu_cu92_mkl
+
+
+** Note: If you want to use a different pip binary (specific mxnet or cuda version, etc), you can edit the last line of the cpu or gpu dockerfile as required.
+
+Refer: https://pypi.org/project/mxnet/
+
+### Usage
+`./build_python_dockerfile.sh <mxnet_version> <path_to_cloned_mxnet_repo>`
+
+For example:
+`./build_python_dockerfile.sh 1.3.0 ~/build-docker/incubator-mxnet`
+
+** Note: The build script picks up the latest pip binaries. This means it uses the latest released mxnet version. The version specified as a parameter to the script is only used to tag the built image correctly.
+
+### Tests run
+* [test_conv.py](https://github.com/apache/incubator-mxnet/blob/master/tests/python/train/test_conv.py)
+* [train_mnist.py](https://github.com/apache/incubator-mxnet/blob/master/example/image-classification/train_mnist.py)
+* [test_mxnet.py](https://github.com/apache/incubator-mxnet/blob/master/docker/docker-python/test_mxnet.py): This script is used to make sure that the docker image builds the expected mxnet version. That is, the version picked by pip is the same as as the version passed as a parameter.
+
+### Dockerhub Credentials
+Dockerhub credentials will be required to push images at the end of this script.
+Credentials can be provided in the following ways:
+* **Interactive Login:** Run the script as is and it will ask you for credentials interactively.
+* **Be Already Logged in:** Login to the mxnet dockerhub account before you run the build script and the script will complete build, test and push.
+* **Set Environment Variables:** Set the following environment variables which the script will pick up to login to dockerhub at runtime -
+ * $MXNET_DOCKERHUB_PASSWORD
+ * $MXNET_DOCKERHUB_USERNAME
diff --git a/docker/docker-python/build_python_dockerfile.sh b/docker/docker-python/build_python_dockerfile.sh
new file mode 100755
index 00000000000..24a44c28970
--- /dev/null
+++ b/docker/docker-python/build_python_dockerfile.sh
@@ -0,0 +1,137 @@
+#!/usr/bin/env bash
+
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -e
+
+# Check Params
+programname=$0
+
+function usage {
+ echo "usage: $programname [version] [path]"
+ echo " [version] Mxnet Version to build"
+ echo " [path] Path to MXNet repository (to run tests)"
+ echo " "
+ exit 1
+}
+
+if [ $# -le 1 ] || [ $# -ge 3 ]
+then
+ usage
+ exit 1
+fi
+
+# Two params provided
+echo "Building Docker Images for Apache MXNet (Incubating) v$1"
+mxnet_version="${1}"
+test_dir="${2}"
+
+docker_build_image(){
+ echo "Building docker image mxnet/python:${1}"
+ docker build -t mxnet/python:${1} -f ${2} .
+}
+
+docker_tag_image(){
+ docker tag mxnet/python:${1} mxnet/python:${2}
+}
+
+docker_test_image_cpu(){
+ echo "Running tests on mxnet/python:${1}"
+ docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}"
+ docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py"
+ docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py"
+}
+
+docker_test_image_gpu(){
+ echo "Running tests on mxnet/python:${1}"
+ nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/docker/docker-python/test_mxnet.py ${mxnet_version}"
+ nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/tests/python/train/test_conv.py --gpu"
+ nvidia-docker run -v ${test_dir}:/mxnet mxnet/python:${1} bash -c "python /mxnet/example/image-classification/train_mnist.py --gpus 2"
+}
+
+# if both $MXNET_DOCKERHUB_PASSWORD and $MXNET_DOCKERHUB_USERNAME environment variables are set, docker will automatically login
+# if env variables are not set, login will be interactive.
+docker_account_login(){
+ if [[ -z $MXNET_DOCKERHUB_PASSWORD ]] || [[ -z $MXNET_DOCKERHUB_USERNAME ]]; then
+ docker login
+ else
+ echo $MXNET_DOCKERHUB_PASSWORD | docker login -u $MXNET_DOCKERHUB_USERNAME --password-stdin
+ fi
+}
+
+docker_account_logout(){
+ docker logout
+}
+
+docker_push_image(){
+ docker push mxnet/python:${1}
+}
+
+
+# Build and Test dockerfiles - CPU
+docker_build_image "${mxnet_version}_cpu" "Dockerfile.mxnet.python.cpu"
+docker_test_image_cpu "${mxnet_version}_cpu"
+
+docker_build_image "${mxnet_version}_cpu_mkl" "Dockerfile.mxnet.python.cpu.mkl"
+docker_test_image_cpu "${mxnet_version}_cpu_mkl"
+
+docker_tag_image "${mxnet_version}_cpu" "latest"
+docker_test_image_cpu "latest"
+
+
+#Build and Test dockerfiles - GPU
+docker_build_image "${mxnet_version}_gpu_cu90" "Dockerfile.mxnet.python.gpu.cu90"
+docker_test_image_gpu "${mxnet_version}_gpu_cu90"
+
+docker_build_image "${mxnet_version}_gpu_cu90_mkl" "Dockerfile.mxnet.python.gpu.cu90.mkl"
+docker_test_image_gpu "${mxnet_version}_gpu_cu90_mkl"
+
+docker_tag_image "${mxnet_version}_gpu_cu90" "gpu"
+docker_test_image_gpu "gpu"
+
+docker_build_image "${mxnet_version}_gpu_cu80" "Dockerfile.mxnet.python.gpu.cu80"
+docker_test_image_gpu "${mxnet_version}_gpu_cu80"
+
+docker_build_image "${mxnet_version}_gpu_cu80_mkl" "Dockerfile.mxnet.python.gpu.cu80.mkl"
+docker_test_image_gpu "${mxnet_version}_gpu_cu80_mkl"
+
+docker_build_image "${mxnet_version}_gpu_cu92" "Dockerfile.mxnet.python.gpu.cu92"
+docker_test_image_gpu "${mxnet_version}_gpu_cu92"
+
+docker_build_image "${mxnet_version}_gpu_cu92_mkl" "Dockerfile.mxnet.python.gpu.cu92.mkl"
+docker_test_image_gpu "${mxnet_version}_gpu_cu92_mkl"
+
+
+# Push dockerfiles
+echo "All images were successfully built. Now login to dockerhub and push images"
+docker_account_login
+
+docker_push_image "${mxnet_version}_cpu"
+docker_push_image "${mxnet_version}_cpu_mkl"
+docker_push_image "latest"
+docker_push_image "${mxnet_version}_gpu_cu90"
+docker_push_image "${mxnet_version}_gpu_cu90_mkl"
+docker_push_image "gpu"
+docker_push_image "${mxnet_version}_gpu_cu80"
+docker_push_image "${mxnet_version}_gpu_cu80_mkl"
+docker_push_image "${mxnet_version}_gpu_cu92"
+docker_push_image "${mxnet_version}_gpu_cu92_mkl"
+
+docker_account_logout
+
+echo "Successfully Built, Tested and Pushed all Images to Dockerhub. Link: https://hub.docker.com/r/mxnet/python/tags/"
diff --git a/docker/docker-python/test_mxnet.py b/docker/docker-python/test_mxnet.py
new file mode 100644
index 00000000000..65720583272
--- /dev/null
+++ b/docker/docker-python/test_mxnet.py
@@ -0,0 +1,26 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This checks that the version of mxnet imported matches the parameter passed to the build script.
+import mxnet as mx
+import sys
+
+pip_version = mx.__version__
+expected_version = sys.argv[1]
+
+if pip_version != expected_version:
+ raise ValueError("ERROR: Incorrect pip version. Please check the parameter passed or pip binary used.")
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services