You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by ns...@apache.org on 2018/08/16 02:44:35 UTC
[incubator-mxnet] branch master updated: [MXNet-744] Fix website
build pipeline Python 3 issues (#12195)
This is an automated email from the ASF dual-hosted git repository.
nswamy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new 7cb28f2 [MXNet-744] Fix website build pipeline Python 3 issues (#12195)
7cb28f2 is described below
commit 7cb28f21e2720afd8dec9f54dd6a61d023a018f2
Author: Aaron Markham <ma...@amazon.com>
AuthorDate: Wed Aug 15 19:44:24 2018 -0700
[MXNet-744] Fix website build pipeline Python 3 issues (#12195)
* Fix website build pipeline Python 3 issues (#12195)
---
ci/docker/Dockerfile.build.ubuntu_cpu | 1 +
ci/docker/Dockerfile.build.ubuntu_gpu | 1 +
ci/docker/Dockerfile.build.ubuntu_nightly_cpu | 1 +
ci/docker/Dockerfile.build.ubuntu_nightly_gpu | 1 +
.../docker/install/docs_requirements | 15 +++---
ci/docker/install/ubuntu_docs.sh | 12 +----
docs/Jenkinsfile-dev | 54 ++++++++++++++++++++++
docs/build_version_doc/setup_docs_ubuntu.sh | 4 +-
docs/mxdoc.py | 11 ++---
9 files changed, 75 insertions(+), 25 deletions(-)
diff --git a/ci/docker/Dockerfile.build.ubuntu_cpu b/ci/docker/Dockerfile.build.ubuntu_cpu
index 76df3df..08fb04d 100644
--- a/ci/docker/Dockerfile.build.ubuntu_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_cpu
@@ -61,6 +61,7 @@ COPY install/ubuntu_onnx.sh /work/
RUN /work/ubuntu_onnx.sh
COPY install/ubuntu_docs.sh /work/
+COPY install/docs_requirements /work/
RUN /work/ubuntu_docs.sh
ARG USER_ID=0
diff --git a/ci/docker/Dockerfile.build.ubuntu_gpu b/ci/docker/Dockerfile.build.ubuntu_gpu
index 8b4cd1e..d99dafb 100644
--- a/ci/docker/Dockerfile.build.ubuntu_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_gpu
@@ -64,6 +64,7 @@ COPY install/ubuntu_onnx.sh /work/
RUN /work/ubuntu_onnx.sh
COPY install/ubuntu_docs.sh /work/
+COPY install/docs_requirements /work/
RUN /work/ubuntu_docs.sh
COPY install/ubuntu_tutorials.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
index c803188..834710c 100644
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_cpu
@@ -58,6 +58,7 @@ COPY install/ubuntu_onnx.sh /work/
RUN /work/ubuntu_onnx.sh
COPY install/ubuntu_docs.sh /work/
+COPY install/docs_requirements /work/
RUN /work/ubuntu_docs.sh
COPY install/ubuntu_nightly_tests.sh /work/
diff --git a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
index c0e31e2..fb34307 100644
--- a/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
+++ b/ci/docker/Dockerfile.build.ubuntu_nightly_gpu
@@ -64,6 +64,7 @@ COPY install/ubuntu_onnx.sh /work/
RUN /work/ubuntu_onnx.sh
COPY install/ubuntu_docs.sh /work/
+COPY install/docs_requirements /work/
RUN /work/ubuntu_docs.sh
COPY install/ubuntu_tutorials.sh /work/
diff --git a/docs/build_version_doc/requirements.txt b/ci/docker/install/docs_requirements
similarity index 51%
rename from docs/build_version_doc/requirements.txt
rename to ci/docker/install/docs_requirements
index 4f3f4d0..7407223 100644
--- a/docs/build_version_doc/requirements.txt
+++ b/ci/docker/install/docs_requirements
@@ -1,15 +1,16 @@
-beautifulsoup4
-breathe
+beautifulsoup4==4.6.3
+breathe==4.10.0
cpplint==1.3.0
CommonMark==0.5.4
h5py==2.8.0rc1
-mock==1.0.1
-nose
-nose-timer
+mock==2.0.0
+nose==1.3.7
+nose-timer==0.7.3
numpy<1.15.0,>=1.8.2
pylint==1.8.3
-pypandoc
-recommonmark==0.4.0
+pypandoc==1.4
+recommonmark==0.4.0
requests<2.19.0,>=2.18.4
scipy==1.0.1
+six==1.11.0
sphinx==1.5.6
diff --git a/ci/docker/install/ubuntu_docs.sh b/ci/docker/install/ubuntu_docs.sh
index ee12196..a709b3d 100755
--- a/ci/docker/install/ubuntu_docs.sh
+++ b/ci/docker/install/ubuntu_docs.sh
@@ -27,15 +27,7 @@ apt-get install -y \
doxygen \
pandoc
-echo 'Installing python packages...'
-pip install --upgrade pip && pip install \
- beautifulsoup4 \
- breathe \
- CommonMark==0.5.4 \
- h5py \
- mock==1.0.1 \
- pypandoc \
- recommonmark==0.4.0 \
- sphinx==1.5.6
+pip3 install -r /work/docs_requirements
+pip2 install -r /work/docs_requirements
echo 'Dependency installation complete.'
diff --git a/docs/Jenkinsfile-dev b/docs/Jenkinsfile-dev
new file mode 100644
index 0000000..b93eba6
--- /dev/null
+++ b/docs/Jenkinsfile-dev
@@ -0,0 +1,54 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 60
+
+node('mxnetlinux-cpu') {
+ // Loading the utilities requires a node context unfortunately
+ checkout scm
+ utils = load('ci/Jenkinsfile_utils.groovy')
+}
+utils.assign_node_labels(linux_cpu: 'mxnetlinux-cpu', linux_gpu: 'mxnetlinux-gpu', linux_gpu_p3: 'mxnetlinux-gpu-p3', windows_cpu: 'mxnetwindows-cpu', windows_gpu: 'mxnetwindows-gpu')
+
+utils.main_wrapper(
+core_logic: {
+ stage('Build Docs') {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/docs') {
+ utils.init_git()
+ timeout(time: max_time, unit: 'MINUTES') {
+ sh "ci/build.py -p ubuntu_cpu --docker-registry ${env.DOCKER_CACHE_REGISTRY} --docker-build-retries 3 /work/runtime_functions.sh build_docs ${params.tags_to_build} ${params.tag_list} ${params.fork} ${params.tag_default} ${params.domain}"
+ archiveArtifacts 'docs/build_version_doc/artifacts.tgz'
+ build 'test-website-publish'
+ }
+ }
+ }
+ }
+}
+,
+failure_handler: {
+ if (currentBuild.result == "FAILURE") {
+ // Do nothing.
+ }
+}
+)
diff --git a/docs/build_version_doc/setup_docs_ubuntu.sh b/docs/build_version_doc/setup_docs_ubuntu.sh
index d00f00c..22b2fe2 100755
--- a/docs/build_version_doc/setup_docs_ubuntu.sh
+++ b/docs/build_version_doc/setup_docs_ubuntu.sh
@@ -110,10 +110,10 @@ sudo apt-get update && sudo apt-get install -y \
wget -nv https://bootstrap.pypa.io/get-pip.py
echo "Installing for Python 3..."
sudo python3 get-pip.py
-pip3 install --user -r requirements.txt
+pip3 install --user -r ../../ci/docker/install/docs_requirements
echo "Installing for Python 2..."
sudo python2 get-pip.py
-pip2 install --user -r requirements.txt
+pip2 install --user -r ../../ci/docker/install/docs_requirements
cd ../../
diff --git a/docs/mxdoc.py b/docs/mxdoc.py
index dabfd3b..33f6475 100644
--- a/docs/mxdoc.py
+++ b/docs/mxdoc.py
@@ -23,16 +23,16 @@ import json
import sys
from recommonmark import transform
import pypandoc
-# import StringIO from io for python3 compatibility
-from io import StringIO
import contextlib
-from ConfigParser import SafeConfigParser
+# Use six for Python 2 / 3 compatibility
+from six import StringIO
+from six.moves import configparser
_BUILD_VER = os.getenv('BUILD_VER', 'default')
print("Building version {}".format(_BUILD_VER))
_DOC_SET = 'document_sets_' + _BUILD_VER
-parser = SafeConfigParser()
+parser = configparser.SafeConfigParser()
parser.read('settings.ini')
if _DOC_SET not in parser.sections():
@@ -41,8 +41,7 @@ if _DOC_SET not in parser.sections():
for section in [ _DOC_SET ]:
print("Document sets to generate:")
for candidate in [ 'scala_docs', 'clojure_docs', 'doxygen_docs', 'r_docs' ]:
- print '%-12s : %s' % (candidate, parser.get(section, candidate))
- print
+ print('%-12s : %s' % (candidate, parser.get(section, candidate)))
_MXNET_DOCS_BUILD_MXNET = parser.getboolean('mxnet', 'build_mxnet')
_SCALA_DOCS = parser.getboolean(_DOC_SET, 'scala_docs')