You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by zh...@apache.org on 2021/03/11 23:38:07 UTC
[incubator-mxnet] branch master updated: [website] Automate website
artifacts uploading (#19955)
This is an automated email from the ASF dual-hosted git repository.
zha0q1 pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new bfa4fdd [website] Automate website artifacts uploading (#19955)
bfa4fdd is described below
commit bfa4fdd664614d677939f6fa9b3b2220ae2908c3
Author: waytrue17 <52...@users.noreply.github.com>
AuthorDate: Thu Mar 11 15:36:05 2021 -0800
[website] Automate website artifacts uploading (#19955)
* website version update
* back up versions
* rename file
* remove test
* add backup version
* remove space
* remove test
* cp between s3 buckets
Co-authored-by: Wei Chu <we...@amazon.com>
---
ci/docker/Dockerfile.build.ubuntu | 1 +
ci/docker/runtime_functions.sh | 24 ++++++++++
ci/jenkins/Jenkins_steps.groovy | 46 ++++++++++++++++++
ci/jenkins/Jenkinsfile_website_version_artifacts | 61 ++++++++++++++++++++++++
4 files changed, 132 insertions(+)
diff --git a/ci/docker/Dockerfile.build.ubuntu b/ci/docker/Dockerfile.build.ubuntu
index 34feb61..7077cac 100644
--- a/ci/docker/Dockerfile.build.ubuntu
+++ b/ci/docker/Dockerfile.build.ubuntu
@@ -47,6 +47,7 @@ RUN export DEBIAN_FRONTEND=noninteractive && \
apt-get install -y \
## Utilities
curl \
+ zip \
unzip \
pandoc \
## Development tools
diff --git a/ci/docker/runtime_functions.sh b/ci/docker/runtime_functions.sh
index 1dc82a2..21e23b6 100755
--- a/ci/docker/runtime_functions.sh
+++ b/ci/docker/runtime_functions.sh
@@ -1214,6 +1214,30 @@ build_docs_beta() {
popd
}
+push_docs() {
+ folder_name=$1
+ set -ex
+ pip3 install --user awscli
+ export PATH=~/.local/bin:$PATH
+ pushd docs/_build
+ tar -xzf full_website.tgz --strip-components 1
+ # check if folder_name already exists in versions
+ pushd versions
+ if [ -d "$folder_name" ]; then
+ echo "Folder $folder_name already exists in versions. Please double check the FOLDER_NAME variable in Jenkens pipeline"
+ exit 1
+ fi
+ mv master $folder_name
+ popd
+ zip -r9 versions.zip versions/.
+ # Upload versions folder
+ aws s3 cp versions.zip s3://mxnet-website-static-artifacts --acl public-read
+ # Backup versions folder with the latest version name
+ backup_file="versions_backup_upto_$folder_name.zip"
+ aws s3 cp s3://mxnet-website-static-artifacts/versions.zip s3://mxnet-website-static-artifacts/$backup_file --acl public-read
+ popd
+}
+
create_repo() {
repo_folder=$1
mxnet_url=$2
diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index b0244c1..03b325a 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -1093,6 +1093,30 @@ def docs_prepare() {
}]
}
+// This is for updateing the new version of website artifact
+// Assumes you have run all of the docs generation functions
+// Called from Jenkins_website_version_artifacts
+def docs_full_website() {
+ return ['Build artifacts full_website.tgz': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/docs') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ utils.init_git()
+
+ unstash 'jekyll-artifacts'
+ unstash 'python-artifacts'
+
+ utils.docker_run('ubuntu_cpu_jekyll', 'build_docs', false)
+
+ utils.pack_lib('full_website', 'docs/_build/full_website.tgz', false)
+
+ // archive so the publish pipeline can access the artifact
+ archiveArtifacts 'docs/_build/full_website.tgz'
+ }
+ }
+ }
+ }]
+}
def docs_prepare_beta() {
return ['Prepare for publication to the staging website': {
@@ -1168,6 +1192,28 @@ def docs_publish_beta() {
}]
}
+// This is for uploading website artifacts to S3 bucket
+// Assumes you have run docs_full_website function
+def docs_upload_s3() {
+ return ['Upload artifacts to s3 bucket': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/docs') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ if(env.FOLDER_NAME) {
+ utils.unpack_and_init('full_website', 'docs/_build/full_website.tgz')
+
+ utils.docker_run('ubuntu_cpu', "push_docs ${env.FOLDER_NAME}", false)
+
+ archiveArtifacts 'docs/_build/versions.zip'
+ } else {
+ sh 'echo Can not find website version for release. Please specify env var FOLDER_NAME in Jenkins pipeline'
+ sh 'exit 1'
+ }
+ }
+ }
+ }
+ }]
+}
def sanity_lint() {
return ['Lint': {
diff --git a/ci/jenkins/Jenkinsfile_website_version_artifacts b/ci/jenkins/Jenkinsfile_website_version_artifacts
new file mode 100644
index 0000000..7f74f1a
--- /dev/null
+++ b/ci/jenkins/Jenkinsfile_website_version_artifacts
@@ -0,0 +1,61 @@
+// -*- mode: groovy -*-
+
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+//
+// Jenkins pipeline
+// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
+
+// timeout in minutes
+max_time = 240
+
+node('restricted-utility') {
+ // Loading the utilities requires a node context unfortunately
+ checkout scm
+ utils = load('ci/Jenkinsfile_utils.groovy')
+ custom_steps = load('ci/jenkins/Jenkins_steps.groovy')
+}
+
+utils.assign_node_labels(utility: 'restricted-utility', linux_cpu: 'restricted-mxnetlinux-cpu', linux_gpu: 'restricted-mxnetlinux-gpu', linux_gpu_p3: 'restricted-mxnetlinux-gpu-p3', windows_cpu: 'restricted-mxnetwindows-cpu', windows_gpu: 'restricted-mxnetwindows-gpu')
+
+utils.main_wrapper(
+core_logic: {
+ utils.parallel_stage('Build', [
+ custom_steps.compile_unix_cpu_openblas('libmxnet')
+ ])
+
+ utils.parallel_stage('Build Docs', [
+ custom_steps.docs_jekyll(),
+ custom_steps.docs_python('libmxnet'),
+ ])
+
+ utils.parallel_stage('Build Full Website', [
+ custom_steps.docs_full_website()
+ ])
+
+ utils.parallel_stage('Upload Docs', [
+ custom_steps.docs_upload_s3()
+ ])
+}
+,
+failure_handler: {
+ // Only send email if master or release branches failed
+ if (currentBuild.result == "FAILURE" && (env.BRANCH_NAME == "master" || env.BRANCH_NAME.startsWith("v"))) {
+ emailext body: 'Build for MXNet branch ${BRANCH_NAME} has broken. Please view the build at ${BUILD_URL}', replyTo: '${EMAIL}', subject: '[BUILD FAILED] Branch ${BRANCH_NAME} build ${BUILD_NUMBER}', to: '${EMAIL}'
+ }
+}
+)