You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by ar...@apache.org on 2022/05/17 22:28:45 UTC
[tvm] 08/34: build arch images in Jenkins
This is an automated email from the ASF dual-hosted git repository.
areusch pushed a commit to branch areusch/freeze-dependencies
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 215a062578cc32acb3d47c0ecc3f8433b90a9789
Author: Andrew Reusch <ar...@gmail.com>
AuthorDate: Thu May 12 15:52:18 2022 -0700
build arch images in Jenkins
---
Jenkinsfile | 153 +++++++++++++++++++++++++++++++++----------------
jenkins/Jenkinsfile.j2 | 86 +++++++++------------------
jenkins/generate.py | 15 ++++-
3 files changed, 143 insertions(+), 111 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 4db9a45e2e..43dbdf0960 100755
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -323,12 +323,27 @@ stage('Lint') {
// a method (so the code can't all be inlined)
lint()
-def build_image(image_name) {
+def build_base_image(arch_name) {
+ hash = sh(
+ returnStdout: true,
+ script: 'git log -1 --format=\'%h\''
+ ).trim()
+ sh(
+ script: "docker/build-base-images.sh ${arch_name}",
+ label: "Build base image for ${arch_name}"
+ )
+ archiveArtifacts artifacts: 'docker/build/base_${arch_name}/**', fingerprint: true
+ def files = findFiles(glob: 'docker/build/base_${arch_name}/**')
+ pack_lib('${arch_name}-lockfiles', files.join(', '))
+}
+
+def build_image(arch_name, image_name) {
hash = sh(
returnStdout: true,
script: 'git log -1 --format=\'%h\''
).trim()
def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}"
+ unpack_lib('${arch_name}-lockfiles')
sh(
script: "${docker_build} ${image_name} --spec ${full_name}",
label: 'Build docker image'
@@ -380,64 +395,102 @@ def build_image(image_name) {
if (rebuild_docker_images) {
stage('Docker Image Build') {
+
// TODO in a follow up PR: Find ecr tag and use in subsequent builds
- parallel 'ci-lint': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_lint')
+ parallel(
+ 'arm64': {
+ node('ARM') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ build_base_image('arm64')
+ }
}
- }
- }, 'ci-cpu': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_cpu')
+ },
+ 'x86': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ build_base_image('x86')
+ }
}
- }
- }, 'ci-gpu': {
- node('GPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_gpu')
+ },
+ 'x86_64': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ build_base_image('x86_64')
+ }
}
- }
- }, 'ci-qemu': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_qemu')
+ },
+ )
+
+ parallel(
+ 'ci_arm': {
+ node('ARM') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ build_image('arm64', 'ci_arm')
+ }
}
- }
- }, 'ci-i386': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_i386')
+ },
+ 'ci_cpu': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ build_image('x86_64', 'ci_cpu')
+ }
}
- }
- }, 'ci-arm': {
- node('ARM') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_arm')
+ },
+ 'ci_gpu': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ build_image('x86_64', 'ci_gpu')
+ }
}
- }
- }, 'ci-wasm': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_wasm')
+ },
+ 'ci_hexagon': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ build_image('x86_64', 'ci_hexagon')
+ }
}
- }
- }, 'ci-hexagon': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_hexagon')
+ },
+ 'ci_i386': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ build_image('x86', 'ci_i386')
+ }
}
- }
- }
+ },
+ 'ci_lint': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ build_image('x86_64', 'ci_lint')
+ }
+ }
+ },
+ 'ci_qemu': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ build_image('x86_64', 'ci_qemu')
+ }
+ }
+ },
+ 'ci_wasm': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ build_image('x86_64', 'ci_wasm')
+ }
+ }
+ },
+ )
+
}
// // TODO: Once we are able to use the built images, enable this step
// // If the docker images changed, we need to run the image build before the lint
diff --git a/jenkins/Jenkinsfile.j2 b/jenkins/Jenkinsfile.j2
index b12ed4cbc0..9aa68d525c 100644
--- a/jenkins/Jenkinsfile.j2
+++ b/jenkins/Jenkinsfile.j2
@@ -246,9 +246,9 @@ def build_base_image(arch_name) {
script: 'git log -1 --format=\'%h\''
).trim()
sh(
- script: 'docker/build-base-images.sh ${arch_name}',
- label: 'Build base image for ${arch_name}'
- }
+ script: "docker/build-base-images.sh ${arch_name}",
+ label: "Build base image for ${arch_name}"
+ )
archiveArtifacts artifacts: 'docker/build/base_${arch_name}/**', fingerprint: true
def files = findFiles(glob: 'docker/build/base_${arch_name}/**')
pack_lib('${arch_name}-lockfiles', files.join(', '))
@@ -312,64 +312,34 @@ def build_image(arch_name, image_name) {
if (rebuild_docker_images) {
stage('Docker Image Build') {
+
// TODO in a follow up PR: Find ecr tag and use in subsequent builds
- parallel 'ci-lint': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_lint')
- }
- }
- }, 'ci-cpu': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_cpu')
- }
- }
- }, 'ci-gpu': {
- node('GPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_gpu')
- }
- }
- }, 'ci-qemu': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_qemu')
- }
- }
- }, 'ci-i386': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_i386')
- }
- }
- }, 'ci-arm': {
- node('ARM') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_arm')
- }
- }
- }, 'ci-wasm': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_wasm')
+ parallel(
+ {% for arch, img in nodes_by_arch(images) %}
+ {{ ' ' }}'{{ arch }}': {
+ node('{{ img.platform }}') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ build_base_image('{{ arch }}')
+ }
}
- }
- }, 'ci-hexagon': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- init_git()
- build_image('ci_hexagon')
+ },
+ {% endfor %}
+ )
+
+ parallel(
+ {% for image in images %}
+ '{{ image.name }}': {
+ node('{{ image.platform }}') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ build_image('{{ image.arch }}', '{{ image.name }}')
+ }
}
- }
- }
+ },
+ {% endfor %}
+ )
+
}
// // TODO: Once we are able to use the built images, enable this step
// // If the docker images changed, we need to run the image build before the lint
diff --git a/jenkins/generate.py b/jenkins/generate.py
index ba7f165925..002bd64bdf 100644
--- a/jenkins/generate.py
+++ b/jenkins/generate.py
@@ -34,37 +34,46 @@ data = {
"images": [
{
"name": "ci_arm",
+ "arch": "arm64",
"platform": "ARM",
},
{
"name": "ci_cpu",
+ "arch": "x86_64",
"platform": "CPU",
},
{
"name": "ci_gpu",
+ "arch": "x86_64",
"platform": "CPU",
},
{
"name": "ci_hexagon",
+ "arch": "x86_64",
"platform": "CPU",
},
{
"name": "ci_i386",
+ "arch": "x86",
"platform": "CPU",
},
{
"name": "ci_lint",
+ "arch": "x86_64",
"platform": "CPU",
},
{
"name": "ci_qemu",
+ "arch": "x86_64",
"platform": "CPU",
},
{
"name": "ci_wasm",
+ "arch": "x86_64",
"platform": "CPU",
},
- ]
+ ],
+ 'nodes_by_arch': lambda images: dict(sorted([(img['arch'], img) for img in images], key=lambda k: k[0])).items()
}
@@ -111,10 +120,10 @@ if __name__ == "__main__":
Newly generated Jenkinsfile did not match the one on disk! If you have made
edits to the Jenkinsfile, move them to 'jenkins/Jenkinsfile.j2' and
regenerate the Jenkinsfile from the template with
-
+
python3 -m pip install -r jenkins/requirements.txt
python3 jenkins/generate.py
-
+
Diffed changes:
"""
).strip()