You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by dr...@apache.org on 2022/12/09 00:08:57 UTC
[tvm] branch main updated: [ci][docker] Read docker image tags during CI runs (#13572)
This is an automated email from the ASF dual-hosted git repository.
driazati pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new c5b8ab2307 [ci][docker] Read docker image tags during CI runs (#13572)
c5b8ab2307 is described below
commit c5b8ab2307e8efe84babbe203e2b6893d57e2a9e
Author: driazati <94...@users.noreply.github.com>
AuthorDate: Thu Dec 8 17:08:51 2022 -0700
[ci][docker] Read docker image tags during CI runs (#13572)
Hardcoding the Docker images in the Jenkinsfiles has the downside that only trusted changes are actually run in CI. Since Docker image updates are pretty frequent, this makes for a pretty bad UX (e.g. CI runs but doesn't actually test the PR). The fix here makes it so the images are read from a different file during CI execution so updates from anyone are picked up.
This PR eliminates the need to use the batch jobs in Jenkins to update Docker images. Docker images are built as part of post-merge CI on `main` and anyone can trigger a validation run by filing a PR with changed Docker tags.
---
ci/jenkins/docker-images.ini | 29 +++++++++++++++++++++
ci/jenkins/generated/arm_jenkinsfile.groovy | 26 ++++++++++---------
ci/jenkins/generated/cortexm_jenkinsfile.groovy | 26 ++++++++++---------
ci/jenkins/generated/cpu_jenkinsfile.groovy | 26 ++++++++++---------
ci/jenkins/generated/docker_jenkinsfile.groovy | 26 ++++++++++---------
ci/jenkins/generated/gpu_jenkinsfile.groovy | 26 ++++++++++---------
ci/jenkins/generated/hexagon_jenkinsfile.groovy | 26 ++++++++++---------
ci/jenkins/generated/i386_jenkinsfile.groovy | 26 ++++++++++---------
ci/jenkins/generated/lint_jenkinsfile.groovy | 26 ++++++++++---------
ci/jenkins/generated/minimal_jenkinsfile.groovy | 26 ++++++++++---------
ci/jenkins/generated/riscv_jenkinsfile.groovy | 26 ++++++++++---------
ci/jenkins/generated/wasm_jenkinsfile.groovy | 26 ++++++++++---------
ci/jenkins/templates/utils/Prepare.groovy.j2 | 2 +-
ci/jenkins/templates/utils/base.groovy.j2 | 23 +++++++++--------
ci/scripts/jenkins/determine_docker_images.py | 25 +++++++++++++-----
docker/bash.sh | 5 +---
docs/contribute/ci.rst | 34 ++++++++++++++++++++-----
tests/lint/check_file_type.py | 2 ++
tests/python/ci/test_ci.py | 12 +++++++--
19 files changed, 257 insertions(+), 161 deletions(-)
diff --git a/ci/jenkins/docker-images.ini b/ci/jenkins/docker-images.ini
new file mode 100644
index 0000000000..119a432186
--- /dev/null
+++ b/ci/jenkins/docker-images.ini
@@ -0,0 +1,29 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# This data file is read during when Jenkins runs job to determine docker images.
+[jenkins]
+ci_arm: tlcpack/ci-arm:20221013-060115-61c9742ea
+ci_cortexm: tlcpack/ci-cortexm:20221013-060115-61c9742ea
+ci_cpu: tlcpack/ci-cpu:20221013-060115-61c9742ea
+ci_gpu: tlcpack/ci-gpu:20221019-060125-0b4836739
+ci_hexagon: tlcpack/ci-hexagon:20221013-060115-61c9742ea
+ci_i386: tlcpack/ci-i386:20221013-060115-61c9742ea
+ci_lint: tlcpack/ci-lint:20221013-060115-61c9742ea
+ci_minimal: tlcpack/ci-minimal:20221013-060115-61c9742ea
+ci_riscv: tlcpack/ci-riscv:20221013-060115-61c9742ea
+ci_wasm: tlcpack/ci-wasm:20221013-060115-61c9742ea
diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy
index f387687528..f1bcc786b7 100644
--- a/ci/jenkins/generated/arm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/arm_jenkinsfile.groovy
@@ -60,19 +60,21 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-12-05T14:48:42.092397
+// Generated at 2022-12-06T20:56:42.365592
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-ci_lint = 'tlcpack/ci-lint:20221013-060115-61c9742ea'
-ci_gpu = 'tlcpack/ci-gpu:20221019-060125-0b4836739'
-ci_cpu = 'tlcpack/ci-cpu:20221013-060115-61c9742ea'
-ci_minimal = 'tlcpack/ci-minimal:20221013-060115-61c9742ea'
-ci_wasm = 'tlcpack/ci-wasm:20221013-060115-61c9742ea'
-ci_i386 = 'tlcpack/ci-i386:20221013-060115-61c9742ea'
-ci_cortexm = 'tlcpack/ci-cortexm:20221013-060115-61c9742ea'
-ci_arm = 'tlcpack/ci-arm:20221013-060115-61c9742ea'
-ci_hexagon = 'tlcpack/ci-hexagon:20221013-060115-61c9742ea'
-ci_riscv = 'tlcpack/ci-riscv:20221013-060115-61c9742ea'
+// These are set at runtime from data in ci/jenkins/docker-images.yml, update
+// image tags in that file
+ci_lint = ''
+ci_gpu = ''
+ci_cpu = ''
+ci_minimal = ''
+ci_wasm = ''
+ci_i386 = ''
+ci_cortexm = ''
+ci_arm = ''
+ci_hexagon = ''
+ci_riscv = ''
// Parameters to allow overriding (in Jenkins UI), the images
// to be used by a given build. When provided, they take precedence
@@ -322,7 +324,7 @@ def prepare() {
if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm=${ci_arm} ci_cortexm=${ci_cortexm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_minimal=${ci_minimal} ci_riscv=${ci_riscv} ci_wasm=${ci_wasm} ",
+ script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
)
// Pull image names from the results of should_rebuild_docker.py
diff --git a/ci/jenkins/generated/cortexm_jenkinsfile.groovy b/ci/jenkins/generated/cortexm_jenkinsfile.groovy
index 76dbbbb7a3..4b5ba2e104 100644
--- a/ci/jenkins/generated/cortexm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/cortexm_jenkinsfile.groovy
@@ -60,19 +60,21 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-12-05T14:48:41.929980
+// Generated at 2022-12-06T20:56:42.204393
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-ci_lint = 'tlcpack/ci-lint:20221013-060115-61c9742ea'
-ci_gpu = 'tlcpack/ci-gpu:20221019-060125-0b4836739'
-ci_cpu = 'tlcpack/ci-cpu:20221013-060115-61c9742ea'
-ci_minimal = 'tlcpack/ci-minimal:20221013-060115-61c9742ea'
-ci_wasm = 'tlcpack/ci-wasm:20221013-060115-61c9742ea'
-ci_i386 = 'tlcpack/ci-i386:20221013-060115-61c9742ea'
-ci_cortexm = 'tlcpack/ci-cortexm:20221013-060115-61c9742ea'
-ci_arm = 'tlcpack/ci-arm:20221013-060115-61c9742ea'
-ci_hexagon = 'tlcpack/ci-hexagon:20221013-060115-61c9742ea'
-ci_riscv = 'tlcpack/ci-riscv:20221013-060115-61c9742ea'
+// These are set at runtime from data in ci/jenkins/docker-images.yml, update
+// image tags in that file
+ci_lint = ''
+ci_gpu = ''
+ci_cpu = ''
+ci_minimal = ''
+ci_wasm = ''
+ci_i386 = ''
+ci_cortexm = ''
+ci_arm = ''
+ci_hexagon = ''
+ci_riscv = ''
// Parameters to allow overriding (in Jenkins UI), the images
// to be used by a given build. When provided, they take precedence
@@ -322,7 +324,7 @@ def prepare() {
if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm=${ci_arm} ci_cortexm=${ci_cortexm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_minimal=${ci_minimal} ci_riscv=${ci_riscv} ci_wasm=${ci_wasm} ",
+ script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
)
// Pull image names from the results of should_rebuild_docker.py
diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy
index ad168c5918..378b20db51 100644
--- a/ci/jenkins/generated/cpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy
@@ -60,19 +60,21 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-12-05T14:48:42.120032
+// Generated at 2022-12-06T20:56:42.393957
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-ci_lint = 'tlcpack/ci-lint:20221013-060115-61c9742ea'
-ci_gpu = 'tlcpack/ci-gpu:20221019-060125-0b4836739'
-ci_cpu = 'tlcpack/ci-cpu:20221013-060115-61c9742ea'
-ci_minimal = 'tlcpack/ci-minimal:20221013-060115-61c9742ea'
-ci_wasm = 'tlcpack/ci-wasm:20221013-060115-61c9742ea'
-ci_i386 = 'tlcpack/ci-i386:20221013-060115-61c9742ea'
-ci_cortexm = 'tlcpack/ci-cortexm:20221013-060115-61c9742ea'
-ci_arm = 'tlcpack/ci-arm:20221013-060115-61c9742ea'
-ci_hexagon = 'tlcpack/ci-hexagon:20221013-060115-61c9742ea'
-ci_riscv = 'tlcpack/ci-riscv:20221013-060115-61c9742ea'
+// These are set at runtime from data in ci/jenkins/docker-images.yml, update
+// image tags in that file
+ci_lint = ''
+ci_gpu = ''
+ci_cpu = ''
+ci_minimal = ''
+ci_wasm = ''
+ci_i386 = ''
+ci_cortexm = ''
+ci_arm = ''
+ci_hexagon = ''
+ci_riscv = ''
// Parameters to allow overriding (in Jenkins UI), the images
// to be used by a given build. When provided, they take precedence
@@ -322,7 +324,7 @@ def prepare() {
if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm=${ci_arm} ci_cortexm=${ci_cortexm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_minimal=${ci_minimal} ci_riscv=${ci_riscv} ci_wasm=${ci_wasm} ",
+ script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
)
// Pull image names from the results of should_rebuild_docker.py
diff --git a/ci/jenkins/generated/docker_jenkinsfile.groovy b/ci/jenkins/generated/docker_jenkinsfile.groovy
index 246b40c6e9..050ef2983e 100644
--- a/ci/jenkins/generated/docker_jenkinsfile.groovy
+++ b/ci/jenkins/generated/docker_jenkinsfile.groovy
@@ -60,19 +60,21 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-12-06T21:25:49.429894
+// Generated at 2022-12-07T07:10:24.637792
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-ci_lint = 'tlcpack/ci-lint:20221013-060115-61c9742ea'
-ci_gpu = 'tlcpack/ci-gpu:20221019-060125-0b4836739'
-ci_cpu = 'tlcpack/ci-cpu:20221013-060115-61c9742ea'
-ci_minimal = 'tlcpack/ci-minimal:20221013-060115-61c9742ea'
-ci_wasm = 'tlcpack/ci-wasm:20221013-060115-61c9742ea'
-ci_i386 = 'tlcpack/ci-i386:20221013-060115-61c9742ea'
-ci_cortexm = 'tlcpack/ci-cortexm:20221013-060115-61c9742ea'
-ci_arm = 'tlcpack/ci-arm:20221013-060115-61c9742ea'
-ci_hexagon = 'tlcpack/ci-hexagon:20221013-060115-61c9742ea'
-ci_riscv = 'tlcpack/ci-riscv:20221013-060115-61c9742ea'
+// These are set at runtime from data in ci/jenkins/docker-images.yml, update
+// image tags in that file
+ci_lint = ''
+ci_gpu = ''
+ci_cpu = ''
+ci_minimal = ''
+ci_wasm = ''
+ci_i386 = ''
+ci_cortexm = ''
+ci_arm = ''
+ci_hexagon = ''
+ci_riscv = ''
// Parameters to allow overriding (in Jenkins UI), the images
// to be used by a given build. When provided, they take precedence
@@ -322,7 +324,7 @@ def prepare() {
if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm=${ci_arm} ci_cortexm=${ci_cortexm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_minimal=${ci_minimal} ci_riscv=${ci_riscv} ci_wasm=${ci_wasm} ",
+ script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
)
// Pull image names from the results of should_rebuild_docker.py
diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy
index ef357f0d7c..48a6619cba 100644
--- a/ci/jenkins/generated/gpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy
@@ -60,19 +60,21 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-12-06T20:30:23.035868
+// Generated at 2022-12-07T07:10:24.840515
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-ci_lint = 'tlcpack/ci-lint:20221013-060115-61c9742ea'
-ci_gpu = 'tlcpack/ci-gpu:20221019-060125-0b4836739'
-ci_cpu = 'tlcpack/ci-cpu:20221013-060115-61c9742ea'
-ci_minimal = 'tlcpack/ci-minimal:20221013-060115-61c9742ea'
-ci_wasm = 'tlcpack/ci-wasm:20221013-060115-61c9742ea'
-ci_i386 = 'tlcpack/ci-i386:20221013-060115-61c9742ea'
-ci_cortexm = 'tlcpack/ci-cortexm:20221013-060115-61c9742ea'
-ci_arm = 'tlcpack/ci-arm:20221013-060115-61c9742ea'
-ci_hexagon = 'tlcpack/ci-hexagon:20221013-060115-61c9742ea'
-ci_riscv = 'tlcpack/ci-riscv:20221013-060115-61c9742ea'
+// These are set at runtime from data in ci/jenkins/docker-images.yml, update
+// image tags in that file
+ci_lint = ''
+ci_gpu = ''
+ci_cpu = ''
+ci_minimal = ''
+ci_wasm = ''
+ci_i386 = ''
+ci_cortexm = ''
+ci_arm = ''
+ci_hexagon = ''
+ci_riscv = ''
// Parameters to allow overriding (in Jenkins UI), the images
// to be used by a given build. When provided, they take precedence
@@ -322,7 +324,7 @@ def prepare() {
if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm=${ci_arm} ci_cortexm=${ci_cortexm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_minimal=${ci_minimal} ci_riscv=${ci_riscv} ci_wasm=${ci_wasm} ",
+ script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
)
// Pull image names from the results of should_rebuild_docker.py
diff --git a/ci/jenkins/generated/hexagon_jenkinsfile.groovy b/ci/jenkins/generated/hexagon_jenkinsfile.groovy
index 6296d0c5c8..e5397eee3a 100644
--- a/ci/jenkins/generated/hexagon_jenkinsfile.groovy
+++ b/ci/jenkins/generated/hexagon_jenkinsfile.groovy
@@ -60,19 +60,21 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-12-05T14:48:42.065368
+// Generated at 2022-12-06T20:56:42.338377
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-ci_lint = 'tlcpack/ci-lint:20221013-060115-61c9742ea'
-ci_gpu = 'tlcpack/ci-gpu:20221019-060125-0b4836739'
-ci_cpu = 'tlcpack/ci-cpu:20221013-060115-61c9742ea'
-ci_minimal = 'tlcpack/ci-minimal:20221013-060115-61c9742ea'
-ci_wasm = 'tlcpack/ci-wasm:20221013-060115-61c9742ea'
-ci_i386 = 'tlcpack/ci-i386:20221013-060115-61c9742ea'
-ci_cortexm = 'tlcpack/ci-cortexm:20221013-060115-61c9742ea'
-ci_arm = 'tlcpack/ci-arm:20221013-060115-61c9742ea'
-ci_hexagon = 'tlcpack/ci-hexagon:20221013-060115-61c9742ea'
-ci_riscv = 'tlcpack/ci-riscv:20221013-060115-61c9742ea'
+// These are set at runtime from data in ci/jenkins/docker-images.yml, update
+// image tags in that file
+ci_lint = ''
+ci_gpu = ''
+ci_cpu = ''
+ci_minimal = ''
+ci_wasm = ''
+ci_i386 = ''
+ci_cortexm = ''
+ci_arm = ''
+ci_hexagon = ''
+ci_riscv = ''
// Parameters to allow overriding (in Jenkins UI), the images
// to be used by a given build. When provided, they take precedence
@@ -322,7 +324,7 @@ def prepare() {
if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm=${ci_arm} ci_cortexm=${ci_cortexm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_minimal=${ci_minimal} ci_riscv=${ci_riscv} ci_wasm=${ci_wasm} ",
+ script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
)
// Pull image names from the results of should_rebuild_docker.py
diff --git a/ci/jenkins/generated/i386_jenkinsfile.groovy b/ci/jenkins/generated/i386_jenkinsfile.groovy
index f0170f5867..876670aceb 100644
--- a/ci/jenkins/generated/i386_jenkinsfile.groovy
+++ b/ci/jenkins/generated/i386_jenkinsfile.groovy
@@ -60,19 +60,21 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-12-05T14:48:42.016799
+// Generated at 2022-12-06T20:56:42.288840
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-ci_lint = 'tlcpack/ci-lint:20221013-060115-61c9742ea'
-ci_gpu = 'tlcpack/ci-gpu:20221019-060125-0b4836739'
-ci_cpu = 'tlcpack/ci-cpu:20221013-060115-61c9742ea'
-ci_minimal = 'tlcpack/ci-minimal:20221013-060115-61c9742ea'
-ci_wasm = 'tlcpack/ci-wasm:20221013-060115-61c9742ea'
-ci_i386 = 'tlcpack/ci-i386:20221013-060115-61c9742ea'
-ci_cortexm = 'tlcpack/ci-cortexm:20221013-060115-61c9742ea'
-ci_arm = 'tlcpack/ci-arm:20221013-060115-61c9742ea'
-ci_hexagon = 'tlcpack/ci-hexagon:20221013-060115-61c9742ea'
-ci_riscv = 'tlcpack/ci-riscv:20221013-060115-61c9742ea'
+// These are set at runtime from data in ci/jenkins/docker-images.yml, update
+// image tags in that file
+ci_lint = ''
+ci_gpu = ''
+ci_cpu = ''
+ci_minimal = ''
+ci_wasm = ''
+ci_i386 = ''
+ci_cortexm = ''
+ci_arm = ''
+ci_hexagon = ''
+ci_riscv = ''
// Parameters to allow overriding (in Jenkins UI), the images
// to be used by a given build. When provided, they take precedence
@@ -322,7 +324,7 @@ def prepare() {
if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm=${ci_arm} ci_cortexm=${ci_cortexm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_minimal=${ci_minimal} ci_riscv=${ci_riscv} ci_wasm=${ci_wasm} ",
+ script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
)
// Pull image names from the results of should_rebuild_docker.py
diff --git a/ci/jenkins/generated/lint_jenkinsfile.groovy b/ci/jenkins/generated/lint_jenkinsfile.groovy
index ee63a1008b..3aaea4436f 100644
--- a/ci/jenkins/generated/lint_jenkinsfile.groovy
+++ b/ci/jenkins/generated/lint_jenkinsfile.groovy
@@ -60,19 +60,21 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-12-05T14:48:42.041376
+// Generated at 2022-12-06T20:56:42.313954
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-ci_lint = 'tlcpack/ci-lint:20221013-060115-61c9742ea'
-ci_gpu = 'tlcpack/ci-gpu:20221019-060125-0b4836739'
-ci_cpu = 'tlcpack/ci-cpu:20221013-060115-61c9742ea'
-ci_minimal = 'tlcpack/ci-minimal:20221013-060115-61c9742ea'
-ci_wasm = 'tlcpack/ci-wasm:20221013-060115-61c9742ea'
-ci_i386 = 'tlcpack/ci-i386:20221013-060115-61c9742ea'
-ci_cortexm = 'tlcpack/ci-cortexm:20221013-060115-61c9742ea'
-ci_arm = 'tlcpack/ci-arm:20221013-060115-61c9742ea'
-ci_hexagon = 'tlcpack/ci-hexagon:20221013-060115-61c9742ea'
-ci_riscv = 'tlcpack/ci-riscv:20221013-060115-61c9742ea'
+// These are set at runtime from data in ci/jenkins/docker-images.yml, update
+// image tags in that file
+ci_lint = ''
+ci_gpu = ''
+ci_cpu = ''
+ci_minimal = ''
+ci_wasm = ''
+ci_i386 = ''
+ci_cortexm = ''
+ci_arm = ''
+ci_hexagon = ''
+ci_riscv = ''
// Parameters to allow overriding (in Jenkins UI), the images
// to be used by a given build. When provided, they take precedence
@@ -322,7 +324,7 @@ def prepare() {
if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm=${ci_arm} ci_cortexm=${ci_cortexm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_minimal=${ci_minimal} ci_riscv=${ci_riscv} ci_wasm=${ci_wasm} ",
+ script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
)
// Pull image names from the results of should_rebuild_docker.py
diff --git a/ci/jenkins/generated/minimal_jenkinsfile.groovy b/ci/jenkins/generated/minimal_jenkinsfile.groovy
index 4c9f469b3b..f8a59ef573 100644
--- a/ci/jenkins/generated/minimal_jenkinsfile.groovy
+++ b/ci/jenkins/generated/minimal_jenkinsfile.groovy
@@ -60,19 +60,21 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-12-05T23:21:03.010229
+// Generated at 2022-12-06T20:56:42.235080
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-ci_lint = 'tlcpack/ci-lint:20221013-060115-61c9742ea'
-ci_gpu = 'tlcpack/ci-gpu:20221019-060125-0b4836739'
-ci_cpu = 'tlcpack/ci-cpu:20221013-060115-61c9742ea'
-ci_minimal = 'tlcpack/ci-minimal:20221013-060115-61c9742ea'
-ci_wasm = 'tlcpack/ci-wasm:20221013-060115-61c9742ea'
-ci_i386 = 'tlcpack/ci-i386:20221013-060115-61c9742ea'
-ci_cortexm = 'tlcpack/ci-cortexm:20221013-060115-61c9742ea'
-ci_arm = 'tlcpack/ci-arm:20221013-060115-61c9742ea'
-ci_hexagon = 'tlcpack/ci-hexagon:20221013-060115-61c9742ea'
-ci_riscv = 'tlcpack/ci-riscv:20221013-060115-61c9742ea'
+// These are set at runtime from data in ci/jenkins/docker-images.yml, update
+// image tags in that file
+ci_lint = ''
+ci_gpu = ''
+ci_cpu = ''
+ci_minimal = ''
+ci_wasm = ''
+ci_i386 = ''
+ci_cortexm = ''
+ci_arm = ''
+ci_hexagon = ''
+ci_riscv = ''
// Parameters to allow overriding (in Jenkins UI), the images
// to be used by a given build. When provided, they take precedence
@@ -322,7 +324,7 @@ def prepare() {
if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm=${ci_arm} ci_cortexm=${ci_cortexm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_minimal=${ci_minimal} ci_riscv=${ci_riscv} ci_wasm=${ci_wasm} ",
+ script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
)
// Pull image names from the results of should_rebuild_docker.py
diff --git a/ci/jenkins/generated/riscv_jenkinsfile.groovy b/ci/jenkins/generated/riscv_jenkinsfile.groovy
index b485e9906f..eb62c3731f 100644
--- a/ci/jenkins/generated/riscv_jenkinsfile.groovy
+++ b/ci/jenkins/generated/riscv_jenkinsfile.groovy
@@ -60,19 +60,21 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-12-05T14:48:42.170796
+// Generated at 2022-12-06T20:56:42.442689
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-ci_lint = 'tlcpack/ci-lint:20221013-060115-61c9742ea'
-ci_gpu = 'tlcpack/ci-gpu:20221019-060125-0b4836739'
-ci_cpu = 'tlcpack/ci-cpu:20221013-060115-61c9742ea'
-ci_minimal = 'tlcpack/ci-minimal:20221013-060115-61c9742ea'
-ci_wasm = 'tlcpack/ci-wasm:20221013-060115-61c9742ea'
-ci_i386 = 'tlcpack/ci-i386:20221013-060115-61c9742ea'
-ci_cortexm = 'tlcpack/ci-cortexm:20221013-060115-61c9742ea'
-ci_arm = 'tlcpack/ci-arm:20221013-060115-61c9742ea'
-ci_hexagon = 'tlcpack/ci-hexagon:20221013-060115-61c9742ea'
-ci_riscv = 'tlcpack/ci-riscv:20221013-060115-61c9742ea'
+// These are set at runtime from data in ci/jenkins/docker-images.yml, update
+// image tags in that file
+ci_lint = ''
+ci_gpu = ''
+ci_cpu = ''
+ci_minimal = ''
+ci_wasm = ''
+ci_i386 = ''
+ci_cortexm = ''
+ci_arm = ''
+ci_hexagon = ''
+ci_riscv = ''
// Parameters to allow overriding (in Jenkins UI), the images
// to be used by a given build. When provided, they take precedence
@@ -322,7 +324,7 @@ def prepare() {
if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm=${ci_arm} ci_cortexm=${ci_cortexm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_minimal=${ci_minimal} ci_riscv=${ci_riscv} ci_wasm=${ci_wasm} ",
+ script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
)
// Pull image names from the results of should_rebuild_docker.py
diff --git a/ci/jenkins/generated/wasm_jenkinsfile.groovy b/ci/jenkins/generated/wasm_jenkinsfile.groovy
index 0c7c2ccf2a..d43c7f9d24 100644
--- a/ci/jenkins/generated/wasm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy
@@ -60,19 +60,21 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-12-05T14:48:42.147157
+// Generated at 2022-12-06T20:56:42.420989
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-ci_lint = 'tlcpack/ci-lint:20221013-060115-61c9742ea'
-ci_gpu = 'tlcpack/ci-gpu:20221019-060125-0b4836739'
-ci_cpu = 'tlcpack/ci-cpu:20221013-060115-61c9742ea'
-ci_minimal = 'tlcpack/ci-minimal:20221013-060115-61c9742ea'
-ci_wasm = 'tlcpack/ci-wasm:20221013-060115-61c9742ea'
-ci_i386 = 'tlcpack/ci-i386:20221013-060115-61c9742ea'
-ci_cortexm = 'tlcpack/ci-cortexm:20221013-060115-61c9742ea'
-ci_arm = 'tlcpack/ci-arm:20221013-060115-61c9742ea'
-ci_hexagon = 'tlcpack/ci-hexagon:20221013-060115-61c9742ea'
-ci_riscv = 'tlcpack/ci-riscv:20221013-060115-61c9742ea'
+// These are set at runtime from data in ci/jenkins/docker-images.yml, update
+// image tags in that file
+ci_lint = ''
+ci_gpu = ''
+ci_cpu = ''
+ci_minimal = ''
+ci_wasm = ''
+ci_i386 = ''
+ci_cortexm = ''
+ci_arm = ''
+ci_hexagon = ''
+ci_riscv = ''
// Parameters to allow overriding (in Jenkins UI), the images
// to be used by a given build. When provided, they take precedence
@@ -322,7 +324,7 @@ def prepare() {
if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm=${ci_arm} ci_cortexm=${ci_cortexm} ci_cpu=${ci_cpu} ci_gpu=${ci_gpu} ci_hexagon=${ci_hexagon} ci_i386=${ci_i386} ci_lint=${ci_lint} ci_minimal=${ci_minimal} ci_riscv=${ci_riscv} ci_wasm=${ci_wasm} ",
+ script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
)
// Pull image names from the results of should_rebuild_docker.py
diff --git a/ci/jenkins/templates/utils/Prepare.groovy.j2 b/ci/jenkins/templates/utils/Prepare.groovy.j2
index 099bde5bc7..b295bb4308 100644
--- a/ci/jenkins/templates/utils/Prepare.groovy.j2
+++ b/ci/jenkins/templates/utils/Prepare.groovy.j2
@@ -194,7 +194,7 @@ def prepare() {
if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py {% for image in images %}{{ image.name }}={% raw %}${{% endraw %}{{ image.name }}{% raw %}}{% endraw %} {% endfor %}",
+ script: "./${jenkins_scripts_root}/determine_docker_images.py {% for image in images %}{{ image.name }} {% endfor %}",
label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
)
// Pull image names from the results of should_rebuild_docker.py
diff --git a/ci/jenkins/templates/utils/base.groovy.j2 b/ci/jenkins/templates/utils/base.groovy.j2
index 0854091c7a..bb00f99300 100644
--- a/ci/jenkins/templates/utils/base.groovy.j2
+++ b/ci/jenkins/templates/utils/base.groovy.j2
@@ -49,16 +49,19 @@
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
{% import 'utils/macros.j2' as m with context -%}
-ci_lint = '{{ ci_lint }}'
-ci_gpu = '{{ ci_gpu }}'
-ci_cpu = '{{ ci_cpu }}'
-ci_minimal = '{{ ci_minimal }}'
-ci_wasm = '{{ ci_wasm }}'
-ci_i386 = '{{ ci_i386 }}'
-ci_cortexm = '{{ ci_cortexm }}'
-ci_arm = '{{ ci_arm }}'
-ci_hexagon = '{{ ci_hexagon }}'
-ci_riscv = '{{ ci_riscv }}'
+
+// These are set at runtime from data in ci/jenkins/docker-images.yml, update
+// image tags in that file
+ci_lint = ''
+ci_gpu = ''
+ci_cpu = ''
+ci_minimal = ''
+ci_wasm = ''
+ci_i386 = ''
+ci_cortexm = ''
+ci_arm = ''
+ci_hexagon = ''
+ci_riscv = ''
// Parameters to allow overriding (in Jenkins UI), the images
// to be used by a given build. When provided, they take precedence
diff --git a/ci/scripts/jenkins/determine_docker_images.py b/ci/scripts/jenkins/determine_docker_images.py
index 82acf2ea46..78da9a3546 100755
--- a/ci/scripts/jenkins/determine_docker_images.py
+++ b/ci/scripts/jenkins/determine_docker_images.py
@@ -20,17 +20,18 @@ import datetime
import json
import logging
import urllib.error
-from pathlib import Path
+import configparser
+from pathlib import Path
from typing import Dict, Any
-
from http_utils import get
-from cmd_utils import init_log
+from cmd_utils import init_log, REPO_ROOT
DOCKER_API_BASE = "https://hub.docker.com/v2/"
PAGE_SIZE = 25
TEST_DATA = None
+IMAGE_TAGS_FILE = REPO_ROOT / "ci" / "jenkins" / "docker-images.ini"
def docker_api(url: str, use_pagination: bool = False) -> Dict[str, Any]:
@@ -77,6 +78,10 @@ if __name__ == "__main__":
"--testing-docker-data",
help="(testing only) JSON data to mock response from Docker Hub API",
)
+ parser.add_argument(
+ "--testing-images-data",
+ help=f"(testing only) JSON data to mock contents of {IMAGE_TAGS_FILE}",
+ )
parser.add_argument(
"--base-dir",
default=".docker-image-names",
@@ -85,10 +90,18 @@ if __name__ == "__main__":
args, other = parser.parse_known_args()
name_dir = Path(args.base_dir)
+ if args.testing_images_data:
+ repo_image_tags = json.loads(args.testing_images_data)
+ else:
+ config = configparser.ConfigParser()
+ config.read(IMAGE_TAGS_FILE)
+ repo_image_tags = {}
+ for name in other:
+ repo_image_tags[name] = config.get("jenkins", name)
+
images = {}
- for item in other:
- name, tag = item.split("=")
- images[name] = tag
+ for name in other:
+ images[name] = repo_image_tags[name]
if args.testing_docker_data is not None:
TEST_DATA = json.loads(args.testing_docker_data)
diff --git a/docker/bash.sh b/docker/bash.sh
index 6919ce9edb..bd5ca70f43 100755
--- a/docker/bash.sh
+++ b/docker/bash.sh
@@ -306,16 +306,13 @@ source "$(dirname $0)/dev_common.sh" || exit 2
DOCKER_MOUNT=( )
DOCKER_DEVICES=( )
-
-
# If the user gave a shortcut defined in the Jenkinsfile, use it.
EXPANDED_SHORTCUT=$(lookup_image_spec "${DOCKER_IMAGE_NAME}")
if [ -n "${EXPANDED_SHORTCUT}" ]; then
if [ "${CI+x}" == "x" ]; then
DOCKER_IMAGE_NAME="${EXPANDED_SHORTCUT}"
else
- python3 ci/scripts/jenkins/determine_docker_images.py "$DOCKER_IMAGE_NAME=$EXPANDED_SHORTCUT" 2> /dev/null
- echo "HERE HERE HERE"
+ python3 ci/scripts/jenkins/determine_docker_images.py "$DOCKER_IMAGE_NAME" 2> /dev/null
DOCKER_IMAGE_NAME=$(cat ".docker-image-names/$DOCKER_IMAGE_NAME")
if [[ "$DOCKER_IMAGE_NAME" == *"tlcpackstaging"* ]]; then
echo "WARNING: resolved docker image to fallback tag in tlcpackstaging" >&2
diff --git a/docs/contribute/ci.rst b/docs/contribute/ci.rst
index 1284fd95fb..428319a99a 100644
--- a/docs/contribute/ci.rst
+++ b/docs/contribute/ci.rst
@@ -168,13 +168,35 @@ Docker Images
^^^^^^^^^^^^^
Each CI job runs most of its work inside a Docker container, built from files
-in the `docker/ <https://github.com/apache/tvm/tree/main/docker>`_ folder. These
-files are built nightly in Jenkins via the `docker-images-ci <https://ci.tlcpack.ai/job/docker-images-ci/>`_ job.
-The images for these containers are hosted in the `tlcpack Docker Hub <https://hub.docker.com/u/tlcpack>`_
-and referenced in the `Jenkinsfile.j2 <https://github.com/apache/tvm/tree/main/Jenkinsfile.j2>`_. These can be inspected and run
-locally via standard Docker commands.
+in the `docker/ <https://github.com/apache/tvm/tree/main/docker>`_ folder.
-Adding a new Docker image
+
+Updating a Docker Image Tag
+"""""""""""""""""""""""""""
+
+To update a tag, a new image needs to be built and uploaded to Docker Hub, then
+the image tags in `docker-images.ini <https://github.com/apache/tvm/tree/main/ci/jenkins/docker-images.ini>`_
+need to be updated to match the image tags on Docker Hub.
+
+Docker images are built automatically nightly via the `docker-images-ci <https://ci.tlcpack.ai/job/docker-images-ci/>`_,
+which uploads the built images to https://hub.docker.com/u/tlcpackstaging once
+they have passed CI. Post-merge CI runs on ``main`` build Docker images ad-hoc
+and upload them to the ``tlcpackstaging`` Docker Hub account as well. There is an
+auto-promotion process for ``tlcpackstaging`` Docker images to be moved to the
+``tlcpack`` account. This means that image tags from ``tlcpackstaging`` can be
+used in CI and they will be automatically moved to ``tlcpack`` after a successful
+post-merge CI run on ``main``. So the steps to update the image are:
+
+1. Merge a PR that changes the Dockerfiles under ``docker/`` or scripts in ``docker/install``.
+2. Do either of:
+
+ a. Wait for the post-merge CI build from the PR to complete and upload the newly built image to the `tlcpackstaging <https://hub.docker.com/u/tlcpackstaging>`_ Docker Hub.
+ b. Wait for the nightly Docker image build to complete and upload the newly built image to the `tlcpackstaging <https://hub.docker.com/u/tlcpackstaging>`_ Docker Hub.
+
+3. Find the newly uploaded image tag on the `tlcpackstaging <https://hub.docker.com/u/tlcpackstaging>`_ Docker Hub, for example ``20221208-070144-22ff38dff`` and update the tag in ``ci/jenkins/docker-images.ini`` to use the tlcpackstaging tag but under the tlcpack account, e.g. ``tlcpack/ci-arm:20221208-070144-22ff38dff``. Send in a PR with these changes and wait for it to run through CI to ensure the new images are valid.
+4. Merge the ``docker-images.ini`` update PR. Once post-merge CI finishes running on ``main`` the ``tlcpackstaging`` tag will be re-uploaded to ``tlcpack`` automatically.
+
+Adding a New Docker Image
"""""""""""""""""""""""""
New docker images can be added to test TVM on a variety of platforms. Here are the steps for adding
diff --git a/tests/lint/check_file_type.py b/tests/lint/check_file_type.py
index f5d5a2f0a3..56fad7a3a8 100644
--- a/tests/lint/check_file_type.py
+++ b/tests/lint/check_file_type.py
@@ -91,6 +91,8 @@ ALLOW_EXTENSION = {
"j2",
# Jenkinsfiles
"groovy",
+ # Python-parseable config files
+ "ini",
}
# List of file names allowed
diff --git a/tests/python/ci/test_ci.py b/tests/python/ci/test_ci.py
index cf31b50b63..f83fa40c61 100644
--- a/tests/python/ci/test_ci.py
+++ b/tests/python/ci/test_ci.py
@@ -1287,14 +1287,14 @@ def test_open_docker_update_pr(
@parameterize_named(
use_tlcpack=dict(
- images=["ci_arm=tlcpack/ci-arm:abc-abc-123", "ci_lint=tlcpack/ci-lint:abc-abc-234"],
+ images=["ci_arm", "ci_lint"],
expected={
"ci_arm": "tlcpack/ci-arm:abc-abc-123",
"ci_lint": "tlcpack/ci-lint:abc-abc-234",
},
),
use_staging=dict(
- images=["ci_arm2=tlcpack/ci-arm2:abc-abc-123"],
+ images=["ci_arm2"],
expected={
"ci_arm2": "tlcpackstaging/ci_arm2:abc-abc-123",
},
@@ -1311,11 +1311,19 @@ def test_determine_docker_images(tmpdir_factory, images, expected):
"repositories/tlcpack/ci-lint/tags/abc-abc-234": {},
}
+ images_data = {
+ "ci_arm": "tlcpack/ci-arm:abc-abc-123",
+ "ci_lint": "tlcpack/ci-lint:abc-abc-234",
+ "ci_arm2": "tlcpack/ci-arm2:abc-abc-123",
+ }
+
run_script(
[
script,
"--testing-docker-data",
json.dumps(docker_data),
+ "--testing-images-data",
+ json.dumps(images_data),
"--base-dir",
git_dir,
]