You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2026/10/15 02:29:12 UTC
(tvm) 02/03: [REFACTOR][CI] Organize shared helpers and retain legacy entry points
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch jenkins-cleanup-trusted-main
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 79cb0c4d40493e165b24088a60354e4b064be22c
Author: Tianqi Chen <tq...@users.noreply.github.com>
AuthorDate: Thu Oct 15 02:16:59 2026 +0000
[REFACTOR][CI] Organize shared helpers and retain legacy entry points
---
ci/jenkins/generated/common.groovy | 251 +++++++++++++++++++------------------
1 file changed, 130 insertions(+), 121 deletions(-)
diff --git a/ci/jenkins/generated/common.groovy b/ci/jenkins/generated/common.groovy
index cd8324b5f2..d4add87448 100644
--- a/ci/jenkins/generated/common.groovy
+++ b/ci/jenkins/generated/common.groovy
@@ -34,11 +34,60 @@ properties([parameters(image_platforms.collect { name, platform ->
string(name: "${name}_param", defaultValue: '')
})])
+// Internal helpers, with dependencies before their callers.
+// Keep these callable for existing pipelines.
+
def per_exec_ws(folder) {
return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
}
-// initialize source codes
+def update_upstream_revision(git_ref) {
+ if (upstream_revision == null) {
+ upstream_revision = sh(
+ script: "git log -1 ${git_ref} --format=\'%H\'",
+ label: 'Determine upstream revision',
+ returnStdout: true,
+ ).trim()
+ }
+}
+
+def merge_with_target() {
+ def target = env.CHANGE_TARGET
+ sh (
+ script: "git fetch origin ${target}",
+ label: "Fetch target branch ${target}",
+ )
+ update_upstream_revision("FETCH_HEAD")
+ sh (
+ script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
+ label: "Merge to origin/${target}"
+ )
+}
+
+def checkout_trusted_files() {
+ // trust everything from branch builds
+ if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
+ return;
+ }
+
+ // Trust contributors listed in the pinned target revision.
+ grep_code = sh(
+ returnStatus: true,
+ script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
+ label: 'Check if change is from a contributor',
+ )
+
+ if (grep_code == 1) {
+ // Any scripts that run on the bare host and not inside a Docker container
+ // (especially those that access secrets) should be checked out here so
+ // only trusted versions are used in CI
+ sh(
+ script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/. ci/jenkins/data.py",
+ label: 'Check out trusted files',
+ )
+ }
+}
+
def init_git() {
retry(5) {
checkout scm
@@ -73,61 +122,6 @@ def init_git() {
checkout_trusted_files()
}
-def update_upstream_revision(git_ref) {
- if (upstream_revision == null) {
- upstream_revision = sh(
- script: "git log -1 ${git_ref} --format=\'%H\'",
- label: 'Determine upstream revision',
- returnStdout: true,
- ).trim()
- }
-}
-
-def merge_with_target() {
- def target = env.CHANGE_TARGET
- sh (
- script: "git fetch origin ${target}",
- label: "Fetch target branch ${target}",
- )
- update_upstream_revision("FETCH_HEAD")
- sh (
- script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
- label: "Merge to origin/${target}"
- )
-}
-
-def docker_init(image) {
- // Clear out all Docker images that aren't going to be used
- sh(
- script: """
- set -eux
- docker image ls --all
- IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}')
-
- echo -e "Found images:\\n\$IMAGES"
- echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
-
- docker image ls --all
- """,
- label: 'Clean old Docker images',
- )
-
- if (image.contains("amazonaws.com")) {
- // If this string is in the image name it's from ECR and needs to be pulled
- // with the right credentials
- ecr_pull(image)
- } else {
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${image}
- """,
- label: 'Pull docker image',
- )
- }
-}
-
def ecr_pull(full_name) {
aws_account_id = sh(
returnStdout: true,
@@ -190,30 +184,6 @@ def is_last_build() {
}
}
-def checkout_trusted_files() {
- // trust everything from branch builds
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- return;
- }
-
- // Trust contributors listed in the pinned target revision.
- grep_code = sh(
- returnStatus: true,
- script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
- label: 'Check if change is from a contributor',
- )
-
- if (grep_code == 1) {
- // Any scripts that run on the bare host and not inside a Docker container
- // (especially those that access secrets) should be checked out here so
- // only trusted versions are used in CI
- sh(
- script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/. ci/jenkins/data.py",
- label: 'Check out trusted files',
- )
- }
-}
-
def should_skip_ci(pr_number) {
if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
// never skip CI on build sourced from a branch
@@ -283,6 +253,85 @@ def prepare(node_type) {
}
}
}
+
+def retry_latest(ex, body) {
+ echo 'Exception during SPOT run ' + ex.toString()
+ if (!is_last_build()) {
+ echo 'Exit since it is not last build'
+ throw ex
+ }
+ echo 'Retry on-demand given it is last build'
+ currentBuild.result = 'SUCCESS'
+ // Do not catch failure of the on-demand retry.
+ body()
+}
+
+def run_on_node(label, name, folder, image, variables, body) {
+ if (skip_ci) {
+ Utils.markStageSkippedForConditional(name)
+ return
+ }
+ echo 'Begin running on node_type ' + label
+ node(label) {
+ ws(per_exec_ws(folder)) {
+ init_git()
+ docker_init(image)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv(variables) { body() }
+ }
+ }
+ }
+ echo 'End running on node_type ' + label
+}
+
+// Public entry points.
+
+def docker_init(image) {
+ // Clear out all Docker images that aren't going to be used
+ sh(
+ script: """
+ set -eux
+ docker image ls --all
+ IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}')
+
+ echo -e "Found images:\\n\$IMAGES"
+ echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
+
+ docker image ls --all
+ """,
+ label: 'Clean old Docker images',
+ )
+
+ if (image.contains("amazonaws.com")) {
+ // If this string is in the image name it's from ECR and needs to be pulled
+ // with the right credentials
+ ecr_pull(image)
+ } else {
+ sh(
+ script: """
+ set -eux
+ . ${jenkins_scripts_root}/retry.sh
+ retry 5 docker pull ${image}
+ """,
+ label: 'Pull docker image',
+ )
+ }
+}
+
+// Preparation retries Exceptions unconditionally; Errors and cancellation of the
+// previous build are deliberately outside this fallback.
+def start() {
+ cancel_previous_build()
+ try {
+ prepare('CPU-SMALL-SPOT')
+ } catch (Exception ex) {
+ prepare('CPU-SMALL')
+ }
+}
+
+// Legacy compatibility functions. Keep names, signatures and behavior until
+// existing pipeline callers have migrated after merge.
+
def python_unittest(image) {
sh (
script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
@@ -310,6 +359,7 @@ def cmake_build(image, path) {
label: 'Run CMake build',
)
}
+
def cpp_unittest(image) {
sh (
script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh",
@@ -317,17 +367,6 @@ def cpp_unittest(image) {
)
}
-// Preparation retries Exceptions unconditionally; Errors and cancellation of the
-// previous build are deliberately outside this fallback.
-def start() {
- cancel_previous_build()
- try {
- prepare('CPU-SMALL-SPOT')
- } catch (Exception ex) {
- prepare('CPU-SMALL')
- }
-}
-
// A failed build command is final. Infrastructure failures retry only while
// this is the newest build; tests intentionally also retry AbortException.
def build(label, name, folder, image, platform, body) {
@@ -345,18 +384,6 @@ def build(label, name, folder, image, platform, body) {
}
}
-def retry_latest(ex, body) {
- echo 'Exception during SPOT run ' + ex.toString()
- if (!is_last_build()) {
- echo 'Exit since it is not last build'
- throw ex
- }
- echo 'Retry on-demand given it is last build'
- currentBuild.result = 'SUCCESS'
- // Do not catch failure of the on-demand retry.
- body()
-}
-
def test(label, name, folder, image, platform, body) {
def variables = ["PLATFORM=${platform}", "TEST_STEP_NAME=${name}"]
try {
@@ -366,24 +393,6 @@ def test(label, name, folder, image, platform, body) {
}
}
-def run_on_node(label, name, folder, image, variables, body) {
- if (skip_ci) {
- Utils.markStageSkippedForConditional(name)
- return
- }
- echo 'Begin running on node_type ' + label
- node(label) {
- ws(per_exec_ws(folder)) {
- init_git()
- docker_init(image)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv(variables) { body() }
- }
- }
- }
- echo 'End running on node_type ' + label
-}
-
def upload(tag, options) {
sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/${tag} ${options}",