You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2026/10/14 23:12:55 UTC
(tvm) 01/02: [CI] Adapt wheel artifacts to native Jenkins pipelines
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch jenkins-wheel-test-artifacts
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 7a8784af3595f2ed91bf4d5291e2ad97229cf1a8
Merge: c748eef52e f216560d1e
Author: Tianqi Chen <tq...@users.noreply.github.com>
AuthorDate: Wed Oct 14 23:02:33 2026 +0000
[CI] Adapt wheel artifacts to native Jenkins pipelines
Build only TVM with the image-provided backend and dependencies, and install its exact wheel for Python consumers. Keep C++ tests in the packaging build tree, pass native parallel limits through Docker, and preserve specialized runtime and Java artifacts.
.github/actions/setup/action.yml | 6 +-
.github/workflows/nightly_docker_update.yml | 24 -
.github/workflows/publish_wheel.yml | 6 +-
.github/workflows/tvmbot.yml | 2 +-
ci/README.md | 15 +-
ci/{scripts => }/github/__init__.py | 0
ci/{scripts/jenkins => github}/git_utils.py | 0
ci/{scripts => }/github/github_tvmbot.py | 12 +-
ci/jenkins/README.md | 91 ++-
ci/jenkins/data.py | 26 +-
ci/jenkins/generate.py | 226 -------
ci/jenkins/generated/arm_jenkinsfile.groovy | 526 ++-------------
.../{wasm_jenkinsfile.groovy => common.groovy} | 339 ++++------
ci/jenkins/generated/cpu_jenkinsfile.groovy | 590 ++---------------
ci/jenkins/generated/docker_jenkinsfile.groovy | 696 +++-----------------
ci/jenkins/generated/gpu_jenkinsfile.groovy | 728 +++------------------
ci/jenkins/generated/wasm_jenkinsfile.groovy | 527 ++-------------
.../scripts}/determine_docker_images.py | 7 +-
.../scripts}/git_change_docker.sh | 0
.../jenkins => jenkins/scripts}/git_skip_ci.py | 51 +-
.../scripts}/git_skip_ci_globs.py | 2 +-
.../jenkins => jenkins/scripts}/http_utils.py | 0
ci/{scripts/jenkins => jenkins/scripts}/retry.sh | 0
ci/{scripts/jenkins => jenkins/scripts}/s3.py | 45 +-
.../scripts}/should_rebuild_docker.py | 19 +-
.../jenkins/scripts/show_node_info.sh | 0
ci/jenkins/templates/arm_jenkinsfile.groovy.j2 | 34 -
ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 | 50 --
ci/jenkins/templates/docker_jenkinsfile.groovy.j2 | 202 ------
ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 | 148 -----
ci/jenkins/templates/utils/Build.groovy.j2 | 43 --
ci/jenkins/templates/utils/Prepare.groovy.j2 | 276 --------
ci/jenkins/templates/utils/base.groovy.j2 | 110 ----
ci/jenkins/templates/utils/macros.j2 | 169 -----
ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 | 37 --
ci/scripts/jenkins/cmd_utils.py | 107 ---
ci/scripts/jenkins/git_change_docs.sh | 39 --
ci/scripts/jenkins/open_docker_update_pr.py | 214 ------
ci/scripts/jenkins/should_run_slow_tests.py | 85 ---
ci/{scripts/package => wheel}/README.md | 0
.../package => wheel}/build-environment.yaml | 0
.../manylinux_build_libtvm_runtime_cuda.sh | 2 +-
.../windows_build_libtvm_runtime_cuda.bat | 4 +-
docker/bash.sh | 2 +-
docker/dev_common.sh | 2 +-
docs/contribute/ci.rst | 2 +-
docs/contribute/testing.rst | 2 +-
pyproject.toml | 2 +-
tests/python/ci/test_ci.py | 122 +---
tests/python/ci/test_jenkins_helpers.py | 80 +++
tests/python/ci/test_tvmbot.py | 6 +-
tests/python/ci/test_utils.py | 4 +-
tests/python/ci/test_wheel_artifacts.py | 40 +-
tests/scripts/ci.py | 23 +-
tests/scripts/release/gather_prs.py | 6 +-
tests/scripts/release/make_notes.py | 5 -
tests/scripts/setup-pytest-env.sh | 2 +-
tests/scripts/task_python_docs.sh | 3 -
tests/scripts/task_python_install_wheels.sh | 36 +-
tests/scripts/task_python_unittest.sh | 5 -
tests/scripts/task_python_unittest_gpuonly.sh | 4 +-
tests/scripts/task_web_wasm.sh | 6 +-
62 files changed, 794 insertions(+), 5016 deletions(-)
diff --cc ci/jenkins/README.md
index dd6b652bee,1294228054..4b98759224
--- a/ci/jenkins/README.md
+++ b/ci/jenkins/README.md
@@@ -52,4 -91,6 +91,30 @@@ No job path or controller reconfigurati
# Infrastructure
- While all TVM tests are contained within the apache/tvm repository, the infrastructure used to run the tests is donated by the TVM Community. To encourage collaboration, the configuration for TVM's CI infrastructure is stored in a public GitHub repository. TVM community members are encouraged to contribute improvements. The configuration, along with documentation of TVM's CI infrastructure, is in the [tlc-pack/ci](https://github.com/tlc-pack/ci) repo.
+ The community's infrastructure configuration and documentation live in
+ [tlc-pack/ci](https://github.com/tlc-pack/ci). Coordinate external callers when
+ changing any retained script interface.
++
++## Wheel build and test artifacts
++
++Normal builds use `uv build --wheel --no-build-isolation --offline` with the
++existing platform CMake configuration. `uv build` builds only TVM; it does not
++resolve runtime dependencies. CPU, ARM and Wasm host builds request `all;cpptest`
++and run C++ tests in that same native tree before uploading anything. Python
++consumers install the exact TVM wheel with `--no-deps --offline`, clear checkout
++path overrides, and verify the package and native-library paths. FFI comes from
++the image. GPU Java tests also receive their existing native runtime libraries;
++the separate runtime-only compile check and Emscripten runtime build remain.
++
++The selected images must already provide the requirements in `pyproject.toml`
++(including scikit-build-core, setuptools-scm and compatible apache-tvm-ffi), the
++platform runtime dependencies, and the test dependencies. The pinned image must
++be checked before rollout: the current image recipe does not provision all these
++wheel prerequisites. Missing dependencies are errors, not permission for a test
++stage to fetch or build replacements. No image publication or Jenkins deployment
++is performed by this change.
++
++`CMAKE_BUILD_PARALLEL_LEVEL` is passed through Docker as an environment variable.
++An explicit value wins; otherwise builds use at least one CPU and divide the
++node CPU count by `CI_NUM_EXECUTORS`. Both the wheel backend and specialized
++native commands use that bound without an additional parallel flag.
diff --cc ci/jenkins/data.py
index ebf29fc91b,fd75107e7d..83773ca2ea
--- a/ci/jenkins/data.py
+++ b/ci/jenkins/data.py
@@@ -31,28 -31,28 +31,10 @@@ data-driven artifact lists
import sys
files_to_stash = {
- # Exact TVM and pinned FFI wheels, installed by Python test/docs consumers.
- # Executables and build files needed to run c++ tests
- "cpptest": ["build/cpptest", "build/build.ninja", "build/CMakeFiles/rules.ninja"],
- # runtime files
- "tvm_runtime": ["build/lib/libtvm_runtime.so", "build/config.cmake"],
- # compiler files
- "tvm_lib": [
- "build/lib/libtvm_compiler.so",
- "build/lib/libtvm_runtime.so",
- "build/lib/libtvm_ffi.so",
- "build/lib/libtvm_runtime_cuda.so",
- "build/lib/libtvm_runtime_vulkan.so",
- "build/lib/libtvm_runtime_opencl.so",
- "build/lib/libtvm_runtime_rocm.so",
- "build/lib/libtvm_runtime_extra.so",
- "build/libtvm_allvisible.so",
- "build/config.cmake",
- ],
- # gpu related compiler files
- "tvm_lib_gpu_extra": [
- "build/3rdparty/libflash_attn/src/libflash_attn.so",
- "build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so",
- ],
++ # Exact TVM wheels, installed by Python test/docs consumers.
+ "wheels": ["build/wheels"],
+ # Maven's native assembly still consumes these paths directly.
+ "tvm_runtime": ["build/lib/libtvm_runtime.so", "build/lib/libtvm_ffi.so"],
- # compiler files
- "tvm_lib": [
- "build/lib/libtvm_compiler.so",
- "build/lib/libtvm_runtime.so",
- "build/lib/libtvm_ffi.so",
- "build/lib/libtvm_runtime_cuda.so",
- "build/lib/libtvm_runtime_vulkan.so",
- "build/lib/libtvm_runtime_opencl.so",
- "build/lib/libtvm_runtime_rocm.so",
- "build/lib/libtvm_runtime_extra.so",
- "build/libtvm_allvisible.so",
- "build/config.cmake",
- ],
- # gpu related compiler files
- "tvm_lib_gpu_extra": [
- "build/3rdparty/libflash_attn/src/libflash_attn.so",
- "build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so",
- ],
}
diff --cc ci/jenkins/generated/arm_jenkinsfile.groovy
index 1670f37db7,e70ce07525..9e5e4a93b8
--- a/ci/jenkins/generated/arm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/arm_jenkinsfile.groovy
@@@ -14,498 -14,53 +14,52 @@@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
- // -*- mode: groovy -*-
- // Licensed to the Apache Software Foundation (ASF) under one
- // or more contributor license agreements. See the NOTICE file
- // distributed with this work for additional information
- // regarding copyright ownership. The ASF licenses this file
- // to you under the Apache License, Version 2.0 (the
- // "License"); you may not use this file except in compliance
- // with the License. You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing,
- // software distributed under the License is distributed on an
- // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- // KIND, either express or implied. See the License for the
- // specific language governing permissions and limitations
- // under the License.
-
- // Jenkins pipeline
- // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
- // Docker env used for testing
- // Different image may have different version tag
- // because some of them are more stable than anoter.
- //
- // Docker images are maintained by PMC, cached in dockerhub
- // and remains relatively stable over the time.
- // Flow for upgrading docker env(need commiter)
- //
- // - Send PR to upgrade build script in the repo
- // - Build the new docker image
- // - Tag the docker image with a new version and push to a binary cache.
- // - Update the version in the Jenkinsfile, send a PR
- // - Fix any issues wrt to the new image version in the PR
- // - Merge the PR and now we are in new version
- // - Tag the new version as the lates
- // - Periodically cleanup the old versions on local workers
- //
-
- // ============================= IMPORTANT NOTE =============================
- // This file is generated by 'jenkins/generate.py'. Do not edit this file directly!
- // Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with
- // 'python3 jenkins/generate.py'
- // Note: This timestamp is here to ensure that updates to the Jenkinsfile are
- // always rebased on main before merging:
- // Generated at 2026-10-14T21:26:39.010162
-
- import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
- // These are set at runtime from data in ci/jenkins/docker-images.yml, update
- // image tags in that file
- ci_gpu = ''
- ci_cpu = ''
- ci_minimal = ''
- ci_wasm = ''
- ci_cortexm = ''
- ci_arm = ''
- ci_riscv = ''
-
- // Parameters to allow overriding (in Jenkins UI), the images
- // to be used by a given build. When provided, they take precedence
- // over default values above.
- properties([
- parameters([
- string(name: 'ci_arm_param', defaultValue: ''),
- string(name: 'ci_cpu_param', defaultValue: ''),
- string(name: 'ci_gpu_param', defaultValue: ''),
- string(name: 'ci_wasm_param', defaultValue: ''),
- ])
- ])
-
- // Placeholders for newly built Docker image names (if rebuild_docker_images
- // is used)
- built_ci_arm = null;
- built_ci_cpu = null;
- built_ci_gpu = null;
- built_ci_wasm = null;
-
- // Global variable assigned during Sanity Check that holds the sha1 which should be
- // merged into the PR in all branches.
- upstream_revision = null
-
- // command to start a docker container
- docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
- docker_build = 'docker/build.sh'
- // timeout in minutes
- max_time = 180
- rebuild_docker_images = false
-
- s3_bucket = 'tvm-jenkins-artifacts-prod'
- s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
-
- // Jenkins script root directory
- jenkins_scripts_root = "ci/scripts/jenkins"
-
-
- // General note: Jenkins has limits on the size of a method (or top level code)
- // that are pretty strict, so most usage of groovy methods in these templates
- // are purely to satisfy the JVM
- def per_exec_ws(folder) {
- return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
- }
-
- // initialize source codes
- def init_git() {
- retry(5) {
- checkout scm
- }
-
- // Add more info about job node
- sh (
- script: './tests/scripts/task_show_node_info.sh',
- label: 'Show executor node info',
- )
-
- // Determine merge commit to use for all stages
- if (env.CHANGE_TARGET) {
- // This is a PR build, so merge with the latest of the PR's target branch
- // (e.g. main or a release branch like v0.25.0).
- merge_with_target()
- } else {
- // This is a branch build (main or a release branch). Only set
- // upstream_revision to HEAD and skip merging to avoid a race with another
- // commit merged to the branch.
- update_upstream_revision("HEAD")
- }
-
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0
- """,
- label: 'Update git submodules',
- )
- checkout_trusted_files()
- }
-
- def update_upstream_revision(git_ref) {
- if (upstream_revision == null) {
- upstream_revision = sh(
- script: "git log -1 ${git_ref} --format=\'%H\'",
- label: 'Determine upstream revision',
- returnStdout: true,
- ).trim()
- }
- }
-
- def merge_with_target() {
- def target = env.CHANGE_TARGET
- sh (
- script: "git fetch origin ${target}",
- label: "Fetch target branch ${target}",
- )
- update_upstream_revision("FETCH_HEAD")
- sh (
- script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
- label: "Merge to origin/${target}"
- )
- }
-
- def docker_init(image) {
- // Clear out all Docker images that aren't going to be used
- sh(
- script: """
- set -eux
- docker image ls --all
- IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}')
-
- echo -e "Found images:\\n\$IMAGES"
- echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
-
- docker image ls --all
- """,
- label: 'Clean old Docker images',
- )
-
- if (image.contains("amazonaws.com")) {
- // If this string is in the image name it's from ECR and needs to be pulled
- // with the right credentials
- ecr_pull(image)
- } else {
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${image}
- """,
- label: 'Pull docker image',
- )
- }
- }
-
- def ecr_pull(full_name) {
- aws_account_id = sh(
- returnStdout: true,
- script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
- label: 'Get AWS ID'
- ).trim()
-
- try {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: '''
- set -eux
- aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
- ''',
- label: 'Log in to ECR'
- )
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${full_name}
- """,
- label: 'Pull image from ECR'
- )
- }
- } finally {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: 'docker logout $AWS_ECR_REPO',
- label: 'Clean up login credentials'
- )
+ // readTrusted pins trusted revisions, but rejects helper edits in untrusted PRs.
+ // In that case use the PR target, pinning the same commit for all source merges.
+ def helper
+ String target_revision = null
+ try {
+ helper = readTrusted('ci/jenkins/generated/common.groovy')
+ } catch (hudson.AbortException | java.io.FileNotFoundException ex) {
+ if (!env.CHANGE_TARGET) throw ex
+ echo 'Loading shared pipeline from the pinned PR target: ' + ex.toString()
+ def from_target = { label ->
+ node(label) {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/pipeline") {
+ // No PR-owned executable or credentials run during this bootstrap.
+ deleteDir()
+ retry(5) { checkout scm }
+ // Quote the branch as data and fetch only from the canonical repository.
+ def target = env.CHANGE_TARGET.replace("'", "'\"'\"'")
+ sh(script: "git fetch https://github.com/apache/tvm.git 'refs/heads/${target}'",
+ label: 'Fetch trusted pipeline target')
+ def revision = sh(script: 'git rev-parse FETCH_HEAD', returnStdout: true).trim()
+ def source = sh(script: "git show ${revision}:ci/jenkins/generated/common.groovy",
+ returnStdout: true)
+ return [revision, source]
+ }
}
}
- }
-
- def should_skip_slow_tests(pr_number) {
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
- result = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
- label: 'Check if CI should run slow tests',
- )
- }
- return result == 0
- }
-
- def cancel_previous_build() {
- // cancel previous build if it is not on main.
- if (env.BRANCH_NAME != 'main') {
- def buildNumber = env.BUILD_NUMBER as int
- // Milestone API allows us to cancel previous build
- // with the same milestone number
- if (buildNumber > 1) milestone(buildNumber - 1)
- milestone(buildNumber)
- }
- }
-
- def is_last_build() {
- // check whether it is last build
+ def pinned
try {
- return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number
- } catch (Throwable ex) {
- echo 'Error during check is_last_build ' + ex.toString()
- return false
- }
- }
-
- def checkout_trusted_files() {
- // trust everything from branch builds
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- return;
- }
-
- // trust peoople listed in CONTRIBUTING.md
- grep_code = sh(
- returnStatus: true,
- script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
- label: 'Check if change is from a contributor',
- )
-
- if (grep_code == 1) {
- // Any scripts that run on the bare host and not inside a Docker container
- // (especially those that access secrets) should be checked out here so
- // only trusted versions are used in CI
- sh(
- script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
- label: 'Check out trusted files',
- )
- }
- }
-
- def should_skip_ci(pr_number) {
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- // never skip CI on build sourced from a branch
- return false
- }
- glob_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
- label: 'Check if CI should be skipped due to changed files',
- )
- if (glob_skip_ci_code == 0) {
- return true
- }
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run full CI (or the script had an error, so run
- // full CI just in case). Exit code of 0 means skip CI.
- git_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
- label: 'Check if CI should be skipped',
- )
- }
- return git_skip_ci_code == 0
- }
-
- def prepare(node_type) {
- stage('Prepare') {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
- init_git()
-
- if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
- sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cpu ci_gpu ci_wasm ",
- label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
- )
- // Pull image names from the results of should_rebuild_docker.py
- ci_arm = sh(
- script: "cat .docker-image-names/ci_arm",
- label: "Find docker image name for ci_arm",
- returnStdout: true,
- ).trim()
- ci_cpu = sh(
- script: "cat .docker-image-names/ci_cpu",
- label: "Find docker image name for ci_cpu",
- returnStdout: true,
- ).trim()
- ci_gpu = sh(
- script: "cat .docker-image-names/ci_gpu",
- label: "Find docker image name for ci_gpu",
- returnStdout: true,
- ).trim()
- ci_wasm = sh(
- script: "cat .docker-image-names/ci_wasm",
- label: "Find docker image name for ci_wasm",
- returnStdout: true,
- ).trim()
- }
-
- ci_arm = params.ci_arm_param ?: ci_arm
- ci_cpu = params.ci_cpu_param ?: ci_cpu
- ci_gpu = params.ci_gpu_param ?: ci_gpu
- ci_wasm = params.ci_wasm_param ?: ci_wasm
-
- sh (script: """
- echo "Docker images being used in this build:"
- echo " ci_arm = ${ci_arm}"
- echo " ci_cpu = ${ci_cpu}"
- echo " ci_gpu = ${ci_gpu}"
- echo " ci_wasm = ${ci_wasm}"
- """, label: 'Docker image names')
-
- is_docs_only_build = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docs.sh",
- label: 'Check for docs only changes',
- )
- skip_ci = should_skip_ci(env.CHANGE_ID)
- skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
- rebuild_docker_images = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docker.sh",
- label: 'Check for any docker changes',
- )
-
- if (skip_ci) {
- // Don't rebuild when skipping CI
- rebuild_docker_images = false
- }
- }
- }
+ pinned = from_target('CPU-SMALL-SPOT')
+ } catch (Exception failure) {
+ pinned = from_target('CPU-SMALL')
}
+ target_revision = pinned[0]
+ helper = pinned[1]
}
- def python_unittest(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
- label: 'Run Python unit tests',
- )
- }
+ // Evaluation is outside the read/fetch fallback: script errors must propagate.
+ ci = evaluate(helper)
+ if (target_revision != null) ci.upstream_revision = target_revision
+ ci.start()
- // Keep compiler caching and executor limits identical for wheels and the
- // separate runtime-only compile check. Build commands remain at their callers.
- def build_environment() {
- return '''
- set -eux
- export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} ))
- if [ "$CMAKE_BUILD_PARALLEL_LEVEL" -lt 1 ]; then export CMAKE_BUILD_PARALLEL_LEVEL=1; fi
- if command -v sccache >/dev/null; then
- export CC=/opt/sccache/cc CXX=/opt/sccache/c++
- if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then
- export SCCACHE_BUCKET=tvm-sccache-prod SCCACHE_REGION=us-west-2
- fi
- sccache --start-server || true
- sccache --show-stats
- trap "sccache --show-stats" EXIT
- fi
- '''
- }
-
- def wheel_build(image, cpp_tests = false) {
- def commands = build_environment() + '''
- rm -rf build/wheels
- uv build --wheel --python python3 --out-dir build/wheels ./3rdparty/tvm-ffi
- uv build --wheel --python python3 --out-dir build/wheels . \
- -Ccmake.build-type=RelWithDebInfo -Cbuild.verbose="${VERBOSE:-true}"
- '''.stripIndent().trim()
- if (cpp_tests) {
- commands += " -Ccmake.define.USE_GTEST=ON -Cbuild.targets=all\\;cpptest"
- // cpptest links the same native tree that scikit-build just packaged.
- commands += "\n./tests/scripts/task_cpp_unittest.sh build"
- }
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} bash -c '${commands}'",
- label: 'Build wheel artifacts' + (cpp_tests ? ' and run C++ tests' : ''),
+ ci.build('ARM-GRAVITON3', 'BUILD: arm', 'tvm/build-arm', images.ci_arm, 'arm') {
+ sh(
+ script: "${docker_run} ${images.ci_arm} ./tests/scripts/task_config_build_arm.sh build",
+ label: 'Create ARM CMake config',
)
- ci.cmake_build(images.ci_arm, 'build')
- ci.make_cpp_tests(images.ci_arm, 'build')
- ci.upload('arm', '--bundle tvm_lib --bundle cpptest')
++ ci.wheel_build(images.ci_arm, true)
++ ci.upload('arm', '--bundle wheels')
}
-
- cancel_previous_build()
-
- try {
- prepare('CPU-SMALL-SPOT')
- } catch(Exception ex) {
- prepare('CPU-SMALL')
- }
- def run_build(node_type) {
- if (!skip_ci && is_docs_only_build != 1) {
- echo 'Begin running node_type ' + node_type
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-arm") {
- init_git()
- docker_init(ci_arm)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=arm',
- ], {
- sh (
- script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build",
- label: 'Create ARM CMake config',
- )
- wheel_build(ci_arm, true)
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/arm --bundle wheels",
- label: 'Upload artifacts to S3',
- )
- })
- }
- }
- }
- echo 'End running node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('BUILD: arm')
- }
- }
- def build() {
- stage('Build') {
- try {
- run_build('ARM-GRAVITON3-SPOT')
- } catch (hudson.AbortException abortEx) {
- echo "Received normal AbortException, exit now. Details:" + abortEx.toString()
- throw abortEx
- } catch (Throwable ex) {
- echo 'Exception during SPOT run ' + ex.toString()
- if (is_last_build()) {
- // retry if we are currently at last build
- // mark the current stage as success
- // and try again via on demand node
- echo 'Retry on-demand given it is last build'
- currentBuild.result = 'SUCCESS'
- run_build('ARM-GRAVITON3')
- } else {
- echo 'Exit since it is not last build'
- throw ex
- }
- }
- }
- }
- build()
diff --cc ci/jenkins/generated/common.groovy
index cd9861e7f6,cd8324b5f2..cebd9a8d08
--- a/ci/jenkins/generated/common.groovy
+++ b/ci/jenkins/generated/common.groovy
@@@ -411,99 -290,56 +290,69 @@@ def python_unittest(image)
)
}
- // Keep compiler caching and executor limits identical for wheels and the
- // separate runtime-only compile check. Build commands remain at their callers.
-def make_cpp_tests(image, build_dir) {
- sh (
- script: """
- set -eux
- ${docker_run} ${image} python3 ./tests/scripts/task_build.py \
- --sccache-bucket tvm-sccache-prod \
- --sccache-region us-west-2 \
- --cmake-target cpptest \
- --build-dir ${build_dir}
- """,
- label: 'Make C++ tests',
- )
++// Share only the compiler/cache environment; build commands stay at their callers.
+def build_environment() {
+ return '''
+ set -eux
- export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} ))
- if [ "$CMAKE_BUILD_PARALLEL_LEVEL" -lt 1 ]; then export CMAKE_BUILD_PARALLEL_LEVEL=1; fi
+ if command -v sccache >/dev/null; then
+ export CC=/opt/sccache/cc CXX=/opt/sccache/c++
+ if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then
+ export SCCACHE_BUCKET=tvm-sccache-prod SCCACHE_REGION=us-west-2
+ fi
+ sccache --start-server || true
+ sccache --show-stats
+ trap "sccache --show-stats" EXIT
+ fi
+ '''
}
-def cmake_build(image, path) {
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir ${path}",
- label: 'Run CMake build',
- )
++// Preserve explicit caller limits, otherwise divide this node among its executors.
++def build_docker() {
++ return '''set -eu
++ parallel=${CMAKE_BUILD_PARALLEL_LEVEL:-$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} ))}
++ if [ "$parallel" -lt 1 ]; then parallel=1; fi
++ ''' + "${docker_run} --env CMAKE_BUILD_PARALLEL_LEVEL=\"\$parallel\" "
+ }
-def cpp_unittest(image) {
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh",
- label: 'Run C++ tests',
- )
++
+def wheel_build(image, cpp_tests = false) {
++ // uv build never resolves runtime dependencies. No isolation plus offline uses
++ // only the backend and FFI already provisioned in the image.
+ def commands = build_environment() + '''
+ rm -rf build/wheels
- uv build --wheel --python python3 --out-dir build/wheels ./3rdparty/tvm-ffi
- uv build --wheel --python python3 --out-dir build/wheels . \
- -Ccmake.build-type=RelWithDebInfo -Cbuild.verbose="${VERBOSE:-true}"
++ uv build --wheel --no-build-isolation --offline --python python3 --out-dir build/wheels . \
++ -Cbuild-dir=build -Ccmake.build-type=RelWithDebInfo -Cbuild.verbose="${VERBOSE:-true}"
+ '''.stripIndent().trim()
+ if (cpp_tests) {
+ commands += " -Ccmake.define.USE_GTEST=ON -Cbuild.targets=all\\;cpptest"
- // cpptest links the same native tree that scikit-build just packaged.
+ commands += "\n./tests/scripts/task_cpp_unittest.sh build"
+ }
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} bash -c '${commands}'",
- label: 'Build wheel artifacts' + (cpp_tests ? ' and run C++ tests' : ''),
- )
++ sh(script: build_docker() + "${image} bash -c '${commands}'",
++ label: 'Build TVM wheel' + (cpp_tests ? ' and run C++ tests' : ''))
}
- cancel_previous_build()
-
- try {
+ // Preparation retries Exceptions unconditionally; Errors and cancellation of the
+ // previous build are deliberately outside this fallback.
+ def start() {
+ cancel_previous_build()
+ try {
prepare('CPU-SMALL-SPOT')
- } catch(Exception ex) {
- prepare('CPU-SMALL')
- }
- def run_build(node_type) {
- if (!skip_ci && is_docs_only_build != 1) {
- echo 'Begin running node_type ' + node_type
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") {
- init_git()
- docker_init(ci_wasm)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=wasm',
- ], {
- sh (
- script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build",
- label: 'Create WASM CMake config',
- )
- wheel_build(ci_wasm, true)
- sh (
- script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh",
- label: 'Run WASM lint and tests',
- )
- })
- }
- }
- }
- echo 'End running node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('BUILD: WASM')
+ } catch (Exception ex) {
+ prepare('CPU-SMALL')
}
}
- def build() {
+
+ // A failed build command is final. Infrastructure failures retry only while
+ // this is the newest build; tests intentionally also retry AbortException.
+ def build(label, name, folder, image, platform, body) {
stage('Build') {
try {
- run_build('CPU-SPOT')
- } catch (hudson.AbortException abortEx) {
- echo "Received normal AbortException, exit now. Details:" + abortEx.toString()
- throw abortEx
+ run_on_node("${label}-SPOT", name, folder, image, ["PLATFORM=${platform}"], body)
+ } catch (hudson.AbortException ex) {
+ echo 'Received normal AbortException, exit now. Details:' + ex.toString()
+ throw ex
} catch (Throwable ex) {
- echo 'Exception during SPOT run ' + ex.toString()
- if (is_last_build()) {
- // retry if we are currently at last build
- // mark the current stage as success
- // and try again via on demand node
- echo 'Retry on-demand given it is last build'
- currentBuild.result = 'SUCCESS'
- run_build('CPU')
- } else {
- echo 'Exit since it is not last build'
- throw ex
+ retry_latest(ex) {
+ run_on_node(label, name, folder, image, ["PLATFORM=${platform}"], body)
}
}
}
diff --cc ci/jenkins/generated/cpu_jenkinsfile.groovy
index 56255dc5fb,e252832c65..c91c3571a5
--- a/ci/jenkins/generated/cpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy
@@@ -14,559 -14,63 +14,61 @@@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
- // -*- mode: groovy -*-
-
- // Licensed to the Apache Software Foundation (ASF) under one
- // or more contributor license agreements. See the NOTICE file
- // distributed with this work for additional information
- // regarding copyright ownership. The ASF licenses this file
- // to you under the Apache License, Version 2.0 (the
- // "License"); you may not use this file except in compliance
- // with the License. You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing,
- // software distributed under the License is distributed on an
- // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- // KIND, either express or implied. See the License for the
- // specific language governing permissions and limitations
- // under the License.
-
- // Jenkins pipeline
- // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
- // Docker env used for testing
- // Different image may have different version tag
- // because some of them are more stable than anoter.
- //
- // Docker images are maintained by PMC, cached in dockerhub
- // and remains relatively stable over the time.
- // Flow for upgrading docker env(need commiter)
- //
- // - Send PR to upgrade build script in the repo
- // - Build the new docker image
- // - Tag the docker image with a new version and push to a binary cache.
- // - Update the version in the Jenkinsfile, send a PR
- // - Fix any issues wrt to the new image version in the PR
- // - Merge the PR and now we are in new version
- // - Tag the new version as the lates
- // - Periodically cleanup the old versions on local workers
- //
-
- // ============================= IMPORTANT NOTE =============================
- // This file is generated by 'jenkins/generate.py'. Do not edit this file directly!
- // Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with
- // 'python3 jenkins/generate.py'
- // Note: This timestamp is here to ensure that updates to the Jenkinsfile are
- // always rebased on main before merging:
- // Generated at 2026-10-14T21:26:39.031061
-
- import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
- // These are set at runtime from data in ci/jenkins/docker-images.yml, update
- // image tags in that file
- ci_gpu = ''
- ci_cpu = ''
- ci_minimal = ''
- ci_wasm = ''
- ci_cortexm = ''
- ci_arm = ''
- ci_riscv = ''
-
- // Parameters to allow overriding (in Jenkins UI), the images
- // to be used by a given build. When provided, they take precedence
- // over default values above.
- properties([
- parameters([
- string(name: 'ci_arm_param', defaultValue: ''),
- string(name: 'ci_cpu_param', defaultValue: ''),
- string(name: 'ci_gpu_param', defaultValue: ''),
- string(name: 'ci_wasm_param', defaultValue: ''),
- ])
- ])
-
- // Placeholders for newly built Docker image names (if rebuild_docker_images
- // is used)
- built_ci_arm = null;
- built_ci_cpu = null;
- built_ci_gpu = null;
- built_ci_wasm = null;
-
- // Global variable assigned during Sanity Check that holds the sha1 which should be
- // merged into the PR in all branches.
- upstream_revision = null
-
- // command to start a docker container
- docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
- docker_build = 'docker/build.sh'
- // timeout in minutes
- max_time = 180
- rebuild_docker_images = false
-
- s3_bucket = 'tvm-jenkins-artifacts-prod'
- s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
-
- // Jenkins script root directory
- jenkins_scripts_root = "ci/scripts/jenkins"
-
-
- // General note: Jenkins has limits on the size of a method (or top level code)
- // that are pretty strict, so most usage of groovy methods in these templates
- // are purely to satisfy the JVM
- def per_exec_ws(folder) {
- return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
- }
-
- // initialize source codes
- def init_git() {
- retry(5) {
- checkout scm
- }
-
- // Add more info about job node
- sh (
- script: './tests/scripts/task_show_node_info.sh',
- label: 'Show executor node info',
- )
-
- // Determine merge commit to use for all stages
- if (env.CHANGE_TARGET) {
- // This is a PR build, so merge with the latest of the PR's target branch
- // (e.g. main or a release branch like v0.25.0).
- merge_with_target()
- } else {
- // This is a branch build (main or a release branch). Only set
- // upstream_revision to HEAD and skip merging to avoid a race with another
- // commit merged to the branch.
- update_upstream_revision("HEAD")
- }
-
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0
- """,
- label: 'Update git submodules',
- )
- checkout_trusted_files()
- }
-
- def update_upstream_revision(git_ref) {
- if (upstream_revision == null) {
- upstream_revision = sh(
- script: "git log -1 ${git_ref} --format=\'%H\'",
- label: 'Determine upstream revision',
- returnStdout: true,
- ).trim()
- }
- }
-
- def merge_with_target() {
- def target = env.CHANGE_TARGET
- sh (
- script: "git fetch origin ${target}",
- label: "Fetch target branch ${target}",
- )
- update_upstream_revision("FETCH_HEAD")
- sh (
- script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
- label: "Merge to origin/${target}"
- )
- }
-
- def docker_init(image) {
- // Clear out all Docker images that aren't going to be used
- sh(
- script: """
- set -eux
- docker image ls --all
- IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}')
-
- echo -e "Found images:\\n\$IMAGES"
- echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
-
- docker image ls --all
- """,
- label: 'Clean old Docker images',
- )
-
- if (image.contains("amazonaws.com")) {
- // If this string is in the image name it's from ECR and needs to be pulled
- // with the right credentials
- ecr_pull(image)
- } else {
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${image}
- """,
- label: 'Pull docker image',
- )
- }
- }
-
- def ecr_pull(full_name) {
- aws_account_id = sh(
- returnStdout: true,
- script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
- label: 'Get AWS ID'
- ).trim()
-
- try {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: '''
- set -eux
- aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
- ''',
- label: 'Log in to ECR'
- )
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${full_name}
- """,
- label: 'Pull image from ECR'
- )
- }
- } finally {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: 'docker logout $AWS_ECR_REPO',
- label: 'Clean up login credentials'
- )
- }
- }
- }
-
- def should_skip_slow_tests(pr_number) {
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
- result = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
- label: 'Check if CI should run slow tests',
- )
- }
- return result == 0
- }
-
- def cancel_previous_build() {
- // cancel previous build if it is not on main.
- if (env.BRANCH_NAME != 'main') {
- def buildNumber = env.BUILD_NUMBER as int
- // Milestone API allows us to cancel previous build
- // with the same milestone number
- if (buildNumber > 1) milestone(buildNumber - 1)
- milestone(buildNumber)
- }
- }
-
- def is_last_build() {
- // check whether it is last build
- try {
- return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number
- } catch (Throwable ex) {
- echo 'Error during check is_last_build ' + ex.toString()
- return false
- }
- }
-
- def checkout_trusted_files() {
- // trust everything from branch builds
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- return;
- }
-
- // trust peoople listed in CONTRIBUTING.md
- grep_code = sh(
- returnStatus: true,
- script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
- label: 'Check if change is from a contributor',
- )
-
- if (grep_code == 1) {
- // Any scripts that run on the bare host and not inside a Docker container
- // (especially those that access secrets) should be checked out here so
- // only trusted versions are used in CI
- sh(
- script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
- label: 'Check out trusted files',
- )
- }
- }
-
- def should_skip_ci(pr_number) {
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- // never skip CI on build sourced from a branch
- return false
- }
- glob_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
- label: 'Check if CI should be skipped due to changed files',
- )
- if (glob_skip_ci_code == 0) {
- return true
- }
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run full CI (or the script had an error, so run
- // full CI just in case). Exit code of 0 means skip CI.
- git_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
- label: 'Check if CI should be skipped',
- )
- }
- return git_skip_ci_code == 0
- }
-
- def prepare(node_type) {
- stage('Prepare') {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
- init_git()
-
- if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
- sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cpu ci_gpu ci_wasm ",
- label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
- )
- // Pull image names from the results of should_rebuild_docker.py
- ci_arm = sh(
- script: "cat .docker-image-names/ci_arm",
- label: "Find docker image name for ci_arm",
- returnStdout: true,
- ).trim()
- ci_cpu = sh(
- script: "cat .docker-image-names/ci_cpu",
- label: "Find docker image name for ci_cpu",
- returnStdout: true,
- ).trim()
- ci_gpu = sh(
- script: "cat .docker-image-names/ci_gpu",
- label: "Find docker image name for ci_gpu",
- returnStdout: true,
- ).trim()
- ci_wasm = sh(
- script: "cat .docker-image-names/ci_wasm",
- label: "Find docker image name for ci_wasm",
- returnStdout: true,
- ).trim()
- }
-
- ci_arm = params.ci_arm_param ?: ci_arm
- ci_cpu = params.ci_cpu_param ?: ci_cpu
- ci_gpu = params.ci_gpu_param ?: ci_gpu
- ci_wasm = params.ci_wasm_param ?: ci_wasm
-
- sh (script: """
- echo "Docker images being used in this build:"
- echo " ci_arm = ${ci_arm}"
- echo " ci_cpu = ${ci_cpu}"
- echo " ci_gpu = ${ci_gpu}"
- echo " ci_wasm = ${ci_wasm}"
- """, label: 'Docker image names')
-
- is_docs_only_build = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docs.sh",
- label: 'Check for docs only changes',
- )
- skip_ci = should_skip_ci(env.CHANGE_ID)
- skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
- rebuild_docker_images = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docker.sh",
- label: 'Check for any docker changes',
- )
-
- if (skip_ci) {
- // Don't rebuild when skipping CI
- rebuild_docker_images = false
- }
- }
- }
- }
- }
- def python_unittest(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
- label: 'Run Python unit tests',
- )
- }
-
- // Keep compiler caching and executor limits identical for wheels and the
- // separate runtime-only compile check. Build commands remain at their callers.
- def build_environment() {
- return '''
- set -eux
- export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} ))
- if [ "$CMAKE_BUILD_PARALLEL_LEVEL" -lt 1 ]; then export CMAKE_BUILD_PARALLEL_LEVEL=1; fi
- if command -v sccache >/dev/null; then
- export CC=/opt/sccache/cc CXX=/opt/sccache/c++
- if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then
- export SCCACHE_BUCKET=tvm-sccache-prod SCCACHE_REGION=us-west-2
- fi
- sccache --start-server || true
- sccache --show-stats
- trap "sccache --show-stats" EXIT
- fi
- '''
- }
-
- def wheel_build(image, cpp_tests = false) {
- def commands = build_environment() + '''
- rm -rf build/wheels
- uv build --wheel --python python3 --out-dir build/wheels ./3rdparty/tvm-ffi
- uv build --wheel --python python3 --out-dir build/wheels . \
- -Ccmake.build-type=RelWithDebInfo -Cbuild.verbose="${VERBOSE:-true}"
- '''.stripIndent().trim()
- if (cpp_tests) {
- commands += " -Ccmake.define.USE_GTEST=ON -Cbuild.targets=all\\;cpptest"
- // cpptest links the same native tree that scikit-build just packaged.
- commands += "\n./tests/scripts/task_cpp_unittest.sh build"
- }
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} bash -c '${commands}'",
- label: 'Build wheel artifacts' + (cpp_tests ? ' and run C++ tests' : ''),
- )
- }
-
- cancel_previous_build()
+ // readTrusted pins trusted revisions, but rejects helper edits in untrusted PRs.
+ // In that case use the PR target, pinning the same commit for all source merges.
+ def helper
+ String target_revision = null
try {
- prepare('CPU-SMALL-SPOT')
- } catch(Exception ex) {
- prepare('CPU-SMALL')
- }
- def run_build(node_type) {
- if (!skip_ci && is_docs_only_build != 1) {
- echo 'Begin running node_type ' + node_type
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") {
- init_git()
- docker_init(ci_cpu)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=cpu',
- ], {
- sh (
- script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build",
- label: 'Create CPU CMake config',
- )
- wheel_build(ci_cpu, true)
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu --bundle wheels",
- label: 'Upload artifacts to S3',
- )
- })
- }
+ helper = readTrusted('ci/jenkins/generated/common.groovy')
+ } catch (hudson.AbortException | java.io.FileNotFoundException ex) {
+ if (!env.CHANGE_TARGET) throw ex
+ echo 'Loading shared pipeline from the pinned PR target: ' + ex.toString()
+ def from_target = { label ->
+ node(label) {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/pipeline") {
+ // No PR-owned executable or credentials run during this bootstrap.
+ deleteDir()
+ retry(5) { checkout scm }
+ // Quote the branch as data and fetch only from the canonical repository.
+ def target = env.CHANGE_TARGET.replace("'", "'\"'\"'")
+ sh(script: "git fetch https://github.com/apache/tvm.git 'refs/heads/${target}'",
+ label: 'Fetch trusted pipeline target')
+ def revision = sh(script: 'git rev-parse FETCH_HEAD', returnStdout: true).trim()
+ def source = sh(script: "git show ${revision}:ci/jenkins/generated/common.groovy",
+ returnStdout: true)
+ return [revision, source]
}
}
- echo 'End running node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('BUILD: CPU')
}
- }
- def build() {
- stage('Build') {
- try {
- run_build('CPU-SPOT')
- } catch (hudson.AbortException abortEx) {
- echo "Received normal AbortException, exit now. Details:" + abortEx.toString()
- throw abortEx
- } catch (Throwable ex) {
- echo 'Exception during SPOT run ' + ex.toString()
- if (is_last_build()) {
- // retry if we are currently at last build
- // mark the current stage as success
- // and try again via on demand node
- echo 'Retry on-demand given it is last build'
- currentBuild.result = 'SUCCESS'
- run_build('CPU')
- } else {
- echo 'Exit since it is not last build'
- throw ex
- }
- }
+ def pinned
+ try {
+ pinned = from_target('CPU-SMALL-SPOT')
+ } catch (Exception failure) {
+ pinned = from_target('CPU-SMALL')
}
+ target_revision = pinned[0]
+ helper = pinned[1]
}
- build()
+ // Evaluation is outside the read/fetch fallback: script errors must propagate.
+ ci = evaluate(helper)
+ if (target_revision != null) ci.upstream_revision = target_revision
+ ci.start()
-
-
- def run_unittest_CPU(node_type) {
- echo 'Begin running on node_type ' + node_type
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") {
- // NOTE: if exception happens, it will be caught outside
- init_git()
- docker_init(ci_cpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cpu',
- 'TEST_STEP_NAME=unittest: CPU',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
- label: 'Download artifacts from S3',
- )
-
- python_unittest(ci_cpu)
- })
- }
- }
- }
- echo 'End running on node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('unittest: CPU')
- }
+ ci.build('CPU', 'BUILD: CPU', 'tvm/build-cpu', images.ci_cpu, 'cpu') {
+ sh(
+ script: "${docker_run} ${images.ci_cpu} ./tests/scripts/task_config_build_cpu.sh build",
+ label: 'Create CPU CMake config',
+ )
- ci.cmake_build(images.ci_cpu, 'build')
- ci.make_cpp_tests(images.ci_cpu, 'build')
- ci.upload('cpu', '--bundle tvm_lib --bundle cpptest')
++ ci.wheel_build(images.ci_cpu, true)
++ ci.upload('cpu', '--bundle wheels')
}
-
- def test() {
- stage('Test') {
- environment {
- SKIP_SLOW_TESTS = "${skip_slow_tests}"
+ stage('Test') {
+ parallel('unittest: CPU': {
+ ci.test('CPU-SMALL', 'unittest: CPU', 'tvm/ut-python-cpu', images.ci_cpu, 'cpu') {
+ ci.download('cpu')
- ci.cpp_unittest(images.ci_cpu)
+ ci.python_unittest(images.ci_cpu)
}
- parallel(
- 'unittest: CPU': {
- try {
- run_unittest_CPU('CPU-SMALL-SPOT')
- } catch (Throwable ex) {
- echo 'Exception during SPOT run ' + ex.toString()
- if (is_last_build()) {
- // retry if at last build
- // mark the current stage as success
- // and try again via on demand node
- echo 'Retry on-demand given it is last build'
- currentBuild.result = 'SUCCESS'
- run_unittest_CPU('CPU-SMALL')
- } else {
- echo 'Exit since it is not last build'
- throw ex
- }
- }
- },
- )
- }
+ })
}
- test()
diff --cc ci/jenkins/generated/gpu_jenkinsfile.groovy
index 2421da1881,8c98cd16cb..0f81b529b6
--- a/ci/jenkins/generated/gpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy
@@@ -14,613 -14,71 +14,75 @@@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
- // -*- mode: groovy -*-
-
- // Licensed to the Apache Software Foundation (ASF) under one
- // or more contributor license agreements. See the NOTICE file
- // distributed with this work for additional information
- // regarding copyright ownership. The ASF licenses this file
- // to you under the Apache License, Version 2.0 (the
- // "License"); you may not use this file except in compliance
- // with the License. You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing,
- // software distributed under the License is distributed on an
- // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- // KIND, either express or implied. See the License for the
- // specific language governing permissions and limitations
- // under the License.
-
- // Jenkins pipeline
- // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
- // Docker env used for testing
- // Different image may have different version tag
- // because some of them are more stable than anoter.
- //
- // Docker images are maintained by PMC, cached in dockerhub
- // and remains relatively stable over the time.
- // Flow for upgrading docker env(need commiter)
- //
- // - Send PR to upgrade build script in the repo
- // - Build the new docker image
- // - Tag the docker image with a new version and push to a binary cache.
- // - Update the version in the Jenkinsfile, send a PR
- // - Fix any issues wrt to the new image version in the PR
- // - Merge the PR and now we are in new version
- // - Tag the new version as the lates
- // - Periodically cleanup the old versions on local workers
- //
-
- // ============================= IMPORTANT NOTE =============================
- // This file is generated by 'jenkins/generate.py'. Do not edit this file directly!
- // Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with
- // 'python3 jenkins/generate.py'
- // Note: This timestamp is here to ensure that updates to the Jenkinsfile are
- // always rebased on main before merging:
- // Generated at 2026-10-14T21:26:39.068100
-
- import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
- // These are set at runtime from data in ci/jenkins/docker-images.yml, update
- // image tags in that file
- ci_gpu = ''
- ci_cpu = ''
- ci_minimal = ''
- ci_wasm = ''
- ci_cortexm = ''
- ci_arm = ''
- ci_riscv = ''
-
- // Parameters to allow overriding (in Jenkins UI), the images
- // to be used by a given build. When provided, they take precedence
- // over default values above.
- properties([
- parameters([
- string(name: 'ci_arm_param', defaultValue: ''),
- string(name: 'ci_cpu_param', defaultValue: ''),
- string(name: 'ci_gpu_param', defaultValue: ''),
- string(name: 'ci_wasm_param', defaultValue: ''),
- ])
- ])
-
- // Placeholders for newly built Docker image names (if rebuild_docker_images
- // is used)
- built_ci_arm = null;
- built_ci_cpu = null;
- built_ci_gpu = null;
- built_ci_wasm = null;
-
- // Global variable assigned during Sanity Check that holds the sha1 which should be
- // merged into the PR in all branches.
- upstream_revision = null
-
- // command to start a docker container
- docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
- docker_build = 'docker/build.sh'
- // timeout in minutes
- max_time = 180
- rebuild_docker_images = false
-
- s3_bucket = 'tvm-jenkins-artifacts-prod'
- s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
-
- // Jenkins script root directory
- jenkins_scripts_root = "ci/scripts/jenkins"
-
-
- // General note: Jenkins has limits on the size of a method (or top level code)
- // that are pretty strict, so most usage of groovy methods in these templates
- // are purely to satisfy the JVM
- def per_exec_ws(folder) {
- return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
- }
-
- // initialize source codes
- def init_git() {
- retry(5) {
- checkout scm
- }
-
- // Add more info about job node
- sh (
- script: './tests/scripts/task_show_node_info.sh',
- label: 'Show executor node info',
- )
-
- // Determine merge commit to use for all stages
- if (env.CHANGE_TARGET) {
- // This is a PR build, so merge with the latest of the PR's target branch
- // (e.g. main or a release branch like v0.25.0).
- merge_with_target()
- } else {
- // This is a branch build (main or a release branch). Only set
- // upstream_revision to HEAD and skip merging to avoid a race with another
- // commit merged to the branch.
- update_upstream_revision("HEAD")
- }
-
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0
- """,
- label: 'Update git submodules',
- )
- checkout_trusted_files()
- }
-
- def update_upstream_revision(git_ref) {
- if (upstream_revision == null) {
- upstream_revision = sh(
- script: "git log -1 ${git_ref} --format=\'%H\'",
- label: 'Determine upstream revision',
- returnStdout: true,
- ).trim()
- }
- }
-
- def merge_with_target() {
- def target = env.CHANGE_TARGET
- sh (
- script: "git fetch origin ${target}",
- label: "Fetch target branch ${target}",
- )
- update_upstream_revision("FETCH_HEAD")
- sh (
- script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
- label: "Merge to origin/${target}"
- )
- }
-
- def docker_init(image) {
- // Clear out all Docker images that aren't going to be used
- sh(
- script: """
- set -eux
- docker image ls --all
- IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}')
-
- echo -e "Found images:\\n\$IMAGES"
- echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
-
- docker image ls --all
- """,
- label: 'Clean old Docker images',
- )
-
- if (image.contains("amazonaws.com")) {
- // If this string is in the image name it's from ECR and needs to be pulled
- // with the right credentials
- ecr_pull(image)
- } else {
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${image}
- """,
- label: 'Pull docker image',
- )
- }
- }
-
- def ecr_pull(full_name) {
- aws_account_id = sh(
- returnStdout: true,
- script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
- label: 'Get AWS ID'
- ).trim()
-
- try {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: '''
- set -eux
- aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
- ''',
- label: 'Log in to ECR'
- )
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${full_name}
- """,
- label: 'Pull image from ECR'
- )
- }
- } finally {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: 'docker logout $AWS_ECR_REPO',
- label: 'Clean up login credentials'
- )
- }
- }
- }
-
- def should_skip_slow_tests(pr_number) {
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
- result = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
- label: 'Check if CI should run slow tests',
- )
- }
- return result == 0
- }
-
- def cancel_previous_build() {
- // cancel previous build if it is not on main.
- if (env.BRANCH_NAME != 'main') {
- def buildNumber = env.BUILD_NUMBER as int
- // Milestone API allows us to cancel previous build
- // with the same milestone number
- if (buildNumber > 1) milestone(buildNumber - 1)
- milestone(buildNumber)
- }
- }
-
- def is_last_build() {
- // check whether it is last build
- try {
- return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number
- } catch (Throwable ex) {
- echo 'Error during check is_last_build ' + ex.toString()
- return false
- }
- }
-
- def checkout_trusted_files() {
- // trust everything from branch builds
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- return;
- }
-
- // trust peoople listed in CONTRIBUTING.md
- grep_code = sh(
- returnStatus: true,
- script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
- label: 'Check if change is from a contributor',
- )
-
- if (grep_code == 1) {
- // Any scripts that run on the bare host and not inside a Docker container
- // (especially those that access secrets) should be checked out here so
- // only trusted versions are used in CI
- sh(
- script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
- label: 'Check out trusted files',
- )
- }
- }
-
- def should_skip_ci(pr_number) {
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- // never skip CI on build sourced from a branch
- return false
- }
- glob_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
- label: 'Check if CI should be skipped due to changed files',
- )
- if (glob_skip_ci_code == 0) {
- return true
- }
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run full CI (or the script had an error, so run
- // full CI just in case). Exit code of 0 means skip CI.
- git_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
- label: 'Check if CI should be skipped',
- )
- }
- return git_skip_ci_code == 0
- }
-
- def prepare(node_type) {
- stage('Prepare') {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
- init_git()
-
- if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
- sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cpu ci_gpu ci_wasm ",
- label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
- )
- // Pull image names from the results of should_rebuild_docker.py
- ci_arm = sh(
- script: "cat .docker-image-names/ci_arm",
- label: "Find docker image name for ci_arm",
- returnStdout: true,
- ).trim()
- ci_cpu = sh(
- script: "cat .docker-image-names/ci_cpu",
- label: "Find docker image name for ci_cpu",
- returnStdout: true,
- ).trim()
- ci_gpu = sh(
- script: "cat .docker-image-names/ci_gpu",
- label: "Find docker image name for ci_gpu",
- returnStdout: true,
- ).trim()
- ci_wasm = sh(
- script: "cat .docker-image-names/ci_wasm",
- label: "Find docker image name for ci_wasm",
- returnStdout: true,
- ).trim()
- }
-
- ci_arm = params.ci_arm_param ?: ci_arm
- ci_cpu = params.ci_cpu_param ?: ci_cpu
- ci_gpu = params.ci_gpu_param ?: ci_gpu
- ci_wasm = params.ci_wasm_param ?: ci_wasm
-
- sh (script: """
- echo "Docker images being used in this build:"
- echo " ci_arm = ${ci_arm}"
- echo " ci_cpu = ${ci_cpu}"
- echo " ci_gpu = ${ci_gpu}"
- echo " ci_wasm = ${ci_wasm}"
- """, label: 'Docker image names')
-
- is_docs_only_build = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docs.sh",
- label: 'Check for docs only changes',
- )
- skip_ci = should_skip_ci(env.CHANGE_ID)
- skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
- rebuild_docker_images = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docker.sh",
- label: 'Check for any docker changes',
- )
-
- if (skip_ci) {
- // Don't rebuild when skipping CI
- rebuild_docker_images = false
- }
- }
- }
- }
- }
- def python_unittest(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
- label: 'Run Python unit tests',
- )
- }
-
- // Keep compiler caching and executor limits identical for wheels and the
- // separate runtime-only compile check. Build commands remain at their callers.
- def build_environment() {
- return '''
- set -eux
- export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} ))
- if [ "$CMAKE_BUILD_PARALLEL_LEVEL" -lt 1 ]; then export CMAKE_BUILD_PARALLEL_LEVEL=1; fi
- if command -v sccache >/dev/null; then
- export CC=/opt/sccache/cc CXX=/opt/sccache/c++
- if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then
- export SCCACHE_BUCKET=tvm-sccache-prod SCCACHE_REGION=us-west-2
- fi
- sccache --start-server || true
- sccache --show-stats
- trap "sccache --show-stats" EXIT
- fi
- '''
- }
-
- def wheel_build(image, cpp_tests = false) {
- def commands = build_environment() + '''
- rm -rf build/wheels
- uv build --wheel --python python3 --out-dir build/wheels ./3rdparty/tvm-ffi
- uv build --wheel --python python3 --out-dir build/wheels . \
- -Ccmake.build-type=RelWithDebInfo -Cbuild.verbose="${VERBOSE:-true}"
- '''.stripIndent().trim()
- if (cpp_tests) {
- commands += " -Ccmake.define.USE_GTEST=ON -Cbuild.targets=all\\;cpptest"
- // cpptest links the same native tree that scikit-build just packaged.
- commands += "\n./tests/scripts/task_cpp_unittest.sh build"
- }
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} bash -c '${commands}'",
- label: 'Build wheel artifacts' + (cpp_tests ? ' and run C++ tests' : ''),
- )
- }
-
- cancel_previous_build()
+ // readTrusted pins trusted revisions, but rejects helper edits in untrusted PRs.
+ // In that case use the PR target, pinning the same commit for all source merges.
+ def helper
+ String target_revision = null
try {
- prepare('CPU-SMALL-SPOT')
- } catch(Exception ex) {
- prepare('CPU-SMALL')
- }
- def run_build(node_type) {
- if (!skip_ci) {
- echo 'Begin running node_type ' + node_type
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=gpu',
- ], {
- sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
- wheel_build("${ci_gpu} --no-gpu")
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu --bundle wheels --bundle tvm_runtime",
- label: 'Upload artifacts to S3',
- )
-
-
- // compiler test
- sh "rm -rf build"
- sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build"
- def runtime_commands = build_environment() + '''
- cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo
- cmake --build build --target tvm_runtime --verbose
- '''
- sh "${docker_run} --env CI_NUM_EXECUTORS ${ci_gpu} --no-gpu bash -c '${runtime_commands}'"
- })
- }
+ helper = readTrusted('ci/jenkins/generated/common.groovy')
+ } catch (hudson.AbortException | java.io.FileNotFoundException ex) {
+ if (!env.CHANGE_TARGET) throw ex
+ echo 'Loading shared pipeline from the pinned PR target: ' + ex.toString()
+ def from_target = { label ->
+ node(label) {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/pipeline") {
+ // No PR-owned executable or credentials run during this bootstrap.
+ deleteDir()
+ retry(5) { checkout scm }
+ // Quote the branch as data and fetch only from the canonical repository.
+ def target = env.CHANGE_TARGET.replace("'", "'\"'\"'")
+ sh(script: "git fetch https://github.com/apache/tvm.git 'refs/heads/${target}'",
+ label: 'Fetch trusted pipeline target')
+ def revision = sh(script: 'git rev-parse FETCH_HEAD', returnStdout: true).trim()
+ def source = sh(script: "git show ${revision}:ci/jenkins/generated/common.groovy",
+ returnStdout: true)
+ return [revision, source]
}
}
- echo 'End running node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('BUILD: GPU')
- }
- }
- def build() {
- stage('Build') {
- try {
- run_build('CPU-SPOT')
- } catch (hudson.AbortException abortEx) {
- echo "Received normal AbortException, exit now. Details:" + abortEx.toString()
- throw abortEx
- } catch (Throwable ex) {
- echo 'Exception during SPOT run ' + ex.toString()
- if (is_last_build()) {
- // retry if we are currently at last build
- // mark the current stage as success
- // and try again via on demand node
- echo 'Retry on-demand given it is last build'
- currentBuild.result = 'SUCCESS'
- run_build('CPU')
- } else {
- echo 'Exit since it is not last build'
- throw ex
- }
- }
- }
- }
- build()
-
-
- def run_unittest_GPU(node_type) {
- echo 'Begin running on node_type ' + node_type
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
- // NOTE: if exception happens, it will be caught outside
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=unittest: GPU',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
- label: 'Download artifacts from S3',
- )
-
- sh (
- script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh",
- label: 'Run Java unit tests',
- )
- sh (
- script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh",
- label: 'Run Python GPU unit tests',
- )
- })
- }
- }
- }
- echo 'End running on node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('unittest: GPU')
- }
- }
-
-
-
- def run_docs_GPU(node_type) {
- echo 'Begin running on node_type ' + node_type
- if (!skip_ci) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/docs-python-gpu") {
- // NOTE: if exception happens, it will be caught outside
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=docs: GPU',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
- label: 'Download artifacts from S3',
- )
-
- sh (
- script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh",
- label: 'Build docs',
- )
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/docs --items docs.tgz",
- label: 'Upload artifacts to S3',
- )
-
- sh(
- script: "aws s3 cp --no-progress _docs s3://${s3_bucket}/${s3_prefix}/docs --recursive",
- label: 'Upload docs to S3',
- )
- })
- }
- }
- }
- echo 'End running on node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('docs: GPU')
}
+ def pinned
+ try {
+ pinned = from_target('CPU-SMALL-SPOT')
+ } catch (Exception failure) {
+ pinned = from_target('CPU-SMALL')
+ }
+ target_revision = pinned[0]
+ helper = pinned[1]
+ }
+ // Evaluation is outside the read/fetch fallback: script errors must propagate.
+ ci = evaluate(helper)
+ if (target_revision != null) ci.upstream_revision = target_revision
+ ci.start()
+
+ ci.build('CPU', 'BUILD: GPU', 'tvm/build-gpu', images.ci_gpu, 'gpu') {
+ sh "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
- ci.cmake_build("${images.ci_gpu} --no-gpu", 'build')
- ci.upload('gpu', '--bundle tvm_lib --bundle tvm_lib_gpu_extra')
++ ci.wheel_build("--no-gpu ${images.ci_gpu}")
++ ci.upload('gpu', '--bundle wheels --bundle tvm_runtime')
+
+ // Check the compiler-only GPU configuration after publishing the test build.
+ sh 'rm -rf build'
+ sh "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build"
- sh "${docker_run} --env CI_NUM_EXECUTORS ${images.ci_gpu} --no-gpu ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --cmake-target tvm_runtime --build-dir build"
++ def commands = ci.build_environment() + '''
++ cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo
++ cmake --build build --target tvm_runtime
++ '''
++ sh(ci.build_docker() + "--no-gpu ${images.ci_gpu} bash -c '${commands}'")
}
-
-
- def test() {
- stage('Test') {
- environment {
- SKIP_SLOW_TESTS = "${skip_slow_tests}"
- }
- parallel(
+ stage('Test') {
+ parallel(
'unittest: GPU': {
- try {
- run_unittest_GPU('GPU-SPOT')
- } catch (Throwable ex) {
- echo 'Exception during SPOT run ' + ex.toString()
- if (is_last_build()) {
- // retry if at last build
- // mark the current stage as success
- // and try again via on demand node
- echo 'Retry on-demand given it is last build'
- currentBuild.result = 'SUCCESS'
- run_unittest_GPU('GPU')
- } else {
- echo 'Exit since it is not last build'
- throw ex
- }
+ ci.test('GPU', 'unittest: GPU', 'tvm/ut-python-gpu', images.ci_gpu, 'gpu') {
+ ci.download('gpu')
+ sh(
+ script: "${docker_run} ${images.ci_gpu} ./tests/scripts/task_java_unittest.sh",
+ label: 'Run Java unit tests',
+ )
+ sh(
+ script: "${docker_run} ${images.ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh",
+ label: 'Run Python GPU unit tests',
+ )
}
},
'docs: GPU': {
diff --cc ci/jenkins/generated/wasm_jenkinsfile.groovy
index cd9861e7f6,1887926335..3b8ad52929
--- a/ci/jenkins/generated/wasm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy
@@@ -14,498 -14,57 +14,55 @@@
// KIND, either express or implied. See the License for the
// specific language governing permissions and limitations
// under the License.
- // -*- mode: groovy -*-
- // Licensed to the Apache Software Foundation (ASF) under one
- // or more contributor license agreements. See the NOTICE file
- // distributed with this work for additional information
- // regarding copyright ownership. The ASF licenses this file
- // to you under the Apache License, Version 2.0 (the
- // "License"); you may not use this file except in compliance
- // with the License. You may obtain a copy of the License at
- //
- // http://www.apache.org/licenses/LICENSE-2.0
- //
- // Unless required by applicable law or agreed to in writing,
- // software distributed under the License is distributed on an
- // "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- // KIND, either express or implied. See the License for the
- // specific language governing permissions and limitations
- // under the License.
-
- // Jenkins pipeline
- // See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
- // Docker env used for testing
- // Different image may have different version tag
- // because some of them are more stable than anoter.
- //
- // Docker images are maintained by PMC, cached in dockerhub
- // and remains relatively stable over the time.
- // Flow for upgrading docker env(need commiter)
- //
- // - Send PR to upgrade build script in the repo
- // - Build the new docker image
- // - Tag the docker image with a new version and push to a binary cache.
- // - Update the version in the Jenkinsfile, send a PR
- // - Fix any issues wrt to the new image version in the PR
- // - Merge the PR and now we are in new version
- // - Tag the new version as the lates
- // - Periodically cleanup the old versions on local workers
- //
-
- // ============================= IMPORTANT NOTE =============================
- // This file is generated by 'jenkins/generate.py'. Do not edit this file directly!
- // Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with
- // 'python3 jenkins/generate.py'
- // Note: This timestamp is here to ensure that updates to the Jenkinsfile are
- // always rebased on main before merging:
- // Generated at 2026-10-14T21:26:39.087178
-
- import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
- // These are set at runtime from data in ci/jenkins/docker-images.yml, update
- // image tags in that file
- ci_gpu = ''
- ci_cpu = ''
- ci_minimal = ''
- ci_wasm = ''
- ci_cortexm = ''
- ci_arm = ''
- ci_riscv = ''
-
- // Parameters to allow overriding (in Jenkins UI), the images
- // to be used by a given build. When provided, they take precedence
- // over default values above.
- properties([
- parameters([
- string(name: 'ci_arm_param', defaultValue: ''),
- string(name: 'ci_cpu_param', defaultValue: ''),
- string(name: 'ci_gpu_param', defaultValue: ''),
- string(name: 'ci_wasm_param', defaultValue: ''),
- ])
- ])
-
- // Placeholders for newly built Docker image names (if rebuild_docker_images
- // is used)
- built_ci_arm = null;
- built_ci_cpu = null;
- built_ci_gpu = null;
- built_ci_wasm = null;
-
- // Global variable assigned during Sanity Check that holds the sha1 which should be
- // merged into the PR in all branches.
- upstream_revision = null
-
- // command to start a docker container
- docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
- docker_build = 'docker/build.sh'
- // timeout in minutes
- max_time = 180
- rebuild_docker_images = false
-
- s3_bucket = 'tvm-jenkins-artifacts-prod'
- s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
-
- // Jenkins script root directory
- jenkins_scripts_root = "ci/scripts/jenkins"
-
-
- // General note: Jenkins has limits on the size of a method (or top level code)
- // that are pretty strict, so most usage of groovy methods in these templates
- // are purely to satisfy the JVM
- def per_exec_ws(folder) {
- return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
- }
-
- // initialize source codes
- def init_git() {
- retry(5) {
- checkout scm
- }
-
- // Add more info about job node
- sh (
- script: './tests/scripts/task_show_node_info.sh',
- label: 'Show executor node info',
- )
-
- // Determine merge commit to use for all stages
- if (env.CHANGE_TARGET) {
- // This is a PR build, so merge with the latest of the PR's target branch
- // (e.g. main or a release branch like v0.25.0).
- merge_with_target()
- } else {
- // This is a branch build (main or a release branch). Only set
- // upstream_revision to HEAD and skip merging to avoid a race with another
- // commit merged to the branch.
- update_upstream_revision("HEAD")
- }
-
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0
- """,
- label: 'Update git submodules',
- )
- checkout_trusted_files()
- }
-
- def update_upstream_revision(git_ref) {
- if (upstream_revision == null) {
- upstream_revision = sh(
- script: "git log -1 ${git_ref} --format=\'%H\'",
- label: 'Determine upstream revision',
- returnStdout: true,
- ).trim()
- }
- }
-
- def merge_with_target() {
- def target = env.CHANGE_TARGET
- sh (
- script: "git fetch origin ${target}",
- label: "Fetch target branch ${target}",
- )
- update_upstream_revision("FETCH_HEAD")
- sh (
- script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
- label: "Merge to origin/${target}"
- )
- }
-
- def docker_init(image) {
- // Clear out all Docker images that aren't going to be used
- sh(
- script: """
- set -eux
- docker image ls --all
- IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}')
-
- echo -e "Found images:\\n\$IMAGES"
- echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
-
- docker image ls --all
- """,
- label: 'Clean old Docker images',
- )
-
- if (image.contains("amazonaws.com")) {
- // If this string is in the image name it's from ECR and needs to be pulled
- // with the right credentials
- ecr_pull(image)
- } else {
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${image}
- """,
- label: 'Pull docker image',
- )
- }
- }
-
- def ecr_pull(full_name) {
- aws_account_id = sh(
- returnStdout: true,
- script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
- label: 'Get AWS ID'
- ).trim()
-
- try {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: '''
- set -eux
- aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
- ''',
- label: 'Log in to ECR'
- )
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${full_name}
- """,
- label: 'Pull image from ECR'
- )
- }
- } finally {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: 'docker logout $AWS_ECR_REPO',
- label: 'Clean up login credentials'
- )
+ // readTrusted pins trusted revisions, but rejects helper edits in untrusted PRs.
+ // In that case use the PR target, pinning the same commit for all source merges.
+ def helper
+ String target_revision = null
+ try {
+ helper = readTrusted('ci/jenkins/generated/common.groovy')
+ } catch (hudson.AbortException | java.io.FileNotFoundException ex) {
+ if (!env.CHANGE_TARGET) throw ex
+ echo 'Loading shared pipeline from the pinned PR target: ' + ex.toString()
+ def from_target = { label ->
+ node(label) {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/pipeline") {
+ // No PR-owned executable or credentials run during this bootstrap.
+ deleteDir()
+ retry(5) { checkout scm }
+ // Quote the branch as data and fetch only from the canonical repository.
+ def target = env.CHANGE_TARGET.replace("'", "'\"'\"'")
+ sh(script: "git fetch https://github.com/apache/tvm.git 'refs/heads/${target}'",
+ label: 'Fetch trusted pipeline target')
+ def revision = sh(script: 'git rev-parse FETCH_HEAD', returnStdout: true).trim()
+ def source = sh(script: "git show ${revision}:ci/jenkins/generated/common.groovy",
+ returnStdout: true)
+ return [revision, source]
+ }
}
}
- }
-
- def should_skip_slow_tests(pr_number) {
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
- result = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
- label: 'Check if CI should run slow tests',
- )
- }
- return result == 0
- }
-
- def cancel_previous_build() {
- // cancel previous build if it is not on main.
- if (env.BRANCH_NAME != 'main') {
- def buildNumber = env.BUILD_NUMBER as int
- // Milestone API allows us to cancel previous build
- // with the same milestone number
- if (buildNumber > 1) milestone(buildNumber - 1)
- milestone(buildNumber)
- }
- }
-
- def is_last_build() {
- // check whether it is last build
+ def pinned
try {
- return currentBuild.number == currentBuild.rawBuild.project.getLastBuild().number
- } catch (Throwable ex) {
- echo 'Error during check is_last_build ' + ex.toString()
- return false
+ pinned = from_target('CPU-SMALL-SPOT')
+ } catch (Exception failure) {
+ pinned = from_target('CPU-SMALL')
}
+ target_revision = pinned[0]
+ helper = pinned[1]
}
+ // Evaluation is outside the read/fetch fallback: script errors must propagate.
+ ci = evaluate(helper)
+ if (target_revision != null) ci.upstream_revision = target_revision
+ ci.start()
- def checkout_trusted_files() {
- // trust everything from branch builds
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- return;
- }
-
- // trust peoople listed in CONTRIBUTING.md
- grep_code = sh(
- returnStatus: true,
- script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
- label: 'Check if change is from a contributor',
- )
-
- if (grep_code == 1) {
- // Any scripts that run on the bare host and not inside a Docker container
- // (especially those that access secrets) should be checked out here so
- // only trusted versions are used in CI
- sh(
- script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
- label: 'Check out trusted files',
- )
- }
- }
-
- def should_skip_ci(pr_number) {
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- // never skip CI on build sourced from a branch
- return false
- }
- glob_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
- label: 'Check if CI should be skipped due to changed files',
- )
- if (glob_skip_ci_code == 0) {
- return true
- }
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run full CI (or the script had an error, so run
- // full CI just in case). Exit code of 0 means skip CI.
- git_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
- label: 'Check if CI should be skipped',
- )
- }
- return git_skip_ci_code == 0
- }
-
- def prepare(node_type) {
- stage('Prepare') {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
- init_git()
-
- if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
- sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cpu ci_gpu ci_wasm ",
- label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
- )
- // Pull image names from the results of should_rebuild_docker.py
- ci_arm = sh(
- script: "cat .docker-image-names/ci_arm",
- label: "Find docker image name for ci_arm",
- returnStdout: true,
- ).trim()
- ci_cpu = sh(
- script: "cat .docker-image-names/ci_cpu",
- label: "Find docker image name for ci_cpu",
- returnStdout: true,
- ).trim()
- ci_gpu = sh(
- script: "cat .docker-image-names/ci_gpu",
- label: "Find docker image name for ci_gpu",
- returnStdout: true,
- ).trim()
- ci_wasm = sh(
- script: "cat .docker-image-names/ci_wasm",
- label: "Find docker image name for ci_wasm",
- returnStdout: true,
- ).trim()
- }
-
- ci_arm = params.ci_arm_param ?: ci_arm
- ci_cpu = params.ci_cpu_param ?: ci_cpu
- ci_gpu = params.ci_gpu_param ?: ci_gpu
- ci_wasm = params.ci_wasm_param ?: ci_wasm
-
- sh (script: """
- echo "Docker images being used in this build:"
- echo " ci_arm = ${ci_arm}"
- echo " ci_cpu = ${ci_cpu}"
- echo " ci_gpu = ${ci_gpu}"
- echo " ci_wasm = ${ci_wasm}"
- """, label: 'Docker image names')
-
- is_docs_only_build = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docs.sh",
- label: 'Check for docs only changes',
- )
- skip_ci = should_skip_ci(env.CHANGE_ID)
- skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
- rebuild_docker_images = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docker.sh",
- label: 'Check for any docker changes',
- )
-
- if (skip_ci) {
- // Don't rebuild when skipping CI
- rebuild_docker_images = false
- }
- }
- }
- }
- }
- def python_unittest(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
- label: 'Run Python unit tests',
+ ci.build('CPU', 'BUILD: WASM', 'tvm/build-wasm', images.ci_wasm, 'wasm') {
+ sh(
+ script: "${docker_run} ${images.ci_wasm} ./tests/scripts/task_config_build_wasm.sh build",
+ label: 'Create WASM CMake config',
)
- }
-
- // Keep compiler caching and executor limits identical for wheels and the
- // separate runtime-only compile check. Build commands remain at their callers.
- def build_environment() {
- return '''
- set -eux
- export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(getconf _NPROCESSORS_ONLN) / ${CI_NUM_EXECUTORS:-1} ))
- if [ "$CMAKE_BUILD_PARALLEL_LEVEL" -lt 1 ]; then export CMAKE_BUILD_PARALLEL_LEVEL=1; fi
- if command -v sccache >/dev/null; then
- export CC=/opt/sccache/cc CXX=/opt/sccache/c++
- if [ -n "${AWS_ACCESS_KEY_ID:-}" ]; then
- export SCCACHE_BUCKET=tvm-sccache-prod SCCACHE_REGION=us-west-2
- fi
- sccache --start-server || true
- sccache --show-stats
- trap "sccache --show-stats" EXIT
- fi
- '''
- }
-
- def wheel_build(image, cpp_tests = false) {
- def commands = build_environment() + '''
- rm -rf build/wheels
- uv build --wheel --python python3 --out-dir build/wheels ./3rdparty/tvm-ffi
- uv build --wheel --python python3 --out-dir build/wheels . \
- -Ccmake.build-type=RelWithDebInfo -Cbuild.verbose="${VERBOSE:-true}"
- '''.stripIndent().trim()
- if (cpp_tests) {
- commands += " -Ccmake.define.USE_GTEST=ON -Cbuild.targets=all\\;cpptest"
- // cpptest links the same native tree that scikit-build just packaged.
- commands += "\n./tests/scripts/task_cpp_unittest.sh build"
- }
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} bash -c '${commands}'",
- label: 'Build wheel artifacts' + (cpp_tests ? ' and run C++ tests' : ''),
- ci.cmake_build(images.ci_wasm, 'build')
- ci.make_cpp_tests(images.ci_wasm, 'build')
- ci.cpp_unittest(images.ci_wasm)
++ ci.wheel_build(images.ci_wasm, true)
+ sh(
+ script: "${docker_run} ${images.ci_wasm} ./tests/scripts/task_web_wasm.sh",
+ label: 'Run WASM lint and tests',
)
}
-
- cancel_previous_build()
-
- try {
- prepare('CPU-SMALL-SPOT')
- } catch(Exception ex) {
- prepare('CPU-SMALL')
- }
- def run_build(node_type) {
- if (!skip_ci && is_docs_only_build != 1) {
- echo 'Begin running node_type ' + node_type
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") {
- init_git()
- docker_init(ci_wasm)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=wasm',
- ], {
- sh (
- script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build",
- label: 'Create WASM CMake config',
- )
- wheel_build(ci_wasm, true)
- sh (
- script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh",
- label: 'Run WASM lint and tests',
- )
- })
- }
- }
- }
- echo 'End running node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('BUILD: WASM')
- }
- }
- def build() {
- stage('Build') {
- try {
- run_build('CPU-SPOT')
- } catch (hudson.AbortException abortEx) {
- echo "Received normal AbortException, exit now. Details:" + abortEx.toString()
- throw abortEx
- } catch (Throwable ex) {
- echo 'Exception during SPOT run ' + ex.toString()
- if (is_last_build()) {
- // retry if we are currently at last build
- // mark the current stage as success
- // and try again via on demand node
- echo 'Retry on-demand given it is last build'
- currentBuild.result = 'SUCCESS'
- run_build('CPU')
- } else {
- echo 'Exit since it is not last build'
- throw ex
- }
- }
- }
- }
- build()
diff --cc tests/python/ci/test_jenkins_helpers.py
index 0000000000,b9d26a78a0..23c0e42383
mode 000000,100644..100644
--- a/tests/python/ci/test_jenkins_helpers.py
+++ b/tests/python/ci/test_jenkins_helpers.py
@@@ -1,0 -1,134 +1,80 @@@
+ # Licensed to the Apache Software Foundation (ASF) under one
+ # or more contributor license agreements. See the NOTICE file
+ # distributed with this work for additional information
+ # regarding copyright ownership. The ASF licenses this file
+ # to you under the Apache License, Version 2.0 (the
+ # "License"); you may not use this file except in compliance
+ # with the License. You may obtain a copy of the License at
+ #
+ # http://www.apache.org/licenses/LICENSE-2.0
+ #
+ # Unless required by applicable law or agreed to in writing,
+ # software distributed under the License is distributed on an
+ # "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ # KIND, either express or implied. See the License for the
+ # specific language governing permissions and limitations
+ # under the License.
+ """Host-side Jenkins commands preserve retries, paths and build configuration."""
+
+ import importlib.util
+ import json
+ import os
-import shutil
+ import subprocess
-import sys
+
+ import pytest
+
-from .test_utils import JENKINS_SCRIPT_ROOT, REPO_ROOT
++from .test_utils import JENKINS_SCRIPT_ROOT
+
+
+ def load_s3():
+ spec = importlib.util.spec_from_file_location("jenkins_s3", JENKINS_SCRIPT_ROOT / "s3.py")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+ def executable(path, body):
+ path.write_text("#!/usr/bin/env python3\n" + body)
+ path.chmod(0o755)
+
+
+ @pytest.mark.parametrize("failures", [2, 3])
+ def test_s3_transfer_retries(tmp_path, monkeypatch, failures):
+ # Run the real retry.sh with fake AWS and sleep, without network or delays.
+ executable(
+ tmp_path / "aws",
+ """
+ import json, os, pathlib, sys
+ log = pathlib.Path(os.environ["AWS_LOG"])
+ with log.open("a") as out:
+ out.write(json.dumps(sys.argv[1:]) + "\\n")
+ attempt = len(log.read_text().splitlines())
+ print("download: s3://bucket/file to file name")
+ sys.exit(1 if attempt <= int(os.environ["FAILURES"]) else 0)
+ """,
+ )
+ executable(
+ tmp_path / "sleep",
+ """
+ import os, pathlib, sys
+ with pathlib.Path(os.environ["SLEEP_LOG"]).open("a") as out:
+ out.write(sys.argv[1] + "\\n")
+ """,
+ )
+ monkeypatch.setenv("PATH", f"{tmp_path}:{os.environ['PATH']}")
+ monkeypatch.setenv("FAILURES", str(failures))
+ monkeypatch.setenv("SLEEP_LOG", str(tmp_path / "sleeps"))
+ s3 = load_s3()
+ for transfer in range(2):
+ log = tmp_path / f"aws-{transfer}"
+ monkeypatch.setenv("AWS_LOG", str(log))
+ if failures == 3:
+ with pytest.raises(subprocess.CalledProcessError):
+ s3.s3("s3://bucket/file", "file name", False)
+ else:
+ assert s3.parse_output_files(s3.s3("s3://bucket/file", "file name", False))
+ commands = [json.loads(line) for line in log.read_text().splitlines()]
+ assert commands == [["s3", "cp", "--no-progress", "s3://bucket/file", "file name"]] * 3
+ delays = (tmp_path / "sleeps").read_text().splitlines()
+ assert len(delays) == 4
+ assert all(30 <= int(delay) <= 200 for delay in delays)
-
-
-def test_build_environment_and_target(tmp_path):
- # Run the entry point in a minimal checkout, capturing the tool boundary.
- script = tmp_path / "tests/scripts/task_build.py"
- script.parent.mkdir(parents=True)
- shutil.copy(REPO_ROOT / "tests/scripts/task_build.py", script)
- (tmp_path / "build").mkdir()
- executable(
- tmp_path / "cmake",
- """
-import json, os, pathlib, sys
-with pathlib.Path(os.environ["COMMAND_LOG"]).open("a") as out:
- entry = [sys.argv[1:], os.getcwd(), os.environ["SCCACHE_BUCKET"], os.environ["CC"]]
- out.write(json.dumps(entry) + "\\n")
-""",
- )
- executable(tmp_path / "sccache", "pass\n")
- log = tmp_path / "commands"
- env = dict(
- os.environ,
- PATH=f"{tmp_path}:{os.environ['PATH']}",
- COMMAND_LOG=str(log),
- AWS_ACCESS_KEY_ID="test",
- CI_NUM_EXECUTORS="2",
- VERBOSE="false",
- )
- subprocess.run(
- [
- sys.executable,
- script,
- "--debug",
- "--cmake-target",
- "cpptest",
- "--sccache-bucket",
- "test-cache",
- ],
- cwd=tmp_path,
- env=env,
- check=True,
- )
- configure, build = [json.loads(line) for line in log.read_text().splitlines()]
- assert configure == [
- ["-GNinja", "-DCMAKE_BUILD_TYPE=Debug", ".."],
- str(tmp_path / "build"),
- "test-cache",
- "/opt/sccache/cc",
- ]
- assert build[0][:4] == ["--build", ".", "--", "cpptest"]
- assert build[0][4].startswith("-j") and int(build[0][4][2:]) >= 1
- assert len(build[0]) == 5
- assert build[1:] == configure[1:]
diff --cc tests/python/ci/test_wheel_artifacts.py
index 9eaa8613bc,0000000000..e764f3b27e
mode 100644,000000..100644
--- a/tests/python/ci/test_wheel_artifacts.py
+++ b/tests/python/ci/test_wheel_artifacts.py
@@@ -1,167 -1,0 +1,189 @@@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Wheel artifact transport and local CI command contracts, without a TVM import."""
+
+import importlib.util
+import json
+import os
+import shutil
+import subprocess
+import sys
+from pathlib import Path
+
+import pytest
+
+ROOT = Path(__file__).resolve().parents[3]
+
+
+def load_local_ci():
+ spec = importlib.util.spec_from_file_location("local_ci", ROOT / "tests/scripts/ci.py")
+ module = importlib.util.module_from_spec(spec)
+ spec.loader.exec_module(module)
+ return module
+
+
+@pytest.mark.parametrize("platform", ["cpu", "gpu", "arm", "wasm"])
+def test_local_wheel_build(monkeypatch, platform):
+ ci = load_local_ci()
+ calls = []
+ monkeypatch.setattr(ci, "docker", lambda **kwargs: calls.append(kwargs))
+ command, _, _ = ci.generate_command(platform, {}, "wheel build")
+ command(cpp=True)
+ scripts = calls[0]["scripts"]
+ wheel = next(script for script in scripts if "-Cbuild-dir=" in script)
+ assert "-Ccmake.build-type=RelWithDebInfo" in wheel
+ assert "all;cpptest" in wheel
- assert any("./3rdparty/tvm-ffi" in script for script in scripts)
- assert any("task_python_install_wheels.sh" in script for script in scripts) == (
- platform != "wasm"
- )
++ assert not any("./3rdparty/tvm-ffi" in script for script in scripts)
++ assert "--no-build-isolation --offline" in wheel
++ assert any("task_python_install_wheels.sh" in script for script in scripts)
+ calls.clear()
+ command(skip_build=True, tests=["tests/python/example.py"])
+ assert not any("uv build" in script for script in calls[0]["scripts"])
+
+
+@pytest.mark.parametrize("succeed", [True, False])
+def test_wheel_directory_upload_retries_each_transfer(tmp_path, succeed):
+ # Exercise the existing S3 CLI/retry boundary with a directory bundle.
+ for relative in [
- "ci/scripts/jenkins/s3.py",
- "ci/scripts/jenkins/cmd_utils.py",
- "ci/scripts/jenkins/retry.sh",
++ "ci/jenkins/scripts/s3.py",
++ "ci/jenkins/scripts/retry.sh",
+ "ci/jenkins/data.py",
+ ]:
+ target = tmp_path / relative
+ target.parent.mkdir(parents=True, exist_ok=True)
+ shutil.copy(ROOT / relative, target)
+ wheels = tmp_path / "build/wheels"
+ wheels.mkdir(parents=True)
+ (wheels / "apache_tvm.whl").write_text("tvm")
- (wheels / "apache_tvm_ffi.whl").write_text("ffi")
+ binaries = tmp_path / "bin"
+ binaries.mkdir()
+ aws = binaries / "aws"
+ aws.write_text(
+ f"#!{sys.executable}\n"
+ "import json, pathlib, sys\n"
+ "path = pathlib.Path('attempts.json')\n"
+ "calls = json.loads(path.read_text()) if path.exists() else []\n"
+ "calls.append(sys.argv[1:])\n"
+ "path.write_text(json.dumps(calls))\n"
+ f"sys.exit(0 if {succeed!r} and len(calls) == 3 else 1)\n"
+ )
+ aws.chmod(0o755)
+ sleep = binaries / "sleep"
+ sleep.write_text('#!/bin/sh\necho "$1" >> delays.txt\n')
+ sleep.chmod(0o755)
+ result = subprocess.run(
+ [
+ sys.executable,
- "ci/scripts/jenkins/s3.py",
++ "ci/jenkins/scripts/s3.py",
+ "--action",
+ "upload",
+ "--bucket",
+ "test",
+ "--prefix",
+ "cpu",
+ "--bundle",
+ "wheels",
+ ],
+ cwd=tmp_path,
+ env={**os.environ, "PATH": f"{binaries}:{os.environ['PATH']}"},
+ capture_output=True,
+ text=True,
+ )
+ assert (result.returncode == 0) == succeed, result.stdout + result.stderr
+ attempts = json.loads((tmp_path / "attempts.json").read_text())
+ assert len(attempts) == 3
+ assert all(
+ call[-3:] == ["--recursive", "build/wheels", "s3://test/cpu/build/wheels"]
+ for call in attempts
+ )
+ delays = [int(line) for line in (tmp_path / "delays.txt").read_text().splitlines()]
+ assert len(delays) == 2 and all(30 <= delay <= 200 for delay in delays)
+
+
+@pytest.mark.parametrize("fail", [False, True])
+def test_local_interactive_keeps_cache_stats(tmp_path, monkeypatch, fail):
+ ci = load_local_ci()
+ monkeypatch.setattr(ci, "check_docker", lambda: None)
+ monkeypatch.setattr(ci, "cmd", lambda *args, **kwargs: None)
+ monkeypatch.setattr(ci, "SCRIPT_DIR", tmp_path / "scripts")
+ monkeypatch.setattr(ci, "REPO_ROOT", tmp_path)
+ monkeypatch.delenv("USE_SCCACHE", raising=False)
+ monkeypatch.setenv("DEBUG", "1")
+ binaries = tmp_path / "bin"
+ binaries.mkdir()
+ for name in ["sccache", "bash"]:
+ binary = binaries / name
+ binary.write_text(f'#!/bin/sh\necho "{name} $*" >> "$CALL_LOG"\n')
+ binary.chmod(0o755)
+ ci.docker("interactive", "ci_cpu", ["false" if fail else "true"], {}, True)
+ result = subprocess.run(
+ [shutil.which("bash"), str(ci.SCRIPT_DIR / "interactive.sh")],
+ env={
+ **os.environ,
+ "PATH": f"{binaries}:{os.environ['PATH']}",
+ "CALL_LOG": str(tmp_path / "calls"),
+ },
+ capture_output=True,
+ text=True,
+ )
+ assert (result.returncode != 0) == fail
+ calls = (tmp_path / "calls").read_text().splitlines()
+ assert calls == [
+ "sccache --start-server",
+ "sccache --show-stats",
+ "sccache --show-stats",
+ "bash ",
+ ]
+
+
+@pytest.mark.parametrize("wheel_docs", [False, True])
+def test_docs_import_path(tmp_path, monkeypatch, wheel_docs):
+ # Run the real Sphinx path bootstrap without requiring Sphinx extensions.
+ # A stale --target=python FFI package must not precede Jenkins' wheel install.
+ monkeypatch.chdir(ROOT / "docs")
+ source_python = str(ROOT / "python")
+ monkeypatch.setattr(sys, "path", [path for path in sys.path if path != source_python])
+ monkeypatch.setenv("TVM_DOCS_USE_WHEEL", "1" if wheel_docs else "0")
+ config = ROOT / "docs/conf.py"
+ bootstrap = config.read_text().split("# -- General configuration")[0]
+ exec(compile(bootstrap, str(config), "exec"), {"__file__": str(config)})
+ assert (source_python in sys.path) != wheel_docs
++
++
++@pytest.mark.parametrize(
++ "host_limit,caller_limit,expected", [(None, None, "2"), ("7", None, "7"), ("7", "9", "9")]
++)
++def test_local_parallel_limit_crosses_docker_boundary(
++ tmp_path, monkeypatch, host_limit, caller_limit, expected
++):
++ ci = load_local_ci()
++ calls = []
++ monkeypatch.setattr(ci, "check_docker", lambda: None)
++ monkeypatch.setattr(ci, "cmd", lambda command: calls.append(command))
++ monkeypatch.setattr(ci, "SCRIPT_DIR", tmp_path / "scripts")
++ monkeypatch.setattr(ci, "REPO_ROOT", tmp_path)
++ monkeypatch.setattr(ci, "NPROC", 8)
++ monkeypatch.setenv("CI_NUM_EXECUTORS", "3")
++ if host_limit is None:
++ monkeypatch.delenv("CMAKE_BUILD_PARALLEL_LEVEL", raising=False)
++ else:
++ monkeypatch.setenv("CMAKE_BUILD_PARALLEL_LEVEL", host_limit)
++ env = {} if caller_limit is None else {"CMAKE_BUILD_PARALLEL_LEVEL": caller_limit}
++ ci.docker("build", "ci_cpu", ["true"], env, False)
++ args = calls[0]
++ index = args.index(f"CMAKE_BUILD_PARALLEL_LEVEL={expected}")
++ assert args[index - 1] == "--env"
diff --cc tests/scripts/ci.py
index f242f110b5,fa027c150a..ec51674b05
--- a/tests/scripts/ci.py
+++ b/tests/scripts/ci.py
@@@ -177,12 -183,9 +177,14 @@@ def docker
env["SCCACHE_CACHE_SIZE"] = os.getenv("SCCACHE_CACHE_SIZE", "50G")
env["SCCACHE_SERVER_PORT"] = os.getenv("SCCACHE_SERVER_PORT", "4226")
- # Match Jenkins' per-executor native build budget, including wheel builds.
- scripts = [
- "export CMAKE_BUILD_PARALLEL_LEVEL=$(( $(getconf _NPROCESSORS_ONLN) / "
- "${CI_NUM_EXECUTORS:-1} ))",
- 'if [ "$CMAKE_BUILD_PARALLEL_LEVEL" -lt 1 ]; then export CMAKE_BUILD_PARALLEL_LEVEL=1; fi',
- ] + scripts
++ # Pass the bound through Docker; CMake and scikit-build honor it directly.
++ env.setdefault(
++ "CMAKE_BUILD_PARALLEL_LEVEL",
++ os.environ.get(
++ "CMAKE_BUILD_PARALLEL_LEVEL",
++ str(max(NPROC // int(os.environ.get("CI_NUM_EXECUTORS", "1")), 1)),
++ ),
++ )
env["PLATFORM"] = name
docker_bash = REPO_ROOT / "docker" / "bash.sh"
@@@ -400,22 -402,9 +402,20 @@@ def generate_command
else:
scripts = [
f"./tests/scripts/task_config_build_{name}.sh {build_dir}",
- f"./tests/scripts/task_build.py --build-dir {build_dir}",
+ f"rm -rf {build_dir}/wheels",
- f"uv build --wheel --python python3 --out-dir {build_dir}/wheels ./3rdparty/tvm-ffi",
- f"uv build --wheel --python python3 --out-dir {build_dir}/wheels . "
++ f"uv build --wheel --no-build-isolation --offline --python python3 --out-dir {build_dir}/wheels . "
+ f"-Cbuild-dir={build_dir} -Ccmake.build-type=RelWithDebInfo "
+ f"-Cbuild.verbose={'true' if verbose else 'false'}"
+ + (
+ " -Ccmake.define.USE_GTEST=ON '-Cbuild.targets=all;cpptest'"
+ if kwargs.get("cpp")
+ else ""
+ ),
]
+ # Direct --tests and interactive Python use the same installed artifacts.
- if name != "wasm":
- scripts.append("source ./tests/scripts/task_python_install_wheels.sh")
++ scripts.append("source ./tests/scripts/task_python_install_wheels.sh")
+
if post_build is not None:
scripts += post_build
@@@ -433,10 -422,10 +433,7 @@@
scripts.extend(script.format(build_dir=build_dir) for script in extra_scripts)
docker_env = {
- # The specialized Wasm source tests retain a native build path;
- # the wheel installer clears it for Python wheel consumers.
- # Need to specify the library path manually or else TVM can't
- # determine which build directory to use (i.e. if there are
- # multiple copies of libtvm_compiler.so / libtvm_runtime.so laying around)
-- "TVM_LIBRARY_PATH": str(REPO_ROOT / get_build_dir(name)),
+ "TVM_WHEEL_DIR": str(REPO_ROOT / build_dir / "wheels"),
"VERBOSE": "true" if verbose else "false",
}
diff --cc tests/scripts/task_python_docs.sh
index 73d2662465,e45d439929..878996fa12
--- a/tests/scripts/task_python_docs.sh
+++ b/tests/scripts/task_python_docs.sh
@@@ -18,14 -18,7 +18,13 @@@
set -euxo pipefail
-export PYTHONPATH="$(pwd)/python"
+# Local docs retain source builds; Jenkins imports the exact build-node wheels.
+if [ "${IS_LOCAL:-0}" != "1" ]; then
+ source "$(dirname "$0")/task_python_install_wheels.sh"
+ export TVM_DOCS_USE_WHEEL=1
+else
- uv pip install -v --target=python ./3rdparty/tvm-ffi/
+ export PYTHONPATH="$(pwd)/python"
+fi
# to avoid CI CPU thread throttling.
export TVM_BIND_THREADS=0
@@@ -133,8 -129,11 +132,6 @@@ clean_file
find . -type f -path "*.log" | xargs rm -f
find . -type f -path "*.pyc" | xargs rm -f
-# setup tvm-ffi into python folder
-uv pip install -v --target=python ./3rdparty/tvm-ffi/
-
--# LLVMModule JIT execution, needed by the tutorials this build runs.
--uv pip install --no-deps apache-tvm-ffi-orcjit==0.1.2
cd docs
diff --cc tests/scripts/task_python_install_wheels.sh
index f54c57c9c5,0000000000..0138187eac
mode 100644,000000..100644
--- a/tests/scripts/task_python_install_wheels.sh
+++ b/tests/scripts/task_python_install_wheels.sh
@@@ -1,54 -1,0 +1,72 @@@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+# Source this file so subsequent Python invocations use the wheel environment.
+# Docker defaults PYTHONPATH to the checkout; never let it shadow the artifacts.
+unset PYTHONPATH TVM_LIBRARY_PATH TVM_FFI_LIBRARY_PATH
+export PYTHONNOUSERSITE=1
+TVM_WHEEL_DIR="$(realpath "${TVM_WHEEL_DIR:-$(pwd)/build/wheels}")"
+# Preserve the image's active environment (often itself a venv), including when
+# local ci.py sources this setup again for a named test suite.
+export TVM_IMAGE_PYTHON="${TVM_IMAGE_PYTHON:-$(command -v python3)}"
+export TVM_IMAGE_SITE_PACKAGES="${TVM_IMAGE_SITE_PACKAGES:-$("$TVM_IMAGE_PYTHON" -I -c 'import site; print("\n".join(site.getsitepackages()))')}"
+export VIRTUAL_ENV="${TVM_WHEEL_DIR%/}/../wheel-venv"
- uv venv --python "$TVM_IMAGE_PYTHON" --clear "$VIRTUAL_ENV"
++uv venv --offline --python "$TVM_IMAGE_PYTHON" --clear "$VIRTUAL_ENV"
+# Append the image dependencies after our wheels without executing its .pth files:
+# system-site-packages can install editable-import hooks ahead of the wheel.
+TVM_WHEEL_SITE="$("$VIRTUAL_ENV/bin/python3" -c 'import sysconfig; print(sysconfig.get_path("purelib"))')"
+printf '%s\n' "$TVM_IMAGE_SITE_PACKAGES" > "$TVM_WHEEL_SITE/image-dependencies.pth"
+export PATH="$VIRTUAL_ENV/bin:$PATH"
- uv pip install --python "$VIRTUAL_ENV/bin/python3" --no-deps --reinstall \
- "$TVM_WHEEL_DIR"/apache_tvm-*.whl "$TVM_WHEEL_DIR"/apache_tvm_ffi-*.whl
++uv pip install --python "$VIRTUAL_ENV/bin/python3" --no-deps --offline --reinstall \
++ "$TVM_WHEEL_DIR"/apache_tvm-*.whl
+
+# Fail before pytest if a source/editable install or native override leaked in.
+python3 - <<'PYTHON'
- from pathlib import Path
++import os
+import sys
++from importlib.metadata import requires, version
++from pathlib import Path
++from packaging.requirements import Requirement
++
++# --no-deps leaves dependency ownership with the image. Check the wheel's own
++# requirements without invoking a resolver or accepting a stale hardcoded floor.
++for text in requires("apache-tvm") or []:
++ requirement = Requirement(text)
++ if requirement.marker and not requirement.marker.evaluate({"extra": ""}):
++ continue
++ installed = version(requirement.name)
++ assert requirement.specifier.contains(installed, prereleases=True), (text, installed)
++
+import tvm
+import tvm_ffi
+
+prefix = Path(sys.prefix).resolve()
- for module in (tvm, tvm_ffi):
- path = Path(module.__file__).resolve()
- assert path.is_relative_to(prefix), path
- print(f"Installed {module.__name__}: {path}")
- for name, library in {"tvm_ffi": tvm_ffi.LIB, **tvm.base._LOADED_LIBS}.items():
++path = Path(tvm.__file__).resolve()
++assert path.is_relative_to(prefix), path
++print(f"Installed tvm: {path}")
++# FFI is supplied by the image, not rebuilt or transferred with TVM.
++image_sites = [Path(p).resolve() for p in os.environ["TVM_IMAGE_SITE_PACKAGES"].splitlines()]
++for path in (Path(tvm_ffi.__file__).resolve(), Path(tvm_ffi.LIB._name).resolve()):
++ assert any(path.is_relative_to(site) for site in image_sites), path
++ print(f"Image FFI: {path}")
++assert {"tvm_runtime", "tvm_compiler"} <= tvm.base._LOADED_LIBS.keys()
++for name, library in tvm.base._LOADED_LIBS.items():
+ path = Path(library._name).resolve()
+ assert path.is_relative_to(prefix), (name, path)
+ print(f"Installed {name}: {path}")
+PYTHON
diff --cc tests/scripts/task_python_unittest.sh
index 8472b142b8,511b3ec2a5..e5e5603429
--- a/tests/scripts/task_python_unittest.sh
+++ b/tests/scripts/task_python_unittest.sh
@@@ -18,12 -18,15 +18,7 @@@
set -euxo pipefail
-export PYTHONPATH="$(pwd)/python"
+source "$(dirname "$0")/task_python_install_wheels.sh"
export PYTEST_ADDOPTS="${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}"
-# setup tvm-ffi into python folder
-uv pip install -v --target=python ./3rdparty/tvm-ffi/
-
--# LLVMModule JIT execution. Installed here as well as in the CI image because a docker/
--# change does not reach the test containers in the same run (they use the pinned tag).
--# --no-deps keeps the submodule-built tvm-ffi above from being replaced by a PyPI build.
--uv pip install --no-deps apache-tvm-ffi-orcjit==0.1.2
--
-python3 -m pytest -vvs -n auto -m "${TVM_TEST_MARKER:-not gpu}" tests/python
+python3 -m pytest -vvs -n auto -m "${TVM_TEST_MARKER:-not gpu}" "${@:-tests/python}"
diff --cc tests/scripts/task_python_unittest_gpuonly.sh
index 9e4b95ec3d,ac9e112b84..9aa95cb201
--- a/tests/scripts/task_python_unittest_gpuonly.sh
+++ b/tests/scripts/task_python_unittest_gpuonly.sh
@@@ -31,4 -31,9 +31,6 @@@ export TVM_TEST_TARGETS='cuda;metal;roc
# only minimal functionality of Vulkan runtime.
export TVM_TEST_TARGETS='{"kind":"vulkan","from_device":0}'
- ./tests/scripts/task_python_unittest.sh tests/python/codegen/test_target_codegen_vulkan.py
-export PYTHONPATH="$(pwd)/python"
-export PYTEST_ADDOPTS="${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}"
-
-python3 -m pytest -vvs -n auto -m "${TVM_TEST_MARKER}" \
++./tests/scripts/task_python_unittest.sh \
+ tests/python/backend/vulkan/codegen/test_vulkan.py \
+ tests/python/backend/vulkan/test_vulkan_module_export.py
diff --cc tests/scripts/task_web_wasm.sh
index 5f4d8d309e,5f4d8d309e..e2f6d6c4db
--- a/tests/scripts/task_web_wasm.sh
+++ b/tests/scripts/task_web_wasm.sh
@@@ -18,10 -18,10 +18,8 @@@
set -euxo pipefail
--export PYTHONPATH=`pwd`/python
--
--# setup tvm-ffi into python folder
--uv pip install -v --target=python ./3rdparty/tvm-ffi/
++# Host Python uses the built wheel; Emscripten still builds its separate runtime.
++source "$(dirname "$0")/task_python_install_wheels.sh"
rm -rf .emscripten_cache
cd web