You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2026/10/15 12:35:12 UTC
(tvm) branch jenkins-wheel-test-artifacts updated (5bb9f9d791 -> 01f05952fc)
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a change to branch jenkins-wheel-test-artifacts
in repository https://gitbox.apache.org/repos/asf/tvm.git
from 5bb9f9d791 Locate minRPC sources through TVM_HOME for installed wheels
add 96c0200bac [CI] Use explicit node fallback and artifact upload helpers
add a1490e3f25 [CI] Retain script and bot coverage with a minimal inline fixture
new c9db658e7d Merge explicit Jenkins node and artifact helpers into wheel pipelines
new d795529e71 [Python] Add lightweight runtime build configuration queries
new 01f05952fc [CI] Transfer TVM and pinned FFI wheels to Python and Java consumers
The 3 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
ci/jenkins/README.md | 50 ++--
ci/jenkins/artifacts.py | 4 +-
ci/jenkins/generated/arm_jenkinsfile.groovy | 18 +-
ci/jenkins/generated/common.groovy | 39 ++-
ci/jenkins/generated/cpu_jenkinsfile.groovy | 23 +-
ci/jenkins/generated/gpu_jenkinsfile.groovy | 40 ++-
ci/jenkins/generated/wasm_jenkinsfile.groovy | 24 +-
.../linux-x86_64/src/main/assembly/assembly.xml | 7 +-
.../osx-x86_64/src/main/assembly/assembly.xml | 7 +-
jvm/core/pom.xml | 2 +-
jvm/native/linux-x86_64/pom.xml | 6 +-
jvm/native/osx-x86_64/pom.xml | 6 +-
jvm/pom.xml | 4 +
pyproject.toml | 3 +
python/tvm/__init__.py | 182 ++++++------
python/tvm/config.py | 63 ++++
python/tvm/libinfo.py | 39 +--
tests/python/ci/jenkins_loader_test.groovy | 84 ------
tests/python/ci/sample_prs/pr10786.json | 130 ---------
tests/python/ci/test_ci.py | 55 +++-
tests/python/ci/test_jenkins_helpers.py | 318 ---------------------
tests/python/ci/test_jenkins_loader.py | 37 ---
tests/python/ci/test_tvmbot.py | 106 +++++--
tests/python/ci/test_utils.py | 80 ------
tests/python/ci/test_wheel_artifacts.py | 309 --------------------
tests/python/runtime/test_config.py | 68 +++++
tests/scripts/ci.py | 1 +
tests/scripts/task_java_unittest.sh | 8 +-
tests/scripts/task_jvm_build.sh | 22 +-
tests/scripts/task_python_install_wheels.sh | 55 +++-
30 files changed, 630 insertions(+), 1160 deletions(-)
create mode 100644 python/tvm/config.py
delete mode 100644 tests/python/ci/jenkins_loader_test.groovy
delete mode 100644 tests/python/ci/sample_prs/pr10786.json
delete mode 100644 tests/python/ci/test_jenkins_helpers.py
delete mode 100644 tests/python/ci/test_jenkins_loader.py
delete mode 100644 tests/python/ci/test_utils.py
delete mode 100644 tests/python/ci/test_wheel_artifacts.py
create mode 100644 tests/python/runtime/test_config.py
(tvm) 03/03: [CI] Transfer TVM and pinned FFI wheels to Python and Java consumers
Posted by tq...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch jenkins-wheel-test-artifacts
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 01f05952fc964e67821500c2fa099cffbb999904
Author: Tianqi Chen <tq...@users.noreply.github.com>
AuthorDate: Thu Oct 15 12:34:58 2026 +0000
[CI] Transfer TVM and pinned FFI wheels to Python and Java consumers
---
ci/jenkins/README.md | 34 +--
ci/jenkins/artifacts.py | 4 +-
ci/jenkins/generated/common.groovy | 3 +-
ci/jenkins/generated/cpu_jenkinsfile.groovy | 2 +-
ci/jenkins/generated/gpu_jenkinsfile.groovy | 2 +-
.../linux-x86_64/src/main/assembly/assembly.xml | 7 +-
.../osx-x86_64/src/main/assembly/assembly.xml | 7 +-
jvm/core/pom.xml | 2 +-
jvm/native/linux-x86_64/pom.xml | 6 +-
jvm/native/osx-x86_64/pom.xml | 6 +-
jvm/pom.xml | 4 +
tests/python/ci/test_wheel_artifacts.py | 309 ---------------------
tests/scripts/ci.py | 1 +
tests/scripts/task_java_unittest.sh | 8 +-
tests/scripts/task_jvm_build.sh | 22 +-
tests/scripts/task_python_install_wheels.sh | 55 +++-
16 files changed, 118 insertions(+), 354 deletions(-)
diff --git a/ci/jenkins/README.md b/ci/jenkins/README.md
index 5121cb5a67..dbb68032b7 100644
--- a/ci/jenkins/README.md
+++ b/ci/jenkins/README.md
@@ -127,14 +127,17 @@ changing any retained script interface.
## Wheel build and test artifacts
Normal builds use `uv build --wheel --no-build-isolation --offline` with the
-existing platform CMake configuration. `uv build` builds only TVM; it does not
-resolve runtime dependencies. CPU, ARM and Wasm host builds request `all;cpptest`
+existing platform CMake configuration. `uv build` does not resolve runtime dependencies. The build stage also builds
+`apache-tvm-ffi` from the pinned submodule, using its declared isolated build
+requirements. CPU, ARM and Wasm host builds request `all;cpptest`
and run C++ tests in that same native tree before uploading anything. Python
-consumers install the exact TVM wheel with `--no-deps --offline`, clear checkout
-path overrides, and verify the package and native-library paths. FFI comes from
-the checked-out submodule. GPU Java tests also receive their existing native
-runtime libraries; the separate runtime-only compile check and Emscripten runtime
-build remain.
+consumers install the exact TVM and FFI wheels with `--no-deps --offline`, clear
+checkout path overrides, and verify artifact hashes and package/native-library
+paths. Only `build/wheels/*.whl` is uploaded. GPU Java consumers discover headers,
+linker paths and packaged native libraries through `tvm.config` and
+`tvm_ffi.config`; no separate native files are transferred. The configuration
+commands avoid loading TVM's Python compiler, backends or native libraries.
+The separate runtime-only compile check and Emscripten runtime build remain.
Immediately before building, the same container and Python install the declared
`pyproject.toml` build requirements (scikit-build-core, setuptools-scm and
@@ -146,17 +149,16 @@ modifying the image environment or exposing the build overlay to test consumers.
`uv build` then runs offline without isolation.
The selected images must already provide compatible runtime and test dependencies,
-apart from core FFI and its ORCJIT addon. Consumers install the checkout's pinned
-FFI source into the writable test environment with `--no-deps`, using its declared
-isolated build
-requirements, plus the explicit `apache-tvm-ffi-orcjit==0.1.2 --no-deps` prerequisite
-used by legacy Python tests. They then install the transferred TVM wheel with
-`--no-deps --offline`.
+apart from core FFI and its ORCJIT addon. Consumers install the explicit
+`apache-tvm-ffi-orcjit==0.1.2 --no-deps` prerequisite used by legacy Python tests,
+then install exactly one transferred wheel for each of TVM and FFI with
+`--no-deps --offline`. Consumers do not compile FFI.
The checkout remains available for tests and resources through `TVM_HOME`, including
Wasm assets under `web/dist`, without adding its Python package ahead of the
-wheel. Provenance checks verify both TVM/native libraries and source-built FFI
-load from the test environment; other missing runtime dependencies
-remain errors rather than permission to fetch replacements.
+wheel. Provenance checks bind installed distributions to the transferred wheel
+filenames and the loaded files' SHA-256 hashes from each wheel RECORD, and verify
+TVM/FFI Python and native libraries load
+from the test environment. Other missing runtime dependencies remain errors.
`CMAKE_BUILD_PARALLEL_LEVEL` is passed through Docker as an environment variable.
An explicit value wins; otherwise builds use at least one CPU and divide the
diff --git a/ci/jenkins/artifacts.py b/ci/jenkins/artifacts.py
index b222d0dd4d..4f468c5cc8 100644
--- a/ci/jenkins/artifacts.py
+++ b/ci/jenkins/artifacts.py
@@ -34,8 +34,8 @@ artifact prefix.
import sys
ARTIFACTS = {
- # New pipelines install the exact TVM wheel and retain only JVM native files.
- "wheels": ["build/wheels"],
+ # New pipelines transfer only wheels; legacy raw-library bundles stay available.
+ "wheels": ["build/wheels/*.whl"],
"jvm_runtime": ["build/lib/libtvm_runtime.so", "build/lib/libtvm_ffi.so"],
# Executables and build files needed to run c++ tests
"cpptest": ["build/cpptest", "build/build.ninja", "build/CMakeFiles/rules.ninja"],
diff --git a/ci/jenkins/generated/common.groovy b/ci/jenkins/generated/common.groovy
index 933bb11f0a..0eb042c301 100644
--- a/ci/jenkins/generated/common.groovy
+++ b/ci/jenkins/generated/common.groovy
@@ -391,9 +391,10 @@ def build_docker() {
def wheel_build(image, cpp_tests = false) {
// Overlay build requirements without modifying the image Python environment.
- // The build itself stays offline; runtime dependencies belong to the image.
+ // FFI uses its declared build requirements; neither build resolves runtime dependencies.
def commands = build_environment() + '''
rm -rf build/wheels build/wheel-build-deps
+ uv build --wheel --python python3 --out-dir build/wheels ./3rdparty/tvm-ffi
uv pip install --python python3 --target build/wheel-build-deps "scikit-build-core>=0.11" "setuptools-scm>=8" "mlc-z3-static>=4.16.0"
PYTHONPATH="$PWD/build/wheel-build-deps${PYTHONPATH:+:$PYTHONPATH}" \
uv build --wheel --no-build-isolation --offline --python python3 --out-dir build/wheels . \
diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy
index 8be385ac42..14d3daae2d 100644
--- a/ci/jenkins/generated/cpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy
@@ -41,7 +41,7 @@ stage('Test') {
variables: ['PLATFORM=cpu', 'TEST_STEP_NAME=unittest: CPU']) {
ci.docker_init(images.ci_cpu)
ci.download('cpu')
- ci.python_unittest(images.ci_cpu)
+ ci.python_unittest(images.ci_cpu)
}
})
}
diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy
index 7f2be47292..e8a3b3d76d 100644
--- a/ci/jenkins/generated/gpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy
@@ -27,7 +27,7 @@ stage('Build') {
ci.docker_init(images.ci_gpu)
sh "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
ci.wheel_build("--no-gpu ${images.ci_gpu}")
- ci.upload_artifacts('gpu', '--artifacts wheels --artifacts jvm_runtime')
+ ci.upload_artifacts('gpu', '--artifacts wheels')
// Check the compiler-only GPU configuration after publishing the test build.
sh 'rm -rf build'
diff --git a/jvm/assembly/linux-x86_64/src/main/assembly/assembly.xml b/jvm/assembly/linux-x86_64/src/main/assembly/assembly.xml
index 7345c496e9..ea7396f8dc 100644
--- a/jvm/assembly/linux-x86_64/src/main/assembly/assembly.xml
+++ b/jvm/assembly/linux-x86_64/src/main/assembly/assembly.xml
@@ -26,7 +26,12 @@ under the License.
<includeBaseDirectory>false</includeBaseDirectory>
<files>
<file>
- <source>../../../build/lib/libtvm_runtime.so</source>
+ <source>${tvm.lib.dir}/libtvm_runtime.so</source>
+ <outputDirectory>lib/native</outputDirectory>
+ <fileMode>0644</fileMode>
+ </file>
+ <file>
+ <source>${tvm.ffi.lib.dir}/libtvm_ffi.so</source>
<outputDirectory>lib/native</outputDirectory>
<fileMode>0644</fileMode>
</file>
diff --git a/jvm/assembly/osx-x86_64/src/main/assembly/assembly.xml b/jvm/assembly/osx-x86_64/src/main/assembly/assembly.xml
index 739852e474..70f5754983 100644
--- a/jvm/assembly/osx-x86_64/src/main/assembly/assembly.xml
+++ b/jvm/assembly/osx-x86_64/src/main/assembly/assembly.xml
@@ -26,7 +26,12 @@ under the License.
<includeBaseDirectory>false</includeBaseDirectory>
<files>
<file>
- <source>../../../build/lib/libtvm_runtime.dylib</source>
+ <source>${tvm.lib.dir}/libtvm_runtime.dylib</source>
+ <outputDirectory>lib/native</outputDirectory>
+ <fileMode>0644</fileMode>
+ </file>
+ <file>
+ <source>${tvm.ffi.lib.dir}/libtvm_ffi.dylib</source>
<outputDirectory>lib/native</outputDirectory>
<fileMode>0644</fileMode>
</file>
diff --git a/jvm/core/pom.xml b/jvm/core/pom.xml
index eeb55f5cfd..26ba6dcd78 100644
--- a/jvm/core/pom.xml
+++ b/jvm/core/pom.xml
@@ -104,7 +104,7 @@ under the License.
<threadCount>1</threadCount>
<argLine>
-Djava.library.path=${project.parent.basedir}/native/${platform}/target
- -Dlibtvm.so.path=${project.parent.basedir}/../build/lib/${libtvm.so.filename}
+ -Dlibtvm.so.path=${tvm.lib.dir}/${libtvm.so.filename}
-Dlog4j.configuration=file://${project.parent.basedir}/conf/log4j.properties
</argLine>
</configuration>
diff --git a/jvm/native/linux-x86_64/pom.xml b/jvm/native/linux-x86_64/pom.xml
index 31e120bc58..dcda4c6ce1 100644
--- a/jvm/native/linux-x86_64/pom.xml
+++ b/jvm/native/linux-x86_64/pom.xml
@@ -117,8 +117,8 @@ under the License.
<compilerStartOption>-std=c++17</compilerStartOption>
</compilerStartOptions>
<compilerEndOptions>
- <compilerEndOption>-I../../../include</compilerEndOption>
- <compilerEndOption>-I../../../3rdparty/tvm-ffi/include</compilerEndOption>
+ <compilerEndOption>-I${tvm.include.dir}</compilerEndOption>
+ <compilerEndOption>-I${tvm.ffi.include.dir}</compilerEndOption>
<compilerEndOption>-I${JAVA_HOME}/include</compilerEndOption>
<compilerEndOption>-I${JAVA_HOME}/include/linux</compilerEndOption>
<compilerEndOption>${cflags}</compilerEndOption>
@@ -127,7 +127,7 @@ under the License.
<linkerStartOption>-shared</linkerStartOption>
</linkerStartOptions>
<linkerEndOptions>
- <linkerEndOption>-L${project.parent.basedir}/../../build/lib/</linkerEndOption>
+ <linkerEndOption>-L${tvm.lib.dir}</linkerEndOption>
<linkerEndOption>-ltvm_runtime</linkerEndOption>
<linkerEndOption>${ldflags}</linkerEndOption>
</linkerEndOptions>
diff --git a/jvm/native/osx-x86_64/pom.xml b/jvm/native/osx-x86_64/pom.xml
index 4f9d70d60d..ccb79f5bbb 100644
--- a/jvm/native/osx-x86_64/pom.xml
+++ b/jvm/native/osx-x86_64/pom.xml
@@ -118,8 +118,8 @@ under the License.
<compilerStartOption>-std=c++17</compilerStartOption>
</compilerStartOptions>
<compilerEndOptions>
- <compilerEndOption>-I../../../include</compilerEndOption>
- <compilerEndOption>-I../../../3rdparty/tvm-ffi/include</compilerEndOption>
+ <compilerEndOption>-I${tvm.include.dir}</compilerEndOption>
+ <compilerEndOption>-I${tvm.ffi.include.dir}</compilerEndOption>
<compilerEndOption>-I${JAVA_HOME}/include</compilerEndOption>
<compilerEndOption>-I${JAVA_HOME}/include/darwin</compilerEndOption>
<compilerEndOption>${cflags}</compilerEndOption>
@@ -134,7 +134,7 @@ under the License.
<linkerMiddleOption>-Wl,-x</linkerMiddleOption>
</linkerMiddleOptions>
<linkerEndOptions>
- <linkerEndOption>-L${project.parent.basedir}/../../build/lib/</linkerEndOption>
+ <linkerEndOption>-L${tvm.lib.dir}</linkerEndOption>
<linkerEndOption>-ltvm_runtime</linkerEndOption>
<linkerEndOption>${ldflags}</linkerEndOption>
</linkerEndOptions>
diff --git a/jvm/pom.xml b/jvm/pom.xml
index e5efce0bf5..2a2f53cd73 100644
--- a/jvm/pom.xml
+++ b/jvm/pom.xml
@@ -44,6 +44,10 @@
<properties>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
+ <tvm.include.dir>${maven.multiModuleProjectDirectory}/../include</tvm.include.dir>
+ <tvm.ffi.include.dir>${maven.multiModuleProjectDirectory}/../3rdparty/tvm-ffi/include</tvm.ffi.include.dir>
+ <tvm.lib.dir>${maven.multiModuleProjectDirectory}/../build/lib</tvm.lib.dir>
+ <tvm.ffi.lib.dir>${tvm.lib.dir}</tvm.ffi.lib.dir>
</properties>
<packaging>pom</packaging>
diff --git a/tests/python/ci/test_wheel_artifacts.py b/tests/python/ci/test_wheel_artifacts.py
deleted file mode 100644
index a8adeef068..0000000000
--- a/tests/python/ci/test_wheel_artifacts.py
+++ /dev/null
@@ -1,309 +0,0 @@
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-"""Wheel artifact transport and local CI command contracts, without a TVM import."""
-
-import ast
-import importlib.util
-import json
-import os
-import re
-import shutil
-import subprocess
-import sys
-from pathlib import Path
-
-import pytest
-
-ROOT = Path(__file__).resolve().parents[3]
-
-
-def load_local_ci():
- spec = importlib.util.spec_from_file_location("local_ci", ROOT / "tests/scripts/ci.py")
- module = importlib.util.module_from_spec(spec)
- spec.loader.exec_module(module)
- return module
-
-
-@pytest.mark.parametrize("platform", ["cpu", "gpu", "arm", "wasm"])
-def test_local_wheel_build(monkeypatch, platform):
- ci = load_local_ci()
- calls = []
- monkeypatch.setattr(ci, "docker", lambda **kwargs: calls.append(kwargs))
- command, _, _ = ci.generate_command(platform, {}, "wheel build")
- command(cpp=True)
- scripts = calls[0]["scripts"]
- wheel = next(script for script in scripts if "-Cbuild-dir=" in script)
- assert "-Ccmake.build-type=RelWithDebInfo" in wheel
- assert "all;cpptest" in wheel
- assert not any("./3rdparty/tvm-ffi" in script for script in scripts)
- assert "--no-build-isolation --offline" in wheel
- assert any("task_python_install_wheels.sh" in script for script in scripts)
- calls.clear()
- command(skip_build=True, tests=["tests/python/example.py"])
- assert not any("uv build" in script for script in calls[0]["scripts"])
-
-
-@pytest.mark.parametrize("succeed", [True, False])
-def test_wheel_directory_upload_retries_each_transfer(tmp_path, succeed):
- # Exercise the existing S3 CLI/retry boundary with a directory bundle.
- for relative in [
- "ci/jenkins/scripts/s3.py",
- "ci/jenkins/scripts/retry.sh",
- "ci/jenkins/artifacts.py",
- ]:
- target = tmp_path / relative
- target.parent.mkdir(parents=True, exist_ok=True)
- shutil.copy(ROOT / relative, target)
- wheels = tmp_path / "build/wheels"
- wheels.mkdir(parents=True)
- (wheels / "apache_tvm.whl").write_text("tvm")
- binaries = tmp_path / "bin"
- binaries.mkdir()
- aws = binaries / "aws"
- aws.write_text(
- f"#!{sys.executable}\n"
- "import json, pathlib, sys\n"
- "path = pathlib.Path('attempts.json')\n"
- "calls = json.loads(path.read_text()) if path.exists() else []\n"
- "calls.append(sys.argv[1:])\n"
- "path.write_text(json.dumps(calls))\n"
- f"sys.exit(0 if {succeed!r} and len(calls) == 3 else 1)\n"
- )
- aws.chmod(0o755)
- sleep = binaries / "sleep"
- sleep.write_text('#!/bin/sh\necho "$1" >> delays.txt\n')
- sleep.chmod(0o755)
- result = subprocess.run(
- [
- sys.executable,
- "ci/jenkins/scripts/s3.py",
- "--action",
- "upload",
- "--bucket",
- "test",
- "--prefix",
- "cpu",
- "--bundle",
- "wheels",
- ],
- cwd=tmp_path,
- env={**os.environ, "PATH": f"{binaries}:{os.environ['PATH']}"},
- capture_output=True,
- text=True,
- )
- assert (result.returncode == 0) == succeed, result.stdout + result.stderr
- attempts = json.loads((tmp_path / "attempts.json").read_text())
- assert len(attempts) == 3
- assert all(
- call[-3:] == ["--recursive", "build/wheels", "s3://test/cpu/build/wheels"]
- for call in attempts
- )
- delays = [int(line) for line in (tmp_path / "delays.txt").read_text().splitlines()]
- assert len(delays) == 2 and all(30 <= delay <= 200 for delay in delays)
-
-
-@pytest.mark.parametrize("fail", [False, True])
-def test_local_interactive_keeps_cache_stats(tmp_path, monkeypatch, fail):
- ci = load_local_ci()
- monkeypatch.setattr(ci, "check_docker", lambda: None)
- monkeypatch.setattr(ci, "cmd", lambda *args, **kwargs: None)
- monkeypatch.setattr(ci, "SCRIPT_DIR", tmp_path / "scripts")
- monkeypatch.setattr(ci, "REPO_ROOT", tmp_path)
- monkeypatch.delenv("USE_SCCACHE", raising=False)
- monkeypatch.setenv("DEBUG", "1")
- binaries = tmp_path / "bin"
- binaries.mkdir()
- for name in ["sccache", "bash"]:
- binary = binaries / name
- binary.write_text(f'#!/bin/sh\necho "{name} $*" >> "$CALL_LOG"\n')
- binary.chmod(0o755)
- ci.docker("interactive", "ci_cpu", ["false" if fail else "true"], {}, True)
- result = subprocess.run(
- [shutil.which("bash"), str(ci.SCRIPT_DIR / "interactive.sh")],
- env={
- **os.environ,
- "PATH": f"{binaries}:{os.environ['PATH']}",
- "CALL_LOG": str(tmp_path / "calls"),
- },
- capture_output=True,
- text=True,
- )
- assert (result.returncode != 0) == fail
- calls = (tmp_path / "calls").read_text().splitlines()
- assert calls == [
- "sccache --start-server",
- "sccache --show-stats",
- "sccache --show-stats",
- "bash ",
- ]
-
-
-@pytest.mark.parametrize("wheel_docs", [False, True])
-def test_docs_import_path(tmp_path, monkeypatch, wheel_docs):
- # Run the real Sphinx path bootstrap without requiring Sphinx extensions.
- # A stale --target=python FFI package must not precede Jenkins' wheel install.
- monkeypatch.chdir(ROOT / "docs")
- source_python = str(ROOT / "python")
- monkeypatch.setattr(sys, "path", [path for path in sys.path if path != source_python])
- monkeypatch.setenv("TVM_DOCS_USE_WHEEL", "1" if wheel_docs else "0")
- config = ROOT / "docs/conf.py"
- bootstrap = config.read_text().split("# -- General configuration")[0]
- exec(compile(bootstrap, str(config), "exec"), {"__file__": str(config)})
- assert (source_python in sys.path) != wheel_docs
-
-
-@pytest.mark.parametrize(
- "host_limit,caller_limit,expected", [(None, None, "2"), ("7", None, "7"), ("7", "9", "9")]
-)
-def test_local_parallel_limit_crosses_docker_boundary(
- tmp_path, monkeypatch, host_limit, caller_limit, expected
-):
- ci = load_local_ci()
- calls = []
- monkeypatch.setattr(ci, "check_docker", lambda: None)
- monkeypatch.setattr(ci, "cmd", lambda command: calls.append(command))
- monkeypatch.setattr(ci, "SCRIPT_DIR", tmp_path / "scripts")
- monkeypatch.setattr(ci, "REPO_ROOT", tmp_path)
- monkeypatch.setattr(ci, "NPROC", 8)
- monkeypatch.setenv("CI_NUM_EXECUTORS", "3")
- if host_limit is None:
- monkeypatch.delenv("CMAKE_BUILD_PARALLEL_LEVEL", raising=False)
- else:
- monkeypatch.setenv("CMAKE_BUILD_PARALLEL_LEVEL", host_limit)
- env = {} if caller_limit is None else {"CMAKE_BUILD_PARALLEL_LEVEL": caller_limit}
- ci.docker("build", "ci_cpu", ["true"], env, False)
- args = calls[0]
- index = args.index(f"CMAKE_BUILD_PARALLEL_LEVEL={expected}")
- assert args[index - 1] == "--env"
-
-
-@pytest.mark.parametrize(
- "script", ["task_python_unittest.sh", "task_python_docs.sh", "task_web_wasm.sh"]
-)
-@pytest.mark.parametrize("wheel_mode", [False, True])
-def test_shared_scripts_preserve_legacy_and_wheel_modes(tmp_path, script, wheel_mode):
- scripts = tmp_path / "tests/scripts"
- scripts.mkdir(parents=True)
- shutil.copy(ROOT / "tests/scripts" / script, scripts / script)
- # Substitute only the install boundary; execute the real caller's mode routing.
- (scripts / "task_python_install_wheels.sh").write_text('echo WHEEL_SETUP >> "$CALL_LOG"\n')
- for directory in ["docs", "web", "bin"]:
- (tmp_path / directory).mkdir()
- for tool in ["uv", "python3", "make"]:
- binary = tmp_path / "bin" / tool
- binary.write_text(
- f'#!/bin/sh\necho "{tool}:$*" >> "$CALL_LOG"\n'
- + ("exit 77\n" if tool == "make" else "")
- )
- binary.chmod(0o755)
- result = subprocess.run(
- [shutil.which("bash"), str(scripts / script)],
- cwd=tmp_path,
- env={
- **os.environ,
- "PATH": f"{tmp_path / 'bin'}:{os.environ['PATH']}",
- "CALL_LOG": str(tmp_path / "calls"),
- "TVM_TEST_USE_WHEEL": "1" if wheel_mode else "0",
- "IS_LOCAL": "0",
- },
- capture_output=True,
- text=True,
- )
- assert result.returncode == (0 if script == "task_python_unittest.sh" else 77), result.stderr
- calls = (tmp_path / "calls").read_text().splitlines()
- assert ("WHEEL_SETUP" in calls) == wheel_mode
- dependency_calls = [call for call in calls if call.startswith("uv:")]
- if wheel_mode:
- assert not dependency_calls
- else:
- assert "uv:pip install -v --target=python ./3rdparty/tvm-ffi/" in dependency_calls
-
-
-def test_wheel_mode_does_not_fall_back_when_install_fails(tmp_path):
- scripts = tmp_path / "tests/scripts"
- scripts.mkdir(parents=True)
- shutil.copy(ROOT / "tests/scripts/task_python_unittest.sh", scripts)
- (scripts / "task_python_install_wheels.sh").write_text("return 37\n")
- result = subprocess.run(
- [shutil.which("bash"), str(scripts / "task_python_unittest.sh")],
- cwd=tmp_path,
- env={**os.environ, "TVM_TEST_USE_WHEEL": "1"},
- capture_output=True,
- text=True,
- )
- assert result.returncode == 37
- assert "uv pip install" not in result.stderr
-
-
-@pytest.mark.parametrize("bootstrap_fails", [False, True])
-def test_build_bootstrap_precedes_offline_build(tmp_path, monkeypatch, bootstrap_fails):
- import shlex
-
- ci = load_local_ci()
- calls = []
- monkeypatch.setattr(ci, "docker", lambda **kwargs: calls.append(kwargs))
- command, _, _ = ci.generate_command("cpu", {}, "wheel build")
- command()
- commands = [
- script
- for script in calls[0]["scripts"]
- if "uv pip install" in script or "uv build" in script
- ]
- bootstrap, build = commands
- arguments = shlex.split(bootstrap)
- # The declared build requirements are a flat string list; keep this test usable on Python 3.10.
- requires = re.search(r"(?ms)^requires = (\[.*?\])", (ROOT / "pyproject.toml").read_text())
- expected = ast.literal_eval(requires[1])
- assert arguments[:5] == ["uv", "pip", "install", "--python", "python3"]
- assert arguments[5] == "--target"
- assert arguments[7:] == expected
- assert "--no-build-isolation --offline --python python3" in build
- assert build.startswith(f'PYTHONPATH="$PWD/{arguments[6]}${{PYTHONPATH:+:$PYTHONPATH}}" ')
- binary = tmp_path / "uv"
- binary.write_text(
- f"#!{sys.executable}\n"
- "import json, os, pathlib, sys\n"
- "with pathlib.Path('calls').open('a') as out: out.write(json.dumps(sys.argv[1:]) + '\\n')\n"
- "with pathlib.Path('paths').open('a') as out: out.write(os.environ['PYTHONPATH'] + '\\n')\n"
- f"sys.exit(17 if {bootstrap_fails!r} and sys.argv[1] == 'pip' else 0)\n"
- )
- binary.chmod(0o755)
- result = subprocess.run(
- [
- shutil.which("bash"),
- "-ec",
- "\n".join([*commands, 'printf "%s" "$PYTHONPATH" > after-path']),
- ],
- cwd=tmp_path,
- env={
- **os.environ,
- "PATH": f"{tmp_path}:{os.environ['PATH']}",
- "PYTHONPATH": "image-marker",
- },
- capture_output=True,
- text=True,
- )
- assert result.returncode == (17 if bootstrap_fails else 0)
- recorded = [json.loads(line) for line in (tmp_path / "calls").read_text().splitlines()]
- assert [call[0] for call in recorded] == (["pip"] if bootstrap_fails else ["pip", "build"])
- paths = (tmp_path / "paths").read_text().splitlines()
- assert paths[0] == "image-marker"
- if not bootstrap_fails:
- assert paths[1] == f"{tmp_path}/{arguments[6]}:image-marker"
- assert (tmp_path / "after-path").read_text() == "image-marker"
- else:
- assert not (tmp_path / "after-path").exists()
diff --git a/tests/scripts/ci.py b/tests/scripts/ci.py
index 5c49251f4f..084dc9865b 100755
--- a/tests/scripts/ci.py
+++ b/tests/scripts/ci.py
@@ -403,6 +403,7 @@ def generate_command(
scripts = [
f"./tests/scripts/task_config_build_{name}.sh {build_dir}",
f"rm -rf {build_dir}/wheels {build_dir}/wheel-build-deps",
+ f"uv build --wheel --python python3 --out-dir {build_dir}/wheels ./3rdparty/tvm-ffi",
f'uv pip install --python python3 --target {build_dir}/wheel-build-deps "scikit-build-core>=0.11" "setuptools-scm>=8" "mlc-z3-static>=4.16.0"',
f'PYTHONPATH="$PWD/{build_dir}/wheel-build-deps${{PYTHONPATH:+:$PYTHONPATH}}" '
f"uv build --wheel --no-build-isolation --offline --python python3 --out-dir {build_dir}/wheels . "
diff --git a/tests/scripts/task_java_unittest.sh b/tests/scripts/task_java_unittest.sh
index 35621a5a13..07effb03e4 100755
--- a/tests/scripts/task_java_unittest.sh
+++ b/tests/scripts/task_java_unittest.sh
@@ -18,8 +18,12 @@
set -euxo pipefail
-export PYTHONPATH=python
-export LD_LIBRARY_PATH="lib:${LD_LIBRARY_PATH:-}"
+if [ "${TVM_TEST_USE_WHEEL:-0}" = "1" ]; then
+ source "$(dirname "$0")/task_python_install_wheels.sh"
+else
+ export PYTHONPATH=python
+ export LD_LIBRARY_PATH="lib:${LD_LIBRARY_PATH:-}"
+fi
# to avoid CI CPU thread throttling.
export TVM_BIND_THREADS=0
diff --git a/tests/scripts/task_jvm_build.sh b/tests/scripts/task_jvm_build.sh
index 468fea4ec6..c5f4dac03f 100755
--- a/tests/scripts/task_jvm_build.sh
+++ b/tests/scripts/task_jvm_build.sh
@@ -33,13 +33,32 @@ fi
ROOTDIR="$(cd "$(dirname "$0")/../.." && pwd)"
TVM_BUILD_PATH="${TVM_BUILD_PATH:-${ROOTDIR}/build}"
-TVM_BUILD_PATH="$(realpath "${TVM_BUILD_PATH}")"
+if [ "${TVM_TEST_USE_WHEEL:-0}" != "1" ]; then
+ TVM_BUILD_PATH="$(realpath "${TVM_BUILD_PATH}")"
+fi
DLPACK_PATH="${DLPACK_PATH:-${ROOTDIR}/3rdparty/tvm-ffi/3rdparty/dlpack}"
INCLUDE_FLAGS="-I${ROOTDIR}/include -I${DLPACK_PATH}/include"
PKG_CFLAGS="-Wall -O3 ${INCLUDE_FLAGS} -fPIC"
PKG_LDFLAGS=""
+NATIVE_ARGS=()
+if [ "${TVM_TEST_USE_WHEEL:-0}" = "1" ]; then
+ TVM_INCLUDE_DIR="$(python3 -m tvm.config --includedir)"
+ FFI_INCLUDE_DIR="$(python3 -m tvm_ffi.config --includedir)"
+ TVM_LIB_DIR="$(python3 -m tvm.config --libdir)"
+ FFI_LIB_DIR="$(python3 -m tvm_ffi.config --libdir)"
+ PKG_CFLAGS="-Wall -O3 -fPIC $(python3 -m tvm.config --cflags) $(python3 -m tvm_ffi.config --cflags)"
+ PKG_LDFLAGS="$(python3 -m tvm.config --ldflags --libs) $(python3 -m tvm_ffi.config --ldflags --libs)"
+ # Each config flag prints one line; Maven expects one compiler/linker argument list.
+ PKG_LDFLAGS="${PKG_LDFLAGS//$'\n'/ }"
+ NATIVE_ARGS=(
+ "-Dtvm.include.dir=${TVM_INCLUDE_DIR}"
+ "-Dtvm.ffi.include.dir=${FFI_INCLUDE_DIR}"
+ "-Dtvm.lib.dir=${TVM_LIB_DIR}"
+ "-Dtvm.ffi.lib.dir=${FFI_LIB_DIR}"
+ )
+fi
if [ "$(uname -s)" = "Darwin" ]; then
JVM_PKG_PROFILE="osx-x86_64"
@@ -64,5 +83,6 @@ mvn ${GOAL} \
"-Dcflags=${PKG_CFLAGS}" \
"-Dldflags=${PKG_LDFLAGS}" \
"-Dcurrent_libdir=${TVM_BUILD_PATH}" \
+ "${NATIVE_ARGS[@]}" \
${JVM_TEST_ARGS} \
"$@"
diff --git a/tests/scripts/task_python_install_wheels.sh b/tests/scripts/task_python_install_wheels.sh
index 9fde7c743c..6791f36780 100644
--- a/tests/scripts/task_python_install_wheels.sh
+++ b/tests/scripts/task_python_install_wheels.sh
@@ -22,7 +22,7 @@ unset PYTHONPATH TVM_LIBRARY_PATH TVM_FFI_LIBRARY_PATH
export PYTHONNOUSERSITE=1
# Installed TVM still needs checkout resources such as web/dist for Wasm linking.
export TVM_HOME="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
-TVM_WHEEL_DIR="$(realpath "${TVM_WHEEL_DIR:-$(pwd)/build/wheels}")"
+export TVM_WHEEL_DIR="$(realpath "${TVM_WHEEL_DIR:-$(pwd)/build/wheels}")"
# Preserve the image's active environment (often itself a venv), including when
# local ci.py sources this setup again for a named test suite.
export TVM_IMAGE_PYTHON="${TVM_IMAGE_PYTHON:-$(command -v python3)}"
@@ -34,26 +34,36 @@ uv venv --offline --python "$TVM_IMAGE_PYTHON" --clear "$VIRTUAL_ENV"
TVM_WHEEL_SITE="$("$VIRTUAL_ENV/bin/python3" -c 'import sysconfig; print(sysconfig.get_path("purelib"))')"
printf '%s\n' "$TVM_IMAGE_SITE_PACKAGES" > "$TVM_WHEEL_SITE/image-dependencies.pth"
export PATH="$VIRTUAL_ENV/bin:$PATH"
-# Keep the checkout available to tests and build its pinned FFI into this environment.
-# FFI declares its own isolated build requirements; do not reuse TVM's build overlay.
-export TVM_FFI_SOURCE="$TVM_HOME/3rdparty/tvm-ffi"
-uv pip install --python "$VIRTUAL_ENV/bin/python3" --no-deps --reinstall "$TVM_FFI_SOURCE"
+# Require one artifact for each distribution before creating an installation.
+shopt -s nullglob
+tvm_wheels=("$TVM_WHEEL_DIR"/apache_tvm-*.whl)
+ffi_wheels=("$TVM_WHEEL_DIR"/apache_tvm_ffi-*.whl)
+shopt -u nullglob
+if [ "${#tvm_wheels[@]}" -ne 1 ] || [ "${#ffi_wheels[@]}" -ne 1 ]; then
+ echo "Expected exactly one TVM wheel and one pinned-source FFI wheel" >&2
+ return 1
+fi
# Match the explicit prerequisite in the legacy Python test setup.
uv pip install --python "$VIRTUAL_ENV/bin/python3" --no-deps apache-tvm-ffi-orcjit==0.1.2
uv pip install --python "$VIRTUAL_ENV/bin/python3" --no-deps --offline --reinstall \
- "$TVM_WHEEL_DIR"/apache_tvm-*.whl
+ "${tvm_wheels[0]}" "${ffi_wheels[0]}"
# Fail before pytest if a source/editable install or native override leaked in.
python3 - <<'PYTHON'
+import base64
+import csv
+import hashlib
+import io
import json
import os
import sys
from importlib.metadata import distribution, requires, version
from pathlib import Path
from urllib.parse import unquote, urlparse
+from zipfile import ZipFile
from packaging.requirements import Requirement
-# Core FFI comes from pinned source and ORCJIT is pinned explicitly.
+# Core FFI comes from the build-stage wheel and ORCJIT is pinned explicitly.
# Check the wheel requirements without invoking a resolver or hardcoding a floor.
for text in requires("apache-tvm") or []:
requirement = Requirement(text)
@@ -69,13 +79,34 @@ prefix = Path(sys.prefix).resolve()
path = Path(tvm.__file__).resolve()
assert path.is_relative_to(prefix), path
print(f"Installed tvm: {path}")
-# Verify FFI was installed from this checkout and loads entirely from this environment.
-source = json.loads(distribution("apache-tvm-ffi").read_text("direct_url.json"))
-assert Path(unquote(urlparse(source["url"]).path)).resolve() == Path(os.environ["TVM_FFI_SOURCE"]).resolve()
-assert not source.get("dir_info", {}).get("editable", False), source
+# Bind each distribution and its loaded Python/native files to the exact wheel.
+def verify_wheel(name, pattern, paths):
+ wheel, = Path(os.environ["TVM_WHEEL_DIR"]).glob(pattern)
+ dist = distribution(name)
+ source = json.loads(dist.read_text("direct_url.json"))
+ assert Path(unquote(urlparse(source["url"]).path)).resolve() == wheel.resolve(), source
+ assert "archive_info" in source and "dir_info" not in source, source
+ with ZipFile(wheel) as archive:
+ record, = (name for name in archive.namelist() if name.endswith(".dist-info/RECORD"))
+ hashes = {row[0]: row[1] for row in csv.reader(io.StringIO(archive.read(record).decode()))}
+ for path in paths:
+ path = Path(path).resolve()
+ relative = path.relative_to(Path(dist.locate_file("")).resolve()).as_posix()
+ digest = hashlib.sha256()
+ with path.open("rb") as stream:
+ for chunk in iter(lambda: stream.read(1024 * 1024), b""):
+ digest.update(chunk)
+ actual = "sha256=" + base64.urlsafe_b64encode(digest.digest()).decode().rstrip("=")
+ assert actual == hashes[relative], (name, path)
+ print(f"Installed {name} wheel: {wheel.name}")
+
+verify_wheel("apache-tvm", "apache_tvm-*.whl",
+ [tvm.__file__, *(lib._name for lib in tvm.base._LOADED_LIBS.values())])
+verify_wheel("apache-tvm-ffi", "apache_tvm_ffi-*.whl",
+ [tvm_ffi.__file__, tvm_ffi.core.__file__, tvm_ffi.LIB._name])
for path in (Path(tvm_ffi.__file__).resolve(), Path(tvm_ffi.LIB._name).resolve()):
assert path.is_relative_to(prefix), path
- print(f"Source-built FFI: {path}")
+ print(f"Installed FFI: {path}")
assert {"tvm_runtime", "tvm_compiler"} <= tvm.base._LOADED_LIBS.keys()
for name, library in tvm.base._LOADED_LIBS.items():
path = Path(library._name).resolve()
(tvm) 02/03: [Python] Add lightweight runtime build configuration queries
Posted by tq...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch jenkins-wheel-test-artifacts
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit d795529e717d2cc92ec2163648ce4ebbede92d1f
Author: Tianqi Chen <tq...@users.noreply.github.com>
AuthorDate: Thu Oct 15 12:34:58 2026 +0000
[Python] Add lightweight runtime build configuration queries
---
pyproject.toml | 3 +
python/tvm/__init__.py | 182 +++++++++++++++++++-----------------
python/tvm/config.py | 63 +++++++++++++
python/tvm/libinfo.py | 39 ++++----
tests/python/runtime/test_config.py | 68 ++++++++++++++
5 files changed, 255 insertions(+), 100 deletions(-)
diff --git a/pyproject.toml b/pyproject.toml
index 5c7512032f..3a19c57d50 100644
--- a/pyproject.toml
+++ b/pyproject.toml
@@ -57,6 +57,9 @@ dependencies = [
"typing_extensions",
]
+[project.scripts]
+tvm-config = "tvm.config:__main__"
+
[project.optional-dependencies]
torch = ["torch"]
cuda = ["cuda-bindings"]
diff --git a/python/tvm/__init__.py b/python/tvm/__init__.py
index e6381cbe3b..2c35d829f2 100644
--- a/python/tvm/__init__.py
+++ b/python/tvm/__init__.py
@@ -18,120 +18,134 @@
# pylint: disable=redefined-builtin, wildcard-import
"""TVM: Open Deep Learning Compiler Stack."""
-import multiprocessing
import os
import sys
-# ffi module must load first
-from tvm_ffi import register_object, register_global_func, get_global_func
+from typing import TYPE_CHECKING
-# top-level alias
-from .libinfo import __version__
-from .base import _RUNTIME_ONLY
-# tvm.runtime
-from .runtime import Object
-from .runtime._tensor import device, cpu, cuda, opencl, vulkan, metal
-from .runtime._tensor import vpi, rocm, ext_dev, hexagon
-from .runtime import DataType, DataTypeCode
+def _is_config_mode() -> bool:
+ """Recognize the config CLI before loading Python or native TVM components."""
+ if sys.argv[0].endswith("tvm-config"):
+ return True
+ argv = getattr(sys, "orig_argv", [])
+ return any(arg == "-m" and argv[i + 1 : i + 2] == ["tvm.config"] for i, arg in enumerate(argv))
-# tvm.error
-from . import error
-# tvm.ir
-from .ir import IRModule, transform
-from . import ir
+if TYPE_CHECKING or not _is_config_mode():
+ import multiprocessing
-# tvm.script — must be imported before any dialect package so that
-# tvm.script.register_dialect is reachable when dialect __init__.py files run.
-from . import script
+ # ffi module must load first
+ from tvm_ffi import register_object, register_global_func, get_global_func
-# tvm.tirx — registers itself via tvm.script.register_dialect in its __init__
-from . import tirx
-from .ir import prim
+ # top-level alias
+ from .libinfo import __version__
+ from .base import _RUNTIME_ONLY
-# tvm.backend — owns backend Python load hooks
-from . import backend
+ # tvm.runtime
+ from .runtime import Object
+ from .runtime._tensor import device, cpu, cuda, opencl, vulkan, metal
+ from .runtime._tensor import vpi, rocm, ext_dev, hexagon
+ from .runtime import DataType, DataTypeCode
-# tvm.target
-from . import target
-from .target import device_from_target
+ # tvm.error
+ from . import error
-# Shared code generation helpers
-from . import codegen
+ # tvm.ir
+ from .ir import IRModule, transform
+ from . import ir
-# tvm.te
-from . import te
+ # tvm.script — must be imported before any dialect package so that
+ # tvm.script.register_dialect is reachable when dialect __init__.py files run.
+ from . import script
-# tvm.driver
-from .driver import compile
+ # tvm.tirx — registers itself via tvm.script.register_dialect in its __init__
+ from . import tirx
+ from .ir import prim
-# others
-from . import sym
+ # tvm.backend — owns backend Python load hooks
+ from . import backend
-# support infra
-from . import support
+ # tvm.target
+ from . import target
+ from .target import device_from_target
-# Side-effect imports: register CUDA/ROCm FFI callbacks at TVM startup
-from .support import rocm as _rocm, nvcc as _nvcc
+ # Shared code generation helpers
+ from . import codegen
-# Relax contain modules that are only available in compiler package
-# Do not import them if TVM is built with runtime only
-if not _RUNTIME_ONLY:
- # tvm.relax — registers itself via tvm.script.register_dialect in its __init__
- from . import relax
+ # tvm.te
+ from . import te
-# NOTE: This file should be python2 compatible so we can
-# raise proper error message when user run the package using
-# an older version of the python
+ # tvm.driver
+ from .driver import compile
+ # others
+ from . import sym
-def _should_print_backtrace():
- in_pytest = "PYTEST_CURRENT_TEST" in os.environ
- tvm_backtrace = os.environ.get("TVM_BACKTRACE", "0")
+ # support infra
+ from . import support
- try:
- tvm_backtrace = bool(int(tvm_backtrace))
- except ValueError:
- raise ValueError(f"invalid value for TVM_BACKTRACE {tvm_backtrace}, please set to 0 or 1.")
+ # Side-effect imports: register CUDA/ROCm FFI callbacks at TVM startup
+ from .support import rocm as _rocm, nvcc as _nvcc
+
+ # Relax contain modules that are only available in compiler package
+ # Do not import them if TVM is built with runtime only
+ if not _RUNTIME_ONLY:
+ # tvm.relax — registers itself via tvm.script.register_dialect in its __init__
+ from . import relax
- return in_pytest or tvm_backtrace
+ # NOTE: This file should be python2 compatible so we can
+ # raise proper error message when user run the package using
+ # an older version of the python
+ def _should_print_backtrace():
+ in_pytest = "PYTEST_CURRENT_TEST" in os.environ
+ tvm_backtrace = os.environ.get("TVM_BACKTRACE", "0")
-def tvm_wrap_excepthook(exception_hook):
- """Wrap given excepthook with TVM additional work."""
+ try:
+ tvm_backtrace = bool(int(tvm_backtrace))
+ except ValueError:
+ raise ValueError(
+ f"invalid value for TVM_BACKTRACE {tvm_backtrace}, please set to 0 or 1."
+ )
- def wrapper(exctype, value, trbk):
- """Clean subprocesses when TVM is interrupted."""
- if exctype is error.DiagnosticError and not _should_print_backtrace():
- # TODO(@jroesch): consider moving to C++?
- print("note: run with `TVM_BACKTRACE=1` environment variable to display a backtrace.")
- else:
- exception_hook(exctype, value, trbk)
+ return in_pytest or tvm_backtrace
- if hasattr(multiprocessing, "active_children"):
- # pylint: disable=not-callable
- for p in multiprocessing.active_children():
- p.terminate()
+ def tvm_wrap_excepthook(exception_hook):
+ """Wrap given excepthook with TVM additional work."""
- return wrapper
+ def wrapper(exctype, value, trbk):
+ """Clean subprocesses when TVM is interrupted."""
+ if exctype is error.DiagnosticError and not _should_print_backtrace():
+ # TODO(@jroesch): consider moving to C++?
+ print(
+ "note: run with `TVM_BACKTRACE=1` environment variable to display a backtrace."
+ )
+ else:
+ exception_hook(exctype, value, trbk)
+ if hasattr(multiprocessing, "active_children"):
+ # pylint: disable=not-callable
+ for p in multiprocessing.active_children():
+ p.terminate()
-sys.excepthook = tvm_wrap_excepthook(sys.excepthook)
+ return wrapper
-# Autoload loads built-in and out-of-tree backends. Out-of-tree extensions opt
-# into being loaded automatically at ``import tvm`` time by declaring an entry
-# point in the ``tvm.backends`` group:
-# [project.entry-points."tvm.backends"] tvm_foo = "tvm_foo:_autoload".
-# Autoload can be disabled via ``TVM_DEVICE_BACKEND_AUTOLOAD=0``.
-from .backend._autoload_backends import _autoload_backends
+ sys.excepthook = tvm_wrap_excepthook(sys.excepthook)
-_autoload_backends()
+ # Autoload loads built-in and out-of-tree backends. Out-of-tree extensions opt
+ # into being loaded automatically at ``import tvm`` time by declaring an entry
+ # point in the ``tvm.backends`` group:
+ # [project.entry-points."tvm.backends"] tvm_foo = "tvm_foo:_autoload".
+ # Autoload can be disabled via ``TVM_DEVICE_BACKEND_AUTOLOAD=0``.
+ from .backend._autoload_backends import _autoload_backends
-# Load the optional ORC JIT extension for its global FFI registrations. LLVM
-# function lookup reports a targeted installation error if this import fails;
-# importing TVM itself remains silent for AOT-only workflows.
-try:
- import tvm_ffi_orcjit as _tvm_ffi_orcjit
-except Exception: # pylint: disable=broad-exception-caught
- pass
+ _autoload_backends()
+
+ # Load the optional ORC JIT extension for its global FFI registrations. LLVM
+ # function lookup reports a targeted installation error if this import fails;
+ # importing TVM itself remains silent for AOT-only workflows.
+ try:
+ import tvm_ffi_orcjit as _tvm_ffi_orcjit
+ except Exception: # pylint: disable=broad-exception-caught
+ pass
diff --git a/python/tvm/config.py b/python/tvm/config.py
new file mode 100644
index 0000000000..8a75f4469b
--- /dev/null
+++ b/python/tvm/config.py
@@ -0,0 +1,63 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""Lightweight compiler and linker configuration for the TVM runtime."""
+
+import argparse
+import sys
+from pathlib import Path
+
+from . import libinfo
+
+
+def __main__() -> None:
+ """Print paths and flags using the same conventions as tvm_ffi.config."""
+ parser = argparse.ArgumentParser(description="Get TVM runtime build configuration")
+ for name, help_text in (
+ ("includedir", "Print include directory"),
+ ("libdir", "Print library directory"),
+ ("libfiles", "Fully qualified library filenames"),
+ ("libs", "Libraries to be linked"),
+ ("cflags", "Print C flags"),
+ ("cxxflags", "Print C++ flags"),
+ ("ldflags", "Print linker flags"),
+ ):
+ parser.add_argument(f"--{name}", action="store_true", help=help_text)
+ args = parser.parse_args()
+ if len(sys.argv) == 1:
+ parser.print_help()
+ return
+ if args.includedir:
+ print(libinfo.find_include_path())
+ if args.libdir or args.libfiles or args.libs or args.ldflags:
+ library = Path(libinfo.find_libtvm_runtime())
+ link_file = library.with_suffix(".lib") if sys.platform == "win32" else library
+ if args.libdir:
+ print(library.parent)
+ if args.libfiles:
+ print(link_file)
+ if args.cxxflags:
+ print(f"-I{libinfo.find_include_path()} -std=c++17")
+ if args.cflags:
+ print(f"-I{libinfo.find_include_path()}")
+ if args.libs:
+ print(link_file if sys.platform == "win32" else "-ltvm_runtime")
+ if args.ldflags and sys.platform != "win32":
+ print(f"-L{library.parent}")
+
+
+if __name__ == "__main__":
+ __main__()
diff --git a/python/tvm/libinfo.py b/python/tvm/libinfo.py
index 312d769f7e..e651277aec 100644
--- a/python/tvm/libinfo.py
+++ b/python/tvm/libinfo.py
@@ -19,10 +19,9 @@
from __future__ import annotations
import os
+import sys
from pathlib import Path
-from tvm_ffi import libinfo as tvm_ffi_libinfo
-
def _rel_top_directory() -> Path:
"""Top directory in the installed (wheel) layout: ``python/tvm/``."""
@@ -69,24 +68,32 @@ def find_libtvm_runtime() -> str:
RuntimeError
If the runtime library cannot be found.
"""
- candidate = tvm_ffi_libinfo._find_library_by_basename(
- "tvm", "tvm_runtime", extra_lib_paths=package_lib_paths()
- )
- if ret := tvm_ffi_libinfo._resolve_and_validate([candidate], cond=lambda _: True):
- return ret
- raise RuntimeError("Cannot find libtvm_runtime")
+ if sys.platform.startswith("win32"):
+ names = ["tvm_runtime.dll"]
+ search_var = "PATH"
+ elif sys.platform == "darwin":
+ names = ["libtvm_runtime.dylib", "libtvm_runtime.so"]
+ search_var = "DYLD_LIBRARY_PATH"
+ else:
+ names = ["libtvm_runtime.so"]
+ search_var = "LD_LIBRARY_PATH"
+ paths = package_lib_paths()
+ paths.extend(Path(p) for p in os.environ.get(search_var, "").split(os.pathsep) if p)
+ if search_var != "PATH":
+ paths.extend(Path(p) for p in os.environ.get("PATH", "").split(os.pathsep) if p)
+ for directory in paths:
+ for name in names:
+ candidate = directory / name
+ if candidate.is_file():
+ return str(candidate.resolve())
+ raise RuntimeError(f"Cannot find libtvm_runtime in {paths}")
def find_include_path() -> str:
"""Return the path to TVM's own ``include/`` directory."""
- if ret := tvm_ffi_libinfo._resolve_and_validate(
- paths=[
- _rel_top_directory() / "include",
- _dev_top_directory() / "include",
- ],
- cond=lambda p: (p / "tvm" / "runtime").is_dir(),
- ):
- return ret
+ for path in (_rel_top_directory() / "include", _dev_top_directory() / "include"):
+ if (path / "tvm" / "runtime").is_dir():
+ return str(path.resolve())
raise RuntimeError("Cannot find TVM include path.")
diff --git a/tests/python/runtime/test_config.py b/tests/python/runtime/test_config.py
new file mode 100644
index 0000000000..238c083e7d
--- /dev/null
+++ b/tests/python/runtime/test_config.py
@@ -0,0 +1,68 @@
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+"""The config command must work without importing native runtime dependencies."""
+
+import os
+import subprocess
+import sys
+from pathlib import Path
+
+import tvm
+
+
+def test_config_without_native_imports(tmp_path):
+ # A startup hook rejects native loads and dependencies even when they are installed.
+ (tmp_path / "sitecustomize.py").write_text(
+ "import sys\n"
+ "class BlockNative:\n"
+ " def find_spec(self, fullname, path=None, target=None):\n"
+ " if fullname.split('.')[0] in ('tvm_ffi', 'torch', 'numpy') or "
+ "fullname in ('tvm.base', 'tvm.runtime', 'tvm.backend', 'tvm.driver'):\n"
+ " raise AssertionError(fullname)\n"
+ "sys.meta_path.insert(0, BlockNative())\n"
+ "def audit(event, args):\n"
+ " if event == 'ctypes.dlopen': raise AssertionError(args)\n"
+ "sys.addaudithook(audit)\n"
+ )
+ env = dict(os.environ)
+ env["PYTHONPATH"] = os.pathsep.join((str(tmp_path), str(Path(tvm.__file__).parent.parent)))
+ command = [sys.executable, "-m", "tvm.config"]
+ include = subprocess.check_output([*command, "--includedir"], env=env, text=True).strip()
+ assert (Path(include) / "tvm" / "runtime").is_dir()
+ library = subprocess.check_output([*command, "--libfiles"], env=env, text=True).strip()
+ assert Path(library).is_file()
+ flags = subprocess.check_output(
+ [*command, "--cflags", "--cxxflags", "--libs", "--ldflags"], env=env, text=True
+ )
+ assert f"-I{include}" in flags
+ assert "-std=c++17" in flags
+
+
+def test_runtime_library_path_fallback(tmp_path, monkeypatch):
+ library = tmp_path / Path(tvm.libinfo.find_libtvm_runtime()).name
+ library.touch()
+ monkeypatch.setattr(tvm.libinfo, "package_lib_paths", lambda: [])
+ monkeypatch.delenv("LD_LIBRARY_PATH", raising=False)
+ monkeypatch.delenv("DYLD_LIBRARY_PATH", raising=False)
+ monkeypatch.setenv("PATH", str(tmp_path))
+ assert tvm.libinfo.find_libtvm_runtime() == str(library)
+
+
+if __name__ == "__main__":
+ import tvm.testing
+
+ tvm.testing.main()
(tvm) 01/03: Merge explicit Jenkins node and artifact helpers into wheel pipelines
Posted by tq...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch jenkins-wheel-test-artifacts
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit c9db658e7d9987279e899755fc0ecec0a5b58ef5
Merge: 5bb9f9d791 a1490e3f25
Author: Tianqi Chen <tq...@users.noreply.github.com>
AuthorDate: Thu Oct 15 11:55:13 2026 +0000
Merge explicit Jenkins node and artifact helpers into wheel pipelines
ci/jenkins/README.md | 16 +-
ci/jenkins/generated/arm_jenkinsfile.groovy | 18 +-
ci/jenkins/generated/common.groovy | 36 +++
ci/jenkins/generated/cpu_jenkinsfile.groovy | 25 ++-
ci/jenkins/generated/gpu_jenkinsfile.groovy | 40 ++--
ci/jenkins/generated/wasm_jenkinsfile.groovy | 24 +-
tests/python/ci/jenkins_loader_test.groovy | 84 -------
tests/python/ci/sample_prs/pr10786.json | 130 -----------
tests/python/ci/test_ci.py | 55 ++++-
tests/python/ci/test_jenkins_helpers.py | 318 ---------------------------
tests/python/ci/test_jenkins_loader.py | 37 ----
tests/python/ci/test_tvmbot.py | 106 +++++++--
tests/python/ci/test_utils.py | 80 -------
13 files changed, 260 insertions(+), 709 deletions(-)
diff --cc ci/jenkins/generated/arm_jenkinsfile.groovy
index b376794a3c,c826e2d104..2c1db5d936
--- a/ci/jenkins/generated/arm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/arm_jenkinsfile.groovy
@@@ -19,13 -19,16 +19,17 @@@
ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy'))
ci.start()
+docker_run += ' --env TVM_TEST_USE_WHEEL=1'
+
- ci.build('ARM-GRAVITON3', 'BUILD: arm', 'tvm/build-arm', images.ci_arm, 'arm') {
- sh(
- script: "${docker_run} ${images.ci_arm} ./tests/scripts/task_config_build_arm.sh build",
- label: 'Create ARM CMake config',
- )
- ci.wheel_build(images.ci_arm, true)
- ci.upload('arm', '--artifacts wheels')
+ stage('Build') {
+ ci.with_node(instance: 'ARM-GRAVITON3-SPOT', retry_instance: 'ARM-GRAVITON3',
+ name: 'BUILD: arm', folder: 'tvm/build-arm', variables: ['PLATFORM=arm']) {
+ ci.docker_init(images.ci_arm)
+ sh(
+ script: "${docker_run} ${images.ci_arm} ./tests/scripts/task_config_build_arm.sh build",
+ label: 'Create ARM CMake config',
+ )
- ci.cmake_build(images.ci_arm, 'build')
- ci.make_cpp_tests(images.ci_arm, 'build')
- ci.upload_artifacts('arm', '--artifacts tvm_lib --artifacts cpptest')
++ ci.wheel_build(images.ci_arm, true)
++ ci.upload_artifacts('arm', '--artifacts wheels')
+ }
}
diff --cc ci/jenkins/generated/cpu_jenkinsfile.groovy
index 72ed6e4e6f,edadf43273..8be385ac42
--- a/ci/jenkins/generated/cpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy
@@@ -19,22 -19,29 +19,29 @@@
ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy'))
ci.start()
+docker_run += ' --env TVM_TEST_USE_WHEEL=1'
+
- ci.build('CPU', 'BUILD: CPU', 'tvm/build-cpu', images.ci_cpu, 'cpu') {
- sh(
- script: "${docker_run} ${images.ci_cpu} ./tests/scripts/task_config_build_cpu.sh build",
- label: 'Create CPU CMake config',
- )
- ci.wheel_build(images.ci_cpu, true)
- ci.upload('cpu', '--artifacts wheels')
+ stage('Build') {
+ ci.with_node(instance: 'CPU-SPOT', retry_instance: 'CPU',
+ name: 'BUILD: CPU', folder: 'tvm/build-cpu', variables: ['PLATFORM=cpu']) {
+ ci.docker_init(images.ci_cpu)
+ sh(
+ script: "${docker_run} ${images.ci_cpu} ./tests/scripts/task_config_build_cpu.sh build",
+ label: 'Create CPU CMake config',
+ )
- ci.cmake_build(images.ci_cpu, 'build')
- ci.make_cpp_tests(images.ci_cpu, 'build')
- ci.upload_artifacts('cpu', '--artifacts tvm_lib --artifacts cpptest')
++ ci.wheel_build(images.ci_cpu, true)
++ ci.upload_artifacts('cpu', '--artifacts wheels')
+ }
}
stage('Test') {
parallel('unittest: CPU': {
- ci.test('CPU', 'unittest: CPU', 'tvm/ut-python-cpu', images.ci_cpu, 'cpu') {
+ ci.with_node(instance: 'CPU-SPOT', retry_instance: 'CPU',
+ name: 'unittest: CPU', folder: 'tvm/ut-python-cpu',
+ variables: ['PLATFORM=cpu', 'TEST_STEP_NAME=unittest: CPU']) {
+ ci.docker_init(images.ci_cpu)
ci.download('cpu')
- ci.cpp_unittest(images.ci_cpu)
-- ci.python_unittest(images.ci_cpu)
++ ci.python_unittest(images.ci_cpu)
}
})
}
diff --cc ci/jenkins/generated/gpu_jenkinsfile.groovy
index b5afbe0b42,668c0dadd2..7f2be47292
--- a/ci/jenkins/generated/gpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy
@@@ -19,21 -19,19 +19,25 @@@
ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy'))
ci.start()
+docker_run += ' --env TVM_TEST_USE_WHEEL=1'
+
- ci.build('CPU', 'BUILD: GPU', 'tvm/build-gpu', images.ci_gpu, 'gpu') {
- sh "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
- ci.wheel_build("--no-gpu ${images.ci_gpu}")
- ci.upload('gpu', '--artifacts wheels --artifacts jvm_runtime')
+ stage('Build') {
+ ci.with_node(instance: 'CPU-SPOT', retry_instance: 'CPU',
+ name: 'BUILD: GPU', folder: 'tvm/build-gpu', variables: ['PLATFORM=gpu']) {
+ ci.docker_init(images.ci_gpu)
+ sh "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
- ci.cmake_build("${images.ci_gpu} --no-gpu", 'build')
- ci.upload_artifacts('gpu', '--artifacts tvm_lib --artifacts tvm_lib_gpu_extra')
++ ci.wheel_build("--no-gpu ${images.ci_gpu}")
++ ci.upload_artifacts('gpu', '--artifacts wheels --artifacts jvm_runtime')
- // Check the compiler-only GPU configuration after publishing the test build.
- sh 'rm -rf build'
- sh "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build"
- def commands = ci.build_environment() + '''
- cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo
- cmake --build build --target tvm_runtime
- '''
- sh(ci.build_docker() + "--no-gpu ${images.ci_gpu} bash -c '${commands}'")
+ // Check the compiler-only GPU configuration after publishing the test build.
+ sh 'rm -rf build'
+ sh "${docker_run} --no-gpu ${images.ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build"
- sh "${docker_run} --env CI_NUM_EXECUTORS ${images.ci_gpu} --no-gpu ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --cmake-target tvm_runtime --build-dir build"
++ def commands = ci.build_environment() + '''
++ cmake -S . -B build -GNinja -DCMAKE_BUILD_TYPE=RelWithDebInfo
++ cmake --build build --target tvm_runtime
++ '''
++ sh(ci.build_docker() + "--no-gpu ${images.ci_gpu} bash -c '${commands}'")
+ }
}
stage('Test') {
diff --cc ci/jenkins/generated/wasm_jenkinsfile.groovy
index f514772dda,da38b1c237..7b67ee4d99
--- a/ci/jenkins/generated/wasm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy
@@@ -19,16 -19,20 +19,20 @@@
ci = evaluate(readTrusted('ci/jenkins/generated/common.groovy'))
ci.start()
+docker_run += ' --env TVM_TEST_USE_WHEEL=1'
+
- ci.build('CPU', 'BUILD: WASM', 'tvm/build-wasm', images.ci_wasm, 'wasm') {
- sh(
- script: "${docker_run} ${images.ci_wasm} ./tests/scripts/task_config_build_wasm.sh build",
- label: 'Create WASM CMake config',
- )
- ci.wheel_build(images.ci_wasm, true)
- sh(
- script: "${docker_run} ${images.ci_wasm} ./tests/scripts/task_web_wasm.sh",
- label: 'Run WASM lint and tests',
- )
+ stage('Build') {
+ ci.with_node(instance: 'CPU-SPOT', retry_instance: 'CPU',
+ name: 'BUILD: WASM', folder: 'tvm/build-wasm', variables: ['PLATFORM=wasm']) {
+ ci.docker_init(images.ci_wasm)
+ sh(
+ script: "${docker_run} ${images.ci_wasm} ./tests/scripts/task_config_build_wasm.sh build",
+ label: 'Create WASM CMake config',
+ )
- ci.cmake_build(images.ci_wasm, 'build')
- ci.make_cpp_tests(images.ci_wasm, 'build')
- ci.cpp_unittest(images.ci_wasm)
++ ci.wheel_build(images.ci_wasm, true)
+ sh(
+ script: "${docker_run} ${images.ci_wasm} ./tests/scripts/task_web_wasm.sh",
+ label: 'Run WASM lint and tests',
+ )
+ }
}