You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2026/07/05 01:44:22 UTC
(tvm) branch tvm-simplify-jenkins-pytest-sharding updated (9f4feee101 -> c73279e84b)
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a change to branch tvm-simplify-jenkins-pytest-sharding
in repository https://gitbox.apache.org/repos/asf/tvm.git
discard 9f4feee101 [CI] Remove pytest JUnit reporting
discard 0992b4955b [CI] Simplify Jenkins pytest execution
new c73279e84b [CI] Simplify Jenkins pytest execution
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (9f4feee101)
\
N -- N -- N refs/heads/tvm-simplify-jenkins-pytest-sharding (c73279e84b)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
(tvm) 01/01: [CI] Simplify Jenkins pytest execution
Posted by tq...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch tvm-simplify-jenkins-pytest-sharding
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit c73279e84b444ff67644a23f05e2758f9fc2c9ec
Author: Tianqi Chen <tq...@users.noreply.github.com>
AuthorDate: Sun Jul 5 01:43:50 2026 +0000
[CI] Simplify Jenkins pytest execution
Remove external test sharding, obsolete pytest wrapper machinery, JUnit reporting and publication, and the skipped-test XML consumer. Repair broad-suite targets and empty-selection failure propagation, while bounding active nested MetaSchedule test parallelism without blanket serialization.
---
ci/jenkins/generated/arm_jenkinsfile.groovy | 11 +-
ci/jenkins/generated/cpu_jenkinsfile.groovy | 98 +-------
ci/jenkins/generated/docker_jenkinsfile.groovy | 11 +-
ci/jenkins/generated/gpu_jenkinsfile.groovy | 129 +---------
ci/jenkins/generated/wasm_jenkinsfile.groovy | 12 +-
ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 | 4 +-
ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 | 10 +-
ci/jenkins/templates/utils/Build.groovy.j2 | 7 -
ci/jenkins/templates/utils/base.groovy.j2 | 2 +-
ci/jenkins/templates/utils/macros.j2 | 30 +--
ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 | 1 -
ci/scripts/github/__init__.py | 2 +-
ci/scripts/github/github_pr_comment.py | 4 -
ci/scripts/github/github_skipped_tests_comment.py | 263 -------------------
ci/scripts/jenkins/pytest_ids.py | 43 ----
ci/scripts/jenkins/pytest_wrapper.py | 137 ----------
conftest.py | 54 ----
docker/install/ubuntu_install_python_package.sh | 2 -
python/tvm/testing/plugin.py | 9 -
tests/python/ci/test_ci.py | 280 +--------------------
.../meta_schedule/test_meta_schedule_cost_model.py | 11 +-
.../test_meta_schedule_search_strategy.py | 1 +
.../test_meta_schedule_task_scheduler.py | 13 +
tests/python/testing/test_tvm_testing_features.py | 42 +++-
tests/scripts/task_clear_pytest.sh | 24 --
tests/scripts/task_python_nightly.sh | 39 ---
tests/scripts/task_python_unittest.sh | 34 ++-
tests/scripts/task_python_unittest_gpuonly.sh | 14 +-
28 files changed, 117 insertions(+), 1170 deletions(-)
diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy
index d6826878bf..3442caded7 100644
--- a/ci/jenkins/generated/arm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/arm_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2026-06-23T03:28:14.520193
+// Generated at 2026-07-05T01:05:57.719405
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -97,7 +97,7 @@ properties([
upstream_revision = null
// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
+docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
docker_build = 'docker/build.sh'
// timeout in minutes
max_time = 180
@@ -404,13 +404,6 @@ def prepare(node_type) {
}
}
}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
def python_unittest(image) {
sh (
script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy
index 879cd8b79f..0d3e408235 100644
--- a/ci/jenkins/generated/cpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2026-06-23T03:28:14.505504
+// Generated at 2026-07-05T01:05:57.741181
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -97,7 +97,7 @@ properties([
upstream_revision = null
// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
+docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
docker_build = 'docker/build.sh'
// timeout in minutes
max_time = 180
@@ -404,13 +404,6 @@ def prepare(node_type) {
}
}
}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
def python_unittest(image) {
sh (
script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
@@ -510,52 +503,7 @@ build()
-
-def shard_run_unittest_CPU_1_of_2(node_type) {
- echo 'Begin running on node_type ' + node_type
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") {
- // NOTE: if exception happens, it will be caught outside
- init_git()
- docker_init(ci_cpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cpu',
- 'TEST_STEP_NAME=unittest: CPU',
- 'TVM_NUM_SHARDS=2',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_cpu)
- cpp_unittest(ci_cpu)
- python_unittest(ci_cpu)
- })
- }
- // only run upload if things are successful
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- echo 'End running on node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('unittest: CPU 1 of 2')
- }
-}
-
-def shard_run_unittest_CPU_2_of_2(node_type) {
+def run_unittest_CPU(node_type) {
echo 'Begin running on node_type ' + node_type
if (!skip_ci && is_docs_only_build != 1) {
node(node_type) {
@@ -567,35 +515,21 @@ def shard_run_unittest_CPU_2_of_2(node_type) {
withEnv([
'PLATFORM=cpu',
'TEST_STEP_NAME=unittest: CPU',
- 'TVM_NUM_SHARDS=2',
- 'TVM_SHARD_INDEX=1',
"SKIP_SLOW_TESTS=${skip_slow_tests}"], {
sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
label: 'Download artifacts from S3',
)
- ci_setup(ci_cpu)
cpp_unittest(ci_cpu)
python_unittest(ci_cpu)
})
}
- // only run upload if things are successful
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
}
}
echo 'End running on node_type ' + node_type
} else {
- Utils.markStageSkippedForConditional('unittest: CPU 2 of 2')
+ Utils.markStageSkippedForConditional('unittest: CPU')
}
}
@@ -606,27 +540,9 @@ def test() {
SKIP_SLOW_TESTS = "${skip_slow_tests}"
}
parallel(
- 'unittest: CPU 1 of 2': {
- try {
- shard_run_unittest_CPU_1_of_2('CPU-SMALL-SPOT')
- } catch (Throwable ex) {
- echo 'Exception during SPOT run ' + ex.toString()
- if (is_last_build()) {
- // retry if at last build
- // mark the current stage as success
- // and try again via on demand node
- echo 'Retry on-demand given it is last build'
- currentBuild.result = 'SUCCESS'
- shard_run_unittest_CPU_1_of_2('CPU-SMALL')
- } else {
- echo 'Exit since it is not last build'
- throw ex
- }
- }
- },
- 'unittest: CPU 2 of 2': {
+ 'unittest: CPU': {
try {
- shard_run_unittest_CPU_2_of_2('CPU-SMALL-SPOT')
+ run_unittest_CPU('CPU-SMALL-SPOT')
} catch (Throwable ex) {
echo 'Exception during SPOT run ' + ex.toString()
if (is_last_build()) {
@@ -635,7 +551,7 @@ def test() {
// and try again via on demand node
echo 'Retry on-demand given it is last build'
currentBuild.result = 'SUCCESS'
- shard_run_unittest_CPU_2_of_2('CPU-SMALL')
+ run_unittest_CPU('CPU-SMALL')
} else {
echo 'Exit since it is not last build'
throw ex
diff --git a/ci/jenkins/generated/docker_jenkinsfile.groovy b/ci/jenkins/generated/docker_jenkinsfile.groovy
index f5409e23aa..8ffe78942a 100644
--- a/ci/jenkins/generated/docker_jenkinsfile.groovy
+++ b/ci/jenkins/generated/docker_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2026-06-23T03:28:14.487314
+// Generated at 2026-07-05T01:05:57.759905
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -97,7 +97,7 @@ properties([
upstream_revision = null
// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
+docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
docker_build = 'docker/build.sh'
// timeout in minutes
max_time = 180
@@ -404,13 +404,6 @@ def prepare(node_type) {
}
}
}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
def python_unittest(image) {
sh (
script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy
index 28c9a6040e..792b7eadc4 100644
--- a/ci/jenkins/generated/gpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2026-06-23T03:28:14.533751
+// Generated at 2026-07-05T01:05:57.780838
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -97,7 +97,7 @@ properties([
upstream_revision = null
// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
+docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
docker_build = 'docker/build.sh'
// timeout in minutes
max_time = 180
@@ -404,13 +404,6 @@ def prepare(node_type) {
}
}
}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
def python_unittest(image) {
sh (
script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
@@ -515,8 +508,7 @@ def build() {
build()
-
-def shard_run_unittest_GPU_1_of_2(node_type) {
+def run_unittest_GPU(node_type) {
echo 'Begin running on node_type ' + node_type
if (!skip_ci && is_docs_only_build != 1) {
node(node_type) {
@@ -528,65 +520,12 @@ def shard_run_unittest_GPU_1_of_2(node_type) {
withEnv([
'PLATFORM=gpu',
'TEST_STEP_NAME=unittest: GPU',
- 'TVM_NUM_SHARDS=2',
- 'TVM_SHARD_INDEX=0',
"SKIP_SLOW_TESTS=${skip_slow_tests}"], {
sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
- ci_setup(ci_gpu)
- sh (
- script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh",
- label: 'Run Python GPU unit tests',
- )
- sh (
- script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh",
- label: 'Run Python GPU integration tests',
- )
- })
- }
- // only run upload if things are successful
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- echo 'End running on node_type ' + node_type
- } else {
- Utils.markStageSkippedForConditional('unittest: GPU 1 of 2')
- }
-}
-
-def shard_run_unittest_GPU_2_of_2(node_type) {
- echo 'Begin running on node_type ' + node_type
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
- // NOTE: if exception happens, it will be caught outside
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=unittest: GPU',
- 'TVM_NUM_SHARDS=2',
- 'TVM_SHARD_INDEX=1',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_gpu)
sh (
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh",
label: 'Run Java unit tests',
@@ -601,29 +540,17 @@ def shard_run_unittest_GPU_2_of_2(node_type) {
)
})
}
- // only run upload if things are successful
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
}
}
echo 'End running on node_type ' + node_type
} else {
- Utils.markStageSkippedForConditional('unittest: GPU 2 of 2')
+ Utils.markStageSkippedForConditional('unittest: GPU')
}
}
-
-def shard_run_docs_GPU_1_of_1(node_type) {
+def run_docs_GPU(node_type) {
echo 'Begin running on node_type ' + node_type
if (!skip_ci) {
node(node_type) {
@@ -635,15 +562,12 @@ def shard_run_docs_GPU_1_of_1(node_type) {
withEnv([
'PLATFORM=gpu',
'TEST_STEP_NAME=docs: GPU',
- 'TVM_NUM_SHARDS=1',
- 'TVM_SHARD_INDEX=0',
"SKIP_SLOW_TESTS=${skip_slow_tests}"], {
sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
- ci_setup(ci_gpu)
sh (
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh",
label: 'Build docs',
@@ -659,22 +583,11 @@ def shard_run_docs_GPU_1_of_1(node_type) {
)
})
}
- // only run upload if things are successful
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/docs_GPU --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
}
}
echo 'End running on node_type ' + node_type
} else {
- Utils.markStageSkippedForConditional('docs: GPU 1 of 1')
+ Utils.markStageSkippedForConditional('docs: GPU')
}
}
@@ -686,27 +599,9 @@ def test() {
SKIP_SLOW_TESTS = "${skip_slow_tests}"
}
parallel(
- 'unittest: GPU 1 of 2': {
- try {
- shard_run_unittest_GPU_1_of_2('GPU-SPOT')
- } catch (Throwable ex) {
- echo 'Exception during SPOT run ' + ex.toString()
- if (is_last_build()) {
- // retry if at last build
- // mark the current stage as success
- // and try again via on demand node
- echo 'Retry on-demand given it is last build'
- currentBuild.result = 'SUCCESS'
- shard_run_unittest_GPU_1_of_2('GPU')
- } else {
- echo 'Exit since it is not last build'
- throw ex
- }
- }
- },
- 'unittest: GPU 2 of 2': {
+ 'unittest: GPU': {
try {
- shard_run_unittest_GPU_2_of_2('GPU-SPOT')
+ run_unittest_GPU('GPU-SPOT')
} catch (Throwable ex) {
echo 'Exception during SPOT run ' + ex.toString()
if (is_last_build()) {
@@ -715,16 +610,16 @@ def test() {
// and try again via on demand node
echo 'Retry on-demand given it is last build'
currentBuild.result = 'SUCCESS'
- shard_run_unittest_GPU_2_of_2('GPU')
+ run_unittest_GPU('GPU')
} else {
echo 'Exit since it is not last build'
throw ex
}
}
},
- 'docs: GPU 1 of 1': {
+ 'docs: GPU': {
try {
- shard_run_docs_GPU_1_of_1('GPU-SPOT')
+ run_docs_GPU('GPU-SPOT')
} catch (Throwable ex) {
echo 'Exception during SPOT run ' + ex.toString()
if (is_last_build()) {
@@ -733,7 +628,7 @@ def test() {
// and try again via on demand node
echo 'Retry on-demand given it is last build'
currentBuild.result = 'SUCCESS'
- shard_run_docs_GPU_1_of_1('GPU')
+ run_docs_GPU('GPU')
} else {
echo 'Exit since it is not last build'
throw ex
diff --git a/ci/jenkins/generated/wasm_jenkinsfile.groovy b/ci/jenkins/generated/wasm_jenkinsfile.groovy
index ab64444e4a..7421fa86ca 100644
--- a/ci/jenkins/generated/wasm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2026-06-23T03:28:14.550555
+// Generated at 2026-07-05T01:05:57.802104
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -97,7 +97,7 @@ properties([
upstream_revision = null
// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
+docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
docker_build = 'docker/build.sh'
// timeout in minutes
max_time = 180
@@ -404,13 +404,6 @@ def prepare(node_type) {
}
}
}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
def python_unittest(image) {
sh (
script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
@@ -471,7 +464,6 @@ def run_build(node_type) {
cmake_build(ci_wasm, 'build')
make_cpp_tests(ci_wasm, 'build')
cpp_unittest(ci_wasm)
- ci_setup(ci_wasm)
sh (
script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh",
label: 'Run WASM lint and tests',
diff --git a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
index d2e479d5e8..62b7c3dfea 100644
--- a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
@@ -37,16 +37,14 @@
{% set test_method_names = [] %}
-{% call(shard_index, num_shards) m.sharded_test_step(
+{% call m.test_step(
name="unittest: CPU",
ws="tvm/ut-python-cpu",
platform="cpu",
- num_shards=2,
docker_image="ci_cpu",
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='cpu') }}
- ci_setup(ci_cpu)
cpp_unittest(ci_cpu)
python_unittest(ci_cpu)
{% endcall %}
diff --git a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
index 7ab5256419..3a13dfe7f3 100644
--- a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
@@ -38,22 +38,18 @@
{% set test_method_names = [] %}
-{% call(shard_index, num_shards) m.sharded_test_step(
+{% call m.test_step(
name="unittest: GPU",
- num_shards=2,
ws="tvm/ut-python-gpu",
platform="gpu",
docker_image="ci_gpu",
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='gpu') }}
- ci_setup(ci_gpu)
- {% if shard_index == 2 or num_shards < 2 %}
sh (
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_java_unittest.sh",
label: 'Run Java unit tests',
)
- {% endif %}
sh (
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_unittest_gpuonly.sh",
label: 'Run Python GPU unit tests',
@@ -65,9 +61,8 @@
{% endcall %}
-{% call(shard_index, num_shards) m.sharded_test_step(
+{% call m.test_step(
name="docs: GPU",
- num_shards=1,
ws="tvm/docs-python-gpu",
platform="gpu",
docker_image="ci_gpu",
@@ -75,7 +70,6 @@
test_method_names=test_method_names,
) %}
{{ m.download_artifacts(tag='gpu') }}
- ci_setup(ci_gpu)
sh (
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_docs.sh",
label: 'Build docs',
diff --git a/ci/jenkins/templates/utils/Build.groovy.j2 b/ci/jenkins/templates/utils/Build.groovy.j2
index c20a1c5c43..42bd4a991f 100644
--- a/ci/jenkins/templates/utils/Build.groovy.j2
+++ b/ci/jenkins/templates/utils/Build.groovy.j2
@@ -1,10 +1,3 @@
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
def python_unittest(image) {
sh (
script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
diff --git a/ci/jenkins/templates/utils/base.groovy.j2 b/ci/jenkins/templates/utils/base.groovy.j2
index 813ceee886..a8fda460b7 100644
--- a/ci/jenkins/templates/utils/base.groovy.j2
+++ b/ci/jenkins/templates/utils/base.groovy.j2
@@ -82,7 +82,7 @@ properties([
upstream_revision = null
// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
+docker_run = 'docker/bash.sh --env CI --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
docker_build = 'docker/build.sh'
// timeout in minutes
max_time = 180
diff --git a/ci/jenkins/templates/utils/macros.j2 b/ci/jenkins/templates/utils/macros.j2
index b1bd3679ac..959d91725f 100644
--- a/ci/jenkins/templates/utils/macros.j2
+++ b/ci/jenkins/templates/utils/macros.j2
@@ -19,18 +19,8 @@
"workspace/exec_${env.EXECUTOR_NUMBER}/{{ folder }}"
{%- endmacro -%}
-{% macro junit_to_s3(test_dir_name) %}
-sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/{{ test_dir_name }} --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-{% endmacro %}
-
-{% macro sharded_test_step(name, num_shards, ws, docker_image, platform, test_method_names, condition="!skip_ci && is_docs_only_build != 1") %}
-
-{% for shard_index in range(1, num_shards + 1) %}
-{% set method_name = "shard_run_" + name.replace(":", "").replace(" ", "-").replace("-", "_") + "_" + shard_index|string + "_of_" + num_shards|string %}
-{% set test_dir_name = name.replace(":", "").replace(" ", "-").replace("-", "_")|string %}
+{% macro test_step(name, ws, docker_image, platform, test_method_names, condition="!skip_ci && is_docs_only_build != 1") %}
+{% set method_name = "run_" + name.replace(":", "").replace(" ", "-").replace("-", "_") %}
def {{ method_name }}(node_type) {
echo 'Begin running on node_type ' + node_type
if ({{ condition }}) {
@@ -43,29 +33,19 @@ def {{ method_name }}(node_type) {
withEnv([
'PLATFORM={{ platform }}',
'TEST_STEP_NAME={{ name }}',
- 'TVM_NUM_SHARDS={{ num_shards }}',
- 'TVM_SHARD_INDEX={{ shard_index - 1 }}',
"SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- {{ caller(shard_index, num_shards) | trim | indent(width=12) }}
+ {{ caller() | trim | indent(width=12) }}
})
}
- // only run upload if things are successful
- try {
- {{ junit_to_s3(test_dir_name) }}
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
}
}
echo 'End running on node_type ' + node_type
} else {
- Utils.markStageSkippedForConditional('{{ name }} {{ shard_index }} of {{ num_shards }}')
+ Utils.markStageSkippedForConditional('{{ name }}')
}
}
-{% set _ = test_method_names.append((name + " " + shard_index|string + " of " + num_shards|string, method_name)) %}
+{% set _ = test_method_names.append((name, method_name)) %}
-{% endfor %}
{% endmacro %}
{% macro invoke_build(name, condition, node, docker_image, ws, platform) %}
diff --git a/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2 b/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2
index 0d119f043e..8e7d909bd1 100644
--- a/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/wasm_jenkinsfile.groovy.j2
@@ -32,7 +32,6 @@
cmake_build(ci_wasm, 'build')
make_cpp_tests(ci_wasm, 'build')
cpp_unittest(ci_wasm)
- ci_setup(ci_wasm)
sh (
script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh",
label: 'Run WASM lint and tests',
diff --git a/ci/scripts/github/__init__.py b/ci/scripts/github/__init__.py
index a6226fec29..0e55424eee 100644
--- a/ci/scripts/github/__init__.py
+++ b/ci/scripts/github/__init__.py
@@ -17,4 +17,4 @@
# under the License.
"""Package to enable testing of GitHub scripts"""
-from . import github_skipped_tests_comment, github_pr_comment, github_tag_teams, github_docs_comment
+from . import github_docs_comment, github_pr_comment, github_tag_teams
diff --git a/ci/scripts/github/github_pr_comment.py b/ci/scripts/github/github_pr_comment.py
index 2d523b21b3..81347659d9 100755
--- a/ci/scripts/github/github_pr_comment.py
+++ b/ci/scripts/github/github_pr_comment.py
@@ -30,7 +30,6 @@ from cmd_utils import init_log
from git_utils import DRY_RUN, GitHubRepo, git, parse_remote
from github_commenter import BotCommentBuilder
from github_docs_comment import get_doc_url
-from github_skipped_tests_comment import get_skipped_tests_comment
from github_tag_teams import get_tags
PR_QUERY = """
@@ -132,17 +131,14 @@ if __name__ == "__main__":
if args.test_comments is not None:
test_comments = json.loads(args.test_comments)
- skipped_tests = test_comments["skipped-tests"]
ccs = test_comments["ccs"]
docs_info = test_comments["docs"]
else:
- skipped_tests = get_skipped_tests_comment(pr_data, github=github)
ccs = get_tags(pr_data, github, team_issue=10317)
docs_info = get_doc_url(pr_data)
items = {
"ccs": ccs,
- "skipped-tests": skipped_tests,
"docs": docs_info,
}
commenter.post_items(items=items.items())
diff --git a/ci/scripts/github/github_skipped_tests_comment.py b/ci/scripts/github/github_skipped_tests_comment.py
deleted file mode 100755
index 10789f7561..0000000000
--- a/ci/scripts/github/github_skipped_tests_comment.py
+++ /dev/null
@@ -1,263 +0,0 @@
-#!/usr/bin/env python3
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# ruff: noqa: E501
-import json
-import logging
-import os
-import subprocess
-from pathlib import Path
-from typing import Any
-from xml.etree import ElementTree
-
-
-def run_subprocess(command):
- logging.info(f"Running command {command}")
- proc = subprocess.run(command, shell=True, stdout=subprocess.PIPE, encoding="utf-8")
- if proc.returncode != 0:
- raise RuntimeError(f"Command failed {command}:\nstdout:\n{proc.stdout}")
- return proc
-
-
-def retrieve_test_report(s3_url, target_dir):
- command = f"aws --region us-west-2 s3 cp {s3_url} {target_dir} --recursive --no-sign-request"
- run_subprocess(command)
-
-
-def get_common_commit_sha():
- command = "git merge-base origin/main HEAD"
- proc = run_subprocess(command)
- return proc.stdout.strip()
-
-
-def get_main_jenkins_build_number(github, common_commit):
- json = github.get(f"commits/{common_commit}/status")
- for status in reversed(json["statuses"]):
- if status["context"] != "tvm-ci/branch":
- continue
- state = status["state"]
- target_url = str(status["target_url"])
- build_number = (
- target_url[target_url.find("job/main") : len(target_url)]
- .strip("job/main/")
- .strip("/display/redirect")
- )
- assert build_number.isdigit()
- return {"build_number": build_number, "state": state}
- raise RuntimeError(f"Failed to find main build number for commit {common_commit}")
-
-
-def retrieve_test_reports(
- common_main_build, pr_number, build_number, s3_prefix, pr_test_report_dir, main_test_report_dir
-):
- cur_build_s3_link = f"s3://{s3_prefix}/tvm/PR-{pr_number!s}/{build_number!s}/pytest-results"
- retrieve_test_report(cur_build_s3_link, pr_test_report_dir)
-
- common_build_s3_link = f"s3://{s3_prefix}/tvm/main/{common_main_build}/pytest-results"
- retrieve_test_report(common_build_s3_link, main_test_report_dir)
-
-
-def get_pr_and_build_numbers(target_url):
- target_url = target_url[target_url.find("PR-") : len(target_url)]
- split = target_url.split("/")
- pr_number = split[0].strip("PR-")
- build_number = split[1]
- return {"pr_number": pr_number, "build_number": build_number}
-
-
-def build_test_set(directory):
- directory = Path(directory)
- subdir_to_skipped = {}
- subdirs = [
- item for item in os.listdir(directory) if os.path.isdir(os.path.join(directory, item))
- ]
- for subdir in subdirs:
- subdir_to_skipped[subdir] = set()
- for root, _, files in os.walk(directory / subdir):
- for file in files:
- test_report = ElementTree.parse(Path(root) / file)
- for testcase in test_report.iter("testcase"):
- skipped = testcase.find("skipped")
- if skipped is not None:
- key = testcase.attrib["classname"] + "#" + testcase.attrib["name"]
- subdir_to_skipped[subdir].add(key)
- return subdir_to_skipped
-
-
-def to_node_name(dir_name: str):
- return dir_name.replace("_", ": ", 1)
-
-
-def build_diff_comment_with_main(
- common_commit_sha,
- skipped_list,
- commit_sha,
-):
- if len(skipped_list) == 0:
- return f"No diff in skipped tests with main found in this branch for commit {commit_sha}.\n"
-
- text = (
- f"The list below shows tests that ran in main {common_commit_sha} but were "
- f"skipped in the CI build of {commit_sha}:\n"
- f"```\n"
- )
- for skip in skipped_list:
- text += skip + "\n"
- text += "```\n"
- return text
-
-
-def build_comment(
- common_commit_sha,
- common_main_build,
- skipped_list,
- additional_skipped_list,
- pr_number,
- build_number,
- commit_sha,
- jenkins_prefix,
-):
- if common_main_build["state"] != "success":
- return f"Unable to run tests bot because main failed to pass CI at {common_commit_sha}."
-
- text = build_diff_comment_with_main(common_commit_sha, skipped_list, commit_sha)
-
- if len(additional_skipped_list) != 0:
- text += "\n"
- text += (
- "Additional tests that were skipped in the CI build and present in the [`required_tests_to_run`]"
- "(https://github.com/apache/tvm/blob/main/ci/scripts/github/required_tests_to_run.json) file:"
- "\n```\n"
- )
- for skip in additional_skipped_list:
- text += skip + "\n"
- text += "```\n"
-
- text += (
- f"A detailed report of ran tests is [here](https://{jenkins_prefix}/job/tvm/job/PR-{pr_number!s}"
- f"/{build_number!s}/testReport/)."
- )
- return text
-
-
-def find_target_url(pr_head: dict[str, Any]):
- for status in pr_head["statusCheckRollup"]["contexts"]["nodes"]:
- if status.get("context", "") == "tvm-ci/pr-head":
- return status["targetUrl"]
-
- raise RuntimeError(f"Unable to find tvm-ci/pr-head status in {pr_head}")
-
-
-def get_skipped_tests_comment(
- pr: dict[str, Any],
- github,
- s3_prefix: str = "tvm-jenkins-artifacts-prod",
- jenkins_prefix: str = "ci.tlcpack.ai",
- pr_test_report_dir: str = "pr-reports",
- main_test_report_dir: str = "main-reports",
- common_commit_sha: str | None = None,
- common_main_build: dict[str, Any] | None = None,
- additional_tests_to_check_file: str = "required_tests_to_run.json",
-) -> str:
- pr_head = pr["commits"]["nodes"][0]["commit"]
- target_url = find_target_url(pr_head)
- pr_and_build = get_pr_and_build_numbers(target_url)
- logging.info(f"Getting comment for {pr_head} with target {target_url}")
-
- commit_sha = pr_head["oid"]
-
- is_dry_run = common_commit_sha is not None
-
- if not is_dry_run:
- logging.info("Fetching common commit sha and build info")
- common_commit_sha = get_common_commit_sha()
- common_main_build = get_main_jenkins_build_number(github, common_commit_sha)
-
- retrieve_test_reports(
- common_main_build=common_main_build["build_number"],
- pr_number=pr_and_build["pr_number"],
- build_number=pr_and_build["build_number"],
- s3_prefix=s3_prefix,
- main_test_report_dir=main_test_report_dir,
- pr_test_report_dir=pr_test_report_dir,
- )
- else:
- logging.info("Dry run, expecting PR and main reports on disk")
-
- main_tests = build_test_set(main_test_report_dir)
- build_tests = build_test_set(pr_test_report_dir)
-
- skipped_list = []
- for subdir, skipped_set in build_tests.items():
- skipped_main = main_tests[subdir]
- if skipped_main is None:
- logging.warning(f"Could not find directory {subdir} in main.")
- continue
-
- diff_set = skipped_set - skipped_main
- if len(diff_set) != 0:
- for test in diff_set:
- skipped_list.append(f"{to_node_name(subdir)} -> {test}")
-
- # Sort the list to maintain an order in the output. Helps when validating the output in tests.
- skipped_list.sort()
-
- if len(skipped_list) == 0:
- logging.info("No skipped tests found.")
-
- if not is_dry_run:
- current_file = Path(__file__).resolve()
- additional_tests_to_check_file = Path(current_file).parent / "required_tests_to_run.json"
-
- logging.info(
- f"Checking additional tests in file {additional_tests_to_check_file} are not skipped."
- )
- try:
- with open(additional_tests_to_check_file) as f:
- additional_tests_to_check = json.load(f)
- except OSError:
- logging.info(
- f"Failed to read additional tests from file: {additional_tests_to_check_file}."
- )
- additional_tests_to_check = {}
-
- # Assert that tests present in "required_tests_to_run.json" are not skipped.
- additional_skipped_tests = []
- for subdir, test_set in additional_tests_to_check.items():
- if subdir not in build_tests.keys():
- logging.warning(f"Could not find directory {subdir} in the build test set.")
- continue
-
- for test in test_set:
- if test in build_tests[subdir]:
- additional_skipped_tests.append(f"{to_node_name(subdir)} -> {test}")
-
- if len(additional_skipped_tests) == 0:
- logging.info("No skipped tests found in the additional list.")
-
- body = build_comment(
- common_commit_sha,
- common_main_build,
- skipped_list,
- additional_skipped_tests,
- pr_and_build["pr_number"],
- pr_and_build["build_number"],
- commit_sha,
- jenkins_prefix,
- )
-
- return body
diff --git a/ci/scripts/jenkins/pytest_ids.py b/ci/scripts/jenkins/pytest_ids.py
deleted file mode 100755
index 548069e855..0000000000
--- a/ci/scripts/jenkins/pytest_ids.py
+++ /dev/null
@@ -1,43 +0,0 @@
-#!/usr/bin/env python3
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-import argparse
-import io
-from contextlib import redirect_stdout
-
-import pytest
-
-
-class NodeidsCollector:
- def pytest_collection_modifyitems(self, items):
- self.nodeids = [item.nodeid for item in items]
-
-
-def main(folder):
- collector = NodeidsCollector()
- f = io.StringIO()
- with redirect_stdout(f):
- pytest.main(["-qq", "--collect-only", folder], plugins=[collector])
- for nodeid in collector.nodeids:
- print(nodeid)
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser(description="List pytest nodeids for a folder")
- parser.add_argument("--folder", required=True, help="test folder to inspect")
- args = parser.parse_args()
- main(args.folder)
diff --git a/ci/scripts/jenkins/pytest_wrapper.py b/ci/scripts/jenkins/pytest_wrapper.py
deleted file mode 100755
index a667810293..0000000000
--- a/ci/scripts/jenkins/pytest_wrapper.py
+++ /dev/null
@@ -1,137 +0,0 @@
-#!/usr/bin/env python3
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-# ruff: noqa: E501
-import argparse
-import logging
-import os
-import textwrap
-import urllib.parse
-from pathlib import Path
-
-import junitparser
-from cmd_utils import init_log
-
-REPO_ROOT = Path(__file__).resolve().parent.parent.parent.parent
-
-
-def lstrip(s: str, prefix: str) -> str:
- if s.startswith(prefix):
- s = s[len(prefix) :]
- return s
-
-
-def classname_to_file(classname: str) -> str:
- classname = lstrip(classname, "cython.")
- classname = lstrip(classname, "ctypes.")
- return classname.replace(".", "/") + ".py"
-
-
-def failed_test_ids() -> list[str]:
- FAILURE_TYPES = (junitparser.Failure, junitparser.Error)
- junit_dir = REPO_ROOT / "build" / "pytest-results"
- failed_node_ids = []
- for junit in junit_dir.glob("*.xml"):
- xml = junitparser.JUnitXml.fromfile(str(junit))
- for suite in xml:
- # handle suites
- for case in suite:
- if case.result is None:
- logging.warn(f"Incorrectly formatted JUnit found, result was None on {case}")
- continue
-
- if len(case.result) > 0 and isinstance(case.result[0], FAILURE_TYPES):
- node_id = classname_to_file(case.classname) + "::" + case.name
- failed_node_ids.append(node_id)
-
- return list(set(failed_node_ids))
-
-
-def repro_command(build_type: str, failed_node_ids: list[str]) -> str | None:
- """
- Parse available JUnit XML files and output a command that users can run to
- reproduce CI failures locally
- """
- test_args = [f"--tests {node_id}" for node_id in failed_node_ids]
- test_args_str = " ".join(test_args)
- return f"python3 tests/scripts/ci.py {build_type} {test_args_str}"
-
-
-def make_issue_url(failed_node_ids: list[str]) -> str:
- names = [f"`{node_id}`" for node_id in failed_node_ids]
- run_url = os.getenv("RUN_DISPLAY_URL", "<insert run URL>")
- test_bullets = [f" - `{node_id}`" for node_id in failed_node_ids]
- params = {
- "labels": "test: flaky",
- "title": "[Flaky Test] " + ", ".join(names),
- "body": textwrap.dedent(
- """
- These tests were found to be flaky (intermittently failing on `main` or failed in a PR with unrelated changes). See [the docs](https://github.com/apache/tvm/blob/main/docs/contribute/ci.rst#handling-flaky-failures) for details.
-
- ### Tests(s)\n
- """
- )
- + "\n".join(test_bullets)
- + f"\n\n### Jenkins Links\n\n - {run_url}",
- }
- return "https://github.com/apache/tvm/issues/new?" + urllib.parse.urlencode(params)
-
-
-def show_failure_help(failed_suites: list[str]) -> None:
- failed_node_ids = failed_test_ids()
-
- if len(failed_node_ids) == 0:
- return
-
- build_type = os.getenv("PLATFORM")
-
- if build_type is None:
- raise RuntimeError("build type was None, cannot show command")
-
- repro = repro_command(build_type=build_type, failed_node_ids=failed_node_ids)
- if repro is None:
- print("No test failures detected")
- return
-
- print(f"Report flaky test shortcut: {make_issue_url(failed_node_ids)}")
- print("=============================== PYTEST FAILURES ================================")
- print(
- "These pytest suites failed to execute. The results can be found in the "
- "Jenkins 'Tests' tab or by scrolling up through the raw logs here. "
- "If there is no test listed below, the failure likely came from a segmentation "
- "fault which you can find in the logs above.\n"
- )
- if failed_suites is not None and len(failed_suites) > 0:
- print("\n".join([f" - {suite}" for suite in failed_suites]))
- print("")
-
- print("You can reproduce these specific failures locally with this command:\n")
- print(textwrap.indent(repro, prefix=" "))
- print("")
-
-
-if __name__ == "__main__":
- parser = argparse.ArgumentParser(description="Print information about a failed pytest run")
- args, other = parser.parse_known_args()
- init_log()
-
- try:
- show_failure_help(failed_suites=other)
- except Exception as e:
- # This script shouldn't ever introduce failures since it's just there to
- # add extra information, so ignore any errors
- logging.exception(e)
diff --git a/conftest.py b/conftest.py
index dd32937620..dd294ea548 100644
--- a/conftest.py
+++ b/conftest.py
@@ -14,8 +14,6 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# ruff: noqa: E501
-import hashlib
import os
import sys
from pathlib import Path
@@ -25,58 +23,6 @@ IS_IN_CI = os.getenv("CI", "") == "true"
REPO_ROOT = Path(__file__).resolve().parent
-# These are long running tests (manually curated and extracted from CI logs)
-# that should be allocated to test shards in a round-robin fashion. These are
-# taken from the 20 (arbitrary number) of tests as from
-# https://ci.tlcpack.ai/job/tvm/job/main/2907/testReport
-_slowest_tests = []
-HARDCODED_ALLOCATIONS = {}
-for idx, test in enumerate(_slowest_tests):
- HARDCODED_ALLOCATIONS[test] = idx
-
-# These rely on running on the same node to pass successfully
-FIXED_ALLOCATION_PREFIXES = {
- "tests/python/testing/test_tvm_testing_features.py": 0,
-}
-
-
-def find_shard_index(nodeid: str, num_shards: int) -> int:
- """
- Return the index of the shard that should run this test
- """
- for prefix, target_shard_idx in FIXED_ALLOCATION_PREFIXES.items():
- if nodeid.startswith(prefix):
- if target_shard_idx >= num_shards:
- raise RuntimeError(
- f"Cannot collect sharded tests, {nodeid} has hardcoded shard index {target_shard_idx} among only {num_shards} shards"
- )
- return target_shard_idx
-
- if nodeid in HARDCODED_ALLOCATIONS:
- hash = HARDCODED_ALLOCATIONS[nodeid]
- else:
- hash = hashlib.md5(nodeid.encode())
- hash = int(hash.hexdigest(), 16)
-
- return hash % num_shards
-
-
-def pytest_collection_modifyitems(config, items):
- if not all(k in os.environ for k in ["CI", "TVM_NUM_SHARDS", "TVM_SHARD_INDEX"]):
- # Only apportion tests if in CI and in a job that is set up for it
- return
-
- num_shards = int(os.environ["TVM_NUM_SHARDS"])
- shard_index = int(os.environ["TVM_SHARD_INDEX"])
-
- print(f"Marking tests for shard {shard_index} of {num_shards}")
- items_copy = list(items)
- for item in items_copy:
- item_shard_index = find_shard_index(item.nodeid, num_shards=num_shards)
- if item_shard_index != shard_index:
- items.remove(item)
-
-
def pytest_sessionstart():
if IS_IN_CI:
hook_script_dir = REPO_ROOT / "tests" / "scripts" / "request_hook"
diff --git a/docker/install/ubuntu_install_python_package.sh b/docker/install/ubuntu_install_python_package.sh
index d07f19ed3c..4445dfd83e 100755
--- a/docker/install/ubuntu_install_python_package.sh
+++ b/docker/install/ubuntu_install_python_package.sh
@@ -32,13 +32,11 @@ uv pip install --upgrade \
Pillow==12.1.1 \
"psutil~=7.0" \
"pytest~=8.3" \
- "pytest-profiling~=1.8" \
"pytest-xdist~=3.6" \
pytest-rerunfailures==16.1 \
"requests~=2.32" \
"scipy~=1.13" \
"Jinja2~=3.1" \
- junitparser==4.0.2 \
"six~=1.17" \
"tornado~=6.4" \
"ml_dtypes~=0.5" \
diff --git a/python/tvm/testing/plugin.py b/python/tvm/testing/plugin.py
index dfb858f1b8..2869664842 100644
--- a/python/tvm/testing/plugin.py
+++ b/python/tvm/testing/plugin.py
@@ -33,7 +33,6 @@ directory as the test scripts.
"""
import _pytest
-import pytest
def pytest_collection_modifyitems(config, items):
@@ -44,14 +43,6 @@ def pytest_collection_modifyitems(config, items):
_sort_tests(items)
-def pytest_sessionfinish(session, exitstatus):
- # Don't exit with an error if we select a subset of tests that doesn't
- # include anything
- if session.config.option.markexpr != "":
- if exitstatus == pytest.ExitCode.NO_TESTS_COLLECTED:
- session.exitstatus = pytest.ExitCode.OK
-
-
def _count_num_fixture_uses(items):
# Helper function, counts the number of tests that use each cached
# fixture. Should be called from pytest_collection_modifyitems().
diff --git a/tests/python/ci/test_ci.py b/tests/python/ci/test_ci.py
index bfd4b32491..e152d8cb41 100644
--- a/tests/python/ci/test_ci.py
+++ b/tests/python/ci/test_ci.py
@@ -18,8 +18,6 @@
"""Test various CI scripts and GitHub Actions workflows"""
import json
-import logging
-import shutil
import subprocess
import sys
import textwrap
@@ -50,277 +48,6 @@ def parameterize_named(**kwargs):
)
-# pylint: disable=line-too-long
-TEST_DATA_SKIPPED_BOT = {
- "found-diff-no-additional": {
- "main_xml_file": "unittest/file1.xml",
- "main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
- <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
- name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "pr_xml_file": "unittest/file2.xml",
- "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
- <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
- name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- <skipped message="This test is skipped" type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- <testcase classname="ctypes.tests.python.unittest.test_roofline"
- name="test_estimate_peak_bandwidth[cuda]" time="4.679">
- <skipped message="This is another skippe test" type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "additional_tests_to_check": """{
- "unittest": ["dummy_class#dummy_test"],
- "unittest_GPU": ["another_dummy_class#another_dummy_test"]
- }
- """,
- "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
- "s3_prefix": "tvm-jenkins-artifacts-prod",
- "jenkins_prefix": "ci.tlcpack.ai",
- "common_main_build": """{"build_number": "4115", "state": "success"}""",
- "commit_sha": "sha1234",
- "expected_body": "The list below shows tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
- },
- "found-diff-skipped-additional": {
- "main_xml_file": "unittest/file1.xml",
- "main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
- <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
- name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "pr_xml_file": "unittest/file2.xml",
- "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
- <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
- name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- <skipped message="This test is skipped" type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- <testcase classname="ctypes.tests.python.unittest.test_roofline"
- name="test_estimate_peak_bandwidth[cuda]" time="4.679">
- <skipped message="This is another skippe test" type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "additional_tests_to_check": """{
- "unittest": ["ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner", "dummy_class#dummy_test"],
- "unittest_GPU": ["another_dummy_class#another_dummy_test"]
- }
- """,
- "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
- "s3_prefix": "tvm-jenkins-artifacts-prod",
- "jenkins_prefix": "ci.tlcpack.ai",
- "common_main_build": """{"build_number": "4115", "state": "success"}""",
- "commit_sha": "sha1234",
- "expected_body": "The list below shows tests that ran in main sha1234 but were skipped in the CI build of sha1234:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\nunittest -> ctypes.tests.python.unittest.test_roofline#test_estimate_peak_bandwidth[cuda]\n```\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/ [...]
- },
- "no-diff": {
- "main_xml_file": "unittest/file1.xml",
- "main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
- <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
- name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- <skipped message="This test is skipped" type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "pr_xml_file": "unittest/file2.xml",
- "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
- <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
- name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- <skipped message="This test is skipped" type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "additional_tests_to_check": """{
- }
- """,
- "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
- "s3_prefix": "tvm-jenkins-artifacts-prod",
- "jenkins_prefix": "ci.tlcpack.ai",
- "common_main_build": """{"build_number": "4115", "state": "success"}""",
- "commit_sha": "sha1234",
- "expected_body": "No diff in skipped tests with main found in this branch for commit sha1234.\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/testReport/).",
- },
- "no-diff-skipped-additional": {
- "main_xml_file": "unittest/file1.xml",
- "main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
- <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
- name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- <skipped message="This test is skipped" type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "pr_xml_file": "unittest/file2.xml",
- "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- <testsuite errors="0" failures="0" hostname="13e7c5f749d8" name="python-unittest-gpu-0-shard-1-ctypes" skipped="102"
- tests="165" time="79.312" timestamp="2022-08-10T22:39:36.673781">
- <testcase classname="ctypes.tests.python.unittest.test_auto_scheduler_search_policy"
- name="test_sketch_search_policy_cuda_rpc_runner" time="9.679">
- <skipped message="This test is skipped" type="pytest.skip">
- Skipped
- </skipped>
- </testcase>
- </testsuite>
- </testsuites>
- """,
- "additional_tests_to_check": """{
- "unittest": ["dummy_class#dummy_test", "ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner"],
- "unittest_GPU": ["another_dummy_class#another_dummy_test"]
- }
- """,
- "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
- "s3_prefix": "tvm-jenkins-artifacts-prod",
- "jenkins_prefix": "ci.tlcpack.ai",
- "common_main_build": """{"build_number": "4115", "state": "success"}""",
- "commit_sha": "sha1234",
- "expected_body": "No diff in skipped tests with main found in this branch for commit sha1234.\n\nAdditional tests that were skipped in the CI build and present in the [`required_tests_to_run`](https://github.com/apache/tvm/blob/main/ci/scripts/github/required_tests_to_run.json) file:\n```\nunittest -> ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner\n```\nA detailed report of ran tests is [here](https://ci.tlcpack.ai/job/tvm [...]
- },
- "unable-to-run": {
- "main_xml_file": "unittest/file1.xml",
- "main_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- </testsuites>
- """,
- "pr_xml_file": "unittest/file2.xml",
- "pr_xml_content": """<?xml version="1.0" encoding="utf-8"?>
- <testsuites>
- </testsuites>
- """,
- "additional_tests_to_check": """{
- "unittest": ["ctypes.tests.python.unittest.test_auto_scheduler_search_policy#test_sketch_search_policy_cuda_rpc_runner", "dummy_class#dummy_test"],
- "unittest_GPU": ["another_dummy_class#another_dummy_test"]
- }
- """,
- "target_url": "https://ci.tlcpack.ai/job/tvm/job/PR-11594/3/display/redirect",
- "s3_prefix": "tvm-jenkins-artifacts-prod",
- "jenkins_prefix": "ci.tlcpack.ai",
- "common_main_build": """{"build_number": "4115", "state": "failed"}""",
- "commit_sha": "sha1234",
- "expected_body": "Unable to run tests bot because main failed to pass CI at sha1234.",
- },
-}
-# pylint: enable=line-too-long
-
-
-@tvm.testing.skip_if_wheel_test
-@parameterize_named(**TEST_DATA_SKIPPED_BOT)
-# pylint: enable=line-too-long
-def test_skipped_tests_comment(
- caplog,
- tmpdir_factory,
- main_xml_file,
- main_xml_content,
- pr_xml_file,
- pr_xml_content,
- additional_tests_to_check,
- target_url,
- s3_prefix,
- jenkins_prefix,
- common_main_build,
- commit_sha,
- expected_body,
-):
- """
- Test that a comment with a link to the docs is successfully left on PRs
- """
-
- def write_xml_file(root_dir, xml_file, xml_content):
- shutil.rmtree(root_dir, ignore_errors=True)
- file = root_dir / xml_file
- file.parent.mkdir(parents=True)
- with open(file, "w") as f:
- f.write(textwrap.dedent(xml_content))
-
- git = TempGit(tmpdir_factory.mktemp("tmp_git_dir"))
- pr_test_report_dir = Path(git.cwd) / "pr-reports"
- write_xml_file(pr_test_report_dir, pr_xml_file, pr_xml_content)
- main_test_report_dir = Path(git.cwd) / "main-reports"
- write_xml_file(main_test_report_dir, main_xml_file, main_xml_content)
- with open(Path(git.cwd) / "required_tests_to_run.json", "w") as f:
- f.write(additional_tests_to_check)
-
- pr_data = {
- "commits": {
- "nodes": [
- {
- "commit": {
- "oid": commit_sha,
- "statusCheckRollup": {
- "contexts": {
- "nodes": [
- {
- "context": "tvm-ci/pr-head",
- "targetUrl": target_url,
- }
- ]
- }
- },
- }
- }
- ]
- }
- }
- with caplog.at_level(logging.INFO):
- comment = scripts.github.github_skipped_tests_comment.get_skipped_tests_comment(
- pr=pr_data,
- github=None,
- s3_prefix=s3_prefix,
- jenkins_prefix=jenkins_prefix,
- common_commit_sha=commit_sha,
- pr_test_report_dir=pr_test_report_dir,
- main_test_report_dir=main_test_report_dir,
- common_main_build=json.loads(common_main_build),
- additional_tests_to_check_file=Path(git.cwd) / "required_tests_to_run.json",
- )
- assert_in(expected_body, comment)
- assert_in(f"with target {target_url}", caplog.text)
-
-
@tvm.testing.skip_if_wheel_test
@parameterize_named(
doc_link=dict(
@@ -548,8 +275,7 @@ def test_update_branch(tmpdir_factory, statuses, expected_rc, expected_output):
Thanks for contributing to TVM! Please refer to the contributing guidelines https://tvm.apache.org/docs/contribute/ for useful information and tips. Please request code reviews from [Reviewers](https://github.com/apache/incubator-tvm/blob/master/CONTRIBUTORS.md#reviewers) by @-ing them in a comment.
<!--bot-comment-ccs-start-->
- * the cc<!--bot-comment-ccs-end--><!--bot-comment-skipped-tests-start-->
- * the skipped tests<!--bot-comment-skipped-tests-end--><!--bot-comment-docs-start-->
+ * the cc<!--bot-comment-ccs-end--><!--bot-comment-docs-start-->
* the docs<!--bot-comment-docs-end-->
"""
).strip(),
@@ -584,8 +310,7 @@ def test_update_branch(tmpdir_factory, statuses, expected_rc, expected_output):
<!--bot-comment-ccs-start-->
* the cc<!--bot-comment-ccs-end--><!--bot-comment-something-tests-start-->
* something else<!--bot-comment-something-tests-end--><!--bot-comment-docs-start-->
- * the docs<!--bot-comment-docs-end--><!--bot-comment-skipped-tests-start-->
- * the skipped tests<!--bot-comment-skipped-tests-end-->
+ * the docs<!--bot-comment-docs-end-->
"""
).strip(),
),
@@ -638,7 +363,6 @@ def test_pr_comment(tmpdir_factory, pr_author, comments, expected):
comments = {
"ccs": "the cc",
"docs": "the docs",
- "skipped-tests": "the skipped tests",
}
proc = run_script(
[
diff --git a/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py b/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py
index 1e209cacbc..97c9469ccf 100644
--- a/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py
+++ b/tests/python/s_tir/meta_schedule/test_meta_schedule_cost_model.py
@@ -149,7 +149,7 @@ def _dummy_result(num_samples: int = 4, max_run_sec: int = 10):
@requires_xgboost
def test_meta_schedule_xgb_model():
extractor = RandomFeatureExtractor()
- model = XGBModel(extractor=extractor, num_warmup_samples=2)
+ model = XGBModel(extractor=extractor, num_warmup_samples=2, num_tuning_cores=1)
update_sample_count = 10
predict_sample_count = 100
model.update(
@@ -162,9 +162,10 @@ def test_meta_schedule_xgb_model():
@requires_xgboost
def test_meta_schedule_xgb_model_no_feature():
- model = XGBModel(num_warmup_samples=0)
+ model = XGBModel(num_warmup_samples=0, num_tuning_cores=1)
tune_ctx = TuneContext(
FullModule,
+ num_threads=1,
target={"kind": "llvm", "num-cores": 16},
space_generator="post-order-apply",
search_strategy="evolutionary",
@@ -177,7 +178,7 @@ def test_meta_schedule_xgb_model_no_feature():
@requires_xgboost
def test_meta_schedule_xgb_model_reload():
extractor = RandomFeatureExtractor()
- model = XGBModel(extractor=extractor, num_warmup_samples=10)
+ model = XGBModel(extractor=extractor, num_warmup_samples=10, num_tuning_cores=1)
update_sample_count = 20
predict_sample_count = 30
model.update(
@@ -221,7 +222,7 @@ def test_meta_schedule_xgb_model_reload():
@requires_xgboost
def test_meta_schedule_xgb_model_reupdate():
extractor = RandomFeatureExtractor()
- model = XGBModel(extractor=extractor, num_warmup_samples=2)
+ model = XGBModel(extractor=extractor, num_warmup_samples=2, num_tuning_cores=1)
update_sample_count = 60
predict_sample_count = 100
model.update(
@@ -252,7 +253,7 @@ def test_meta_schedule_xgb_model_callback_as_function():
# pylint: enable=import-outside-toplevel
extractor = RandomFeatureExtractor()
- model = XGBModel(extractor=extractor, num_warmup_samples=10)
+ model = XGBModel(extractor=extractor, num_warmup_samples=10, num_tuning_cores=1)
update_sample_count = 20
predict_sample_count = 30
diff --git a/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py b/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py
index 002741c6bf..6451a94ec9 100644
--- a/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py
+++ b/tests/python/s_tir/meta_schedule/test_meta_schedule_search_strategy.py
@@ -103,6 +103,7 @@ def test_meta_schedule_replay_func(
context = ms.TuneContext(
mod=Matmul,
+ num_threads=1,
space_generator=ms.space_generator.ScheduleFn(sch_fn=_schedule_matmul, postprocs=[]),
search_strategy=TestClass(),
)
diff --git a/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py b/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py
index 61f5583c2a..6dc351761a 100644
--- a/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py
+++ b/tests/python/s_tir/meta_schedule/test_meta_schedule_task_scheduler.py
@@ -157,6 +157,7 @@ def test_meta_schedule_task_scheduler_single():
[
ms.TuneContext(
MatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -183,6 +184,7 @@ def test_meta_schedule_task_scheduler_multiple():
tasks = [
ms.TuneContext(
MatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -191,6 +193,7 @@ def test_meta_schedule_task_scheduler_multiple():
),
ms.TuneContext(
MatmulReluModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -199,6 +202,7 @@ def test_meta_schedule_task_scheduler_multiple():
),
ms.TuneContext(
BatchMatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_batch_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -255,6 +259,7 @@ def test_meta_schedule_task_scheduler_override_next_task_id_only(): # pylint: d
tasks = [
ms.TuneContext(
MatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -263,6 +268,7 @@ def test_meta_schedule_task_scheduler_override_next_task_id_only(): # pylint: d
),
ms.TuneContext(
MatmulReluModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -271,6 +277,7 @@ def test_meta_schedule_task_scheduler_override_next_task_id_only(): # pylint: d
),
ms.TuneContext(
BatchMatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_batch_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -310,6 +317,7 @@ def test_meta_schedule_task_scheduler_multiple_gradient_based():
tasks = [
ms.TuneContext(
MatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -318,6 +326,7 @@ def test_meta_schedule_task_scheduler_multiple_gradient_based():
),
ms.TuneContext(
MatmulReluModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -326,6 +335,7 @@ def test_meta_schedule_task_scheduler_multiple_gradient_based():
),
ms.TuneContext(
BatchMatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_batch_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
@@ -394,6 +404,7 @@ def test_meta_schedule_task_scheduler_gradient_based_with_null_search_strategy()
tasks = [
ms.TuneContext(
MatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=NullSearchStrategy(rounds_with_empty_candidates=5),
@@ -402,6 +413,7 @@ def test_meta_schedule_task_scheduler_gradient_based_with_null_search_strategy()
),
ms.TuneContext(
BatchMatmulModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_batch_matmul,
search_strategy=NullSearchStrategy(rounds_with_empty_candidates=0),
@@ -410,6 +422,7 @@ def test_meta_schedule_task_scheduler_gradient_based_with_null_search_strategy()
),
ms.TuneContext(
MatmulReluModule,
+ num_threads=1,
target=tvm.target.Target("llvm"),
space_generator=_schedule_matmul,
search_strategy=ms.search_strategy.ReplayTrace(),
diff --git a/tests/python/testing/test_tvm_testing_features.py b/tests/python/testing/test_tvm_testing_features.py
index 549aedda6a..ba2faf2a45 100644
--- a/tests/python/testing/test_tvm_testing_features.py
+++ b/tests/python/testing/test_tvm_testing_features.py
@@ -14,9 +14,10 @@
# KIND, either express or implied. See the License for the
# specific language governing permissions and limitations
# under the License.
-# ruff: noqa: F401, RUF012
+# ruff: noqa: RUF012
import os
+import subprocess
import sys
import pytest
@@ -189,5 +190,44 @@ class TestPytestCache:
assert cached_fixture == param * param
+def test_empty_marker_selection_is_an_error(tmp_path):
+ test_file = tmp_path / "test_no_gpu.py"
+ test_file.write_text("def test_cpu_only():\n pass\n")
+ env = os.environ.copy()
+ env.pop("PYTEST_ADDOPTS", None)
+
+ result = subprocess.run(
+ [
+ sys.executable,
+ "-m",
+ "pytest",
+ "-q",
+ "-p",
+ "tvm.testing.plugin",
+ "-m",
+ "gpu",
+ str(test_file),
+ ],
+ env=env,
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == pytest.ExitCode.NO_TESTS_COLLECTED, result.stdout + result.stderr
+
+ test_file.write_text("def test_failure():\n assert False\n")
+ result = subprocess.run(
+ [sys.executable, "-m", "pytest", "-q", "-p", "tvm.testing.plugin", str(test_file)],
+ env=env,
+ check=False,
+ capture_output=True,
+ text=True,
+ )
+
+ assert result.returncode == pytest.ExitCode.TESTS_FAILED, result.stdout + result.stderr
+ assert not list(tmp_path.glob("*.xml"))
+
+
if __name__ == "__main__":
tvm.testing.main()
diff --git a/tests/scripts/task_clear_pytest.sh b/tests/scripts/task_clear_pytest.sh
deleted file mode 100755
index 0a24f62531..0000000000
--- a/tests/scripts/task_clear_pytest.sh
+++ /dev/null
@@ -1,24 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -euxo pipefail
-
-# Ensure no stale pytest-results remain from a previous test run.
-pushd build
-rm -rf pytest-results
-popd
diff --git a/tests/scripts/task_python_nightly.sh b/tests/scripts/task_python_nightly.sh
deleted file mode 100755
index 05070f0a04..0000000000
--- a/tests/scripts/task_python_nightly.sh
+++ /dev/null
@@ -1,39 +0,0 @@
-#!/usr/bin/env bash
-# Licensed to the Apache Software Foundation (ASF) under one
-# or more contributor license agreements. See the NOTICE file
-# distributed with this work for additional information
-# regarding copyright ownership. The ASF licenses this file
-# to you under the Apache License, Version 2.0 (the
-# "License"); you may not use this file except in compliance
-# with the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing,
-# software distributed under the License is distributed on an
-# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-# KIND, either express or implied. See the License for the
-# specific language governing permissions and limitations
-# under the License.
-
-set -euxo pipefail
-
-export PYTHONPATH="$(pwd)/python"
-export PYTEST_ADDOPTS="-s -vv ${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}"
-mkdir -p build/pytest-results
-
-# setup tvm-ffi into python folder
-uv pip install -v --target=python ./3rdparty/tvm-ffi/
-
-# cleanup pycache
-find . -type f -path "*.pyc" | xargs rm -f
-
-python3 -m pytest -n auto \
- -o junit_suite_name=python-topi-nightly \
- --junit-xml=build/pytest-results/python-topi-nightly.xml \
- --junit-prefix=cython \
- tests/python/topi/nightly
-
-# Tensorflow device verification and network tests on nightly
-export CI_ENV_NIGHTLY
-python3 tests/python/relax/test_frontend_tflite.py
diff --git a/tests/scripts/task_python_unittest.sh b/tests/scripts/task_python_unittest.sh
index 14675bf8a0..c05f8503d3 100755
--- a/tests/scripts/task_python_unittest.sh
+++ b/tests/scripts/task_python_unittest.sh
@@ -20,7 +20,6 @@ set -euxo pipefail
export PYTHONPATH="$(pwd)/python"
export PYTEST_ADDOPTS="-s -vv ${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}"
-mkdir -p build/pytest-results
# cleanup pycache
find . -type f -path "*.pyc" | xargs rm -f
@@ -28,28 +27,20 @@ find . -type f -path "*.pyc" | xargs rm -f
# setup tvm-ffi into python folder
uv pip install -v --target=python ./3rdparty/tvm-ffi/
-# NOTE: also set by task_python_unittest_gpuonly.sh.
-if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then
- TVM_UNITTEST_TESTSUITE_NAME=python-unittest
-fi
-PYTEST_SHARD_SUFFIX="${TVM_SHARD_INDEX:+-shard-${TVM_SHARD_INDEX}}"
-
# First run the minimal platform test. A GPU-only run can select no tests here.
-python3 -m pytest -n auto \
- -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test${PYTEST_SHARD_SUFFIX}" \
- "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test${PYTEST_SHARD_SUFFIX}.xml" \
- --junit-prefix=cython \
- tests/python/all-platform-minimal-test || [ "$?" -eq 5 ]
+if [ ! -d tests/python/all-platform-minimal-test ]; then
+ echo "Missing pytest target: tests/python/all-platform-minimal-test" >&2
+ exit 1
+fi
+python3 -m pytest -n auto tests/python/all-platform-minimal-test || [ "$?" -eq 5 ]
# Then run all unit tests.
TEST_FILES=(
- "ffi"
"arith"
"ci"
"codegen"
"driver"
"ir"
- "meta_schedule"
"runtime"
"target"
"te"
@@ -58,6 +49,7 @@ TEST_FILES=(
"s_tir/schedule"
"s_tir/dlight"
"s_tir/analysis"
+ "s_tir/meta_schedule"
"s_tir/transform"
"tirx-analysis"
"tirx-base"
@@ -69,11 +61,13 @@ TEST_FILES=(
PYTEST_TARGETS=()
for TEST_FILE in "${TEST_FILES[@]}"; do
- PYTEST_TARGETS+=("tests/python/${TEST_FILE}")
+ TEST_PATH="tests/python/${TEST_FILE}"
+ if [ ! -d "${TEST_PATH}" ]; then
+ echo "Missing pytest target: ${TEST_PATH}" >&2
+ exit 1
+ fi
+ PYTEST_TARGETS+=("${TEST_PATH}")
done
-python3 -m pytest -n auto --dist=loadgroup \
- -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}${PYTEST_SHARD_SUFFIX}" \
- "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}${PYTEST_SHARD_SUFFIX}.xml" \
- --junit-prefix=cython \
- "${PYTEST_TARGETS[@]}"
+# Do not mask pytest's exit 5: an unexpectedly empty broad suite must fail CI.
+python3 -m pytest -n auto --dist=loadgroup "${PYTEST_TARGETS[@]}"
diff --git a/tests/scripts/task_python_unittest_gpuonly.sh b/tests/scripts/task_python_unittest_gpuonly.sh
index 9753a1a8a4..8194b23ac4 100755
--- a/tests/scripts/task_python_unittest_gpuonly.sh
+++ b/tests/scripts/task_python_unittest_gpuonly.sh
@@ -24,22 +24,18 @@ export PYTEST_ADDOPTS="-m gpu ${PYTEST_ADDOPTS:-}"
# Test most of the enabled runtimes here.
# TODO: disabled opencl tests due to segmentation fault.
export TVM_TEST_TARGETS='cuda;metal;rocm;nvptx'
-export TVM_UNITTEST_TESTSUITE_NAME=python-unittest-gpu
./tests/scripts/task_python_unittest.sh
# Kept separate to avoid increasing time needed to run CI, testing
# only minimal functionality of Vulkan runtime.
export TVM_TEST_TARGETS='{"kind":"vulkan","from_device":0}'
-export TVM_UNITTEST_TESTSUITE_NAME=python-codegen-vulkan
export PYTHONPATH="$(pwd)/python"
export PYTEST_ADDOPTS="-s -vv ${CI_PYTEST_ADD_OPTIONS:-} ${PYTEST_ADDOPTS:-}"
-mkdir -p build/pytest-results
-PYTEST_SHARD_SUFFIX="${TVM_SHARD_INDEX:+-shard-${TVM_SHARD_INDEX}}"
-python3 -m pytest -n auto \
- -o "junit_suite_name=${TVM_UNITTEST_TESTSUITE_NAME}${PYTEST_SHARD_SUFFIX}" \
- "--junit-xml=build/pytest-results/${TVM_UNITTEST_TESTSUITE_NAME}${PYTEST_SHARD_SUFFIX}.xml" \
- --junit-prefix=cython \
- tests/python/codegen/test_target_codegen_vulkan.py
+if [ ! -f tests/python/codegen/test_target_codegen_vulkan.py ]; then
+ echo "Missing pytest target: tests/python/codegen/test_target_codegen_vulkan.py" >&2
+ exit 1
+fi
+python3 -m pytest -n auto tests/python/codegen/test_target_codegen_vulkan.py