You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by tq...@apache.org on 2025/02/07 00:19:33 UTC
(tvm) branch ci updated (0f41a9006f -> 2e57c12e8b)
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a change to branch ci
in repository https://gitbox.apache.org/repos/asf/tvm.git
discard 0f41a9006f update
discard 1101b1bd5b update
discard 4594375323 [CI] Robustify CI for SPOT failure
new 2e57c12e8b [CI] Robustify CI for SPOT failure
This update added new revisions after undoing existing revisions.
That is to say, some revisions that were in the old version of the
branch are not in the new version. This situation occurs
when a user --force pushes a change and generates a repository
containing something like this:
* -- * -- B -- O -- O -- O (0f41a9006f)
\
N -- N -- N refs/heads/ci (2e57c12e8b)
You should already have received notification emails for all of the O
revisions, and so the following emails describe only the N revisions
from the common base, B.
Any revisions marked "omit" are not gone; other references still
refer to them. Any revisions marked "discard" are gone forever.
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
Summary of changes:
ci/jenkins/generated/arm_jenkinsfile.groovy | 40 ++++++++---------
ci/jenkins/generated/cpu_jenkinsfile.groovy | 32 +++++++------
ci/jenkins/generated/gpu_jenkinsfile.groovy | 60 ++++++++++++-------------
ci/jenkins/generated/hexagon_jenkinsfile.groovy | 40 ++++++++---------
ci/jenkins/generated/i386_jenkinsfile.groovy | 20 ++++-----
ci/jenkins/generated/wasm_jenkinsfile.groovy | 8 ++--
ci/jenkins/templates/utils/macros.j2 | 10 ++---
7 files changed, 98 insertions(+), 112 deletions(-)
(tvm) 01/01: [CI] Robustify CI for SPOT failure
Posted by tq...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
tqchen pushed a commit to branch ci
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 2e57c12e8b2c37be46e969e6284d92ac36059615
Author: tqchen <tq...@gmail.com>
AuthorDate: Thu Feb 6 12:13:25 2025 -0500
[CI] Robustify CI for SPOT failure
This PR cleans up some legacy CI files and robustifies
the CI for spot failure by marking a stage as success then retry
on the on demand node.
---
ci/jenkins/data.py | 12 +-
ci/jenkins/generated/arm_jenkinsfile.groovy | 417 ++++---
ci/jenkins/generated/cortexm_jenkinsfile.groovy | 1231 --------------------
ci/jenkins/generated/cpu_jenkinsfile.groovy | 325 +++---
ci/jenkins/generated/docker_jenkinsfile.groovy | 2 +-
ci/jenkins/generated/gpu_jenkinsfile.groovy | 647 +++++-----
ci/jenkins/generated/hexagon_jenkinsfile.groovy | 415 ++++---
ci/jenkins/generated/i386_jenkinsfile.groovy | 188 +--
ci/jenkins/generated/lint_jenkinsfile.groovy | 2 +-
.../generated/minimal_cross_isa_jenkinsfile.groovy | 622 ----------
ci/jenkins/generated/minimal_jenkinsfile.groovy | 620 ----------
ci/jenkins/generated/riscv_jenkinsfile.groovy | 624 ----------
ci/jenkins/generated/wasm_jenkinsfile.groovy | 47 +-
ci/jenkins/templates/arm_jenkinsfile.groovy.j2 | 2 +-
ci/jenkins/templates/cortexm_jenkinsfile.groovy.j2 | 63 -
ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 | 2 +-
ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 | 6 +-
ci/jenkins/templates/i386_jenkinsfile.groovy.j2 | 2 +-
.../minimal_cross_isa_jenkinsfile.groovy.j2 | 57 -
ci/jenkins/templates/minimal_jenkinsfile.groovy.j2 | 55 -
ci/jenkins/templates/riscv_jenkinsfile.groovy.j2 | 61 -
ci/jenkins/templates/utils/macros.j2 | 87 +-
ci/jenkins/unity_jenkinsfile.groovy | 2 +-
23 files changed, 1171 insertions(+), 4318 deletions(-)
diff --git a/ci/jenkins/data.py b/ci/jenkins/data.py
index 8cf762f161..6eeb0bc731 100644
--- a/ci/jenkins/data.py
+++ b/ci/jenkins/data.py
@@ -21,8 +21,6 @@ import sys
files_to_stash = {
# Executables and build files needed to run c++ tests
"cpptest": ["build/cpptest", "build/build.ninja", "build/CMakeFiles/rules.ninja"],
- # Executables and build files needed to c runtime tests
- "crttest": ["build/crttest"],
# Folder for hexagon build
"hexagon_api": [
"build/hexagon_api_output",
@@ -33,12 +31,10 @@ files_to_stash = {
"tvm_runtime": ["build/libtvm_runtime.so", "build/config.cmake"],
# compiler files
"tvm_lib": ["build/libtvm.so", "build/libtvm_runtime.so", "build/config.cmake"],
- # compiler files and fsim
- "tvm_multilib": [
- "build/libtvm.so",
- "build/libvta_fsim.so",
- "build/libtvm_runtime.so",
- "build/config.cmake",
+ # gpu related compiler files
+ "tvm_lib_gpu_extra": [
+ "build/3rdparty/libflash_attn/src/libflash_attn.so",
+ "build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so",
],
}
diff --git a/ci/jenkins/generated/arm_jenkinsfile.groovy b/ci/jenkins/generated/arm_jenkinsfile.groovy
index c33be22a6f..2f20d7ea83 100644
--- a/ci/jenkins/generated/arm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/arm_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.226391
+// Generated at 2025-02-06T19:18:57.625018
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -515,40 +515,45 @@ try {
prepare('CPU-SMALL')
}
def build(node_type) {
- stage('Build') {
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-arm") {
- init_git()
- docker_init(ci_arm)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=arm',
- ], {
- sh (
+ if (!skip_ci && is_docs_only_build != 1) {
+ echo 'Run build on node_type=' + node_type
+ node(node_type) {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-arm") {
+ init_git()
+ docker_init(ci_arm)
+ timeout(time: max_time, unit: 'MINUTES') {
+
+ withEnv([
+ 'PLATFORM=arm',
+ ], {
+ sh (
script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build",
label: 'Create ARM cmake config',
)
cmake_build(ci_arm, 'build', '-j4')
make_cpp_tests(ci_arm, 'build')
sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/arm --items build/libtvm.so build/libvta_fsim.so build/libtvm_runtime.so build/config.cmake build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/crttest build/build.ninja build/microtvm_template_projects",
+ script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/arm --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/cpptest build/build.ninja build/CMakeFiles/rules.ninja",
label: 'Upload artifacts to S3',
)
- })
- }
+ })
}
}
- } else {
- Utils.markStageSkippedForConditional('BUILD: arm')
}
+ } else {
+ Utils.markStageSkippedForConditional('BUILD: arm')
}
}
-try {
- build('ARM-GRAVITON3-SPOT')
-} catch (Exception ex) {
- build('ARM-GRAVITON3')
+stage('Build') {
+ try {
+ build('ARM-GRAVITON3-SPOT')
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
+ build('ARM-GRAVITON3')
+ }
}
@@ -558,19 +563,20 @@ def shard_run_integration_aarch64_1_of_4(node_type='ARM-GRAVITON3-SPOT', on_dema
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'ARM-GRAVITON3'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
- try {
- init_git()
- docker_init(ci_arm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=arm',
- 'TEST_STEP_NAME=integration: aarch64',
- 'TVM_NUM_SHARDS=4',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_arm)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=arm',
+ 'TEST_STEP_NAME=integration: aarch64',
+ 'TVM_NUM_SHARDS=4',
+ 'TVM_SHARD_INDEX=0',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
label: 'Download artifacts from S3',
)
@@ -581,19 +587,18 @@ def shard_run_integration_aarch64_1_of_4(node_type='ARM-GRAVITON3-SPOT', on_dema
script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh",
label: 'Run CPU integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_aarch64 --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -607,19 +612,20 @@ def shard_run_integration_aarch64_2_of_4(node_type='ARM-GRAVITON3-SPOT', on_dema
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'ARM-GRAVITON3'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
- try {
- init_git()
- docker_init(ci_arm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=arm',
- 'TEST_STEP_NAME=integration: aarch64',
- 'TVM_NUM_SHARDS=4',
- 'TVM_SHARD_INDEX=1',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_arm)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=arm',
+ 'TEST_STEP_NAME=integration: aarch64',
+ 'TVM_NUM_SHARDS=4',
+ 'TVM_SHARD_INDEX=1',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
label: 'Download artifacts from S3',
)
@@ -630,19 +636,18 @@ def shard_run_integration_aarch64_2_of_4(node_type='ARM-GRAVITON3-SPOT', on_dema
script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh",
label: 'Run CPU integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_aarch64 --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -656,19 +661,20 @@ def shard_run_integration_aarch64_3_of_4(node_type='ARM-GRAVITON3-SPOT', on_dema
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'ARM-GRAVITON3'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
- try {
- init_git()
- docker_init(ci_arm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=arm',
- 'TEST_STEP_NAME=integration: aarch64',
- 'TVM_NUM_SHARDS=4',
- 'TVM_SHARD_INDEX=2',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_arm)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=arm',
+ 'TEST_STEP_NAME=integration: aarch64',
+ 'TVM_NUM_SHARDS=4',
+ 'TVM_SHARD_INDEX=2',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
label: 'Download artifacts from S3',
)
@@ -679,19 +685,18 @@ def shard_run_integration_aarch64_3_of_4(node_type='ARM-GRAVITON3-SPOT', on_dema
script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh",
label: 'Run CPU integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_aarch64 --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -705,19 +710,20 @@ def shard_run_integration_aarch64_4_of_4(node_type='ARM-GRAVITON3-SPOT', on_dema
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'ARM-GRAVITON3'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
- try {
- init_git()
- docker_init(ci_arm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=arm',
- 'TEST_STEP_NAME=integration: aarch64',
- 'TVM_NUM_SHARDS=4',
- 'TVM_SHARD_INDEX=3',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_arm)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=arm',
+ 'TEST_STEP_NAME=integration: aarch64',
+ 'TVM_NUM_SHARDS=4',
+ 'TVM_SHARD_INDEX=3',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
label: 'Download artifacts from S3',
)
@@ -728,19 +734,18 @@ def shard_run_integration_aarch64_4_of_4(node_type='ARM-GRAVITON3-SPOT', on_dema
script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_integration.sh",
label: 'Run CPU integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_aarch64 --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -756,19 +761,20 @@ def shard_run_topi_aarch64_1_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=fals
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'ARM-GRAVITON3'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
- try {
- init_git()
- docker_init(ci_arm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=arm',
- 'TEST_STEP_NAME=topi: aarch64',
- 'TVM_NUM_SHARDS=2',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_arm)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=arm',
+ 'TEST_STEP_NAME=topi: aarch64',
+ 'TVM_NUM_SHARDS=2',
+ 'TVM_SHARD_INDEX=0',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
label: 'Download artifacts from S3',
)
@@ -783,19 +789,18 @@ def shard_run_topi_aarch64_1_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=fals
script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_topi.sh",
label: 'Run TOPI tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_aarch64 --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -809,19 +814,20 @@ def shard_run_topi_aarch64_2_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=fals
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'ARM-GRAVITON3'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-arm") {
- try {
- init_git()
- docker_init(ci_arm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=arm',
- 'TEST_STEP_NAME=topi: aarch64',
- 'TVM_NUM_SHARDS=2',
- 'TVM_SHARD_INDEX=1',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_arm)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=arm',
+ 'TEST_STEP_NAME=topi: aarch64',
+ 'TVM_NUM_SHARDS=2',
+ 'TVM_SHARD_INDEX=1',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
label: 'Download artifacts from S3',
)
@@ -835,19 +841,18 @@ def shard_run_topi_aarch64_2_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=fals
script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_topi.sh",
label: 'Run TOPI tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_aarch64 --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -863,19 +868,20 @@ def shard_run_frontend_aarch64_1_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'ARM-GRAVITON3'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") {
- try {
- init_git()
- docker_init(ci_arm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=arm',
- 'TEST_STEP_NAME=frontend: aarch64',
- 'TVM_NUM_SHARDS=2',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_arm)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=arm',
+ 'TEST_STEP_NAME=frontend: aarch64',
+ 'TVM_NUM_SHARDS=2',
+ 'TVM_SHARD_INDEX=0',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
label: 'Download artifacts from S3',
)
@@ -885,19 +891,18 @@ def shard_run_frontend_aarch64_1_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=
script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh",
label: 'Run Python frontend tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_aarch64 --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -911,19 +916,20 @@ def shard_run_frontend_aarch64_2_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'ARM-GRAVITON3'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-arm") {
- try {
- init_git()
- docker_init(ci_arm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=arm',
- 'TEST_STEP_NAME=frontend: aarch64',
- 'TVM_NUM_SHARDS=2',
- 'TVM_SHARD_INDEX=1',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_arm)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=arm',
+ 'TEST_STEP_NAME=frontend: aarch64',
+ 'TVM_NUM_SHARDS=2',
+ 'TVM_SHARD_INDEX=1',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/arm",
label: 'Download artifacts from S3',
)
@@ -933,19 +939,18 @@ def shard_run_frontend_aarch64_2_of_2(node_type='ARM-GRAVITON3-SPOT', on_demand=
script: "${docker_run} ${ci_arm} ./tests/scripts/task_python_frontend_cpu.sh",
label: 'Run Python frontend tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_aarch64 --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -964,56 +969,88 @@ def test() {
'integration: aarch64 1 of 4': {
try {
shard_run_integration_aarch64_1_of_4()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_integration_aarch64_1_of_4(on_demand = true)
}
},
'integration: aarch64 2 of 4': {
try {
shard_run_integration_aarch64_2_of_4()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_integration_aarch64_2_of_4(on_demand = true)
}
},
'integration: aarch64 3 of 4': {
try {
shard_run_integration_aarch64_3_of_4()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_integration_aarch64_3_of_4(on_demand = true)
}
},
'integration: aarch64 4 of 4': {
try {
shard_run_integration_aarch64_4_of_4()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_integration_aarch64_4_of_4(on_demand = true)
}
},
'topi: aarch64 1 of 2': {
try {
shard_run_topi_aarch64_1_of_2()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_topi_aarch64_1_of_2(on_demand = true)
}
},
'topi: aarch64 2 of 2': {
try {
shard_run_topi_aarch64_2_of_2()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_topi_aarch64_2_of_2(on_demand = true)
}
},
'frontend: aarch64 1 of 2': {
try {
shard_run_frontend_aarch64_1_of_2()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_frontend_aarch64_1_of_2(on_demand = true)
}
},
'frontend: aarch64 2 of 2': {
try {
shard_run_frontend_aarch64_2_of_2()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_frontend_aarch64_2_of_2(on_demand = true)
}
},
diff --git a/ci/jenkins/generated/cortexm_jenkinsfile.groovy b/ci/jenkins/generated/cortexm_jenkinsfile.groovy
deleted file mode 100644
index 6bd4b2e092..0000000000
--- a/ci/jenkins/generated/cortexm_jenkinsfile.groovy
+++ /dev/null
@@ -1,1231 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-// -*- mode: groovy -*-
-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Jenkins pipeline
-// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
-// Docker env used for testing
-// Different image may have different version tag
-// because some of them are more stable than anoter.
-//
-// Docker images are maintained by PMC, cached in dockerhub
-// and remains relatively stable over the time.
-// Flow for upgrading docker env(need commiter)
-//
-// - Send PR to upgrade build script in the repo
-// - Build the new docker image
-// - Tag the docker image with a new version and push to a binary cache.
-// - Update the version in the Jenkinsfile, send a PR
-// - Fix any issues wrt to the new image version in the PR
-// - Merge the PR and now we are in new version
-// - Tag the new version as the lates
-// - Periodically cleanup the old versions on local workers
-//
-
-// ============================= IMPORTANT NOTE =============================
-// This file is generated by 'jenkins/generate.py'. Do not edit this file directly!
-// Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with
-// 'python3 jenkins/generate.py'
-// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
-// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.121865
-
-import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-// These are set at runtime from data in ci/jenkins/docker-images.yml, update
-// image tags in that file
-ci_lint = ''
-ci_gpu = ''
-ci_cpu = ''
-ci_minimal = ''
-ci_wasm = ''
-ci_i386 = ''
-ci_cortexm = ''
-ci_arm = ''
-ci_hexagon = ''
-ci_riscv = ''
-
-// Parameters to allow overriding (in Jenkins UI), the images
-// to be used by a given build. When provided, they take precedence
-// over default values above.
-properties([
- parameters([
- string(name: 'ci_arm_param', defaultValue: ''),
- string(name: 'ci_cortexm_param', defaultValue: ''),
- string(name: 'ci_cpu_param', defaultValue: ''),
- string(name: 'ci_gpu_param', defaultValue: ''),
- string(name: 'ci_hexagon_param', defaultValue: ''),
- string(name: 'ci_i386_param', defaultValue: ''),
- string(name: 'ci_lint_param', defaultValue: ''),
- string(name: 'ci_minimal_param', defaultValue: ''),
- string(name: 'ci_riscv_param', defaultValue: ''),
- string(name: 'ci_wasm_param', defaultValue: ''),
- ])
-])
-
-// Placeholders for newly built Docker image names (if rebuild_docker_images
-// is used)
- built_ci_arm = null;
- built_ci_cortexm = null;
- built_ci_cpu = null;
- built_ci_gpu = null;
- built_ci_hexagon = null;
- built_ci_i386 = null;
- built_ci_lint = null;
- built_ci_minimal = null;
- built_ci_riscv = null;
- built_ci_wasm = null;
-
-// Global variable assigned during Sanity Check that holds the sha1 which should be
-// merged into the PR in all branches.
-upstream_revision = null
-
-// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
-docker_build = 'docker/build.sh'
-// timeout in minutes
-max_time = 180
-rebuild_docker_images = false
-
-s3_bucket = 'tvm-jenkins-artifacts-prod'
-s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
-
-// Jenkins script root directory
-jenkins_scripts_root = "ci/scripts/jenkins"
-
-
-// General note: Jenkins has limits on the size of a method (or top level code)
-// that are pretty strict, so most usage of groovy methods in these templates
-// are purely to satisfy the JVM
-def per_exec_ws(folder) {
- return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
-}
-
-// initialize source codes
-def init_git() {
- retry(5) {
- checkout scm
- }
-
- // Add more info about job node
- sh (
- script: './tests/scripts/task_show_node_info.sh',
- label: 'Show executor node info',
- )
-
- // Determine merge commit to use for all stages
- if (env.BRANCH_NAME == 'main') {
- // Only set upstream_revision to HEAD and skip merging to avoid a race with another commit merged to main.
- update_upstream_revision("HEAD")
- } else {
- // This is PR branch so merge with latest main.
- merge_with_main()
- }
-
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0
- """,
- label: 'Update git submodules',
- )
- checkout_trusted_files()
-}
-
-def update_upstream_revision(git_ref) {
- if (upstream_revision == null) {
- upstream_revision = sh(
- script: "git log -1 ${git_ref} --format=\'%H\'",
- label: 'Determine upstream revision',
- returnStdout: true,
- ).trim()
- }
-}
-
-def merge_with_main() {
- sh (
- script: 'git fetch origin main',
- label: 'Fetch upstream',
- )
- update_upstream_revision("FETCH_HEAD")
- sh (
- script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
- label: 'Merge to origin/main'
- )
-}
-
-def docker_init(image) {
- // Clear out all Docker images that aren't going to be used
- sh(
- script: """
- set -eux
- docker image ls --all
- IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}')
-
- echo -e "Found images:\\n\$IMAGES"
- echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
-
- docker image ls --all
- """,
- label: 'Clean old Docker images',
- )
-
- if (image.contains("amazonaws.com")) {
- // If this string is in the image name it's from ECR and needs to be pulled
- // with the right credentials
- ecr_pull(image)
- } else {
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${image}
- """,
- label: 'Pull docker image',
- )
- }
-}
-
-def ecr_pull(full_name) {
- aws_account_id = sh(
- returnStdout: true,
- script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
- label: 'Get AWS ID'
- ).trim()
-
- try {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: '''
- set -eux
- aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
- ''',
- label: 'Log in to ECR'
- )
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${full_name}
- """,
- label: 'Pull image from ECR'
- )
- }
- } finally {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: 'docker logout $AWS_ECR_REPO',
- label: 'Clean up login credentials'
- )
- }
- }
-}
-
-def should_skip_slow_tests(pr_number) {
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
- result = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
- label: 'Check if CI should run slow tests',
- )
- }
- return result == 0
-}
-
-def cancel_previous_build() {
- // cancel previous build if it is not on main.
- if (env.BRANCH_NAME != 'main') {
- def buildNumber = env.BUILD_NUMBER as int
- // Milestone API allows us to cancel previous build
- // with the same milestone number
- if (buildNumber > 1) milestone(buildNumber - 1)
- milestone(buildNumber)
- }
-}
-
-def checkout_trusted_files() {
- // trust everything from branch builds
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- return;
- }
-
- // trust peoople listed in CONTRIBUTING.md
- grep_code = sh(
- returnStatus: true,
- script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
- label: 'Check if change is from a contributor',
- )
-
- if (grep_code == 1) {
- // Any scripts that run on the bare host and not inside a Docker container
- // (especially those that access secrets) should be checked out here so
- // only trusted versions are used in CI
- sh(
- script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
- label: 'Check out trusted files',
- )
- }
-}
-
-def should_skip_ci(pr_number) {
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- // never skip CI on build sourced from a branch
- return false
- }
- glob_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
- label: 'Check if CI should be skipped due to changed files',
- )
- if (glob_skip_ci_code == 0) {
- return true
- }
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run full CI (or the script had an error, so run
- // full CI just in case). Exit code of 0 means skip CI.
- git_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
- label: 'Check if CI should be skipped',
- )
- }
- return git_skip_ci_code == 0
-}
-
-def check_pr(pr_number) {
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- // never skip CI on build sourced from a branch
- return false
- }
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- sh (
- script: "python3 ${jenkins_scripts_root}/check_pr.py --pr ${pr_number}",
- label: 'Check PR title and body',
- )
- }
-
-}
-
-def prepare(node_type) {
- stage('Prepare') {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
- init_git()
-
- check_pr(env.CHANGE_ID)
-
- if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
- sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
- label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
- )
- // Pull image names from the results of should_rebuild_docker.py
- ci_arm = sh(
- script: "cat .docker-image-names/ci_arm",
- label: "Find docker image name for ci_arm",
- returnStdout: true,
- ).trim()
- ci_cortexm = sh(
- script: "cat .docker-image-names/ci_cortexm",
- label: "Find docker image name for ci_cortexm",
- returnStdout: true,
- ).trim()
- ci_cpu = sh(
- script: "cat .docker-image-names/ci_cpu",
- label: "Find docker image name for ci_cpu",
- returnStdout: true,
- ).trim()
- ci_gpu = sh(
- script: "cat .docker-image-names/ci_gpu",
- label: "Find docker image name for ci_gpu",
- returnStdout: true,
- ).trim()
- ci_hexagon = sh(
- script: "cat .docker-image-names/ci_hexagon",
- label: "Find docker image name for ci_hexagon",
- returnStdout: true,
- ).trim()
- ci_i386 = sh(
- script: "cat .docker-image-names/ci_i386",
- label: "Find docker image name for ci_i386",
- returnStdout: true,
- ).trim()
- ci_lint = sh(
- script: "cat .docker-image-names/ci_lint",
- label: "Find docker image name for ci_lint",
- returnStdout: true,
- ).trim()
- ci_minimal = sh(
- script: "cat .docker-image-names/ci_minimal",
- label: "Find docker image name for ci_minimal",
- returnStdout: true,
- ).trim()
- ci_riscv = sh(
- script: "cat .docker-image-names/ci_riscv",
- label: "Find docker image name for ci_riscv",
- returnStdout: true,
- ).trim()
- ci_wasm = sh(
- script: "cat .docker-image-names/ci_wasm",
- label: "Find docker image name for ci_wasm",
- returnStdout: true,
- ).trim()
- }
-
- ci_arm = params.ci_arm_param ?: ci_arm
- ci_cortexm = params.ci_cortexm_param ?: ci_cortexm
- ci_cpu = params.ci_cpu_param ?: ci_cpu
- ci_gpu = params.ci_gpu_param ?: ci_gpu
- ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
- ci_i386 = params.ci_i386_param ?: ci_i386
- ci_lint = params.ci_lint_param ?: ci_lint
- ci_minimal = params.ci_minimal_param ?: ci_minimal
- ci_riscv = params.ci_riscv_param ?: ci_riscv
- ci_wasm = params.ci_wasm_param ?: ci_wasm
-
- sh (script: """
- echo "Docker images being used in this build:"
- echo " ci_arm = ${ci_arm}"
- echo " ci_cortexm = ${ci_cortexm}"
- echo " ci_cpu = ${ci_cpu}"
- echo " ci_gpu = ${ci_gpu}"
- echo " ci_hexagon = ${ci_hexagon}"
- echo " ci_i386 = ${ci_i386}"
- echo " ci_lint = ${ci_lint}"
- echo " ci_minimal = ${ci_minimal}"
- echo " ci_riscv = ${ci_riscv}"
- echo " ci_wasm = ${ci_wasm}"
- """, label: 'Docker image names')
-
- is_docs_only_build = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docs.sh",
- label: 'Check for docs only changes',
- )
- skip_ci = should_skip_ci(env.CHANGE_ID)
- skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
- rebuild_docker_images = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docker.sh",
- label: 'Check for any docker changes',
- )
-
- if (skip_ci) {
- // Don't rebuild when skipping CI
- rebuild_docker_images = false
- }
- }
- }
- }
-}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
-def python_unittest(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
- label: 'Run Python unit tests',
- )
-}
-
-def make_cpp_tests(image, build_dir) {
- sh (
- script: """
- set -eux
- ${docker_run} ${image} python3 ./tests/scripts/task_build.py \
- --sccache-bucket tvm-sccache-prod \
- --sccache-region us-west-2 \
- --cmake-target cpptest \
- --build-dir ${build_dir}
- """,
- label: 'Make C++ tests',
- )
-}
-
-def cmake_build(image, path, make_flag) {
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir ${path}",
- label: 'Run cmake build',
- )
-}
-def cpp_unittest(image) {
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh",
- label: 'Run C++ tests',
- )
-}
-
-cancel_previous_build()
-
-try {
- prepare('CPU-SMALL-SPOT')
-} catch(Exception ex) {
- prepare('CPU-SMALL')
-}
-def build(node_type) {
- stage('Build') {
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cortexm") {
- init_git()
- docker_init(ci_cortexm)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=cortexm',
- ], {
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_config_build_cortexm.sh build",
- label: 'Create Cortex-M cmake config',
- )
- cmake_build(ci_cortexm, 'build', '-j2')
- make_cpp_tests(ci_cortexm, 'build')
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/crttest build/build.ninja build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/microtvm_template_projects",
- label: 'Upload artifacts to S3',
- )
- })
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('BUILD: Cortex-M')
- }
- }
-}
-try {
- build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
- build('CPU-SMALL')
-}
-
-
-
-def shard_run_test_Cortex_M_1_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
- try {
- init_git()
- docker_init(ci_cortexm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cortexm',
- 'TEST_STEP_NAME=test: Cortex-M',
- 'TVM_NUM_SHARDS=12',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_cortexm)
- cpp_unittest(ci_cortexm)
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_demo_microtvm.sh",
- label: 'Run microTVM demos',
- )
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
- label: 'Run microTVM tests',
- )
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('test: Cortex-M 1 of 12')
- }
-}
-
-def shard_run_test_Cortex_M_2_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
- try {
- init_git()
- docker_init(ci_cortexm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cortexm',
- 'TEST_STEP_NAME=test: Cortex-M',
- 'TVM_NUM_SHARDS=12',
- 'TVM_SHARD_INDEX=1',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_cortexm)
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
- label: 'Run microTVM tests',
- )
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('test: Cortex-M 2 of 12')
- }
-}
-
-def shard_run_test_Cortex_M_3_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
- try {
- init_git()
- docker_init(ci_cortexm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cortexm',
- 'TEST_STEP_NAME=test: Cortex-M',
- 'TVM_NUM_SHARDS=12',
- 'TVM_SHARD_INDEX=2',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_cortexm)
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
- label: 'Run microTVM tests',
- )
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('test: Cortex-M 3 of 12')
- }
-}
-
-def shard_run_test_Cortex_M_4_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
- try {
- init_git()
- docker_init(ci_cortexm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cortexm',
- 'TEST_STEP_NAME=test: Cortex-M',
- 'TVM_NUM_SHARDS=12',
- 'TVM_SHARD_INDEX=3',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_cortexm)
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
- label: 'Run microTVM tests',
- )
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('test: Cortex-M 4 of 12')
- }
-}
-
-def shard_run_test_Cortex_M_5_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
- try {
- init_git()
- docker_init(ci_cortexm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cortexm',
- 'TEST_STEP_NAME=test: Cortex-M',
- 'TVM_NUM_SHARDS=12',
- 'TVM_SHARD_INDEX=4',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_cortexm)
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
- label: 'Run microTVM tests',
- )
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('test: Cortex-M 5 of 12')
- }
-}
-
-def shard_run_test_Cortex_M_6_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
- try {
- init_git()
- docker_init(ci_cortexm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cortexm',
- 'TEST_STEP_NAME=test: Cortex-M',
- 'TVM_NUM_SHARDS=12',
- 'TVM_SHARD_INDEX=5',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_cortexm)
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
- label: 'Run microTVM tests',
- )
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('test: Cortex-M 6 of 12')
- }
-}
-
-def shard_run_test_Cortex_M_7_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
- try {
- init_git()
- docker_init(ci_cortexm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cortexm',
- 'TEST_STEP_NAME=test: Cortex-M',
- 'TVM_NUM_SHARDS=12',
- 'TVM_SHARD_INDEX=6',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_cortexm)
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
- label: 'Run microTVM tests',
- )
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('test: Cortex-M 7 of 12')
- }
-}
-
-def shard_run_test_Cortex_M_8_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
- try {
- init_git()
- docker_init(ci_cortexm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cortexm',
- 'TEST_STEP_NAME=test: Cortex-M',
- 'TVM_NUM_SHARDS=12',
- 'TVM_SHARD_INDEX=7',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_cortexm)
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
- label: 'Run microTVM tests',
- )
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('test: Cortex-M 8 of 12')
- }
-}
-
-def shard_run_test_Cortex_M_9_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
- try {
- init_git()
- docker_init(ci_cortexm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cortexm',
- 'TEST_STEP_NAME=test: Cortex-M',
- 'TVM_NUM_SHARDS=12',
- 'TVM_SHARD_INDEX=8',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_cortexm)
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
- label: 'Run microTVM tests',
- )
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('test: Cortex-M 9 of 12')
- }
-}
-
-def shard_run_test_Cortex_M_10_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
- try {
- init_git()
- docker_init(ci_cortexm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cortexm',
- 'TEST_STEP_NAME=test: Cortex-M',
- 'TVM_NUM_SHARDS=12',
- 'TVM_SHARD_INDEX=9',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_cortexm)
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
- label: 'Run microTVM tests',
- )
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('test: Cortex-M 10 of 12')
- }
-}
-
-def shard_run_test_Cortex_M_11_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
- try {
- init_git()
- docker_init(ci_cortexm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cortexm',
- 'TEST_STEP_NAME=test: Cortex-M',
- 'TVM_NUM_SHARDS=12',
- 'TVM_SHARD_INDEX=10',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_cortexm)
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
- label: 'Run microTVM tests',
- )
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('test: Cortex-M 11 of 12')
- }
-}
-
-def shard_run_test_Cortex_M_12_of_12(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-cortexm") {
- try {
- init_git()
- docker_init(ci_cortexm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cortexm',
- 'TEST_STEP_NAME=test: Cortex-M',
- 'TVM_NUM_SHARDS=12',
- 'TVM_SHARD_INDEX=11',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cortexm",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_cortexm)
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
- label: 'Run microTVM tests',
- )
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Cortex_M --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('test: Cortex-M 12 of 12')
- }
-}
-
-
-def test() {
- stage('Test') {
- environment {
- SKIP_SLOW_TESTS = "${skip_slow_tests}"
- }
- parallel(
- 'test: Cortex-M 1 of 12': {
- try {
- shard_run_test_Cortex_M_1_of_12()
- } catch (Exception ex) {
- shard_run_test_Cortex_M_1_of_12(on_demand = true)
- }
- },
- 'test: Cortex-M 2 of 12': {
- try {
- shard_run_test_Cortex_M_2_of_12()
- } catch (Exception ex) {
- shard_run_test_Cortex_M_2_of_12(on_demand = true)
- }
- },
- 'test: Cortex-M 3 of 12': {
- try {
- shard_run_test_Cortex_M_3_of_12()
- } catch (Exception ex) {
- shard_run_test_Cortex_M_3_of_12(on_demand = true)
- }
- },
- 'test: Cortex-M 4 of 12': {
- try {
- shard_run_test_Cortex_M_4_of_12()
- } catch (Exception ex) {
- shard_run_test_Cortex_M_4_of_12(on_demand = true)
- }
- },
- 'test: Cortex-M 5 of 12': {
- try {
- shard_run_test_Cortex_M_5_of_12()
- } catch (Exception ex) {
- shard_run_test_Cortex_M_5_of_12(on_demand = true)
- }
- },
- 'test: Cortex-M 6 of 12': {
- try {
- shard_run_test_Cortex_M_6_of_12()
- } catch (Exception ex) {
- shard_run_test_Cortex_M_6_of_12(on_demand = true)
- }
- },
- 'test: Cortex-M 7 of 12': {
- try {
- shard_run_test_Cortex_M_7_of_12()
- } catch (Exception ex) {
- shard_run_test_Cortex_M_7_of_12(on_demand = true)
- }
- },
- 'test: Cortex-M 8 of 12': {
- try {
- shard_run_test_Cortex_M_8_of_12()
- } catch (Exception ex) {
- shard_run_test_Cortex_M_8_of_12(on_demand = true)
- }
- },
- 'test: Cortex-M 9 of 12': {
- try {
- shard_run_test_Cortex_M_9_of_12()
- } catch (Exception ex) {
- shard_run_test_Cortex_M_9_of_12(on_demand = true)
- }
- },
- 'test: Cortex-M 10 of 12': {
- try {
- shard_run_test_Cortex_M_10_of_12()
- } catch (Exception ex) {
- shard_run_test_Cortex_M_10_of_12(on_demand = true)
- }
- },
- 'test: Cortex-M 11 of 12': {
- try {
- shard_run_test_Cortex_M_11_of_12()
- } catch (Exception ex) {
- shard_run_test_Cortex_M_11_of_12(on_demand = true)
- }
- },
- 'test: Cortex-M 12 of 12': {
- try {
- shard_run_test_Cortex_M_12_of_12()
- } catch (Exception ex) {
- shard_run_test_Cortex_M_12_of_12(on_demand = true)
- }
- },
- )
- }
-}
-test()
diff --git a/ci/jenkins/generated/cpu_jenkinsfile.groovy b/ci/jenkins/generated/cpu_jenkinsfile.groovy
index 04ab19f404..f46eac5907 100644
--- a/ci/jenkins/generated/cpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/cpu_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.103852
+// Generated at 2025-02-06T19:18:57.641343
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -515,25 +515,25 @@ try {
prepare('CPU-SMALL')
}
def build(node_type) {
- stage('Build') {
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") {
- init_git()
- docker_init(ci_cpu)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=cpu',
- ], {
- sh (
+ if (!skip_ci && is_docs_only_build != 1) {
+ echo 'Run build on node_type=' + node_type
+ node(node_type) {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") {
+ init_git()
+ docker_init(ci_cpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+
+ withEnv([
+ 'PLATFORM=cpu',
+ ], {
+ sh (
script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build",
label: 'Create CPU cmake config',
)
cmake_build(ci_cpu, 'build', '-j2')
make_cpp_tests(ci_cpu, 'build')
sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu --items build/libvta_tsim.so build/libtvm.so build/libvta_fsim.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/crttest build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/build.ninja build/microtvm_template_projects",
+ script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/cpptest build/build.ninja build/CMakeFiles/rules.ninja",
label: 'Upload artifacts to S3',
)
@@ -541,19 +541,24 @@ def build(node_type) {
// sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh"
// TODO(@jroesch): need to resolve CI issue will turn back on in follow up patch
sh (script: "${docker_run} ${ci_cpu} ./tests/scripts/task_rust.sh", label: 'Rust build and test')
- })
- }
+ })
}
}
- } else {
- Utils.markStageSkippedForConditional('BUILD: CPU')
}
+ } else {
+ Utils.markStageSkippedForConditional('BUILD: CPU')
}
}
-try {
- build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
- build('CPU-SMALL')
+stage('Build') {
+ try {
+ build('CPU-SMALL-SPOT')
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
+ build('CPU-SMALL')
+ }
}
@@ -563,19 +568,20 @@ def shard_run_integration_CPU_1_of_4(node_type='CPU-SMALL-SPOT', on_demand=false
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
- try {
- init_git()
- docker_init(ci_cpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cpu',
- 'TEST_STEP_NAME=integration: CPU',
- 'TVM_NUM_SHARDS=4',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_cpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=cpu',
+ 'TEST_STEP_NAME=integration: CPU',
+ 'TVM_NUM_SHARDS=4',
+ 'TVM_SHARD_INDEX=0',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
label: 'Download artifacts from S3',
)
@@ -585,19 +591,18 @@ def shard_run_integration_CPU_1_of_4(node_type='CPU-SMALL-SPOT', on_demand=false
script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh",
label: 'Run CPU integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_CPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -611,19 +616,20 @@ def shard_run_integration_CPU_2_of_4(node_type='CPU-SMALL-SPOT', on_demand=false
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
- try {
- init_git()
- docker_init(ci_cpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cpu',
- 'TEST_STEP_NAME=integration: CPU',
- 'TVM_NUM_SHARDS=4',
- 'TVM_SHARD_INDEX=1',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_cpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=cpu',
+ 'TEST_STEP_NAME=integration: CPU',
+ 'TVM_NUM_SHARDS=4',
+ 'TVM_SHARD_INDEX=1',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
label: 'Download artifacts from S3',
)
@@ -633,19 +639,18 @@ def shard_run_integration_CPU_2_of_4(node_type='CPU-SMALL-SPOT', on_demand=false
script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh",
label: 'Run CPU integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_CPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -659,19 +664,20 @@ def shard_run_integration_CPU_3_of_4(node_type='CPU-SMALL-SPOT', on_demand=false
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
- try {
- init_git()
- docker_init(ci_cpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cpu',
- 'TEST_STEP_NAME=integration: CPU',
- 'TVM_NUM_SHARDS=4',
- 'TVM_SHARD_INDEX=2',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_cpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=cpu',
+ 'TEST_STEP_NAME=integration: CPU',
+ 'TVM_NUM_SHARDS=4',
+ 'TVM_SHARD_INDEX=2',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
label: 'Download artifacts from S3',
)
@@ -681,19 +687,18 @@ def shard_run_integration_CPU_3_of_4(node_type='CPU-SMALL-SPOT', on_demand=false
script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh",
label: 'Run CPU integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_CPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -707,19 +712,20 @@ def shard_run_integration_CPU_4_of_4(node_type='CPU-SMALL-SPOT', on_demand=false
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-cpu") {
- try {
- init_git()
- docker_init(ci_cpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cpu',
- 'TEST_STEP_NAME=integration: CPU',
- 'TVM_NUM_SHARDS=4',
- 'TVM_SHARD_INDEX=3',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_cpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=cpu',
+ 'TEST_STEP_NAME=integration: CPU',
+ 'TVM_NUM_SHARDS=4',
+ 'TVM_SHARD_INDEX=3',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
label: 'Download artifacts from S3',
)
@@ -729,19 +735,18 @@ def shard_run_integration_CPU_4_of_4(node_type='CPU-SMALL-SPOT', on_demand=false
script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_integration.sh",
label: 'Run CPU integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/integration_CPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -757,19 +762,20 @@ def shard_run_unittest_CPU_1_of_1(node_type='CPU-SMALL-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu") {
- try {
- init_git()
- docker_init(ci_cpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cpu',
- 'TEST_STEP_NAME=unittest: CPU',
- 'TVM_NUM_SHARDS=1',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_cpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=cpu',
+ 'TEST_STEP_NAME=unittest: CPU',
+ 'TVM_NUM_SHARDS=1',
+ 'TVM_SHARD_INDEX=0',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
label: 'Download artifacts from S3',
)
@@ -777,19 +783,18 @@ def shard_run_unittest_CPU_1_of_1(node_type='CPU-SMALL-SPOT', on_demand=false) {
ci_setup(ci_cpu)
cpp_unittest(ci_cpu)
python_unittest(ci_cpu)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -804,19 +809,20 @@ def shard_run_frontend_CPU_1_of_1(node_type='CPU-SMALL-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-cpu") {
- try {
- init_git()
- docker_init(ci_cpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=cpu',
- 'TEST_STEP_NAME=frontend: CPU',
- 'TVM_NUM_SHARDS=1',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_cpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=cpu',
+ 'TEST_STEP_NAME=frontend: CPU',
+ 'TVM_NUM_SHARDS=1',
+ 'TVM_SHARD_INDEX=0',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu",
label: 'Download artifacts from S3',
)
@@ -826,19 +832,18 @@ def shard_run_frontend_CPU_1_of_1(node_type='CPU-SMALL-SPOT', on_demand=false) {
script: "${docker_run} ${ci_cpu} ./tests/scripts/task_python_frontend_cpu.sh",
label: 'Run Python frontend tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_CPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -857,42 +862,66 @@ def test() {
'integration: CPU 1 of 4': {
try {
shard_run_integration_CPU_1_of_4()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_integration_CPU_1_of_4(on_demand = true)
}
},
'integration: CPU 2 of 4': {
try {
shard_run_integration_CPU_2_of_4()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_integration_CPU_2_of_4(on_demand = true)
}
},
'integration: CPU 3 of 4': {
try {
shard_run_integration_CPU_3_of_4()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_integration_CPU_3_of_4(on_demand = true)
}
},
'integration: CPU 4 of 4': {
try {
shard_run_integration_CPU_4_of_4()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_integration_CPU_4_of_4(on_demand = true)
}
},
'unittest: CPU 1 of 1': {
try {
shard_run_unittest_CPU_1_of_1()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_unittest_CPU_1_of_1(on_demand = true)
}
},
'frontend: CPU 1 of 1': {
try {
shard_run_frontend_CPU_1_of_1()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_frontend_CPU_1_of_1(on_demand = true)
}
},
diff --git a/ci/jenkins/generated/docker_jenkinsfile.groovy b/ci/jenkins/generated/docker_jenkinsfile.groovy
index 796fbe0c2d..553678b38c 100644
--- a/ci/jenkins/generated/docker_jenkinsfile.groovy
+++ b/ci/jenkins/generated/docker_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.207618
+// Generated at 2025-02-06T12:16:02.541252
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
diff --git a/ci/jenkins/generated/gpu_jenkinsfile.groovy b/ci/jenkins/generated/gpu_jenkinsfile.groovy
index 014377f05d..ccb759077f 100644
--- a/ci/jenkins/generated/gpu_jenkinsfile.groovy
+++ b/ci/jenkins/generated/gpu_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.186261
+// Generated at 2025-02-06T19:18:57.667923
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -515,21 +515,21 @@ try {
prepare('CPU-SMALL')
}
def build(node_type) {
- stage('Build') {
- if (!skip_ci) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
+ if (!skip_ci) {
+ echo 'Run build on node_type=' + node_type
+ node(node_type) {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") {
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- ], {
- sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
+ withEnv([
+ 'PLATFORM=gpu',
+ ], {
+ sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
cmake_build("${ci_gpu} --no-gpu", 'build', '-j2')
sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu --items build/libtvm.so build/libvta_fsim.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/microtvm_template_projects build/crttest build/build.ninja build/3rdparty/libflash_attn/src/libflash_attn.so build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so",
+ script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/3rdparty/libflash_attn/src/libflash_attn.so build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so",
label: 'Upload artifacts to S3',
)
@@ -539,22 +539,27 @@ def build(node_type) {
sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build"
cmake_build("${ci_gpu} --no-gpu", 'build', '-j2')
sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu2 --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/crttest build/build.ninja build/3rdparty/libflash_attn/src/libflash_attn.so build/3rdparty/cutlass_fpA_intB_gemm/cutlass_kernels/libfpA_intB_gemm.so",
+ script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu2 --items build/libtvm.so build/libtvm_runtime.so build/config.cmake",
label: 'Upload artifacts to S3',
)
- })
- }
+ })
}
}
- } else {
- Utils.markStageSkippedForConditional('BUILD: GPU')
}
+ } else {
+ Utils.markStageSkippedForConditional('BUILD: GPU')
}
}
-try {
- build('CPU')
-} catch (Exception ex) {
- build('CPU-SMALL')
+stage('Build') {
+ try {
+ build('CPU-SPOT')
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
+ build('CPU')
+ }
}
@@ -564,19 +569,20 @@ def shard_run_unittest_GPU_1_of_3(node_type='GPU-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'GPU'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
- try {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=unittest: GPU',
- 'TVM_NUM_SHARDS=3',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=gpu',
+ 'TEST_STEP_NAME=unittest: GPU',
+ 'TVM_NUM_SHARDS=3',
+ 'TVM_SHARD_INDEX=0',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu2",
label: 'Download artifacts from S3',
)
@@ -612,19 +618,18 @@ def shard_run_unittest_GPU_1_of_3(node_type='GPU-SPOT', on_demand=false) {
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh",
label: 'Run Python GPU integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -638,19 +643,20 @@ def shard_run_unittest_GPU_2_of_3(node_type='GPU-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'GPU'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
- try {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=unittest: GPU',
- 'TVM_NUM_SHARDS=3',
- 'TVM_SHARD_INDEX=1',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=gpu',
+ 'TEST_STEP_NAME=unittest: GPU',
+ 'TVM_NUM_SHARDS=3',
+ 'TVM_SHARD_INDEX=1',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
@@ -668,19 +674,18 @@ def shard_run_unittest_GPU_2_of_3(node_type='GPU-SPOT', on_demand=false) {
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh",
label: 'Run Python GPU integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -694,19 +699,20 @@ def shard_run_unittest_GPU_3_of_3(node_type='GPU-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'GPU'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-gpu") {
- try {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=unittest: GPU',
- 'TVM_NUM_SHARDS=3',
- 'TVM_SHARD_INDEX=2',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=gpu',
+ 'TEST_STEP_NAME=unittest: GPU',
+ 'TVM_NUM_SHARDS=3',
+ 'TVM_SHARD_INDEX=2',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
@@ -720,19 +726,18 @@ def shard_run_unittest_GPU_3_of_3(node_type='GPU-SPOT', on_demand=false) {
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_integration_gpuonly.sh",
label: 'Run Python GPU integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_GPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -748,19 +753,20 @@ def shard_run_topi_GPU_1_of_3(node_type='GPU-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'GPU'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
- try {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=topi: GPU',
- 'TVM_NUM_SHARDS=3',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=gpu',
+ 'TEST_STEP_NAME=topi: GPU',
+ 'TVM_NUM_SHARDS=3',
+ 'TVM_SHARD_INDEX=0',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
@@ -770,19 +776,18 @@ def shard_run_topi_GPU_1_of_3(node_type='GPU-SPOT', on_demand=false) {
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
label: 'Run TOPI tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_GPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -796,19 +801,20 @@ def shard_run_topi_GPU_2_of_3(node_type='GPU-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'GPU'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
- try {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=topi: GPU',
- 'TVM_NUM_SHARDS=3',
- 'TVM_SHARD_INDEX=1',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=gpu',
+ 'TEST_STEP_NAME=topi: GPU',
+ 'TVM_NUM_SHARDS=3',
+ 'TVM_SHARD_INDEX=1',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
@@ -818,19 +824,18 @@ def shard_run_topi_GPU_2_of_3(node_type='GPU-SPOT', on_demand=false) {
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
label: 'Run TOPI tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_GPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -844,19 +849,20 @@ def shard_run_topi_GPU_3_of_3(node_type='GPU-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'GPU'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/topi-python-gpu") {
- try {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=topi: GPU',
- 'TVM_NUM_SHARDS=3',
- 'TVM_SHARD_INDEX=2',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=gpu',
+ 'TEST_STEP_NAME=topi: GPU',
+ 'TVM_NUM_SHARDS=3',
+ 'TVM_SHARD_INDEX=2',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
@@ -866,19 +872,18 @@ def shard_run_topi_GPU_3_of_3(node_type='GPU-SPOT', on_demand=false) {
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_topi.sh",
label: 'Run TOPI tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/topi_GPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -894,19 +899,20 @@ def shard_run_frontend_GPU_1_of_6(node_type='GPU-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'GPU'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
- try {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=frontend: GPU',
- 'TVM_NUM_SHARDS=6',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=gpu',
+ 'TEST_STEP_NAME=frontend: GPU',
+ 'TVM_NUM_SHARDS=6',
+ 'TVM_SHARD_INDEX=0',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
@@ -916,19 +922,18 @@ def shard_run_frontend_GPU_1_of_6(node_type='GPU-SPOT', on_demand=false) {
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
label: 'Run Python frontend tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_GPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -942,19 +947,20 @@ def shard_run_frontend_GPU_2_of_6(node_type='GPU-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'GPU'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
- try {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=frontend: GPU',
- 'TVM_NUM_SHARDS=6',
- 'TVM_SHARD_INDEX=1',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=gpu',
+ 'TEST_STEP_NAME=frontend: GPU',
+ 'TVM_NUM_SHARDS=6',
+ 'TVM_SHARD_INDEX=1',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
@@ -964,19 +970,18 @@ def shard_run_frontend_GPU_2_of_6(node_type='GPU-SPOT', on_demand=false) {
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
label: 'Run Python frontend tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_GPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -990,19 +995,20 @@ def shard_run_frontend_GPU_3_of_6(node_type='GPU-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'GPU'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
- try {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=frontend: GPU',
- 'TVM_NUM_SHARDS=6',
- 'TVM_SHARD_INDEX=2',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=gpu',
+ 'TEST_STEP_NAME=frontend: GPU',
+ 'TVM_NUM_SHARDS=6',
+ 'TVM_SHARD_INDEX=2',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
@@ -1012,19 +1018,18 @@ def shard_run_frontend_GPU_3_of_6(node_type='GPU-SPOT', on_demand=false) {
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
label: 'Run Python frontend tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_GPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -1038,19 +1043,20 @@ def shard_run_frontend_GPU_4_of_6(node_type='GPU-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'GPU'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
- try {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=frontend: GPU',
- 'TVM_NUM_SHARDS=6',
- 'TVM_SHARD_INDEX=3',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=gpu',
+ 'TEST_STEP_NAME=frontend: GPU',
+ 'TVM_NUM_SHARDS=6',
+ 'TVM_SHARD_INDEX=3',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
@@ -1060,19 +1066,18 @@ def shard_run_frontend_GPU_4_of_6(node_type='GPU-SPOT', on_demand=false) {
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
label: 'Run Python frontend tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_GPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -1086,19 +1091,20 @@ def shard_run_frontend_GPU_5_of_6(node_type='GPU-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'GPU'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
- try {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=frontend: GPU',
- 'TVM_NUM_SHARDS=6',
- 'TVM_SHARD_INDEX=4',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=gpu',
+ 'TEST_STEP_NAME=frontend: GPU',
+ 'TVM_NUM_SHARDS=6',
+ 'TVM_SHARD_INDEX=4',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
@@ -1108,19 +1114,18 @@ def shard_run_frontend_GPU_5_of_6(node_type='GPU-SPOT', on_demand=false) {
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
label: 'Run Python frontend tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_GPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -1134,19 +1139,20 @@ def shard_run_frontend_GPU_6_of_6(node_type='GPU-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'GPU'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/frontend-python-gpu") {
- try {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=frontend: GPU',
- 'TVM_NUM_SHARDS=6',
- 'TVM_SHARD_INDEX=5',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=gpu',
+ 'TEST_STEP_NAME=frontend: GPU',
+ 'TVM_NUM_SHARDS=6',
+ 'TVM_SHARD_INDEX=5',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
@@ -1156,19 +1162,18 @@ def shard_run_frontend_GPU_6_of_6(node_type='GPU-SPOT', on_demand=false) {
script: "${docker_run} ${ci_gpu} ./tests/scripts/task_python_frontend.sh",
label: 'Run Python frontend tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/frontend_GPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -1184,19 +1189,20 @@ def shard_run_docs_GPU_1_of_1(node_type='GPU-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'GPU'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/docs-python-gpu") {
- try {
- init_git()
- docker_init(ci_gpu)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=gpu',
- 'TEST_STEP_NAME=docs: GPU',
- 'TVM_NUM_SHARDS=1',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_gpu)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=gpu',
+ 'TEST_STEP_NAME=docs: GPU',
+ 'TVM_NUM_SHARDS=1',
+ 'TVM_SHARD_INDEX=0',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/gpu",
label: 'Download artifacts from S3',
)
@@ -1215,19 +1221,18 @@ def shard_run_docs_GPU_1_of_1(node_type='GPU-SPOT', on_demand=false) {
script: "aws s3 cp --no-progress _docs s3://${s3_bucket}/${s3_prefix}/docs --recursive",
label: 'Upload docs to S3',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/docs_GPU --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -1247,91 +1252,143 @@ def test() {
'unittest: GPU 1 of 3': {
try {
shard_run_unittest_GPU_1_of_3()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_unittest_GPU_1_of_3(on_demand = true)
}
},
'unittest: GPU 2 of 3': {
try {
shard_run_unittest_GPU_2_of_3()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_unittest_GPU_2_of_3(on_demand = true)
}
},
'unittest: GPU 3 of 3': {
try {
shard_run_unittest_GPU_3_of_3()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_unittest_GPU_3_of_3(on_demand = true)
}
},
'topi: GPU 1 of 3': {
try {
shard_run_topi_GPU_1_of_3()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_topi_GPU_1_of_3(on_demand = true)
}
},
'topi: GPU 2 of 3': {
try {
shard_run_topi_GPU_2_of_3()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_topi_GPU_2_of_3(on_demand = true)
}
},
'topi: GPU 3 of 3': {
try {
shard_run_topi_GPU_3_of_3()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_topi_GPU_3_of_3(on_demand = true)
}
},
'frontend: GPU 1 of 6': {
try {
shard_run_frontend_GPU_1_of_6()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_frontend_GPU_1_of_6(on_demand = true)
}
},
'frontend: GPU 2 of 6': {
try {
shard_run_frontend_GPU_2_of_6()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_frontend_GPU_2_of_6(on_demand = true)
}
},
'frontend: GPU 3 of 6': {
try {
shard_run_frontend_GPU_3_of_6()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_frontend_GPU_3_of_6(on_demand = true)
}
},
'frontend: GPU 4 of 6': {
try {
shard_run_frontend_GPU_4_of_6()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_frontend_GPU_4_of_6(on_demand = true)
}
},
'frontend: GPU 5 of 6': {
try {
shard_run_frontend_GPU_5_of_6()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_frontend_GPU_5_of_6(on_demand = true)
}
},
'frontend: GPU 6 of 6': {
try {
shard_run_frontend_GPU_6_of_6()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_frontend_GPU_6_of_6(on_demand = true)
}
},
'docs: GPU 1 of 1': {
try {
shard_run_docs_GPU_1_of_1()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_docs_GPU_1_of_1(on_demand = true)
}
},
diff --git a/ci/jenkins/generated/hexagon_jenkinsfile.groovy b/ci/jenkins/generated/hexagon_jenkinsfile.groovy
index a48b12ecd2..312668806b 100644
--- a/ci/jenkins/generated/hexagon_jenkinsfile.groovy
+++ b/ci/jenkins/generated/hexagon_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.087221
+// Generated at 2025-02-06T19:18:57.609516
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -515,18 +515,18 @@ try {
prepare('CPU-SMALL')
}
def build(node_type) {
- stage('Build') {
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-hexagon") {
- init_git()
- docker_init(ci_hexagon)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=hexagon',
- ], {
- sh (
+ if (!skip_ci && is_docs_only_build != 1) {
+ echo 'Run build on node_type=' + node_type
+ node(node_type) {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-hexagon") {
+ init_git()
+ docker_init(ci_hexagon)
+ timeout(time: max_time, unit: 'MINUTES') {
+
+ withEnv([
+ 'PLATFORM=hexagon',
+ ], {
+ sh (
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build",
label: 'Create Hexagon cmake config',
)
@@ -540,19 +540,24 @@ def build(node_type) {
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/hexagon_api_output",
label: 'Upload artifacts to S3',
)
- })
- }
+ })
}
}
- } else {
- Utils.markStageSkippedForConditional('BUILD: Hexagon')
}
+ } else {
+ Utils.markStageSkippedForConditional('BUILD: Hexagon')
}
}
-try {
- build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
- build('CPU-SMALL')
+stage('Build') {
+ try {
+ build('CPU-SMALL-SPOT')
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
+ build('CPU-SMALL')
+ }
}
@@ -563,19 +568,20 @@ def shard_run_test_Hexagon_1_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
- try {
- init_git()
- docker_init(ci_hexagon)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=hexagon',
- 'TEST_STEP_NAME=test: Hexagon',
- 'TVM_NUM_SHARDS=8',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_hexagon)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=hexagon',
+ 'TEST_STEP_NAME=test: Hexagon',
+ 'TVM_NUM_SHARDS=8',
+ 'TVM_SHARD_INDEX=0',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
label: 'Download artifacts from S3',
)
@@ -586,19 +592,18 @@ def shard_run_test_Hexagon_1_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
label: 'Run Hexagon tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -612,19 +617,20 @@ def shard_run_test_Hexagon_2_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
- try {
- init_git()
- docker_init(ci_hexagon)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=hexagon',
- 'TEST_STEP_NAME=test: Hexagon',
- 'TVM_NUM_SHARDS=8',
- 'TVM_SHARD_INDEX=1',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_hexagon)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=hexagon',
+ 'TEST_STEP_NAME=test: Hexagon',
+ 'TVM_NUM_SHARDS=8',
+ 'TVM_SHARD_INDEX=1',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
label: 'Download artifacts from S3',
)
@@ -634,19 +640,18 @@ def shard_run_test_Hexagon_2_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
label: 'Run Hexagon tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -660,19 +665,20 @@ def shard_run_test_Hexagon_3_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
- try {
- init_git()
- docker_init(ci_hexagon)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=hexagon',
- 'TEST_STEP_NAME=test: Hexagon',
- 'TVM_NUM_SHARDS=8',
- 'TVM_SHARD_INDEX=2',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_hexagon)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=hexagon',
+ 'TEST_STEP_NAME=test: Hexagon',
+ 'TVM_NUM_SHARDS=8',
+ 'TVM_SHARD_INDEX=2',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
label: 'Download artifacts from S3',
)
@@ -682,19 +688,18 @@ def shard_run_test_Hexagon_3_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
label: 'Run Hexagon tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -708,19 +713,20 @@ def shard_run_test_Hexagon_4_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
- try {
- init_git()
- docker_init(ci_hexagon)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=hexagon',
- 'TEST_STEP_NAME=test: Hexagon',
- 'TVM_NUM_SHARDS=8',
- 'TVM_SHARD_INDEX=3',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_hexagon)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=hexagon',
+ 'TEST_STEP_NAME=test: Hexagon',
+ 'TVM_NUM_SHARDS=8',
+ 'TVM_SHARD_INDEX=3',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
label: 'Download artifacts from S3',
)
@@ -730,19 +736,18 @@ def shard_run_test_Hexagon_4_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
label: 'Run Hexagon tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -756,19 +761,20 @@ def shard_run_test_Hexagon_5_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
- try {
- init_git()
- docker_init(ci_hexagon)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=hexagon',
- 'TEST_STEP_NAME=test: Hexagon',
- 'TVM_NUM_SHARDS=8',
- 'TVM_SHARD_INDEX=4',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_hexagon)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=hexagon',
+ 'TEST_STEP_NAME=test: Hexagon',
+ 'TVM_NUM_SHARDS=8',
+ 'TVM_SHARD_INDEX=4',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
label: 'Download artifacts from S3',
)
@@ -778,19 +784,18 @@ def shard_run_test_Hexagon_5_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
label: 'Run Hexagon tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -804,19 +809,20 @@ def shard_run_test_Hexagon_6_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
- try {
- init_git()
- docker_init(ci_hexagon)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=hexagon',
- 'TEST_STEP_NAME=test: Hexagon',
- 'TVM_NUM_SHARDS=8',
- 'TVM_SHARD_INDEX=5',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_hexagon)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=hexagon',
+ 'TEST_STEP_NAME=test: Hexagon',
+ 'TVM_NUM_SHARDS=8',
+ 'TVM_SHARD_INDEX=5',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
label: 'Download artifacts from S3',
)
@@ -826,19 +832,18 @@ def shard_run_test_Hexagon_6_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
label: 'Run Hexagon tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -852,19 +857,20 @@ def shard_run_test_Hexagon_7_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
- try {
- init_git()
- docker_init(ci_hexagon)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=hexagon',
- 'TEST_STEP_NAME=test: Hexagon',
- 'TVM_NUM_SHARDS=8',
- 'TVM_SHARD_INDEX=6',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_hexagon)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=hexagon',
+ 'TEST_STEP_NAME=test: Hexagon',
+ 'TVM_NUM_SHARDS=8',
+ 'TVM_SHARD_INDEX=6',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
label: 'Download artifacts from S3',
)
@@ -874,19 +880,18 @@ def shard_run_test_Hexagon_7_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
label: 'Run Hexagon tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -900,19 +905,20 @@ def shard_run_test_Hexagon_8_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-hexagon") {
- try {
- init_git()
- docker_init(ci_hexagon)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=hexagon',
- 'TEST_STEP_NAME=test: Hexagon',
- 'TVM_NUM_SHARDS=8',
- 'TVM_SHARD_INDEX=7',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_hexagon)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=hexagon',
+ 'TEST_STEP_NAME=test: Hexagon',
+ 'TVM_NUM_SHARDS=8',
+ 'TVM_SHARD_INDEX=7',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/hexagon",
label: 'Download artifacts from S3',
)
@@ -922,19 +928,18 @@ def shard_run_test_Hexagon_8_of_8(node_type='CPU-SMALL-SPOT', on_demand=false) {
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_python_hexagon.sh",
label: 'Run Hexagon tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_Hexagon --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -953,56 +958,88 @@ def test() {
'test: Hexagon 1 of 8': {
try {
shard_run_test_Hexagon_1_of_8()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_test_Hexagon_1_of_8(on_demand = true)
}
},
'test: Hexagon 2 of 8': {
try {
shard_run_test_Hexagon_2_of_8()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_test_Hexagon_2_of_8(on_demand = true)
}
},
'test: Hexagon 3 of 8': {
try {
shard_run_test_Hexagon_3_of_8()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_test_Hexagon_3_of_8(on_demand = true)
}
},
'test: Hexagon 4 of 8': {
try {
shard_run_test_Hexagon_4_of_8()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_test_Hexagon_4_of_8(on_demand = true)
}
},
'test: Hexagon 5 of 8': {
try {
shard_run_test_Hexagon_5_of_8()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_test_Hexagon_5_of_8(on_demand = true)
}
},
'test: Hexagon 6 of 8': {
try {
shard_run_test_Hexagon_6_of_8()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_test_Hexagon_6_of_8(on_demand = true)
}
},
'test: Hexagon 7 of 8': {
try {
shard_run_test_Hexagon_7_of_8()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_test_Hexagon_7_of_8(on_demand = true)
}
},
'test: Hexagon 8 of 8': {
try {
shard_run_test_Hexagon_8_of_8()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_test_Hexagon_8_of_8(on_demand = true)
}
},
diff --git a/ci/jenkins/generated/i386_jenkinsfile.groovy b/ci/jenkins/generated/i386_jenkinsfile.groovy
index 47d2056473..32fba2b7a6 100644
--- a/ci/jenkins/generated/i386_jenkinsfile.groovy
+++ b/ci/jenkins/generated/i386_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.169799
+// Generated at 2025-02-06T19:18:57.582572
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -507,7 +507,6 @@ def cpp_unittest(image) {
)
}
-
cancel_previous_build()
try {
@@ -516,40 +515,45 @@ try {
prepare('CPU-SMALL')
}
def build(node_type) {
- stage('Build') {
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-i386") {
- init_git()
- docker_init(ci_i386)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=i386',
- ], {
- sh (
+ if (!skip_ci && is_docs_only_build != 1) {
+ echo 'Run build on node_type=' + node_type
+ node(node_type) {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-i386") {
+ init_git()
+ docker_init(ci_i386)
+ timeout(time: max_time, unit: 'MINUTES') {
+
+ withEnv([
+ 'PLATFORM=i386',
+ ], {
+ sh (
script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build",
label: 'Create i386 cmake config',
)
cmake_build(ci_i386, 'build', '-j2')
make_cpp_tests(ci_i386, 'build')
sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/i386 --items build/libvta_tsim.so build/libtvm.so build/libvta_fsim.so build/libtvm_runtime.so build/config.cmake build/build.ninja build/crttest build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/microtvm_template_projects",
+ script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/i386 --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/cpptest build/build.ninja build/CMakeFiles/rules.ninja",
label: 'Upload artifacts to S3',
)
- })
- }
+ })
}
}
- } else {
- Utils.markStageSkippedForConditional('BUILD: i386')
}
+ } else {
+ Utils.markStageSkippedForConditional('BUILD: i386')
}
}
-try {
- build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
- build('CPU-SMALL')
+stage('Build') {
+ try {
+ build('CPU-SMALL-SPOT')
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
+ build('CPU-SMALL')
+ }
}
@@ -560,19 +564,20 @@ def shard_run_python_i386_1_of_3(node_type='CPU-SMALL-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
- try {
- init_git()
- docker_init(ci_i386)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=i386',
- 'TEST_STEP_NAME=python: i386',
- 'TVM_NUM_SHARDS=3',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_i386)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=i386',
+ 'TEST_STEP_NAME=python: i386',
+ 'TVM_NUM_SHARDS=3',
+ 'TVM_SHARD_INDEX=0',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/i386",
label: 'Download artifacts from S3',
)
@@ -584,19 +589,18 @@ def shard_run_python_i386_1_of_3(node_type='CPU-SMALL-SPOT', on_demand=false) {
script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh",
label: 'Run i386 integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/python_i386 --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -610,19 +614,20 @@ def shard_run_python_i386_2_of_3(node_type='CPU-SMALL-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
- try {
- init_git()
- docker_init(ci_i386)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=i386',
- 'TEST_STEP_NAME=python: i386',
- 'TVM_NUM_SHARDS=3',
- 'TVM_SHARD_INDEX=1',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_i386)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=i386',
+ 'TEST_STEP_NAME=python: i386',
+ 'TVM_NUM_SHARDS=3',
+ 'TVM_SHARD_INDEX=1',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/i386",
label: 'Download artifacts from S3',
)
@@ -633,19 +638,18 @@ def shard_run_python_i386_2_of_3(node_type='CPU-SMALL-SPOT', on_demand=false) {
script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh",
label: 'Run i386 integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/python_i386 --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -659,19 +663,20 @@ def shard_run_python_i386_3_of_3(node_type='CPU-SMALL-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = 'CPU-SMALL'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/integration-python-i386") {
- try {
- init_git()
- docker_init(ci_i386)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=i386',
- 'TEST_STEP_NAME=python: i386',
- 'TVM_NUM_SHARDS=3',
- 'TVM_SHARD_INDEX=2',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init(ci_i386)
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM=i386',
+ 'TEST_STEP_NAME=python: i386',
+ 'TVM_NUM_SHARDS=3',
+ 'TVM_SHARD_INDEX=2',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/i386",
label: 'Download artifacts from S3',
)
@@ -682,19 +687,18 @@ def shard_run_python_i386_3_of_3(node_type='CPU-SMALL-SPOT', on_demand=false) {
script: "${docker_run} ${ci_i386} ./tests/scripts/task_python_integration_i386only.sh",
label: 'Run i386 integration tests',
)
- })
- }
- } finally {
- try {
- sh(
+ })
+ }
+ // only run upload if things are successful
+ try {
+ sh(
script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/python_i386 --items build/pytest-results",
label: 'Upload JUnits to S3',
)
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -713,21 +717,33 @@ def test() {
'python: i386 1 of 3': {
try {
shard_run_python_i386_1_of_3()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_python_i386_1_of_3(on_demand = true)
}
},
'python: i386 2 of 3': {
try {
shard_run_python_i386_2_of_3()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_python_i386_2_of_3(on_demand = true)
}
},
'python: i386 3 of 3': {
try {
shard_run_python_i386_3_of_3()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
shard_run_python_i386_3_of_3(on_demand = true)
}
},
diff --git a/ci/jenkins/generated/lint_jenkinsfile.groovy b/ci/jenkins/generated/lint_jenkinsfile.groovy
index b1e300d712..bc8118c546 100644
--- a/ci/jenkins/generated/lint_jenkinsfile.groovy
+++ b/ci/jenkins/generated/lint_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.245060
+// Generated at 2025-02-06T12:16:02.571072
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
diff --git a/ci/jenkins/generated/minimal_cross_isa_jenkinsfile.groovy b/ci/jenkins/generated/minimal_cross_isa_jenkinsfile.groovy
deleted file mode 100644
index 3645cf9036..0000000000
--- a/ci/jenkins/generated/minimal_cross_isa_jenkinsfile.groovy
+++ /dev/null
@@ -1,622 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-// -*- mode: groovy -*-
-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Jenkins pipeline
-// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
-// Docker env used for testing
-// Different image may have different version tag
-// because some of them are more stable than anoter.
-//
-// Docker images are maintained by PMC, cached in dockerhub
-// and remains relatively stable over the time.
-// Flow for upgrading docker env(need commiter)
-//
-// - Send PR to upgrade build script in the repo
-// - Build the new docker image
-// - Tag the docker image with a new version and push to a binary cache.
-// - Update the version in the Jenkinsfile, send a PR
-// - Fix any issues wrt to the new image version in the PR
-// - Merge the PR and now we are in new version
-// - Tag the new version as the lates
-// - Periodically cleanup the old versions on local workers
-//
-
-// ============================= IMPORTANT NOTE =============================
-// This file is generated by 'jenkins/generate.py'. Do not edit this file directly!
-// Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with
-// 'python3 jenkins/generate.py'
-// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
-// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.045689
-
-import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-// These are set at runtime from data in ci/jenkins/docker-images.yml, update
-// image tags in that file
-ci_lint = ''
-ci_gpu = ''
-ci_cpu = ''
-ci_minimal = ''
-ci_wasm = ''
-ci_i386 = ''
-ci_cortexm = ''
-ci_arm = ''
-ci_hexagon = ''
-ci_riscv = ''
-
-// Parameters to allow overriding (in Jenkins UI), the images
-// to be used by a given build. When provided, they take precedence
-// over default values above.
-properties([
- parameters([
- string(name: 'ci_arm_param', defaultValue: ''),
- string(name: 'ci_cortexm_param', defaultValue: ''),
- string(name: 'ci_cpu_param', defaultValue: ''),
- string(name: 'ci_gpu_param', defaultValue: ''),
- string(name: 'ci_hexagon_param', defaultValue: ''),
- string(name: 'ci_i386_param', defaultValue: ''),
- string(name: 'ci_lint_param', defaultValue: ''),
- string(name: 'ci_minimal_param', defaultValue: ''),
- string(name: 'ci_riscv_param', defaultValue: ''),
- string(name: 'ci_wasm_param', defaultValue: ''),
- ])
-])
-
-// Placeholders for newly built Docker image names (if rebuild_docker_images
-// is used)
- built_ci_arm = null;
- built_ci_cortexm = null;
- built_ci_cpu = null;
- built_ci_gpu = null;
- built_ci_hexagon = null;
- built_ci_i386 = null;
- built_ci_lint = null;
- built_ci_minimal = null;
- built_ci_riscv = null;
- built_ci_wasm = null;
-
-// Global variable assigned during Sanity Check that holds the sha1 which should be
-// merged into the PR in all branches.
-upstream_revision = null
-
-// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
-docker_build = 'docker/build.sh'
-// timeout in minutes
-max_time = 180
-rebuild_docker_images = false
-
-s3_bucket = 'tvm-jenkins-artifacts-prod'
-s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
-
-// Jenkins script root directory
-jenkins_scripts_root = "ci/scripts/jenkins"
-
-
-// General note: Jenkins has limits on the size of a method (or top level code)
-// that are pretty strict, so most usage of groovy methods in these templates
-// are purely to satisfy the JVM
-def per_exec_ws(folder) {
- return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
-}
-
-// initialize source codes
-def init_git() {
- retry(5) {
- checkout scm
- }
-
- // Add more info about job node
- sh (
- script: './tests/scripts/task_show_node_info.sh',
- label: 'Show executor node info',
- )
-
- // Determine merge commit to use for all stages
- if (env.BRANCH_NAME == 'main') {
- // Only set upstream_revision to HEAD and skip merging to avoid a race with another commit merged to main.
- update_upstream_revision("HEAD")
- } else {
- // This is PR branch so merge with latest main.
- merge_with_main()
- }
-
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0
- """,
- label: 'Update git submodules',
- )
- checkout_trusted_files()
-}
-
-def update_upstream_revision(git_ref) {
- if (upstream_revision == null) {
- upstream_revision = sh(
- script: "git log -1 ${git_ref} --format=\'%H\'",
- label: 'Determine upstream revision',
- returnStdout: true,
- ).trim()
- }
-}
-
-def merge_with_main() {
- sh (
- script: 'git fetch origin main',
- label: 'Fetch upstream',
- )
- update_upstream_revision("FETCH_HEAD")
- sh (
- script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
- label: 'Merge to origin/main'
- )
-}
-
-def docker_init(image) {
- // Clear out all Docker images that aren't going to be used
- sh(
- script: """
- set -eux
- docker image ls --all
- IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}')
-
- echo -e "Found images:\\n\$IMAGES"
- echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
-
- docker image ls --all
- """,
- label: 'Clean old Docker images',
- )
-
- if (image.contains("amazonaws.com")) {
- // If this string is in the image name it's from ECR and needs to be pulled
- // with the right credentials
- ecr_pull(image)
- } else {
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${image}
- """,
- label: 'Pull docker image',
- )
- }
-}
-
-def ecr_pull(full_name) {
- aws_account_id = sh(
- returnStdout: true,
- script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
- label: 'Get AWS ID'
- ).trim()
-
- try {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: '''
- set -eux
- aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
- ''',
- label: 'Log in to ECR'
- )
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${full_name}
- """,
- label: 'Pull image from ECR'
- )
- }
- } finally {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: 'docker logout $AWS_ECR_REPO',
- label: 'Clean up login credentials'
- )
- }
- }
-}
-
-def should_skip_slow_tests(pr_number) {
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
- result = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
- label: 'Check if CI should run slow tests',
- )
- }
- return result == 0
-}
-
-def cancel_previous_build() {
- // cancel previous build if it is not on main.
- if (env.BRANCH_NAME != 'main') {
- def buildNumber = env.BUILD_NUMBER as int
- // Milestone API allows us to cancel previous build
- // with the same milestone number
- if (buildNumber > 1) milestone(buildNumber - 1)
- milestone(buildNumber)
- }
-}
-
-def checkout_trusted_files() {
- // trust everything from branch builds
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- return;
- }
-
- // trust peoople listed in CONTRIBUTING.md
- grep_code = sh(
- returnStatus: true,
- script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
- label: 'Check if change is from a contributor',
- )
-
- if (grep_code == 1) {
- // Any scripts that run on the bare host and not inside a Docker container
- // (especially those that access secrets) should be checked out here so
- // only trusted versions are used in CI
- sh(
- script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
- label: 'Check out trusted files',
- )
- }
-}
-
-def should_skip_ci(pr_number) {
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- // never skip CI on build sourced from a branch
- return false
- }
- glob_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
- label: 'Check if CI should be skipped due to changed files',
- )
- if (glob_skip_ci_code == 0) {
- return true
- }
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run full CI (or the script had an error, so run
- // full CI just in case). Exit code of 0 means skip CI.
- git_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
- label: 'Check if CI should be skipped',
- )
- }
- return git_skip_ci_code == 0
-}
-
-def check_pr(pr_number) {
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- // never skip CI on build sourced from a branch
- return false
- }
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- sh (
- script: "python3 ${jenkins_scripts_root}/check_pr.py --pr ${pr_number}",
- label: 'Check PR title and body',
- )
- }
-
-}
-
-def prepare(node_type) {
- stage('Prepare') {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
- init_git()
-
- check_pr(env.CHANGE_ID)
-
- if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
- sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
- label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
- )
- // Pull image names from the results of should_rebuild_docker.py
- ci_arm = sh(
- script: "cat .docker-image-names/ci_arm",
- label: "Find docker image name for ci_arm",
- returnStdout: true,
- ).trim()
- ci_cortexm = sh(
- script: "cat .docker-image-names/ci_cortexm",
- label: "Find docker image name for ci_cortexm",
- returnStdout: true,
- ).trim()
- ci_cpu = sh(
- script: "cat .docker-image-names/ci_cpu",
- label: "Find docker image name for ci_cpu",
- returnStdout: true,
- ).trim()
- ci_gpu = sh(
- script: "cat .docker-image-names/ci_gpu",
- label: "Find docker image name for ci_gpu",
- returnStdout: true,
- ).trim()
- ci_hexagon = sh(
- script: "cat .docker-image-names/ci_hexagon",
- label: "Find docker image name for ci_hexagon",
- returnStdout: true,
- ).trim()
- ci_i386 = sh(
- script: "cat .docker-image-names/ci_i386",
- label: "Find docker image name for ci_i386",
- returnStdout: true,
- ).trim()
- ci_lint = sh(
- script: "cat .docker-image-names/ci_lint",
- label: "Find docker image name for ci_lint",
- returnStdout: true,
- ).trim()
- ci_minimal = sh(
- script: "cat .docker-image-names/ci_minimal",
- label: "Find docker image name for ci_minimal",
- returnStdout: true,
- ).trim()
- ci_riscv = sh(
- script: "cat .docker-image-names/ci_riscv",
- label: "Find docker image name for ci_riscv",
- returnStdout: true,
- ).trim()
- ci_wasm = sh(
- script: "cat .docker-image-names/ci_wasm",
- label: "Find docker image name for ci_wasm",
- returnStdout: true,
- ).trim()
- }
-
- ci_arm = params.ci_arm_param ?: ci_arm
- ci_cortexm = params.ci_cortexm_param ?: ci_cortexm
- ci_cpu = params.ci_cpu_param ?: ci_cpu
- ci_gpu = params.ci_gpu_param ?: ci_gpu
- ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
- ci_i386 = params.ci_i386_param ?: ci_i386
- ci_lint = params.ci_lint_param ?: ci_lint
- ci_minimal = params.ci_minimal_param ?: ci_minimal
- ci_riscv = params.ci_riscv_param ?: ci_riscv
- ci_wasm = params.ci_wasm_param ?: ci_wasm
-
- sh (script: """
- echo "Docker images being used in this build:"
- echo " ci_arm = ${ci_arm}"
- echo " ci_cortexm = ${ci_cortexm}"
- echo " ci_cpu = ${ci_cpu}"
- echo " ci_gpu = ${ci_gpu}"
- echo " ci_hexagon = ${ci_hexagon}"
- echo " ci_i386 = ${ci_i386}"
- echo " ci_lint = ${ci_lint}"
- echo " ci_minimal = ${ci_minimal}"
- echo " ci_riscv = ${ci_riscv}"
- echo " ci_wasm = ${ci_wasm}"
- """, label: 'Docker image names')
-
- is_docs_only_build = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docs.sh",
- label: 'Check for docs only changes',
- )
- skip_ci = should_skip_ci(env.CHANGE_ID)
- skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
- rebuild_docker_images = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docker.sh",
- label: 'Check for any docker changes',
- )
-
- if (skip_ci) {
- // Don't rebuild when skipping CI
- rebuild_docker_images = false
- }
- }
- }
- }
-}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
-def python_unittest(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
- label: 'Run Python unit tests',
- )
-}
-
-def make_cpp_tests(image, build_dir) {
- sh (
- script: """
- set -eux
- ${docker_run} ${image} python3 ./tests/scripts/task_build.py \
- --sccache-bucket tvm-sccache-prod \
- --sccache-region us-west-2 \
- --cmake-target cpptest \
- --build-dir ${build_dir}
- """,
- label: 'Make C++ tests',
- )
-}
-
-def cmake_build(image, path, make_flag) {
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir ${path}",
- label: 'Run cmake build',
- )
-}
-def cpp_unittest(image) {
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh",
- label: 'Run C++ tests',
- )
-}
-
-cancel_previous_build()
-
-try {
- prepare('CPU-SMALL-SPOT')
-} catch(Exception ex) {
- prepare('CPU-SMALL')
-}
-def build(node_type) {
- stage('Build') {
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu-minimal-cross-isa") {
- init_git()
- docker_init(ci_minimal)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=arm',
- ], {
- sh (
- script: "${docker_run} ${ci_minimal} ./tests/scripts/task_config_build_minimal_cross_isa.sh build",
- label: 'Create CPU minimal cmake config',
- )
- cmake_build(ci_minimal, 'build', '-j2')
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu-minimal-cross-isa --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/build.ninja build/microtvm_template_projects",
- label: 'Upload artifacts to S3',
- )
- })
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('BUILD: CPU MINIMAL CROSS ISA')
- }
- }
-}
-try {
- build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
- build('CPU-SMALL')
-}
-
-
-
-
-def shard_run_unittest_CPU_MINIMAL_CROSS_ISA_1_of_1(node_type='ARM-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'ARM-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-cpp-arm-cross-isa") {
- try {
- init_git()
- docker_init(ci_arm)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=arm',
- 'TEST_STEP_NAME=unittest: CPU MINIMAL CROSS ISA',
- 'TVM_NUM_SHARDS=1',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu-minimal-cross-isa",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_arm)
- sh "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_minimal_cross_isa.sh build"
- make_cpp_tests(ci_arm, 'build')
- cpp_unittest(ci_arm)
- python_unittest(ci_arm)
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU_MINIMAL_CROSS_ISA --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('unittest: CPU MINIMAL CROSS ISA 1 of 1')
- }
-}
-
-
-
-def test() {
- stage('Test') {
- environment {
- SKIP_SLOW_TESTS = "${skip_slow_tests}"
- }
- parallel(
- 'unittest: CPU MINIMAL CROSS ISA 1 of 1': {
- try {
- shard_run_unittest_CPU_MINIMAL_CROSS_ISA_1_of_1()
- } catch (Exception ex) {
- shard_run_unittest_CPU_MINIMAL_CROSS_ISA_1_of_1(on_demand = true)
- }
- },
- )
- }
-}
-test()
diff --git a/ci/jenkins/generated/minimal_jenkinsfile.groovy b/ci/jenkins/generated/minimal_jenkinsfile.groovy
deleted file mode 100644
index 9ae2380122..0000000000
--- a/ci/jenkins/generated/minimal_jenkinsfile.groovy
+++ /dev/null
@@ -1,620 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-// -*- mode: groovy -*-
-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Jenkins pipeline
-// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
-// Docker env used for testing
-// Different image may have different version tag
-// because some of them are more stable than anoter.
-//
-// Docker images are maintained by PMC, cached in dockerhub
-// and remains relatively stable over the time.
-// Flow for upgrading docker env(need commiter)
-//
-// - Send PR to upgrade build script in the repo
-// - Build the new docker image
-// - Tag the docker image with a new version and push to a binary cache.
-// - Update the version in the Jenkinsfile, send a PR
-// - Fix any issues wrt to the new image version in the PR
-// - Merge the PR and now we are in new version
-// - Tag the new version as the lates
-// - Periodically cleanup the old versions on local workers
-//
-
-// ============================= IMPORTANT NOTE =============================
-// This file is generated by 'jenkins/generate.py'. Do not edit this file directly!
-// Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with
-// 'python3 jenkins/generate.py'
-// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
-// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.139264
-
-import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-// These are set at runtime from data in ci/jenkins/docker-images.yml, update
-// image tags in that file
-ci_lint = ''
-ci_gpu = ''
-ci_cpu = ''
-ci_minimal = ''
-ci_wasm = ''
-ci_i386 = ''
-ci_cortexm = ''
-ci_arm = ''
-ci_hexagon = ''
-ci_riscv = ''
-
-// Parameters to allow overriding (in Jenkins UI), the images
-// to be used by a given build. When provided, they take precedence
-// over default values above.
-properties([
- parameters([
- string(name: 'ci_arm_param', defaultValue: ''),
- string(name: 'ci_cortexm_param', defaultValue: ''),
- string(name: 'ci_cpu_param', defaultValue: ''),
- string(name: 'ci_gpu_param', defaultValue: ''),
- string(name: 'ci_hexagon_param', defaultValue: ''),
- string(name: 'ci_i386_param', defaultValue: ''),
- string(name: 'ci_lint_param', defaultValue: ''),
- string(name: 'ci_minimal_param', defaultValue: ''),
- string(name: 'ci_riscv_param', defaultValue: ''),
- string(name: 'ci_wasm_param', defaultValue: ''),
- ])
-])
-
-// Placeholders for newly built Docker image names (if rebuild_docker_images
-// is used)
- built_ci_arm = null;
- built_ci_cortexm = null;
- built_ci_cpu = null;
- built_ci_gpu = null;
- built_ci_hexagon = null;
- built_ci_i386 = null;
- built_ci_lint = null;
- built_ci_minimal = null;
- built_ci_riscv = null;
- built_ci_wasm = null;
-
-// Global variable assigned during Sanity Check that holds the sha1 which should be
-// merged into the PR in all branches.
-upstream_revision = null
-
-// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
-docker_build = 'docker/build.sh'
-// timeout in minutes
-max_time = 180
-rebuild_docker_images = false
-
-s3_bucket = 'tvm-jenkins-artifacts-prod'
-s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
-
-// Jenkins script root directory
-jenkins_scripts_root = "ci/scripts/jenkins"
-
-
-// General note: Jenkins has limits on the size of a method (or top level code)
-// that are pretty strict, so most usage of groovy methods in these templates
-// are purely to satisfy the JVM
-def per_exec_ws(folder) {
- return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
-}
-
-// initialize source codes
-def init_git() {
- retry(5) {
- checkout scm
- }
-
- // Add more info about job node
- sh (
- script: './tests/scripts/task_show_node_info.sh',
- label: 'Show executor node info',
- )
-
- // Determine merge commit to use for all stages
- if (env.BRANCH_NAME == 'main') {
- // Only set upstream_revision to HEAD and skip merging to avoid a race with another commit merged to main.
- update_upstream_revision("HEAD")
- } else {
- // This is PR branch so merge with latest main.
- merge_with_main()
- }
-
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0
- """,
- label: 'Update git submodules',
- )
- checkout_trusted_files()
-}
-
-def update_upstream_revision(git_ref) {
- if (upstream_revision == null) {
- upstream_revision = sh(
- script: "git log -1 ${git_ref} --format=\'%H\'",
- label: 'Determine upstream revision',
- returnStdout: true,
- ).trim()
- }
-}
-
-def merge_with_main() {
- sh (
- script: 'git fetch origin main',
- label: 'Fetch upstream',
- )
- update_upstream_revision("FETCH_HEAD")
- sh (
- script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
- label: 'Merge to origin/main'
- )
-}
-
-def docker_init(image) {
- // Clear out all Docker images that aren't going to be used
- sh(
- script: """
- set -eux
- docker image ls --all
- IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}')
-
- echo -e "Found images:\\n\$IMAGES"
- echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
-
- docker image ls --all
- """,
- label: 'Clean old Docker images',
- )
-
- if (image.contains("amazonaws.com")) {
- // If this string is in the image name it's from ECR and needs to be pulled
- // with the right credentials
- ecr_pull(image)
- } else {
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${image}
- """,
- label: 'Pull docker image',
- )
- }
-}
-
-def ecr_pull(full_name) {
- aws_account_id = sh(
- returnStdout: true,
- script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
- label: 'Get AWS ID'
- ).trim()
-
- try {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: '''
- set -eux
- aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
- ''',
- label: 'Log in to ECR'
- )
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${full_name}
- """,
- label: 'Pull image from ECR'
- )
- }
- } finally {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: 'docker logout $AWS_ECR_REPO',
- label: 'Clean up login credentials'
- )
- }
- }
-}
-
-def should_skip_slow_tests(pr_number) {
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
- result = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
- label: 'Check if CI should run slow tests',
- )
- }
- return result == 0
-}
-
-def cancel_previous_build() {
- // cancel previous build if it is not on main.
- if (env.BRANCH_NAME != 'main') {
- def buildNumber = env.BUILD_NUMBER as int
- // Milestone API allows us to cancel previous build
- // with the same milestone number
- if (buildNumber > 1) milestone(buildNumber - 1)
- milestone(buildNumber)
- }
-}
-
-def checkout_trusted_files() {
- // trust everything from branch builds
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- return;
- }
-
- // trust peoople listed in CONTRIBUTING.md
- grep_code = sh(
- returnStatus: true,
- script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
- label: 'Check if change is from a contributor',
- )
-
- if (grep_code == 1) {
- // Any scripts that run on the bare host and not inside a Docker container
- // (especially those that access secrets) should be checked out here so
- // only trusted versions are used in CI
- sh(
- script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
- label: 'Check out trusted files',
- )
- }
-}
-
-def should_skip_ci(pr_number) {
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- // never skip CI on build sourced from a branch
- return false
- }
- glob_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
- label: 'Check if CI should be skipped due to changed files',
- )
- if (glob_skip_ci_code == 0) {
- return true
- }
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run full CI (or the script had an error, so run
- // full CI just in case). Exit code of 0 means skip CI.
- git_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
- label: 'Check if CI should be skipped',
- )
- }
- return git_skip_ci_code == 0
-}
-
-def check_pr(pr_number) {
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- // never skip CI on build sourced from a branch
- return false
- }
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- sh (
- script: "python3 ${jenkins_scripts_root}/check_pr.py --pr ${pr_number}",
- label: 'Check PR title and body',
- )
- }
-
-}
-
-def prepare(node_type) {
- stage('Prepare') {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
- init_git()
-
- check_pr(env.CHANGE_ID)
-
- if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
- sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
- label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
- )
- // Pull image names from the results of should_rebuild_docker.py
- ci_arm = sh(
- script: "cat .docker-image-names/ci_arm",
- label: "Find docker image name for ci_arm",
- returnStdout: true,
- ).trim()
- ci_cortexm = sh(
- script: "cat .docker-image-names/ci_cortexm",
- label: "Find docker image name for ci_cortexm",
- returnStdout: true,
- ).trim()
- ci_cpu = sh(
- script: "cat .docker-image-names/ci_cpu",
- label: "Find docker image name for ci_cpu",
- returnStdout: true,
- ).trim()
- ci_gpu = sh(
- script: "cat .docker-image-names/ci_gpu",
- label: "Find docker image name for ci_gpu",
- returnStdout: true,
- ).trim()
- ci_hexagon = sh(
- script: "cat .docker-image-names/ci_hexagon",
- label: "Find docker image name for ci_hexagon",
- returnStdout: true,
- ).trim()
- ci_i386 = sh(
- script: "cat .docker-image-names/ci_i386",
- label: "Find docker image name for ci_i386",
- returnStdout: true,
- ).trim()
- ci_lint = sh(
- script: "cat .docker-image-names/ci_lint",
- label: "Find docker image name for ci_lint",
- returnStdout: true,
- ).trim()
- ci_minimal = sh(
- script: "cat .docker-image-names/ci_minimal",
- label: "Find docker image name for ci_minimal",
- returnStdout: true,
- ).trim()
- ci_riscv = sh(
- script: "cat .docker-image-names/ci_riscv",
- label: "Find docker image name for ci_riscv",
- returnStdout: true,
- ).trim()
- ci_wasm = sh(
- script: "cat .docker-image-names/ci_wasm",
- label: "Find docker image name for ci_wasm",
- returnStdout: true,
- ).trim()
- }
-
- ci_arm = params.ci_arm_param ?: ci_arm
- ci_cortexm = params.ci_cortexm_param ?: ci_cortexm
- ci_cpu = params.ci_cpu_param ?: ci_cpu
- ci_gpu = params.ci_gpu_param ?: ci_gpu
- ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
- ci_i386 = params.ci_i386_param ?: ci_i386
- ci_lint = params.ci_lint_param ?: ci_lint
- ci_minimal = params.ci_minimal_param ?: ci_minimal
- ci_riscv = params.ci_riscv_param ?: ci_riscv
- ci_wasm = params.ci_wasm_param ?: ci_wasm
-
- sh (script: """
- echo "Docker images being used in this build:"
- echo " ci_arm = ${ci_arm}"
- echo " ci_cortexm = ${ci_cortexm}"
- echo " ci_cpu = ${ci_cpu}"
- echo " ci_gpu = ${ci_gpu}"
- echo " ci_hexagon = ${ci_hexagon}"
- echo " ci_i386 = ${ci_i386}"
- echo " ci_lint = ${ci_lint}"
- echo " ci_minimal = ${ci_minimal}"
- echo " ci_riscv = ${ci_riscv}"
- echo " ci_wasm = ${ci_wasm}"
- """, label: 'Docker image names')
-
- is_docs_only_build = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docs.sh",
- label: 'Check for docs only changes',
- )
- skip_ci = should_skip_ci(env.CHANGE_ID)
- skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
- rebuild_docker_images = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docker.sh",
- label: 'Check for any docker changes',
- )
-
- if (skip_ci) {
- // Don't rebuild when skipping CI
- rebuild_docker_images = false
- }
- }
- }
- }
-}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
-def python_unittest(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
- label: 'Run Python unit tests',
- )
-}
-
-def make_cpp_tests(image, build_dir) {
- sh (
- script: """
- set -eux
- ${docker_run} ${image} python3 ./tests/scripts/task_build.py \
- --sccache-bucket tvm-sccache-prod \
- --sccache-region us-west-2 \
- --cmake-target cpptest \
- --build-dir ${build_dir}
- """,
- label: 'Make C++ tests',
- )
-}
-
-def cmake_build(image, path, make_flag) {
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir ${path}",
- label: 'Run cmake build',
- )
-}
-def cpp_unittest(image) {
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh",
- label: 'Run C++ tests',
- )
-}
-
-cancel_previous_build()
-
-try {
- prepare('CPU-SMALL-SPOT')
-} catch(Exception ex) {
- prepare('CPU-SMALL')
-}
-def build(node_type) {
- stage('Build') {
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu-minimal") {
- init_git()
- docker_init(ci_minimal)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=minimal',
- ], {
- sh (
- script: "${docker_run} ${ci_minimal} ./tests/scripts/task_config_build_minimal.sh build",
- label: 'Create CPU minimal cmake config',
- )
- cmake_build(ci_minimal, 'build', '-j2')
- make_cpp_tests(ci_minimal, 'build')
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu-minimal --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/crttest build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/build.ninja build/microtvm_template_projects",
- label: 'Upload artifacts to S3',
- )
- })
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('BUILD: CPU MINIMAL')
- }
- }
-}
-try {
- build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
- build('CPU-SMALL')
-}
-
-
-
-
-def shard_run_unittest_CPU_MINIMAL_1_of_1(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/ut-python-cpu-minimal") {
- try {
- init_git()
- docker_init(ci_minimal)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=minimal',
- 'TEST_STEP_NAME=unittest: CPU MINIMAL',
- 'TVM_NUM_SHARDS=1',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/cpu-minimal",
- label: 'Download artifacts from S3',
- )
-
- cpp_unittest(ci_minimal)
- python_unittest(ci_minimal)
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/unittest_CPU_MINIMAL --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('unittest: CPU MINIMAL 1 of 1')
- }
-}
-
-
-
-def test() {
- stage('Test') {
- environment {
- SKIP_SLOW_TESTS = "${skip_slow_tests}"
- }
- parallel(
- 'unittest: CPU MINIMAL 1 of 1': {
- try {
- shard_run_unittest_CPU_MINIMAL_1_of_1()
- } catch (Exception ex) {
- shard_run_unittest_CPU_MINIMAL_1_of_1(on_demand = true)
- }
- },
- )
- }
-}
-test()
diff --git a/ci/jenkins/generated/riscv_jenkinsfile.groovy b/ci/jenkins/generated/riscv_jenkinsfile.groovy
deleted file mode 100644
index f836d71630..0000000000
--- a/ci/jenkins/generated/riscv_jenkinsfile.groovy
+++ /dev/null
@@ -1,624 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-// -*- mode: groovy -*-
-
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-
-// Jenkins pipeline
-// See documents at https://jenkins.io/doc/book/pipeline/jenkinsfile/
-
-// Docker env used for testing
-// Different image may have different version tag
-// because some of them are more stable than anoter.
-//
-// Docker images are maintained by PMC, cached in dockerhub
-// and remains relatively stable over the time.
-// Flow for upgrading docker env(need commiter)
-//
-// - Send PR to upgrade build script in the repo
-// - Build the new docker image
-// - Tag the docker image with a new version and push to a binary cache.
-// - Update the version in the Jenkinsfile, send a PR
-// - Fix any issues wrt to the new image version in the PR
-// - Merge the PR and now we are in new version
-// - Tag the new version as the lates
-// - Periodically cleanup the old versions on local workers
-//
-
-// ============================= IMPORTANT NOTE =============================
-// This file is generated by 'jenkins/generate.py'. Do not edit this file directly!
-// Make edits to 'jenkins/Jenkinsfile.j2' and regenerate this with
-// 'python3 jenkins/generate.py'
-// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
-// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.070888
-
-import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
-// These are set at runtime from data in ci/jenkins/docker-images.yml, update
-// image tags in that file
-ci_lint = ''
-ci_gpu = ''
-ci_cpu = ''
-ci_minimal = ''
-ci_wasm = ''
-ci_i386 = ''
-ci_cortexm = ''
-ci_arm = ''
-ci_hexagon = ''
-ci_riscv = ''
-
-// Parameters to allow overriding (in Jenkins UI), the images
-// to be used by a given build. When provided, they take precedence
-// over default values above.
-properties([
- parameters([
- string(name: 'ci_arm_param', defaultValue: ''),
- string(name: 'ci_cortexm_param', defaultValue: ''),
- string(name: 'ci_cpu_param', defaultValue: ''),
- string(name: 'ci_gpu_param', defaultValue: ''),
- string(name: 'ci_hexagon_param', defaultValue: ''),
- string(name: 'ci_i386_param', defaultValue: ''),
- string(name: 'ci_lint_param', defaultValue: ''),
- string(name: 'ci_minimal_param', defaultValue: ''),
- string(name: 'ci_riscv_param', defaultValue: ''),
- string(name: 'ci_wasm_param', defaultValue: ''),
- ])
-])
-
-// Placeholders for newly built Docker image names (if rebuild_docker_images
-// is used)
- built_ci_arm = null;
- built_ci_cortexm = null;
- built_ci_cpu = null;
- built_ci_gpu = null;
- built_ci_hexagon = null;
- built_ci_i386 = null;
- built_ci_lint = null;
- built_ci_minimal = null;
- built_ci_riscv = null;
- built_ci_wasm = null;
-
-// Global variable assigned during Sanity Check that holds the sha1 which should be
-// merged into the PR in all branches.
-upstream_revision = null
-
-// command to start a docker container
-docker_run = 'docker/bash.sh --env CI --env PLATFORM --env TVM_SHARD_INDEX --env TVM_NUM_SHARDS --env RUN_DISPLAY_URL --env PLATFORM --env SKIP_SLOW_TESTS --env TEST_STEP_NAME'
-docker_build = 'docker/build.sh'
-// timeout in minutes
-max_time = 180
-rebuild_docker_images = false
-
-s3_bucket = 'tvm-jenkins-artifacts-prod'
-s3_prefix = "tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
-
-// Jenkins script root directory
-jenkins_scripts_root = "ci/scripts/jenkins"
-
-
-// General note: Jenkins has limits on the size of a method (or top level code)
-// that are pretty strict, so most usage of groovy methods in these templates
-// are purely to satisfy the JVM
-def per_exec_ws(folder) {
- return "workspace/exec_${env.EXECUTOR_NUMBER}/" + folder
-}
-
-// initialize source codes
-def init_git() {
- retry(5) {
- checkout scm
- }
-
- // Add more info about job node
- sh (
- script: './tests/scripts/task_show_node_info.sh',
- label: 'Show executor node info',
- )
-
- // Determine merge commit to use for all stages
- if (env.BRANCH_NAME == 'main') {
- // Only set upstream_revision to HEAD and skip merging to avoid a race with another commit merged to main.
- update_upstream_revision("HEAD")
- } else {
- // This is PR branch so merge with latest main.
- merge_with_main()
- }
-
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 3 timeout 5m git submodule update --init --recursive -f --jobs 0
- """,
- label: 'Update git submodules',
- )
- checkout_trusted_files()
-}
-
-def update_upstream_revision(git_ref) {
- if (upstream_revision == null) {
- upstream_revision = sh(
- script: "git log -1 ${git_ref} --format=\'%H\'",
- label: 'Determine upstream revision',
- returnStdout: true,
- ).trim()
- }
-}
-
-def merge_with_main() {
- sh (
- script: 'git fetch origin main',
- label: 'Fetch upstream',
- )
- update_upstream_revision("FETCH_HEAD")
- sh (
- script: "git -c user.name=TVM-Jenkins -c user.email=jenkins@tvm.apache.org merge ${upstream_revision}",
- label: 'Merge to origin/main'
- )
-}
-
-def docker_init(image) {
- // Clear out all Docker images that aren't going to be used
- sh(
- script: """
- set -eux
- docker image ls --all
- IMAGES=\$(docker image ls --all --format '{{.Repository}}:{{.Tag}} {{.ID}}')
-
- echo -e "Found images:\\n\$IMAGES"
- echo "\$IMAGES" | { grep -vE '${image}' || test \$? = 1; } | { xargs docker rmi || test \$? = 123; }
-
- docker image ls --all
- """,
- label: 'Clean old Docker images',
- )
-
- if (image.contains("amazonaws.com")) {
- // If this string is in the image name it's from ECR and needs to be pulled
- // with the right credentials
- ecr_pull(image)
- } else {
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${image}
- """,
- label: 'Pull docker image',
- )
- }
-}
-
-def ecr_pull(full_name) {
- aws_account_id = sh(
- returnStdout: true,
- script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
- label: 'Get AWS ID'
- ).trim()
-
- try {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: '''
- set -eux
- aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
- ''',
- label: 'Log in to ECR'
- )
- sh(
- script: """
- set -eux
- . ${jenkins_scripts_root}/retry.sh
- retry 5 docker pull ${full_name}
- """,
- label: 'Pull image from ECR'
- )
- }
- } finally {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2',
- "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
- sh(
- script: 'docker logout $AWS_ECR_REPO',
- label: 'Clean up login credentials'
- )
- }
- }
-}
-
-def should_skip_slow_tests(pr_number) {
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run slow tests, exit code of 0 means skip slow tests
- result = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/should_run_slow_tests.py --pr '${pr_number}'",
- label: 'Check if CI should run slow tests',
- )
- }
- return result == 0
-}
-
-def cancel_previous_build() {
- // cancel previous build if it is not on main.
- if (env.BRANCH_NAME != 'main') {
- def buildNumber = env.BUILD_NUMBER as int
- // Milestone API allows us to cancel previous build
- // with the same milestone number
- if (buildNumber > 1) milestone(buildNumber - 1)
- milestone(buildNumber)
- }
-}
-
-def checkout_trusted_files() {
- // trust everything from branch builds
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- return;
- }
-
- // trust peoople listed in CONTRIBUTING.md
- grep_code = sh(
- returnStatus: true,
- script: "git show '${upstream_revision}:CONTRIBUTORS.md' | grep '@${env.CHANGE_AUTHOR}'",
- label: 'Check if change is from a contributor',
- )
-
- if (grep_code == 1) {
- // Any scripts that run on the bare host and not inside a Docker container
- // (especially those that access secrets) should be checked out here so
- // only trusted versions are used in CI
- sh(
- script: "git checkout ${upstream_revision} ${jenkins_scripts_root}/.",
- label: 'Check out trusted files',
- )
- }
-}
-
-def should_skip_ci(pr_number) {
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- // never skip CI on build sourced from a branch
- return false
- }
- glob_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci_globs.py",
- label: 'Check if CI should be skipped due to changed files',
- )
- if (glob_skip_ci_code == 0) {
- return true
- }
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- // Exit code of 1 means run full CI (or the script had an error, so run
- // full CI just in case). Exit code of 0 means skip CI.
- git_skip_ci_code = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_skip_ci.py --pr '${pr_number}'",
- label: 'Check if CI should be skipped',
- )
- }
- return git_skip_ci_code == 0
-}
-
-def check_pr(pr_number) {
- if (env.BRANCH_NAME == null || !env.BRANCH_NAME.startsWith('PR-')) {
- // never skip CI on build sourced from a branch
- return false
- }
- withCredentials([string(
- credentialsId: 'tvm-bot-jenkins-reader',
- variable: 'GITHUB_TOKEN',
- )]) {
- sh (
- script: "python3 ${jenkins_scripts_root}/check_pr.py --pr ${pr_number}",
- label: 'Check PR title and body',
- )
- }
-
-}
-
-def prepare(node_type) {
- stage('Prepare') {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/prepare") {
- init_git()
-
- check_pr(env.CHANGE_ID)
-
- if (env.DETERMINE_DOCKER_IMAGES == 'yes') {
- sh(
- script: "./${jenkins_scripts_root}/determine_docker_images.py ci_arm ci_cortexm ci_cpu ci_gpu ci_hexagon ci_i386 ci_lint ci_minimal ci_riscv ci_wasm ",
- label: 'Decide whether to use tlcpack or tlcpackstaging for Docker images',
- )
- // Pull image names from the results of should_rebuild_docker.py
- ci_arm = sh(
- script: "cat .docker-image-names/ci_arm",
- label: "Find docker image name for ci_arm",
- returnStdout: true,
- ).trim()
- ci_cortexm = sh(
- script: "cat .docker-image-names/ci_cortexm",
- label: "Find docker image name for ci_cortexm",
- returnStdout: true,
- ).trim()
- ci_cpu = sh(
- script: "cat .docker-image-names/ci_cpu",
- label: "Find docker image name for ci_cpu",
- returnStdout: true,
- ).trim()
- ci_gpu = sh(
- script: "cat .docker-image-names/ci_gpu",
- label: "Find docker image name for ci_gpu",
- returnStdout: true,
- ).trim()
- ci_hexagon = sh(
- script: "cat .docker-image-names/ci_hexagon",
- label: "Find docker image name for ci_hexagon",
- returnStdout: true,
- ).trim()
- ci_i386 = sh(
- script: "cat .docker-image-names/ci_i386",
- label: "Find docker image name for ci_i386",
- returnStdout: true,
- ).trim()
- ci_lint = sh(
- script: "cat .docker-image-names/ci_lint",
- label: "Find docker image name for ci_lint",
- returnStdout: true,
- ).trim()
- ci_minimal = sh(
- script: "cat .docker-image-names/ci_minimal",
- label: "Find docker image name for ci_minimal",
- returnStdout: true,
- ).trim()
- ci_riscv = sh(
- script: "cat .docker-image-names/ci_riscv",
- label: "Find docker image name for ci_riscv",
- returnStdout: true,
- ).trim()
- ci_wasm = sh(
- script: "cat .docker-image-names/ci_wasm",
- label: "Find docker image name for ci_wasm",
- returnStdout: true,
- ).trim()
- }
-
- ci_arm = params.ci_arm_param ?: ci_arm
- ci_cortexm = params.ci_cortexm_param ?: ci_cortexm
- ci_cpu = params.ci_cpu_param ?: ci_cpu
- ci_gpu = params.ci_gpu_param ?: ci_gpu
- ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
- ci_i386 = params.ci_i386_param ?: ci_i386
- ci_lint = params.ci_lint_param ?: ci_lint
- ci_minimal = params.ci_minimal_param ?: ci_minimal
- ci_riscv = params.ci_riscv_param ?: ci_riscv
- ci_wasm = params.ci_wasm_param ?: ci_wasm
-
- sh (script: """
- echo "Docker images being used in this build:"
- echo " ci_arm = ${ci_arm}"
- echo " ci_cortexm = ${ci_cortexm}"
- echo " ci_cpu = ${ci_cpu}"
- echo " ci_gpu = ${ci_gpu}"
- echo " ci_hexagon = ${ci_hexagon}"
- echo " ci_i386 = ${ci_i386}"
- echo " ci_lint = ${ci_lint}"
- echo " ci_minimal = ${ci_minimal}"
- echo " ci_riscv = ${ci_riscv}"
- echo " ci_wasm = ${ci_wasm}"
- """, label: 'Docker image names')
-
- is_docs_only_build = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docs.sh",
- label: 'Check for docs only changes',
- )
- skip_ci = should_skip_ci(env.CHANGE_ID)
- skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
- rebuild_docker_images = sh (
- returnStatus: true,
- script: "./${jenkins_scripts_root}/git_change_docker.sh",
- label: 'Check for any docker changes',
- )
-
- if (skip_ci) {
- // Don't rebuild when skipping CI
- rebuild_docker_images = false
- }
- }
- }
- }
-}
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_clear_pytest.sh",
- label: 'Clean up old workspace',
- )
-}
-
-def python_unittest(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
- label: 'Run Python unit tests',
- )
-}
-
-def make_cpp_tests(image, build_dir) {
- sh (
- script: """
- set -eux
- ${docker_run} ${image} python3 ./tests/scripts/task_build.py \
- --sccache-bucket tvm-sccache-prod \
- --sccache-region us-west-2 \
- --cmake-target cpptest \
- --build-dir ${build_dir}
- """,
- label: 'Make C++ tests',
- )
-}
-
-def cmake_build(image, path, make_flag) {
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod --sccache-region us-west-2 --build-dir ${path}",
- label: 'Run cmake build',
- )
-}
-def cpp_unittest(image) {
- sh (
- script: "${docker_run} --env CI_NUM_EXECUTORS ${image} ./tests/scripts/task_cpp_unittest.sh",
- label: 'Run C++ tests',
- )
-}
-
-cancel_previous_build()
-
-try {
- prepare('CPU-SMALL-SPOT')
-} catch(Exception ex) {
- prepare('CPU-SMALL')
-}
-def build(node_type) {
- stage('Build') {
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-riscv") {
- init_git()
- docker_init(ci_riscv)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=riscv',
- ], {
- sh (
- script: "${docker_run} ${ci_riscv} ./tests/scripts/task_config_build_riscv.sh build",
- label: 'Create RISC-V cmake config',
- )
- cmake_build(ci_riscv, 'build', '-j2')
- make_cpp_tests(ci_riscv, 'build')
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/riscv --items build/libtvm.so build/libtvm_runtime.so build/config.cmake build/libtvm_allvisible.so build/build.ninja build/crttest build/cpptest build/build.ninja build/CMakeFiles/rules.ninja build/microtvm_template_projects",
- label: 'Upload artifacts to S3',
- )
- })
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('BUILD: RISC-V')
- }
- }
-}
-try {
- build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
- build('CPU-SMALL')
-}
-
-
-
-
-
-def shard_run_test_RISC_V_1_of_1(node_type='CPU-SMALL-SPOT', on_demand=false) {
- if (!skip_ci && is_docs_only_build != 1) {
- if (on_demand==true || node_type.contains('ARM')) {
- node_type = 'CPU-SMALL'
- }
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/test-riscv") {
- try {
- init_git()
- docker_init(ci_riscv)
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM=riscv',
- 'TEST_STEP_NAME=test: RISC-V',
- 'TVM_NUM_SHARDS=1',
- 'TVM_SHARD_INDEX=0',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action download --bucket ${s3_bucket} --prefix ${s3_prefix}/riscv",
- label: 'Download artifacts from S3',
- )
-
- ci_setup(ci_riscv)
- cpp_unittest(ci_riscv)
- sh (
- script: "${docker_run} ${ci_riscv} ./tests/scripts/task_riscv_microtvm.sh",
- label: 'Run microTVM tests',
- )
- })
- }
- } finally {
- try {
- sh(
- script: "./${jenkins_scripts_root}/s3.py --action upload --bucket ${s3_bucket} --prefix ${s3_prefix}/pytest-results/test_RISC_V --items build/pytest-results",
- label: 'Upload JUnits to S3',
- )
-
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
- }
- }
- }
- } else {
- Utils.markStageSkippedForConditional('test: RISC-V 1 of 1')
- }
-}
-
-
-def test() {
- stage('Test') {
- environment {
- SKIP_SLOW_TESTS = "${skip_slow_tests}"
- }
- parallel(
- 'test: RISC-V 1 of 1': {
- try {
- shard_run_test_RISC_V_1_of_1()
- } catch (Exception ex) {
- shard_run_test_RISC_V_1_of_1(on_demand = true)
- }
- },
- )
- }
-}
-test()
diff --git a/ci/jenkins/generated/wasm_jenkinsfile.groovy b/ci/jenkins/generated/wasm_jenkinsfile.groovy
index da7cdfd5c1..f31738201d 100644
--- a/ci/jenkins/generated/wasm_jenkinsfile.groovy
+++ b/ci/jenkins/generated/wasm_jenkinsfile.groovy
@@ -60,7 +60,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2024-01-10T13:15:25.155555
+// Generated at 2025-02-06T19:18:57.656055
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// These are set at runtime from data in ci/jenkins/docker-images.yml, update
@@ -515,18 +515,18 @@ try {
prepare('CPU-SMALL')
}
def build(node_type) {
- stage('Build') {
- if (!skip_ci && is_docs_only_build != 1) {
- node(node_type) {
- ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") {
- init_git()
- docker_init(ci_wasm)
- timeout(time: max_time, unit: 'MINUTES') {
-
- withEnv([
- 'PLATFORM=wasm',
- ], {
- sh (
+ if (!skip_ci && is_docs_only_build != 1) {
+ echo 'Run build on node_type=' + node_type
+ node(node_type) {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") {
+ init_git()
+ docker_init(ci_wasm)
+ timeout(time: max_time, unit: 'MINUTES') {
+
+ withEnv([
+ 'PLATFORM=wasm',
+ ], {
+ sh (
script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build",
label: 'Create WASM cmake config',
)
@@ -538,17 +538,22 @@ def build(node_type) {
script: "${docker_run} ${ci_wasm} ./tests/scripts/task_web_wasm.sh",
label: 'Run WASM lint and tests',
)
- })
- }
+ })
}
}
- } else {
- Utils.markStageSkippedForConditional('BUILD: WASM')
}
+ } else {
+ Utils.markStageSkippedForConditional('BUILD: WASM')
}
}
-try {
- build('CPU-SMALL-SPOT')
-} catch (Exception ex) {
- build('CPU-SMALL')
+stage('Build') {
+ try {
+ build('CPU-SMALL-SPOT')
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
+ build('CPU-SMALL')
+ }
}
diff --git a/ci/jenkins/templates/arm_jenkinsfile.groovy.j2 b/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
index c512ddd0ce..e9a5bde25e 100644
--- a/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/arm_jenkinsfile.groovy.j2
@@ -31,7 +31,7 @@
)
cmake_build(ci_arm, 'build', '-j4')
make_cpp_tests(ci_arm, 'build')
- {{ m.upload_artifacts(tag='arm', filenames=tvm_multilib + cpptest + crttest + microtvm_template_projects) }}
+ {{ m.upload_artifacts(tag='arm', filenames=tvm_lib + cpptest) }}
{% endcall %}
{% set test_method_names = [] %}
diff --git a/ci/jenkins/templates/cortexm_jenkinsfile.groovy.j2 b/ci/jenkins/templates/cortexm_jenkinsfile.groovy.j2
deleted file mode 100644
index 4f38306701..0000000000
--- a/ci/jenkins/templates/cortexm_jenkinsfile.groovy.j2
+++ /dev/null
@@ -1,63 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-{% include "utils/base.groovy.j2" with context %}
-{% import 'utils/macros.j2' as m with context -%}
-
-{% call m.invoke_build(
- name='BUILD: Cortex-M',
- node='CPU-SMALL',
- condition='!skip_ci && is_docs_only_build != 1',
- ws='tvm/build-cortexm',
- docker_image='ci_cortexm',
- platform="cortexm",
-) %}
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_config_build_cortexm.sh build",
- label: 'Create Cortex-M cmake config',
- )
- cmake_build(ci_cortexm, 'build', '-j2')
- make_cpp_tests(ci_cortexm, 'build')
- {{ m.upload_artifacts(tag='cortexm', filenames=tvm_lib + tvm_allvisible + crttest + cpptest + microtvm_template_projects) }}
-{% endcall %}
-
-{% set test_method_names = [] %}
-
-{% call(shard_index, num_shards) m.sharded_test_step(
- name="test: Cortex-M",
- node="CPU-SMALL",
- ws="tvm/test-cortexm",
- platform="cortexm",
- docker_image="ci_cortexm",
- num_shards=12,
- test_method_names=test_method_names,
-) %}
- {{ m.download_artifacts(tag='cortexm') }}
- ci_setup(ci_cortexm)
- {% if shard_index == 1%}
- cpp_unittest(ci_cortexm)
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_demo_microtvm.sh",
- label: 'Run microTVM demos',
- )
- {% endif %}
- sh (
- script: "${docker_run} ${ci_cortexm} ./tests/scripts/task_python_microtvm.sh",
- label: 'Run microTVM tests',
- )
-{% endcall %}
-
-{{ m.invoke_tests(test_method_names) -}}
diff --git a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
index 4d6081f3f4..f522196939 100644
--- a/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/cpu_jenkinsfile.groovy.j2
@@ -31,7 +31,7 @@
)
cmake_build(ci_cpu, 'build', '-j2')
make_cpp_tests(ci_cpu, 'build')
- {{ m.upload_artifacts(tag='cpu', filenames=tvm_multilib_tsim + tvm_allvisible + crttest + cpptest + microtvm_template_projects) }}
+ {{ m.upload_artifacts(tag='cpu', filenames=tvm_lib + tvm_allvisible + cpptest) }}
ci_setup(ci_cpu)
// sh "${docker_run} ${ci_cpu} ./tests/scripts/task_golang.sh"
// TODO(@jroesch): need to resolve CI issue will turn back on in follow up patch
diff --git a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2 b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
index b6c7c2cecb..54b2da50cf 100644
--- a/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/gpu_jenkinsfile.groovy.j2
@@ -19,7 +19,7 @@
{% call m.invoke_build(
name='BUILD: GPU',
- node='CPU-SMALL',
+ node='CPU',
condition='!skip_ci',
ws='tvm/build-gpu',
docker_image='ci_gpu',
@@ -27,13 +27,13 @@
) %}
sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
cmake_build("${ci_gpu} --no-gpu", 'build', '-j2')
- {{ m.upload_artifacts(tag='gpu', filenames=tvm_multilib + tvm_allvisible + microtvm_template_projects + crttest) }}
+ {{ m.upload_artifacts(tag='gpu', filenames=tvm_lib + tvm_allvisible + tvm_lib_gpu_extra) }}
// compiler test
sh "rm -rf build"
sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu_other.sh build"
cmake_build("${ci_gpu} --no-gpu", 'build', '-j2')
- {{ m.upload_artifacts(tag='gpu2', filenames=tvm_lib + crttest) }}
+ {{ m.upload_artifacts(tag='gpu2', filenames=tvm_lib) }}
{% endcall %}
{% set test_method_names = [] %}
diff --git a/ci/jenkins/templates/i386_jenkinsfile.groovy.j2 b/ci/jenkins/templates/i386_jenkinsfile.groovy.j2
index 5b18136745..617c3f75e7 100644
--- a/ci/jenkins/templates/i386_jenkinsfile.groovy.j2
+++ b/ci/jenkins/templates/i386_jenkinsfile.groovy.j2
@@ -31,7 +31,7 @@
)
cmake_build(ci_i386, 'build', '-j2')
make_cpp_tests(ci_i386, 'build')
- {{ m.upload_artifacts(tag='i386', filenames=tvm_multilib_tsim + crttest + cpptest + microtvm_template_projects) }}
+ {{ m.upload_artifacts(tag='i386', filenames=tvm_lib + cpptest) }}
{% endcall %}
diff --git a/ci/jenkins/templates/minimal_cross_isa_jenkinsfile.groovy.j2 b/ci/jenkins/templates/minimal_cross_isa_jenkinsfile.groovy.j2
deleted file mode 100644
index 07c3890dfc..0000000000
--- a/ci/jenkins/templates/minimal_cross_isa_jenkinsfile.groovy.j2
+++ /dev/null
@@ -1,57 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-{% include "utils/base.groovy.j2" with context %}
-{% import 'utils/macros.j2' as m with context -%}
-
-{% call m.invoke_build(
- name='BUILD: CPU MINIMAL CROSS ISA',
- node='CPU-SMALL',
- condition='!skip_ci && is_docs_only_build != 1',
- ws='tvm/build-cpu-minimal-cross-isa',
- docker_image='ci_minimal',
- platform="arm",
-) %}
- sh (
- script: "${docker_run} ${ci_minimal} ./tests/scripts/task_config_build_minimal_cross_isa.sh build",
- label: 'Create CPU minimal cmake config',
- )
- cmake_build(ci_minimal, 'build', '-j2')
- {{ m.upload_artifacts(tag='cpu-minimal-cross-isa', filenames=tvm_lib + tvm_allvisible + microtvm_template_projects) }}
-{% endcall %}
-
-
-{% set test_method_names = [] %}
-
-{% call(shard_index, num_shards) m.sharded_test_step(
- name="unittest: CPU MINIMAL CROSS ISA",
- node="ARM-SMALL",
- num_shards=1,
- ws="tvm/ut-cpp-arm-cross-isa",
- platform="arm",
- docker_image="ci_arm",
- test_method_names=test_method_names,
-) %}
- {{ m.download_artifacts(tag='cpu-minimal-cross-isa') }}
- ci_setup(ci_arm)
- sh "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_minimal_cross_isa.sh build"
- make_cpp_tests(ci_arm, 'build')
- cpp_unittest(ci_arm)
- python_unittest(ci_arm)
-{% endcall %}
-
-
-{{ m.invoke_tests(test_method_names) -}}
diff --git a/ci/jenkins/templates/minimal_jenkinsfile.groovy.j2 b/ci/jenkins/templates/minimal_jenkinsfile.groovy.j2
deleted file mode 100644
index dd4dd02fc8..0000000000
--- a/ci/jenkins/templates/minimal_jenkinsfile.groovy.j2
+++ /dev/null
@@ -1,55 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-{% include "utils/base.groovy.j2" with context %}
-{% import 'utils/macros.j2' as m with context -%}
-
-{% call m.invoke_build(
- name='BUILD: CPU MINIMAL',
- node='CPU-SMALL',
- condition='!skip_ci && is_docs_only_build != 1',
- ws='tvm/build-cpu-minimal',
- docker_image='ci_minimal',
- platform="minimal",
-) %}
- sh (
- script: "${docker_run} ${ci_minimal} ./tests/scripts/task_config_build_minimal.sh build",
- label: 'Create CPU minimal cmake config',
- )
- cmake_build(ci_minimal, 'build', '-j2')
- make_cpp_tests(ci_minimal, 'build')
- {{ m.upload_artifacts(tag='cpu-minimal', filenames=tvm_lib + tvm_allvisible + crttest + cpptest + microtvm_template_projects) }}
-{% endcall %}
-
-
-{% set test_method_names = [] %}
-
-{% call(shard_index, num_shards) m.sharded_test_step(
- name="unittest: CPU MINIMAL",
- node="CPU-SMALL",
- num_shards=1,
- ws="tvm/ut-python-cpu-minimal",
- platform="minimal",
- docker_image="ci_minimal",
- test_method_names=test_method_names,
-) %}
- {{ m.download_artifacts(tag='cpu-minimal') }}
- cpp_unittest(ci_minimal)
- python_unittest(ci_minimal)
-{% endcall %}
-
-
-{{ m.invoke_tests(test_method_names) -}}
diff --git a/ci/jenkins/templates/riscv_jenkinsfile.groovy.j2 b/ci/jenkins/templates/riscv_jenkinsfile.groovy.j2
deleted file mode 100644
index 902e912f75..0000000000
--- a/ci/jenkins/templates/riscv_jenkinsfile.groovy.j2
+++ /dev/null
@@ -1,61 +0,0 @@
-// Licensed to the Apache Software Foundation (ASF) under one
-// or more contributor license agreements. See the NOTICE file
-// distributed with this work for additional information
-// regarding copyright ownership. The ASF licenses this file
-// to you under the Apache License, Version 2.0 (the
-// "License"); you may not use this file except in compliance
-// with the License. You may obtain a copy of the License at
-//
-// http://www.apache.org/licenses/LICENSE-2.0
-//
-// Unless required by applicable law or agreed to in writing,
-// software distributed under the License is distributed on an
-// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
-// KIND, either express or implied. See the License for the
-// specific language governing permissions and limitations
-// under the License.
-{% include "utils/base.groovy.j2" with context %}
-{% import 'utils/macros.j2' as m with context -%}
-
-{% call m.invoke_build(
- name='BUILD: RISC-V',
- node='CPU-SMALL',
- condition='!skip_ci && is_docs_only_build != 1',
- ws='tvm/build-riscv',
- docker_image='ci_riscv',
- platform="riscv",
-) %}
- sh (
- script: "${docker_run} ${ci_riscv} ./tests/scripts/task_config_build_riscv.sh build",
- label: 'Create RISC-V cmake config',
- )
- cmake_build(ci_riscv, 'build', '-j2')
- make_cpp_tests(ci_riscv, 'build')
- {{ m.upload_artifacts(tag='riscv', filenames=tvm_lib + tvm_allvisible + crttest + cpptest + microtvm_template_projects) }}
-{% endcall %}
-
-
-
-{% set test_method_names = [] %}
-
-{% call(shard_index, num_shards) m.sharded_test_step(
- name="test: RISC-V",
- node="CPU-SMALL",
- ws="tvm/test-riscv",
- platform="riscv",
- docker_image="ci_riscv",
- num_shards=1,
- test_method_names=test_method_names,
-) %}
- {{ m.download_artifacts(tag='riscv') }}
- ci_setup(ci_riscv)
- {% if shard_index == 1%}
- cpp_unittest(ci_riscv)
- {% endif %}
- sh (
- script: "${docker_run} ${ci_riscv} ./tests/scripts/task_riscv_microtvm.sh",
- label: 'Run microTVM tests',
- )
-{% endcall %}
-
-{{ m.invoke_tests(test_method_names) -}}
diff --git a/ci/jenkins/templates/utils/macros.j2 b/ci/jenkins/templates/utils/macros.j2
index 6de4bd6d65..720e1271bb 100644
--- a/ci/jenkins/templates/utils/macros.j2
+++ b/ci/jenkins/templates/utils/macros.j2
@@ -36,28 +36,28 @@ def {{ method_name }}(node_type='{{ node }}-SPOT', on_demand=false) {
if (on_demand==true || node_type.contains('ARM')) {
node_type = '{{ node }}'
}
+ echo 'Running on node_type=' + node_type
node(node_type) {
ws({{ per_exec_ws(ws) }}) {
+ // NOTE: if exception happens, it will be caught outside
+ init_git()
+ docker_init({{ docker_image }})
+ timeout(time: max_time, unit: 'MINUTES') {
+ withEnv([
+ 'PLATFORM={{ platform }}',
+ 'TEST_STEP_NAME={{ name }}',
+ 'TVM_NUM_SHARDS={{ num_shards }}',
+ 'TVM_SHARD_INDEX={{ shard_index - 1 }}',
+ "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
+ {{ caller(shard_index, num_shards) | trim | indent(width=12) }}
+ })
+ }
+ // only run upload if things are successful
try {
- init_git()
- docker_init({{ docker_image }})
- timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM={{ platform }}',
- 'TEST_STEP_NAME={{ name }}',
- 'TVM_NUM_SHARDS={{ num_shards }}',
- 'TVM_SHARD_INDEX={{ shard_index - 1 }}',
- "SKIP_SLOW_TESTS=${skip_slow_tests}"], {
- {{ caller(shard_index, num_shards) | trim | indent(width=12) }}
- })
- }
- } finally {
- try {
- {{ junit_to_s3(test_dir_name) }}
- junit 'build/pytest-results/*.xml'
- } catch (Exception e) {
- echo 'Exception during JUnit upload: ' + e.toString()
- }
+ {{ junit_to_s3(test_dir_name) }}
+ junit 'build/pytest-results/*.xml'
+ } catch (Exception e) {
+ echo 'Exception during JUnit upload: ' + e.toString()
}
}
}
@@ -72,31 +72,36 @@ def {{ method_name }}(node_type='{{ node }}-SPOT', on_demand=false) {
{% macro invoke_build(name, condition, node, docker_image, ws, platform) %}
def build(node_type) {
- stage('Build') {
- if ({{ condition }}) {
- node(node_type) {
- ws({{ per_exec_ws(ws) }}) {
- init_git()
- docker_init({{ docker_image }})
- timeout(time: max_time, unit: 'MINUTES') {
+ if ({{ condition }}) {
+ echo 'Run build on node_type=' + node_type
+ node(node_type) {
+ ws({{ per_exec_ws(ws) }}) {
+ init_git()
+ docker_init({{ docker_image }})
+ timeout(time: max_time, unit: 'MINUTES') {
- withEnv([
- 'PLATFORM={{ platform }}',
- ], {
- {{ caller() | trim | indent(width=6) }}
- })
- }
+ withEnv([
+ 'PLATFORM={{ platform }}',
+ ], {
+ {{ caller() | trim | indent(width=6) }}
+ })
}
}
- } else {
- Utils.markStageSkippedForConditional('{{ name }}')
}
+ } else {
+ Utils.markStageSkippedForConditional('{{ name }}')
}
}
-try {
- build('{{ node }}-SPOT')
-} catch (Exception ex) {
- build('{{ node }}')
+stage('Build') {
+ try {
+ build('{{ node }}-SPOT')
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
+ build('{{ node }}')
+ }
}
{% endmacro %}
@@ -111,7 +116,11 @@ def test() {
'{{ stage_name }}': {
try {
{{ method_name }}()
- } catch (Exception ex) {
+ } catch (Throwable ex) {
+ // mark the current stage as success
+ // and try again via on demand node
+ echo 'Exception during SPOT run' + ex.toString() + 'retry via on-demand'
+ currentBuild.result = 'SUCCESS'
{{ method_name }}(on_demand = true)
}
},
diff --git a/ci/jenkins/unity_jenkinsfile.groovy b/ci/jenkins/unity_jenkinsfile.groovy
index d6a5d46f6f..a0c1120fa3 100755
--- a/ci/jenkins/unity_jenkinsfile.groovy
+++ b/ci/jenkins/unity_jenkinsfile.groovy
@@ -348,7 +348,7 @@ stage('Build and Test') {
}
},
'BUILD: CPU': {
- node('CPU-SMALL-SPOT') {
+ node('CPU-SMALL') {
ws(per_exec_ws('tvm/build-cpu')) {
init_git()
sh "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build"