You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by dr...@apache.org on 2022/08/08 18:40:08 UTC
[tvm] branch main updated: [CI] Deduplicate and clean XML test reports (#12332)
This is an automated email from the ASF dual-hosted git repository.
driazati pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new 7f800e41de [CI] Deduplicate and clean XML test reports (#12332)
7f800e41de is described below
commit 7f800e41de097c333ff5790feb1a8c42575b0fe1
Author: Florin Blanaru <fl...@gmail.com>
AuthorDate: Mon Aug 8 19:40:03 2022 +0100
[CI] Deduplicate and clean XML test reports (#12332)
This PR does the following:
- When running on a shard, removes the tests that will not run on the current shard from the suite. We'll no longer get the message `Test running on shard X of Y` in the XML reports. This will result in cleaner test reports.
- Adds the shard index or `no-shard` to the generated XML report file name. Currently, the reports generated when sharding is present are overwritten in S3 by the last shard to finish in the CI pipeline. This change is needed as part of #11670.
- Since the same tests might run on different configurations (CPU, GPU), it uploads the result of each configuration in a subdirectory in S3 (e.g `/pytest-results/frontend_aarch64`).
---
Jenkinsfile | 106 +++++++++++++++++++-------------------
ci/jenkins/macros.j2 | 10 ++--
conftest.py | 11 ++--
tests/scripts/setup-pytest-env.sh | 14 ++++-
4 files changed, 75 insertions(+), 66 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 6fe2c85e22..237d314f61 100755
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -1166,7 +1166,7 @@ def shard_run_unittest_GPU_1_of_3() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/unittest_GPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -1246,7 +1246,7 @@ def shard_run_unittest_GPU_2_of_3() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/unittest_GPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -1322,7 +1322,7 @@ def shard_run_unittest_GPU_3_of_3() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/unittest_GPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -1397,7 +1397,7 @@ def shard_run_integration_CPU_1_of_10() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_CPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -1471,7 +1471,7 @@ def shard_run_integration_CPU_2_of_10() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_CPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -1545,7 +1545,7 @@ def shard_run_integration_CPU_3_of_10() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_CPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -1619,7 +1619,7 @@ def shard_run_integration_CPU_4_of_10() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_CPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -1693,7 +1693,7 @@ def shard_run_integration_CPU_5_of_10() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_CPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -1767,7 +1767,7 @@ def shard_run_integration_CPU_6_of_10() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_CPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -1841,7 +1841,7 @@ def shard_run_integration_CPU_7_of_10() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_CPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -1915,7 +1915,7 @@ def shard_run_integration_CPU_8_of_10() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_CPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -1989,7 +1989,7 @@ def shard_run_integration_CPU_9_of_10() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_CPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -2063,7 +2063,7 @@ def shard_run_integration_CPU_10_of_10() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_CPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -2138,7 +2138,7 @@ def shard_run_python_i386_1_of_5() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/python_i386 --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -2212,7 +2212,7 @@ def shard_run_python_i386_2_of_5() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/python_i386 --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -2285,7 +2285,7 @@ def shard_run_python_i386_3_of_5() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/python_i386 --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -2358,7 +2358,7 @@ def shard_run_python_i386_4_of_5() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/python_i386 --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -2431,7 +2431,7 @@ def shard_run_python_i386_5_of_5() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/python_i386 --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -2505,7 +2505,7 @@ def shard_run_test_Hexagon_1_of_7() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Hexagon --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -2577,7 +2577,7 @@ def shard_run_test_Hexagon_2_of_7() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Hexagon --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -2649,7 +2649,7 @@ def shard_run_test_Hexagon_3_of_7() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Hexagon --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -2721,7 +2721,7 @@ def shard_run_test_Hexagon_4_of_7() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Hexagon --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -2793,7 +2793,7 @@ def shard_run_test_Hexagon_5_of_7() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Hexagon --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -2865,7 +2865,7 @@ def shard_run_test_Hexagon_6_of_7() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Hexagon --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -2937,7 +2937,7 @@ def shard_run_test_Hexagon_7_of_7() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Hexagon --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3011,7 +3011,7 @@ def shard_run_integration_aarch64_1_of_4() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_aarch64 --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3084,7 +3084,7 @@ def shard_run_integration_aarch64_2_of_4() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_aarch64 --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3157,7 +3157,7 @@ def shard_run_integration_aarch64_3_of_4() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_aarch64 --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3230,7 +3230,7 @@ def shard_run_integration_aarch64_4_of_4() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/integration_aarch64 --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3303,7 +3303,7 @@ def shard_run_topi_GPU_1_of_4() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/topi_GPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3375,7 +3375,7 @@ def shard_run_topi_GPU_2_of_4() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/topi_GPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3447,7 +3447,7 @@ def shard_run_topi_GPU_3_of_4() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/topi_GPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3519,7 +3519,7 @@ def shard_run_topi_GPU_4_of_4() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/topi_GPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3592,7 +3592,7 @@ def shard_run_frontend_GPU_1_of_6() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/frontend_GPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3664,7 +3664,7 @@ def shard_run_frontend_GPU_2_of_6() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/frontend_GPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3736,7 +3736,7 @@ def shard_run_frontend_GPU_3_of_6() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/frontend_GPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3808,7 +3808,7 @@ def shard_run_frontend_GPU_4_of_6() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/frontend_GPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3880,7 +3880,7 @@ def shard_run_frontend_GPU_5_of_6() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/frontend_GPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -3952,7 +3952,7 @@ def shard_run_frontend_GPU_6_of_6() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/frontend_GPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -4030,7 +4030,7 @@ def shard_run_topi_aarch64_1_of_2() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/topi_aarch64 --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -4106,7 +4106,7 @@ def shard_run_topi_aarch64_2_of_2() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/topi_aarch64 --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -4179,7 +4179,7 @@ def shard_run_frontend_aarch64_1_of_2() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/frontend_aarch64 --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -4251,7 +4251,7 @@ def shard_run_frontend_aarch64_2_of_2() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/frontend_aarch64 --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -4329,7 +4329,7 @@ def shard_run_test_Cortex_M_1_of_8() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Cortex_M --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -4401,7 +4401,7 @@ def shard_run_test_Cortex_M_2_of_8() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Cortex_M --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -4473,7 +4473,7 @@ def shard_run_test_Cortex_M_3_of_8() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Cortex_M --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -4545,7 +4545,7 @@ def shard_run_test_Cortex_M_4_of_8() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Cortex_M --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -4617,7 +4617,7 @@ def shard_run_test_Cortex_M_5_of_8() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Cortex_M --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -4689,7 +4689,7 @@ def shard_run_test_Cortex_M_6_of_8() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Cortex_M --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -4761,7 +4761,7 @@ def shard_run_test_Cortex_M_7_of_8() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Cortex_M --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -4833,7 +4833,7 @@ def shard_run_test_Cortex_M_8_of_8() {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/test_Cortex_M --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -5067,7 +5067,7 @@ stage('Test') {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/unittest_CPU --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -5135,7 +5135,7 @@ stage('Test') {
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/frontend_CPU --recursive
""",
label: 'Upload JUnits to S3',
)
diff --git a/ci/jenkins/macros.j2 b/ci/jenkins/macros.j2
index 9f4e1cd874..22afdd5df6 100644
--- a/ci/jenkins/macros.j2
+++ b/ci/jenkins/macros.j2
@@ -19,11 +19,11 @@
"workspace/exec_${env.EXECUTOR_NUMBER}/{{ folder }}"
{%- endmacro -%}
-{% macro junit_to_s3() %}
+{% macro junit_to_s3(test_dir_name) %}
sh(
script: """
set -eux
- aws s3 cp --no-progress build/pytest-results s3://${s3_prefix}/pytest-results --recursive
+ aws s3 cp --no-progress build/pytest-results/* s3://${s3_prefix}/pytest-results/{{ test_dir_name }} --recursive
""",
label: 'Upload JUnits to S3',
)
@@ -33,6 +33,7 @@ sh(
{% for shard_index in range(1, num_shards + 1) %}
{% set method_name = "shard_run_" + name.replace(":", "").replace(" ", "-").replace("-", "_") + "_" + shard_index|string + "_of_" + num_shards|string %}
+{% set test_dir_name = name.replace(":", "").replace(" ", "-").replace("-", "_")|string %}
def {{ method_name }}() {
if (!skip_ci && is_docs_only_build != 1) {
node('{{ node }}') {
@@ -49,7 +50,7 @@ def {{ method_name }}() {
})
}
} finally {
- {{ junit_to_s3() }}
+ {{ junit_to_s3(test_dir_name) }}
junit 'build/pytest-results/*.xml'
}
}
@@ -85,6 +86,7 @@ def {{ method_name }}() {
{% macro test_step(name, node, ws, docker_image, platform) %}
+{% set test_dir_name = name.replace(":", "").replace(" ", "-").replace("-", "_")|string %}
'{{ name }}': {
if (!skip_ci && is_docs_only_build != 1) {
node('{{ node }}') {
@@ -97,7 +99,7 @@ def {{ method_name }}() {
{{ caller() | indent(width=12) | trim }}
})
} finally {
- {{ junit_to_s3() | indent(width=4) }}
+ {{ junit_to_s3(test_dir_name) | indent(width=4) }}
junit 'build/pytest-results/*.xml'
}
}
diff --git a/conftest.py b/conftest.py
index 78f028052c..0583dac201 100644
--- a/conftest.py
+++ b/conftest.py
@@ -92,14 +92,11 @@ def pytest_collection_modifyitems(config, items):
shard_index = int(os.environ["TVM_SHARD_INDEX"])
print(f"Marking tests for shard {shard_index} of {num_shards}")
- for item in items:
+ items_copy = list(items)
+ for item in items_copy:
item_shard_index = find_shard_index(item.nodeid, num_shards=num_shards)
- item.add_marker(
- pytest.mark.skipif(
- item_shard_index != shard_index,
- reason=f"Test running on shard {item_shard_index} of {num_shards}",
- )
- )
+ if item_shard_index != shard_index:
+ items.remove(item)
def pytest_sessionstart():
diff --git a/tests/scripts/setup-pytest-env.sh b/tests/scripts/setup-pytest-env.sh
index 1e4883f13d..3774ea10f6 100755
--- a/tests/scripts/setup-pytest-env.sh
+++ b/tests/scripts/setup-pytest-env.sh
@@ -58,19 +58,29 @@ function run_pytest() {
exit 2
fi
+ # Allow unbound variable here.
+ set +u
+ if [[ -z "${TVM_SHARD_INDEX}" ]]; then
+ current_shard="no-shard"
+ else
+ current_shard="shard-${TVM_SHARD_INDEX}"
+ fi
+ set -u
+
has_reruns=$(python3 -m pytest --help 2>&1 | grep 'reruns=' || true)
if [ -n "$has_reruns" ]; then
extra_args+=('--reruns=3')
fi
- suite_name="${test_suite_name}-${ffi_type}"
+ suite_name="${test_suite_name}-${current_shard}-${ffi_type}"
exit_code=0
TVM_FFI=${ffi_type} python3 -m pytest \
-o "junit_suite_name=${suite_name}" \
"--junit-xml=${TVM_PYTEST_RESULT_DIR}/${suite_name}.xml" \
"--junit-prefix=${ffi_type}" \
"${extra_args[@]}" || exit_code=$?
- if [ "$exit_code" -ne "0" ]; then
+ # Pytest will return error code -5 if no test is collected.
+ if [ "$exit_code" -ne "0" ] && [ "$exit_code" -ne "5" ]; then
pytest_errors+=("${suite_name}: $@")
fi
}