You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by sy...@apache.org on 2023/11/27 08:38:18 UTC
(tvm) branch unity_ci created (now 17026f06c6)
This is an automated email from the ASF dual-hosted git repository.
syfeng pushed a change to branch unity_ci
in repository https://gitbox.apache.org/repos/asf/tvm.git
at 17026f06c6 upd ci
This branch includes the following new commits:
new 99cd823d8f upd ci
new 17026f06c6 upd ci
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(tvm) 02/02: upd ci
Posted by sy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
syfeng pushed a commit to branch unity_ci
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 17026f06c6bc712abbd7b7ef06c30a1c1146325a
Author: Siyuan Feng <hz...@sjtu.edu.cn>
AuthorDate: Mon Nov 27 16:37:55 2023 +0800
upd ci
---
tests/scripts/unity/task_python_tir.sh | 51 ++++++++++++++++++++++++++
tests/scripts/unity/task_python_tir_gpuonly.sh | 27 ++++++++++++++
2 files changed, 78 insertions(+)
diff --git a/tests/scripts/unity/task_python_tir.sh b/tests/scripts/unity/task_python_tir.sh
new file mode 100755
index 0000000000..30f5e8a4c9
--- /dev/null
+++ b/tests/scripts/unity/task_python_tir.sh
@@ -0,0 +1,51 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euxo pipefail
+
+source tests/scripts/setup-pytest-env.sh
+
+# cleanup pycache
+find . -type f -path "*.pyc" | xargs rm -f
+make cython3
+
+# NOTE: also set by task_python_unittest_gpuonly.sh.
+if [ -z "${TVM_UNITTEST_TESTSUITE_NAME:-}" ]; then
+ TVM_UNITTEST_TESTSUITE_NAME=python-unittest
+fi
+
+# First run minimal test on both ctypes and cython.
+run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test-0 tests/python/all-platform-minimal-test
+run_pytest cython ${TVM_UNITTEST_TESTSUITE_NAME}-platform-minimal-test-1 tests/python/all-platform-minimal-test
+
+# Then run all unittests on both ctypes and cython.
+TEST_FILES=(
+ "tir-analysis"
+ "tir-base"
+ "tir-schedule"
+ "tir-transform"
+ "tir-usmp"
+)
+
+for TEST_FILE in ${TEST_FILES}; do
+ run_pytest ctypes ${TEST_FILE}-0, tests/python/${TEST_FILE}
+ run_pytest cython ${TEST_FILE}-1, tests/python/${TEST_FILE}
+done
+
+# Then run CI tests
+run_pytest ctypes ${TVM_UNITTEST_TESTSUITE_NAME}-ci tests/python/ci
diff --git a/tests/scripts/unity/task_python_tir_gpuonly.sh b/tests/scripts/unity/task_python_tir_gpuonly.sh
new file mode 100755
index 0000000000..a3c0da3296
--- /dev/null
+++ b/tests/scripts/unity/task_python_tir_gpuonly.sh
@@ -0,0 +1,27 @@
+#!/usr/bin/env bash
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+set -euxo pipefail
+
+export PYTEST_ADDOPTS="-m gpu ${PYTEST_ADDOPTS:-}"
+
+# Test most of the enabled runtimes here.
+export TVM_TEST_TARGETS="cuda;opencl;metal;rocm;nvptx;opencl -device=mali,aocl_sw_emu"
+export TVM_UNITTEST_TESTSUITE_NAME=python-unittest-gpu
+
+./tests/scripts/unity/task_python_tir.sh
\ No newline at end of file
(tvm) 01/02: upd ci
Posted by sy...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
syfeng pushed a commit to branch unity_ci
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit 99cd823d8f572e68650dd1b2bf177908300ad376
Author: Siyuan Feng <hz...@sjtu.edu.cn>
AuthorDate: Mon Nov 27 16:37:48 2023 +0800
upd ci
---
ci/jenkins/unity_jenkinsfile.groovy | 240 ++++++++++++++++--------------------
1 file changed, 108 insertions(+), 132 deletions(-)
diff --git a/ci/jenkins/unity_jenkinsfile.groovy b/ci/jenkins/unity_jenkinsfile.groovy
index 99485f7c55..31d3325889 100644
--- a/ci/jenkins/unity_jenkinsfile.groovy
+++ b/ci/jenkins/unity_jenkinsfile.groovy
@@ -26,7 +26,6 @@
// This file is manually updated to maintain unity branch specific builds.
// Please do not send this file to main
-
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. -->
@@ -40,6 +39,25 @@ ci_arm = 'tlcpack/ci-arm:v0.08'
ci_hexagon = 'tlcpackstaging/ci_hexagon:20230504-142417-4d37a0a0'
// <--- End of regex-scanned config.
+tvm_lib = 'build/libtvm.so, build/libtvm_runtime.so, build/config.cmake'
+tvm_lib_gpu = tvm_lib + ", build/libfpA_intB_gemm.so, build/libflash_attn.so"
+
+def pack_lib(name, libs) {
+ sh """
+ echo "Packing ${libs} into ${name}"
+ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+ """
+ stash includes: libs, name: name
+}
+
+def unpack_lib(name, libs) {
+ unstash name
+ sh """
+ echo "Unpacked ${libs} from ${name}"
+ echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
+ """
+}
+
// Parameters to allow overriding (in Jenkins UI), the images
// to be used by a given build. When provided, they take precedence
// over default values above.
@@ -136,42 +154,37 @@ def should_skip_ci(pr_number) {
cancel_previous_build()
def lint() {
-stage('Prepare') {
- node('CPU-SMALL') {
- // When something is provided in ci_*_param, use it, otherwise default with ci_*
- ci_lint = params.ci_lint_param ?: ci_lint
- ci_cpu = params.ci_cpu_param ?: ci_cpu
- ci_gpu = params.ci_gpu_param ?: ci_gpu
- ci_wasm = params.ci_wasm_param ?: ci_wasm
- ci_i386 = params.ci_i386_param ?: ci_i386
- ci_qemu = params.ci_qemu_param ?: ci_qemu
- ci_arm = params.ci_arm_param ?: ci_arm
- ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
-
- sh (script: """
- echo "Docker images being used in this build:"
- echo " ci_lint = ${ci_lint}"
- echo " ci_cpu = ${ci_cpu}"
- echo " ci_gpu = ${ci_gpu}"
- echo " ci_wasm = ${ci_wasm}"
- echo " ci_i386 = ${ci_i386}"
- echo " ci_qemu = ${ci_qemu}"
- echo " ci_arm = ${ci_arm}"
- echo " ci_hexagon = ${ci_hexagon}"
- """, label: 'Docker image names')
+ stage('Prepare') {
+ node('CPU-SMALL') {
+ // When something is provided in ci_*_param, use it, otherwise default with ci_*
+ ci_lint = params.ci_lint_param ?: ci_lint
+ ci_cpu = params.ci_cpu_param ?: ci_cpu
+ ci_gpu = params.ci_gpu_param ?: ci_gpu
+ ci_wasm = params.ci_wasm_param ?: ci_wasm
+ ci_i386 = params.ci_i386_param ?: ci_i386
+ ci_qemu = params.ci_qemu_param ?: ci_qemu
+ ci_arm = params.ci_arm_param ?: ci_arm
+ ci_hexagon = params.ci_hexagon_param ?: ci_hexagon
+
+ sh (script: """
+ echo "Docker images being used in this build:"
+ echo " ci_lint = ${ci_lint}"
+ echo " ci_cpu = ${ci_cpu}"
+ echo " ci_gpu = ${ci_gpu}"
+ echo " ci_wasm = ${ci_wasm}"
+ echo " ci_i386 = ${ci_i386}"
+ echo " ci_qemu = ${ci_qemu}"
+ echo " ci_arm = ${ci_arm}"
+ echo " ci_hexagon = ${ci_hexagon}"
+ """, label: 'Docker image names')
+ }
}
-}
stage('Sanity Check') {
timeout(time: max_time, unit: 'MINUTES') {
node('CPU-SMALL') {
ws(per_exec_ws('tvm/sanity')) {
init_git()
- is_docs_only_build = sh (
- returnStatus: true,
- script: './tests/scripts/git_change_docs.sh',
- label: 'Check for docs only changes',
- )
skip_ci = should_skip_ci(env.CHANGE_ID)
skip_slow_tests = should_skip_slow_tests(env.CHANGE_ID)
sh (
@@ -186,152 +199,115 @@ stage('Sanity Check') {
}
}
}
-}
lint()
+// ********************** BUILD Stage **********************
+
// Run make. First try to do an incremental make from a previous workspace in hope to
// accelerate the compilation. If something is wrong, clean the workspace and then
// build from scratch.
-def make(docker_type, path, make_flag) {
+def make(docker_type) {
timeout(time: max_time, unit: 'MINUTES') {
try {
- cmake_build(docker_type, path, make_flag)
- // always run cpp test when build
- // sh "${docker_run} ${docker_type} ./tests/scripts/task_cpp_unittest.sh"
+ cmake_build(docker_type)
} catch (hudson.AbortException ae) {
// script exited due to user abort, directly throw instead of retry
if (ae.getMessage().contains('script returned exit code 143')) {
throw ae
}
echo 'Incremental compilation failed. Fall back to build from scratch'
- sh (
- script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}",
+ sh(
+ script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh build",
label: 'Clear old cmake workspace',
)
- cmake_build(docker_type, path, make_flag)
- cpp_unittest(docker_type)
+ cmake_build(docker_type)
+ // cpp_unittest(docker_type)
}
}
}
// Specifications to Jenkins "stash" command for use with various pack_ and unpack_ functions.
-tvm_runtime = 'build/libtvm_runtime.so, build/config.cmake' // use libtvm_runtime.so.
-tvm_lib = 'build/libtvm.so, ' + tvm_runtime // use libtvm.so to run the full compiler.
+tvm_runtime = 'build/libtvm_runtime.so, build/config.cmake' // use libtvm_runtime.so.
+tvm_lib = 'build/libtvm.so, ' + tvm_runtime // use libtvm.so to run the full compiler.
// LLVM upstream lib
tvm_multilib = 'build/libtvm.so, ' +
- 'build/libvta_fsim.so, ' +
- tvm_runtime
-
-tvm_multilib_tsim = 'build/libvta_tsim.so, ' +
- tvm_multilib
-
-microtvm_tar_gz = 'build/microtvm_template_projects.tar.gz'
-
-// pack libraries for later use
-def pack_lib(name, libs) {
- sh (script: """
- echo "Packing ${libs} into ${name}"
- echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
- """, label: 'Stash libraries and show md5')
- stash includes: libs, name: name
-}
-
-// unpack libraries saved before
-def unpack_lib(name, libs) {
- unstash name
- sh (script: """
- echo "Unpacked ${libs} from ${name}"
- echo ${libs} | sed -e 's/,/ /g' | xargs md5sum
- """, label: 'Unstash libraries and show md5')
-}
-
-// compress microtvm template projects and pack the tar.
-def pack_microtvm_template_projects(name) {
- sh(
- script: 'cd build && tar -czvf microtvm_template_projects.tar.gz microtvm_template_projects/',
- label: 'Compress microtvm_template_projects'
- )
- pack_lib(name + '-microtvm-libs', microtvm_tar_gz)
-}
+ 'build/libvta_fsim.so, ' +
+ tvm_runtime
-def unpack_microtvm_template_projects(name) {
- unpack_lib(name + '-microtvm-libs', microtvm_tar_gz)
+def cmake_build(image) {
sh(
- script: 'cd build && tar -xzvf microtvm_template_projects.tar.gz',
- label: 'Unpack microtvm_template_projects'
- )
-}
-
-def ci_setup(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_ci_setup.sh",
- label: 'Set up CI environment',
- )
-}
-
-def python_unittest(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_python_unittest.sh",
- label: 'Run Python unit tests',
- )
-}
-
-def fsim_test(image) {
- sh (
- script: "${docker_run} ${image} ./tests/scripts/task_python_vta_fsim.sh",
- label: 'Run VTA tests in FSIM',
- )
-}
-
-def cmake_build(image, path, make_flag) {
- sh (
script: "${docker_run} ${image} ./tests/scripts/task_build.py --sccache-bucket tvm-sccache-prod",
label: 'Run cmake build',
)
}
def cpp_unittest(image) {
- sh (
+ sh(
script: "${docker_run} ${image} ./tests/scripts/task_cpp_unittest.sh",
label: 'Build and run C++ tests',
)
}
-def add_hexagon_permissions() {
- sh(
- script: 'find build/hexagon_api_output -type f | xargs chmod +x',
- label: 'Add execute permissions for hexagon files',
- )
-}
-
-// NOTE: limit tests to relax folder for now to allow us to skip some of the tests
-// that are mostly related to changes in main.
-// This helps to speedup CI time and reduce CI cost.
-stage('Build and Test') {
- if (is_docs_only_build != 1) {
- parallel 'BUILD: GPU': {
- node('GPU') {
+stage('Build') {
+ parallel 'CPU': {
+ node('CPU') {
+ ws(per_exec_ws('tvm/build-cpu')) {
+ init_git()
+ sh "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build"
+ make(ci_cpu)
+ pack_lib('cpu', tvm_lib)
+ }
+ }
+ },
+ 'GPU': {
+ node('CPU') {
ws(per_exec_ws('tvm/build-gpu')) {
init_git()
sh "${docker_run} ${ci_gpu} nvidia-smi"
sh "${docker_run} ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
- make("${ci_gpu}", 'build', '-j2')
- sh "${docker_run} ${ci_gpu} ./tests/scripts/unity/task_python_relax_gpuonly.sh"
+ make(ci_gpu)
+ pack_lib('gpu', tvm_lib_gpu)
+ }
+ }
+ }
+}
+
+stage('Unittest') {
+ parallel 'TIR: CPU': {
+ node('CPU') {
+ ws(per_exec_ws('tvm/unittest/tir-cpu')) {
+ init_git()
+ unpack_lib('cpu', tvm_lib)
+ sh "${docker_run} ${ci_cpu} ./tests/scripts/unity/task_python_tir.sh"
}
}
},
- 'BUILD: CPU': {
- node('CPU-SMALL') {
- ws(per_exec_ws('tvm/build-cpu')) {
+ 'TIR: GPU': {
+ node('GPU') {
+ ws(per_exec_ws('tvm/unittest/tir-gpu')) {
init_git()
- sh "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build"
- make(ci_cpu, 'build', '-j2')
+ unpack_lib('gpu', tvm_lib_gpu)
+ sh "${docker_run} ${ci_gpu} ./tests/scripts/unity/task_python_tir_gpuonly.sh"
+ }
+ }
+ },
+ 'Relax: CPU': {
+ node('CPU') {
+ ws(per_exec_ws('tvm/unittest/relax-cpu')) {
+ init_git()
+ unpack_lib('cpu', tvm_lib)
sh "${docker_run} ${ci_cpu} ./tests/scripts/unity/task_python_relax.sh"
}
}
- }
- } else {
- Utils.markStageSkippedForConditional('BUILD: CPU')
- }
-}
+ },
+ 'Relax: GPU': {
+ node('GPU') {
+ ws(per_exec_ws('tvm/unittest/relax-gpu')) {
+ init_git()
+ unpack_lib('gpu', tvm_lib_gpu)
+ sh "${docker_run} ${ci_gpu} ./tests/scripts/unity/task_python_relax_gpuonly.sh"
+ }
+ }
+ },
+}
\ No newline at end of file