You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by ar...@apache.org on 2022/06/09 19:23:28 UTC
[tvm] 08/25: fix docker images to built ones for testing.
This is an automated email from the ASF dual-hosted git repository.
areusch pushed a commit to branch areusch/freeze-dependencies
in repository https://gitbox.apache.org/repos/asf/tvm.git
commit dc5f159b8210dbe9a485bc9a203b501fa78d7eda
Author: Andrew Reusch <ar...@gmail.com>
AuthorDate: Tue May 24 12:28:47 2022 -0700
fix docker images to built ones for testing.
---
Jenkinsfile | 105 ++++++++++++++++++++++++++++++++++++------
jenkins/Build.groovy.j2 | 7 +++
jenkins/Deploy.groovy.j2 | 1 +
jenkins/DockerBuild.groovy.j2 | 53 +++++++++++++++++++++
jenkins/Jenkinsfile.j2 | 34 +++++++++-----
jenkins/Lint.groovy.j2 | 1 +
jenkins/Test.groovy.j2 | 4 ++
7 files changed, 180 insertions(+), 25 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 347d6725ab..34a24e4d5d 100755
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -45,18 +45,27 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-05-24T12:25:02.041933
+// Generated at 2022-05-25T11:48:25.852378
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. -->
-ci_lint = 'tlcpack/ci-lint:20220513-055910-fa834f67e'
-ci_gpu = 'tlcpack/ci-gpu:20220519-055908-ddfa1da69'
-ci_cpu = 'tlcpack/ci-cpu:20220519-055908-ddfa1da69'
-ci_wasm = 'tlcpack/ci-wasm:20220513-055910-fa834f67e'
-ci_i386 = 'tlcpack/ci-i386:20220513-055910-fa834f67e'
-ci_qemu = 'tlcpack/ci-qemu:20220517-094028-de21c8f2e'
-ci_arm = 'tlcpack/ci-arm:20220513-055910-fa834f67e'
-ci_hexagon = 'tlcpack/ci-hexagon:20220516-190055-672ce3365'
+def aws_account_id = ''
+node("CPU-SMALL") {
+aws_account_id = sh(
+ returnStdout: true,
+ script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
+ label: 'Get AWS ID'
+ ).trim()
+}
+def AWS_DEFAULT_REGION = 'us-west-2'
+ci_lint = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_lint:areusch-2ffreeze-dependencies-4233ca6eb-49"
+ci_gpu = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_gpu:areusch-2ffreeze-dependencies-6799bd6f9-49"
+ci_cpu = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_cpu:areusch-2ffreeze-dependencies-98aa6dde6-49"
+ci_wasm = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_wasm:areusch-2ffreeze-dependencies-c12a78740-49"
+ci_i386 = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_i386:areusch-2ffreeze-dependencies-89e22c948-49"
+ci_qemu = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_qemu:areusch-2ffreeze-dependencies-e839c36c8-48"
+ci_arm = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_arm:areusch-2ffreeze-dependencies-b9d48a35f-49"
+ci_hexagon = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_hexagon:areusch-2ffreeze-dependencies-e7d44544e-49"
// <--- End of regex-scanned config.
// Parameters to allow overriding (in Jenkins UI), the images
@@ -94,6 +103,7 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) {
return
}
+
// Filenames for stashing between build and test steps
s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
@@ -429,6 +439,59 @@ def build_docker_images() {
// }
}
+def ecr_pull(full_name) {
+ aws_account_id = sh(
+ returnStdout: true,
+ script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
+ label: 'Get AWS ID'
+ ).trim()
+
+ try {
+ withEnv([
+ "AWS_ACCOUNT_ID=${aws_account_id}",
+ 'AWS_DEFAULT_REGION=us-west-2',
+ "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
+ sh(
+ script: '''
+ set -eux
+ aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
+ ''',
+ label: 'Log in to ECR'
+ )
+ sh(
+ script: """
+ set -eux
+ docker pull ${full_name}
+ """,
+ label: 'Pull image from ECR'
+ )
+ }
+ } finally {
+ withEnv([
+ "AWS_ACCOUNT_ID=${aws_account_id}",
+ 'AWS_DEFAULT_REGION=us-west-2',
+ "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
+ sh(
+ script: 'docker logout $AWS_ECR_REPO',
+ label: 'Clean up login credentials'
+ )
+ }
+ }
+}
+
+def docker_init(image) {
+ if (image.contains("amazonaws.com")) {
+ // If this string is in the image name it's from ECR and needs to be pulled
+ // with the right credentials
+ ecr_pull(image)
+ } else {
+ sh(
+ script: "docker pull ${image}",
+ label: 'Pull docker image',
+ )
+ }
+}
+
// Run make. First try to do an incremental make from a previous workspace in hope to
// accelerate the compilation. If something is wrong, clean the workspace and then
// build from scratch.
@@ -463,7 +526,8 @@ def lint() {
withEnv([
'TVM_NUM_SHARDS=2',
'TVM_SHARD_INDEX=0'], {
- sh (
+ docker_init(ci_lint)
+ sh (
script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
label: 'Run lint',
)
@@ -481,7 +545,8 @@ def lint() {
withEnv([
'TVM_NUM_SHARDS=2',
'TVM_SHARD_INDEX=1'], {
- sh (
+ docker_init(ci_lint)
+ sh (
script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
label: 'Run lint',
)
@@ -555,6 +620,7 @@ stage('Build') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-gpu") {
docker_init(ci_gpu)
init_git()
+ docker_init(ci_gpu)
sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
make("${ci_gpu} --no-gpu", 'build', '-j2')
sh(
@@ -602,6 +668,7 @@ stage('Build') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-cpu") {
docker_init(ci_cpu)
init_git()
+ docker_init(ci_cpu)
sh (
script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build",
label: 'Create CPU cmake config',
@@ -642,6 +709,7 @@ stage('Build') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-wasm") {
docker_init(ci_wasm)
init_git()
+ docker_init(ci_wasm)
sh (
script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build",
label: 'Create WASM cmake config',
@@ -667,6 +735,7 @@ stage('Build') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-i386") {
docker_init(ci_i386)
init_git()
+ docker_init(ci_i386)
sh (
script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build",
label: 'Create i386 cmake config',
@@ -701,6 +770,7 @@ stage('Build') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-arm") {
docker_init(ci_arm)
init_git()
+ docker_init(ci_arm)
sh (
script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build",
label: 'Create ARM cmake config',
@@ -733,6 +803,7 @@ stage('Build') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-qemu") {
docker_init(ci_qemu)
init_git()
+ docker_init(ci_qemu)
sh (
script: "${docker_run} ${ci_qemu} ./tests/scripts/task_config_build_qemu.sh build",
label: 'Create QEMU cmake config',
@@ -764,6 +835,7 @@ stage('Build') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/build-hexagon") {
docker_init(ci_hexagon)
init_git()
+ docker_init(ci_hexagon)
sh (
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build",
label: 'Create Hexagon cmake config',
@@ -3039,6 +3111,7 @@ stage('Test') {
docker_init(ci_cpu)
init_git()
withEnv(['PLATFORM=cpu'], {
+ docker_init(ci_cpu)
sh(
script: """
set -eux
@@ -3092,6 +3165,7 @@ stage('Test') {
docker_init(ci_qemu)
init_git()
withEnv(['PLATFORM=qemu'], {
+ docker_init(ci_qemu)
sh(
script: """
set -eux
@@ -3145,6 +3219,7 @@ stage('Test') {
docker_init(ci_cpu)
init_git()
withEnv(['PLATFORM=cpu'], {
+ docker_init(ci_cpu)
sh(
script: """
set -eux
@@ -3190,6 +3265,7 @@ stage('Test') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/docs-python-gpu") {
docker_init(ci_gpu)
init_git()
+ docker_init(ci_gpu)
sh(
script: """
set -eux
@@ -3297,6 +3373,7 @@ def deploy() {
if (env.BRANCH_NAME == 'main' && env.DOCS_DEPLOY_ENABLED == 'yes') {
node('CPU') {
ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/deploy-docs") {
+ docker_init(ci_gpu)
sh(
script: """
set -eux
@@ -3318,9 +3395,9 @@ cancel_previous_build()
prepare()
-if (rebuild_docker_images) {
- build_docker_images()
-}
+// if (rebuild_docker_images) {
+// build_docker_images()
+// }
lint()
diff --git a/jenkins/Build.groovy.j2 b/jenkins/Build.groovy.j2
index 62ccc94916..a4bb11f0cf 100644
--- a/jenkins/Build.groovy.j2
+++ b/jenkins/Build.groovy.j2
@@ -64,6 +64,7 @@ stage('Build') {
ws({{ m.per_exec_ws('tvm/build-gpu') }}) {
docker_init(ci_gpu)
init_git()
+ docker_init(ci_gpu)
sh "${docker_run} --no-gpu ${ci_gpu} ./tests/scripts/task_config_build_gpu.sh build"
make("${ci_gpu} --no-gpu", 'build', '-j2')
{{ m.upload_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }}
@@ -82,6 +83,7 @@ stage('Build') {
ws({{ m.per_exec_ws('tvm/build-cpu') }}) {
docker_init(ci_cpu)
init_git()
+ docker_init(ci_cpu)
sh (
script: "${docker_run} ${ci_cpu} ./tests/scripts/task_config_build_cpu.sh build",
label: 'Create CPU cmake config',
@@ -106,6 +108,7 @@ stage('Build') {
ws({{ m.per_exec_ws('tvm/build-wasm') }}) {
docker_init(ci_wasm)
init_git()
+ docker_init(ci_wasm)
sh (
script: "${docker_run} ${ci_wasm} ./tests/scripts/task_config_build_wasm.sh build",
label: 'Create WASM cmake config',
@@ -131,6 +134,7 @@ stage('Build') {
ws({{ m.per_exec_ws('tvm/build-i386') }}) {
docker_init(ci_i386)
init_git()
+ docker_init(ci_i386)
sh (
script: "${docker_run} ${ci_i386} ./tests/scripts/task_config_build_i386.sh build",
label: 'Create i386 cmake config',
@@ -149,6 +153,7 @@ stage('Build') {
ws({{ m.per_exec_ws('tvm/build-arm') }}) {
docker_init(ci_arm)
init_git()
+ docker_init(ci_arm)
sh (
script: "${docker_run} ${ci_arm} ./tests/scripts/task_config_build_arm.sh build",
label: 'Create ARM cmake config',
@@ -167,6 +172,7 @@ stage('Build') {
ws({{ m.per_exec_ws('tvm/build-qemu') }}) {
docker_init(ci_qemu)
init_git()
+ docker_init(ci_qemu)
sh (
script: "${docker_run} ${ci_qemu} ./tests/scripts/task_config_build_qemu.sh build",
label: 'Create QEMU cmake config',
@@ -185,6 +191,7 @@ stage('Build') {
ws({{ m.per_exec_ws('tvm/build-hexagon') }}) {
docker_init(ci_hexagon)
init_git()
+ docker_init(ci_hexagon)
sh (
script: "${docker_run} ${ci_hexagon} ./tests/scripts/task_config_build_hexagon.sh build",
label: 'Create Hexagon cmake config',
diff --git a/jenkins/Deploy.groovy.j2 b/jenkins/Deploy.groovy.j2
index 917f71ded1..02b7f5a40a 100644
--- a/jenkins/Deploy.groovy.j2
+++ b/jenkins/Deploy.groovy.j2
@@ -62,6 +62,7 @@ def deploy() {
if (env.BRANCH_NAME == 'main' && env.DOCS_DEPLOY_ENABLED == 'yes') {
node('CPU') {
ws({{ m.per_exec_ws('tvm/deploy-docs') }}) {
+ docker_init(ci_gpu)
{{ m.download_artifacts(tag='docs', filenames=["docs.tgz"]) }}
deploy_docs()
}
diff --git a/jenkins/DockerBuild.groovy.j2 b/jenkins/DockerBuild.groovy.j2
index 68cbe49413..c7c1a3acdb 100644
--- a/jenkins/DockerBuild.groovy.j2
+++ b/jenkins/DockerBuild.groovy.j2
@@ -154,6 +154,59 @@ def build_docker_images() {
// }
}
+def ecr_pull(full_name) {
+ aws_account_id = sh(
+ returnStdout: true,
+ script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
+ label: 'Get AWS ID'
+ ).trim()
+
+ try {
+ withEnv([
+ "AWS_ACCOUNT_ID=${aws_account_id}",
+ 'AWS_DEFAULT_REGION={{ aws_default_region }}',
+ "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) {
+ sh(
+ script: '''
+ set -eux
+ aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
+ ''',
+ label: 'Log in to ECR'
+ )
+ sh(
+ script: """
+ set -eux
+ docker pull ${full_name}
+ """,
+ label: 'Pull image from ECR'
+ )
+ }
+ } finally {
+ withEnv([
+ "AWS_ACCOUNT_ID=${aws_account_id}",
+ 'AWS_DEFAULT_REGION={{ aws_default_region }}',
+ "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) {
+ sh(
+ script: 'docker logout $AWS_ECR_REPO',
+ label: 'Clean up login credentials'
+ )
+ }
+ }
+}
+
+def docker_init(image) {
+ if (image.contains("amazonaws.com")) {
+ // If this string is in the image name it's from ECR and needs to be pulled
+ // with the right credentials
+ ecr_pull(image)
+ } else {
+ sh(
+ script: "docker pull ${image}",
+ label: 'Pull docker image',
+ )
+ }
+}
+
// Run make. First try to do an incremental make from a previous workspace in hope to
// accelerate the compilation. If something is wrong, clean the workspace and then
// build from scratch.
diff --git a/jenkins/Jenkinsfile.j2 b/jenkins/Jenkinsfile.j2
index c165de964f..e90f161858 100644
--- a/jenkins/Jenkinsfile.j2
+++ b/jenkins/Jenkinsfile.j2
@@ -51,14 +51,23 @@ import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
{% import 'jenkins/macros.j2' as m with context -%}
// NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. -->
-ci_lint = 'tlcpack/ci-lint:20220513-055910-fa834f67e'
-ci_gpu = 'tlcpack/ci-gpu:20220519-055908-ddfa1da69'
-ci_cpu = 'tlcpack/ci-cpu:20220519-055908-ddfa1da69'
-ci_wasm = 'tlcpack/ci-wasm:20220513-055910-fa834f67e'
-ci_i386 = 'tlcpack/ci-i386:20220513-055910-fa834f67e'
-ci_qemu = 'tlcpack/ci-qemu:20220517-094028-de21c8f2e'
-ci_arm = 'tlcpack/ci-arm:20220513-055910-fa834f67e'
-ci_hexagon = 'tlcpack/ci-hexagon:20220516-190055-672ce3365'
+def aws_account_id = ''
+node("CPU-SMALL") {
+aws_account_id = sh(
+ returnStdout: true,
+ script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
+ label: 'Get AWS ID'
+ ).trim()
+}
+def AWS_DEFAULT_REGION = 'us-west-2'
+ci_lint = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_lint:areusch-2ffreeze-dependencies-4233ca6eb-49"
+ci_gpu = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_gpu:areusch-2ffreeze-dependencies-6799bd6f9-49"
+ci_cpu = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_cpu:areusch-2ffreeze-dependencies-98aa6dde6-49"
+ci_wasm = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_wasm:areusch-2ffreeze-dependencies-c12a78740-49"
+ci_i386 = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_i386:areusch-2ffreeze-dependencies-89e22c948-49"
+ci_qemu = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_qemu:areusch-2ffreeze-dependencies-e839c36c8-48"
+ci_arm = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_arm:areusch-2ffreeze-dependencies-b9d48a35f-49"
+ci_hexagon = "${aws_account_id}.dkr.ecr.${AWS_DEFAULT_REGION}.amazonaws.com/ci_hexagon:areusch-2ffreeze-dependencies-e7d44544e-49"
// <--- End of regex-scanned config.
// Parameters to allow overriding (in Jenkins UI), the images
@@ -91,6 +100,9 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) {
return
}
+{% set aws_default_region = "us-west-2" %}
+{% set aws_ecr_url = "dkr.ecr." + aws_default_region + ".amazonaws.com" %}
+
// Filenames for stashing between build and test steps
{% set tvm_runtime = ['build/libtvm_runtime.so', 'build/config.cmake'] %}
{% set tvm_lib = ['build/libtvm.so'] + tvm_runtime %}
@@ -115,9 +127,9 @@ cancel_previous_build()
prepare()
-if (rebuild_docker_images) {
- build_docker_images()
-}
+// if (rebuild_docker_images) {
+// build_docker_images()
+// }
lint()
diff --git a/jenkins/Lint.groovy.j2 b/jenkins/Lint.groovy.j2
index 40dad3aef7..717ba79348 100644
--- a/jenkins/Lint.groovy.j2
+++ b/jenkins/Lint.groovy.j2
@@ -9,6 +9,7 @@ def lint() {
docker_image='ci_lint',
)
%}
+ docker_init(ci_lint)
sh (
script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
label: 'Run lint',
diff --git a/jenkins/Test.groovy.j2 b/jenkins/Test.groovy.j2
index d219b47bc7..393ed6243a 100644
--- a/jenkins/Test.groovy.j2
+++ b/jenkins/Test.groovy.j2
@@ -206,6 +206,7 @@ stage('Test') {
platform="cpu",
docker_image="ci_cpu",
) %}
+ docker_init(ci_cpu)
{{ m.download_artifacts(tag='cpu', filenames=tvm_multilib_tsim) }}
ci_setup(ci_cpu)
cpp_unittest(ci_cpu)
@@ -223,6 +224,7 @@ stage('Test') {
platform="qemu",
docker_image="ci_qemu",
) %}
+ docker_init(ci_qemu)
{{ m.download_artifacts(tag='qemu', filenames=tvm_lib, folders=microtvm_template_projects) }}
add_microtvm_permissions()
ci_setup(ci_qemu)
@@ -243,6 +245,7 @@ stage('Test') {
platform="cpu",
docker_image="ci_cpu",
) %}
+ docker_init(ci_cpu)
{{ m.download_artifacts(tag='cpu', filenames=tvm_multilib) }}
ci_setup(ci_cpu)
sh (
@@ -256,6 +259,7 @@ stage('Test') {
ws({{ m.per_exec_ws('tvm/docs-python-gpu') }}) {
docker_init(ci_gpu)
init_git()
+ docker_init(ci_gpu)
{{ m.download_artifacts(tag='gpu', filenames=tvm_multilib, folders=microtvm_template_projects) }}
add_microtvm_permissions()
timeout(time: 180, unit: 'MINUTES') {