You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tvm.apache.org by mo...@apache.org on 2022/06/09 22:01:54 UTC
[tvm] branch main updated: [ci] Rebuild Docker images if necessary (#11329)
This is an automated email from the ASF dual-hosted git repository.
mousius pushed a commit to branch main
in repository https://gitbox.apache.org/repos/asf/tvm.git
The following commit(s) were added to refs/heads/main by this push:
new 6d557ffae2 [ci] Rebuild Docker images if necessary (#11329)
6d557ffae2 is described below
commit 6d557ffae2db64fcea127b5e34089d9bc8e74fb0
Author: driazati <94...@users.noreply.github.com>
AuthorDate: Thu Jun 9 15:01:48 2022 -0700
[ci] Rebuild Docker images if necessary (#11329)
This rebuilds Docker images and uses them in later stages in the same build. If the build is running on `main`, then the images are uploaded to Docker Hub automatically once the run is complete. Images are always rebuilt, but Docker Hub functions as a cache. If there have been no changes to `docker/` since the last available hash on Docker Hub, then the build will just use the images from Hub.
---
Jenkinsfile | 393 +++++++++++++++++++++------------
jenkins/Build.groovy.j2 | 23 ++
jenkins/Deploy.groovy.j2 | 50 +++++
jenkins/DockerBuild.groovy.j2 | 240 ++++++++------------
jenkins/Jenkinsfile.j2 | 3 +
jenkins/Lint.groovy.j2 | 10 +-
jenkins/Prepare.groovy.j2 | 11 +
tests/python/ci/test_ci.py | 97 +++++++-
tests/scripts/cmd_utils.py | 21 +-
tests/scripts/git_utils.py | 1 +
tests/scripts/http_utils.py | 34 +++
tests/scripts/should_rebuild_docker.py | 154 +++++++++++++
12 files changed, 737 insertions(+), 300 deletions(-)
diff --git a/Jenkinsfile b/Jenkinsfile
index 0205a1e736..ec4cea52d6 100755
--- a/Jenkinsfile
+++ b/Jenkinsfile
@@ -45,7 +45,7 @@
// 'python3 jenkins/generate.py'
// Note: This timestamp is here to ensure that updates to the Jenkinsfile are
// always rebased on main before merging:
-// Generated at 2022-06-02T14:03:43.284817
+// Generated at 2022-06-09T09:42:12.430625
import org.jenkinsci.plugins.pipeline.modeldefinition.Utils
// NOTE: these lines are scanned by docker/dev_common.sh. Please update the regex as needed. -->
@@ -97,6 +97,7 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) {
// Filenames for stashing between build and test steps
s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
+
// General note: Jenkins has limits on the size of a method (or top level code)
// that are pretty strict, so most usage of groovy methods in these templates
// are purely to satisfy the JVM
@@ -171,6 +172,17 @@ def docker_init(image) {
""",
label: 'Clean old Docker images',
)
+
+ if (image.contains("amazonaws.com")) {
+ // If this string is in the image name it's from ECR and needs to be pulled
+ // with the right credentials
+ ecr_pull(image)
+ } else {
+ sh(
+ script: "docker pull ${image}",
+ label: 'Pull docker image',
+ )
+ }
}
def should_skip_slow_tests(pr_number) {
@@ -273,16 +285,50 @@ def prepare() {
}
}
}
-def build_image(image_name) {
- hash = sh(
+def ecr_push(full_name) {
+ aws_account_id = sh(
returnStdout: true,
- script: 'git log -1 --format=\'%h\''
+ script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
+ label: 'Get AWS ID'
).trim()
- def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}"
- sh(
- script: "${docker_build} ${image_name} --spec ${full_name}",
- label: 'Build docker image'
- )
+
+ def ecr_name = "${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com/${full_name}"
+ try {
+ withEnv([
+ "AWS_ACCOUNT_ID=${aws_account_id}",
+ 'AWS_DEFAULT_REGION=us-west-2',
+ "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
+ sh(
+ script: '''
+ set -eux
+ aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
+ ''',
+ label: 'Log in to ECR'
+ )
+ sh(
+ script: """
+ set -x
+ docker tag ${full_name} \$AWS_ECR_REPO/${full_name}
+ docker push \$AWS_ECR_REPO/${full_name}
+ """,
+ label: 'Upload image to ECR'
+ )
+ }
+ } finally {
+ withEnv([
+ "AWS_ACCOUNT_ID=${aws_account_id}",
+ 'AWS_DEFAULT_REGION=us-west-2',
+ "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
+ sh(
+ script: 'docker logout $AWS_ECR_REPO',
+ label: 'Clean up login credentials'
+ )
+ }
+ }
+ return ecr_name
+}
+
+def ecr_pull(full_name) {
aws_account_id = sh(
returnStdout: true,
script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
@@ -290,153 +336,144 @@ def build_image(image_name) {
).trim()
try {
- // Use a credential so Jenkins knows to scrub the AWS account ID which is nice
- // (but so we don't have to rely it being hardcoded in Jenkins)
- withCredentials([string(
- credentialsId: 'aws-account-id',
- variable: '_ACCOUNT_ID_DO_NOT_USE',
- )]) {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2']) {
- sh(
- script: '''
- set -x
- aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
- ''',
- label: 'Log in to ECR'
- )
- sh(
- script: """
- set -x
- docker tag ${full_name} \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name}
- docker push \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name}
- """,
- label: 'Upload image to ECR'
- )
- }
+ withEnv([
+ "AWS_ACCOUNT_ID=${aws_account_id}",
+ 'AWS_DEFAULT_REGION=us-west-2',
+ "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
+ sh(
+ script: '''
+ set -eux
+ aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
+ ''',
+ label: 'Log in to ECR'
+ )
+ sh(
+ script: """
+ set -eux
+ docker pull ${full_name}
+ """,
+ label: 'Pull image from ECR'
+ )
}
} finally {
- sh(
- script: 'rm -f ~/.docker/config.json',
- label: 'Clean up login credentials'
- )
+ withEnv([
+ "AWS_ACCOUNT_ID=${aws_account_id}",
+ 'AWS_DEFAULT_REGION=us-west-2',
+ "AWS_ECR_REPO=${aws_account_id}.dkr.ecr.us-west-2.amazonaws.com"]) {
+ sh(
+ script: 'docker logout $AWS_ECR_REPO',
+ label: 'Clean up login credentials'
+ )
+ }
}
+}
+
+def build_image(image_name) {
+ hash = sh(
+ returnStdout: true,
+ script: 'git log -1 --format=\'%h\''
+ ).trim()
+ def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}"
sh(
- script: "docker rmi ${full_name}",
- label: 'Remove docker image'
+ script: "${docker_build} ${image_name} --spec ${full_name}",
+ label: 'Build docker image'
)
+ return ecr_push(full_name)
}
+
def build_docker_images() {
stage('Docker Image Build') {
- // TODO in a follow up PR: Find ecr tag and use in subsequent builds
- parallel 'ci-lint': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_lint')
+ parallel(
+ 'ci_arm': {
+ node('ARM') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ // We're purposefully not setting the built image here since they
+ // are not yet being uploaded to tlcpack
+ // ci_arm = build_image('ci_arm')
+ build_image('ci_arm')
+ }
}
- }
- }, 'ci-cpu': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_cpu')
+ },
+ 'ci_cpu': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ // We're purposefully not setting the built image here since they
+ // are not yet being uploaded to tlcpack
+ // ci_cpu = build_image('ci_cpu')
+ build_image('ci_cpu')
+ }
}
- }
- }, 'ci-gpu': {
- node('GPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_gpu')
+ },
+ 'ci_gpu': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ // We're purposefully not setting the built image here since they
+ // are not yet being uploaded to tlcpack
+ // ci_gpu = build_image('ci_gpu')
+ build_image('ci_gpu')
+ }
}
- }
- }, 'ci-qemu': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_qemu')
+ },
+ 'ci_hexagon': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ // We're purposefully not setting the built image here since they
+ // are not yet being uploaded to tlcpack
+ // ci_hexagon = build_image('ci_hexagon')
+ build_image('ci_hexagon')
+ }
}
- }
- }, 'ci-i386': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_i386')
+ },
+ 'ci_i386': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ // We're purposefully not setting the built image here since they
+ // are not yet being uploaded to tlcpack
+ // ci_i386 = build_image('ci_i386')
+ build_image('ci_i386')
+ }
}
- }
- }, 'ci-arm': {
- node('ARM') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_arm')
+ },
+ 'ci_lint': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ // We're purposefully not setting the built image here since they
+ // are not yet being uploaded to tlcpack
+ // ci_lint = build_image('ci_lint')
+ build_image('ci_lint')
+ }
}
- }
- }, 'ci-wasm': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_wasm')
+ },
+ 'ci_qemu': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ // We're purposefully not setting the built image here since they
+ // are not yet being uploaded to tlcpack
+ // ci_qemu = build_image('ci_qemu')
+ build_image('ci_qemu')
+ }
}
- }
- }, 'ci-hexagon': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_hexagon')
+ },
+ 'ci_wasm': {
+ node('CPU') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ // We're purposefully not setting the built image here since they
+ // are not yet being uploaded to tlcpack
+ // ci_wasm = build_image('ci_wasm')
+ build_image('ci_wasm')
+ }
}
- }
- }
- }
- // // TODO: Once we are able to use the built images, enable this step
- // // If the docker images changed, we need to run the image build before the lint
- // // can run since it requires a base docker image. Most of the time the images
- // // aren't build though so it's faster to use the same node that checks for
- // // docker changes to run the lint in the usual case.
- // stage('Sanity Check (re-run)') {
- // timeout(time: max_time, unit: 'MINUTES') {
- // node('CPU') {
- // ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/sanity") {
- // init_git()
- // sh (
- // script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
- // label: 'Run lint',
- // )
- // }
- // }
- // }
- // }
-}
-
-// Run make. First try to do an incremental make from a previous workspace in hope to
-// accelerate the compilation. If something is wrong, clean the workspace and then
-// build from scratch.
-def make(docker_type, path, make_flag) {
- timeout(time: max_time, unit: 'MINUTES') {
- try {
- cmake_build(docker_type, path, make_flag)
- // always run cpp test when build
- } catch (hudson.AbortException ae) {
- // script exited due to user abort, directly throw instead of retry
- if (ae.getMessage().contains('script returned exit code 143')) {
- throw ae
- }
- echo 'Incremental compilation failed. Fall back to build from scratch'
- sh (
- script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}",
- label: 'Clear old cmake workspace',
- )
- cmake_build(docker_type, path, make_flag)
- }
+ },
+ )
}
}
def lint() {
@@ -531,6 +568,29 @@ def add_hexagon_permissions() {
)
}
+// Run make. First try to do an incremental make from a previous workspace in hope to
+// accelerate the compilation. If something is wrong, clean the workspace and then
+// build from scratch.
+def make(docker_type, path, make_flag) {
+ timeout(time: max_time, unit: 'MINUTES') {
+ try {
+ cmake_build(docker_type, path, make_flag)
+ } catch (hudson.AbortException ae) {
+ // script exited due to user abort, directly throw instead of retry
+ if (ae.getMessage().contains('script returned exit code 143')) {
+ throw ae
+ }
+ echo 'Incremental compilation failed. Fall back to build from scratch'
+ sh (
+ script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}",
+ label: 'Clear old cmake workspace',
+ )
+ cmake_build(docker_type, path, make_flag)
+ }
+ }
+}
+
+
def build() {
stage('Build') {
environment {
@@ -3239,6 +3299,25 @@ stage('Build packages') {
}
*/
+
+def update_docker(ecr_image, hub_image) {
+ if (!ecr_image.contains("amazonaws.com")) {
+ sh("echo Skipping '${ecr_image}' since it doesn't look like an ECR image")
+ return
+ }
+ docker_init(ecr_image)
+ sh(
+ script: """
+ set -eux
+ docker tag \
+ ${ecr_image} \
+ ${hub_image}
+ docker push ${hub_image}
+ """,
+ label: "Update ${hub_image} on Docker Hub",
+ )
+}
+
def deploy_docs() {
// Note: This code must stay in the Jenkinsfile to ensure that it runs
// from a trusted context only
@@ -3298,6 +3377,42 @@ def deploy() {
}
}
}
+ if (env.BRANCH_NAME == 'main' && env.DEPLOY_DOCKER_IMAGES == 'yes' && rebuild_docker_images && upstream_revision != null) {
+ node('CPU') {
+ ws("workspace/exec_${env.EXECUTOR_NUMBER}/tvm/deploy-docker") {
+ try {
+ withCredentials([string(
+ credentialsId: 'dockerhub-tlcpackstaging-key',
+ variable: 'DOCKERHUB_KEY',
+ )]) {
+ sh(
+ script: 'docker login -u tlcpackstaging -p ${DOCKERHUB_KEY}',
+ label: 'Log in to Docker Hub',
+ )
+ }
+ def date_Ymd_HMS = sh(
+ script: 'python3 -c \'import datetime; print(datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))\'',
+ label: 'Determine date',
+ returnStdout: true,
+ ).trim()
+ def tag = "${date_Ymd_HMS}-${upstream_revision.substring(0, 8)}"
+ update_docker(ci_arm, "tlcpackstaging/test_ci_arm:${tag}")
+ update_docker(ci_cpu, "tlcpackstaging/test_ci_cpu:${tag}")
+ update_docker(ci_gpu, "tlcpackstaging/test_ci_gpu:${tag}")
+ update_docker(ci_hexagon, "tlcpackstaging/test_ci_hexagon:${tag}")
+ update_docker(ci_i386, "tlcpackstaging/test_ci_i386:${tag}")
+ update_docker(ci_lint, "tlcpackstaging/test_ci_lint:${tag}")
+ update_docker(ci_qemu, "tlcpackstaging/test_ci_qemu:${tag}")
+ update_docker(ci_wasm, "tlcpackstaging/test_ci_wasm:${tag}")
+ } finally {
+ sh(
+ script: 'docker logout',
+ label: 'Clean up login credentials'
+ )
+ }
+ }
+ }
+ }
}
}
diff --git a/jenkins/Build.groovy.j2 b/jenkins/Build.groovy.j2
index 62ccc94916..fcde53f559 100644
--- a/jenkins/Build.groovy.j2
+++ b/jenkins/Build.groovy.j2
@@ -52,6 +52,29 @@ def add_hexagon_permissions() {
{% endfor %}
}
+// Run make. First try to do an incremental make from a previous workspace in hope to
+// accelerate the compilation. If something is wrong, clean the workspace and then
+// build from scratch.
+def make(docker_type, path, make_flag) {
+ timeout(time: max_time, unit: 'MINUTES') {
+ try {
+ cmake_build(docker_type, path, make_flag)
+ } catch (hudson.AbortException ae) {
+ // script exited due to user abort, directly throw instead of retry
+ if (ae.getMessage().contains('script returned exit code 143')) {
+ throw ae
+ }
+ echo 'Incremental compilation failed. Fall back to build from scratch'
+ sh (
+ script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}",
+ label: 'Clear old cmake workspace',
+ )
+ cmake_build(docker_type, path, make_flag)
+ }
+ }
+}
+
+
def build() {
stage('Build') {
environment {
diff --git a/jenkins/Deploy.groovy.j2 b/jenkins/Deploy.groovy.j2
index 917f71ded1..3a049c5141 100644
--- a/jenkins/Deploy.groovy.j2
+++ b/jenkins/Deploy.groovy.j2
@@ -16,6 +16,25 @@ stage('Build packages') {
}
*/
+
+def update_docker(ecr_image, hub_image) {
+ if (!ecr_image.contains("amazonaws.com")) {
+ sh("echo Skipping '${ecr_image}' since it doesn't look like an ECR image")
+ return
+ }
+ docker_init(ecr_image)
+ sh(
+ script: """
+ set -eux
+ docker tag \
+ ${ecr_image} \
+ ${hub_image}
+ docker push ${hub_image}
+ """,
+ label: "Update ${hub_image} on Docker Hub",
+ )
+}
+
def deploy_docs() {
// Note: This code must stay in the Jenkinsfile to ensure that it runs
// from a trusted context only
@@ -67,5 +86,36 @@ def deploy() {
}
}
}
+ if (env.BRANCH_NAME == 'main' && env.DEPLOY_DOCKER_IMAGES == 'yes' && rebuild_docker_images && upstream_revision != null) {
+ node('CPU') {
+ ws({{ m.per_exec_ws('tvm/deploy-docker') }}) {
+ try {
+ withCredentials([string(
+ credentialsId: 'dockerhub-tlcpackstaging-key',
+ variable: 'DOCKERHUB_KEY',
+ )]) {
+ sh(
+ script: 'docker login -u tlcpackstaging -p ${DOCKERHUB_KEY}',
+ label: 'Log in to Docker Hub',
+ )
+ }
+ def date_Ymd_HMS = sh(
+ script: 'python3 -c \'import datetime; print(datetime.datetime.now().strftime("%Y%m%d-%H%M%S"))\'',
+ label: 'Determine date',
+ returnStdout: true,
+ ).trim()
+ def tag = "${date_Ymd_HMS}-${upstream_revision.substring(0, 8)}"
+ {% for image in images %}
+ update_docker({{ image.name }}, "tlcpackstaging/test_{{ image.name }}:${tag}")
+ {% endfor %}
+ } finally {
+ sh(
+ script: 'docker logout',
+ label: 'Clean up login credentials'
+ )
+ }
+ }
+ }
+ }
}
}
diff --git a/jenkins/DockerBuild.groovy.j2 b/jenkins/DockerBuild.groovy.j2
index e9d80801a9..a0ff666773 100644
--- a/jenkins/DockerBuild.groovy.j2
+++ b/jenkins/DockerBuild.groovy.j2
@@ -1,13 +1,47 @@
-def build_image(image_name) {
- hash = sh(
+def ecr_push(full_name) {
+ aws_account_id = sh(
returnStdout: true,
- script: 'git log -1 --format=\'%h\''
+ script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
+ label: 'Get AWS ID'
).trim()
- def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}"
- sh(
- script: "${docker_build} ${image_name} --spec ${full_name}",
- label: 'Build docker image'
- )
+
+ def ecr_name = "${aws_account_id}.{{ aws_ecr_url }}/${full_name}"
+ try {
+ withEnv([
+ "AWS_ACCOUNT_ID=${aws_account_id}",
+ 'AWS_DEFAULT_REGION={{ aws_default_region }}',
+ "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) {
+ sh(
+ script: '''
+ set -eux
+ aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
+ ''',
+ label: 'Log in to ECR'
+ )
+ sh(
+ script: """
+ set -x
+ docker tag ${full_name} \$AWS_ECR_REPO/${full_name}
+ docker push \$AWS_ECR_REPO/${full_name}
+ """,
+ label: 'Upload image to ECR'
+ )
+ }
+ } finally {
+ withEnv([
+ "AWS_ACCOUNT_ID=${aws_account_id}",
+ 'AWS_DEFAULT_REGION={{ aws_default_region }}',
+ "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) {
+ sh(
+ script: 'docker logout $AWS_ECR_REPO',
+ label: 'Clean up login credentials'
+ )
+ }
+ }
+ return ecr_name
+}
+
+def ecr_pull(full_name) {
aws_account_id = sh(
returnStdout: true,
script: 'aws sts get-caller-identity | grep Account | cut -f4 -d\\"',
@@ -15,152 +49,68 @@ def build_image(image_name) {
).trim()
try {
- // Use a credential so Jenkins knows to scrub the AWS account ID which is nice
- // (but so we don't have to rely it being hardcoded in Jenkins)
- withCredentials([string(
- credentialsId: 'aws-account-id',
- variable: '_ACCOUNT_ID_DO_NOT_USE',
- )]) {
- withEnv([
- "AWS_ACCOUNT_ID=${aws_account_id}",
- 'AWS_DEFAULT_REGION=us-west-2']) {
- sh(
- script: '''
- set -x
- aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ACCOUNT_ID.dkr.ecr.$AWS_DEFAULT_REGION.amazonaws.com
- ''',
- label: 'Log in to ECR'
- )
- sh(
- script: """
- set -x
- docker tag ${full_name} \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name}
- docker push \$AWS_ACCOUNT_ID.dkr.ecr.\$AWS_DEFAULT_REGION.amazonaws.com/${full_name}
- """,
- label: 'Upload image to ECR'
- )
- }
+ withEnv([
+ "AWS_ACCOUNT_ID=${aws_account_id}",
+ 'AWS_DEFAULT_REGION={{ aws_default_region }}',
+ "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) {
+ sh(
+ script: '''
+ set -eux
+ aws ecr get-login-password --region $AWS_DEFAULT_REGION | docker login --username AWS --password-stdin $AWS_ECR_REPO
+ ''',
+ label: 'Log in to ECR'
+ )
+ sh(
+ script: """
+ set -eux
+ docker pull ${full_name}
+ """,
+ label: 'Pull image from ECR'
+ )
}
} finally {
- sh(
- script: 'rm -f ~/.docker/config.json',
- label: 'Clean up login credentials'
- )
+ withEnv([
+ "AWS_ACCOUNT_ID=${aws_account_id}",
+ 'AWS_DEFAULT_REGION={{ aws_default_region }}',
+ "AWS_ECR_REPO=${aws_account_id}.{{ aws_ecr_url }}"]) {
+ sh(
+ script: 'docker logout $AWS_ECR_REPO',
+ label: 'Clean up login credentials'
+ )
+ }
}
+}
+
+def build_image(image_name) {
+ hash = sh(
+ returnStdout: true,
+ script: 'git log -1 --format=\'%h\''
+ ).trim()
+ def full_name = "${image_name}:${env.BRANCH_NAME}-${hash}-${env.BUILD_NUMBER}"
sh(
- script: "docker rmi ${full_name}",
- label: 'Remove docker image'
+ script: "${docker_build} ${image_name} --spec ${full_name}",
+ label: 'Build docker image'
)
+ return ecr_push(full_name)
}
+
def build_docker_images() {
stage('Docker Image Build') {
- // TODO in a follow up PR: Find ecr tag and use in subsequent builds
- parallel 'ci-lint': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_lint')
- }
- }
- }, 'ci-cpu': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_cpu')
+ parallel(
+ {% for image in images %}
+ '{{ image.name }}': {
+ node('{{ image.platform }}') {
+ timeout(time: max_time, unit: 'MINUTES') {
+ init_git()
+ // We're purposefully not setting the built image here since they
+ // are not yet being uploaded to tlcpack
+ // {{ image.name }} = build_image('{{ image.name }}')
+ build_image('{{ image.name }}')
+ }
}
- }
- }, 'ci-gpu': {
- node('GPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_gpu')
- }
- }
- }, 'ci-qemu': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_qemu')
- }
- }
- }, 'ci-i386': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_i386')
- }
- }
- }, 'ci-arm': {
- node('ARM') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_arm')
- }
- }
- }, 'ci-wasm': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_wasm')
- }
- }
- }, 'ci-hexagon': {
- node('CPU') {
- timeout(time: max_time, unit: 'MINUTES') {
- docker_init('none')
- init_git()
- build_image('ci_hexagon')
- }
- }
- }
- }
- // // TODO: Once we are able to use the built images, enable this step
- // // If the docker images changed, we need to run the image build before the lint
- // // can run since it requires a base docker image. Most of the time the images
- // // aren't build though so it's faster to use the same node that checks for
- // // docker changes to run the lint in the usual case.
- // stage('Sanity Check (re-run)') {
- // timeout(time: max_time, unit: 'MINUTES') {
- // node('CPU') {
- // ws({{ m.per_exec_ws('tvm/sanity') }}) {
- // init_git()
- // sh (
- // script: "${docker_run} ${ci_lint} ./tests/scripts/task_lint.sh",
- // label: 'Run lint',
- // )
- // }
- // }
- // }
- // }
-}
-
-// Run make. First try to do an incremental make from a previous workspace in hope to
-// accelerate the compilation. If something is wrong, clean the workspace and then
-// build from scratch.
-def make(docker_type, path, make_flag) {
- timeout(time: max_time, unit: 'MINUTES') {
- try {
- cmake_build(docker_type, path, make_flag)
- // always run cpp test when build
- } catch (hudson.AbortException ae) {
- // script exited due to user abort, directly throw instead of retry
- if (ae.getMessage().contains('script returned exit code 143')) {
- throw ae
- }
- echo 'Incremental compilation failed. Fall back to build from scratch'
- sh (
- script: "${docker_run} ${docker_type} ./tests/scripts/task_clean.sh ${path}",
- label: 'Clear old cmake workspace',
- )
- cmake_build(docker_type, path, make_flag)
- }
+ },
+ {% endfor %}
+ )
}
}
diff --git a/jenkins/Jenkinsfile.j2 b/jenkins/Jenkinsfile.j2
index c165de964f..4e344c56d7 100644
--- a/jenkins/Jenkinsfile.j2
+++ b/jenkins/Jenkinsfile.j2
@@ -100,6 +100,9 @@ if (currentBuild.getBuildCauses().toString().contains('BranchIndexingCause')) {
{% set hexagon_api = ['build/hexagon_api_output',] %}
s3_prefix = "tvm-jenkins-artifacts-prod/tvm/${env.BRANCH_NAME}/${env.BUILD_NUMBER}"
+{% set aws_default_region = "us-west-2" %}
+{% set aws_ecr_url = "dkr.ecr." + aws_default_region + ".amazonaws.com" %}
+
// General note: Jenkins has limits on the size of a method (or top level code)
// that are pretty strict, so most usage of groovy methods in these templates
// are purely to satisfy the JVM
diff --git a/jenkins/Lint.groovy.j2 b/jenkins/Lint.groovy.j2
index 40dad3aef7..3ede64301c 100644
--- a/jenkins/Lint.groovy.j2
+++ b/jenkins/Lint.groovy.j2
@@ -2,11 +2,11 @@ def lint() {
stage('Lint') {
parallel(
{% call m.sharded_lint_step(
- name='Lint',
- num_shards=2,
- node='CPU-SMALL',
- ws='tvm/lint',
- docker_image='ci_lint',
+ name='Lint',
+ num_shards=2,
+ node='CPU-SMALL',
+ ws='tvm/lint',
+ docker_image='ci_lint',
)
%}
sh (
diff --git a/jenkins/Prepare.groovy.j2 b/jenkins/Prepare.groovy.j2
index 2900775f49..894ddc72ee 100644
--- a/jenkins/Prepare.groovy.j2
+++ b/jenkins/Prepare.groovy.j2
@@ -69,6 +69,17 @@ def docker_init(image) {
""",
label: 'Clean old Docker images',
)
+
+ if (image.contains("amazonaws.com")) {
+ // If this string is in the image name it's from ECR and needs to be pulled
+ // with the right credentials
+ ecr_pull(image)
+ } else {
+ sh(
+ script: "docker pull ${image}",
+ label: 'Pull docker image',
+ )
+ }
}
def should_skip_slow_tests(pr_number) {
diff --git a/tests/python/ci/test_ci.py b/tests/python/ci/test_ci.py
index 042c109dd9..7ef2f0cd58 100644
--- a/tests/python/ci/test_ci.py
+++ b/tests/python/ci/test_ci.py
@@ -18,9 +18,11 @@
import subprocess
import sys
import json
+from tempfile import tempdir
import textwrap
import pytest
import tvm.testing
+from pathlib import Path
from test_utils import REPO_ROOT
@@ -29,11 +31,13 @@ class TempGit:
def __init__(self, cwd):
self.cwd = cwd
- def run(self, *args):
- proc = subprocess.run(["git"] + list(args), cwd=self.cwd)
+ def run(self, *args, **kwargs):
+ proc = subprocess.run(["git"] + list(args), encoding="utf-8", cwd=self.cwd, **kwargs)
if proc.returncode != 0:
raise RuntimeError(f"git command failed: '{args}'")
+ return proc
+
def test_cc_reviewers(tmpdir_factory):
reviewers_script = REPO_ROOT / "tests" / "scripts" / "github_cc_reviewers.py"
@@ -747,5 +751,94 @@ def test_github_tag_teams(tmpdir_factory):
)
+@pytest.mark.parametrize(
+ "changed_files,name,check,expected_code",
+ [
+ d.values()
+ for d in [
+ dict(
+ changed_files=[],
+ name="abc",
+ check="Image abc is not using new naming scheme",
+ expected_code=1,
+ ),
+ dict(
+ changed_files=[], name="123-123-abc", check="No extant hash found", expected_code=1
+ ),
+ dict(
+ changed_files=[["test.txt"]],
+ name=None,
+ check="Did not find changes, no rebuild necessary",
+ expected_code=0,
+ ),
+ dict(
+ changed_files=[["test.txt"], ["docker/test.txt"]],
+ name=None,
+ check="Found docker changes",
+ expected_code=2,
+ ),
+ ]
+ ],
+)
+def test_should_rebuild_docker(tmpdir_factory, changed_files, name, check, expected_code):
+ tag_script = REPO_ROOT / "tests" / "scripts" / "should_rebuild_docker.py"
+
+ git = TempGit(tmpdir_factory.mktemp("tmp_git_dir"))
+ git.run("init")
+ git.run("config", "user.name", "ci")
+ git.run("config", "user.email", "email@example.com")
+ git.run("checkout", "-b", "main")
+ git.run("remote", "add", "origin", "https://github.com/apache/tvm.git")
+
+ git_path = Path(git.cwd)
+ for i, commits in enumerate(changed_files):
+ for filename in commits:
+ path = git_path / filename
+ path.parent.mkdir(exist_ok=True, parents=True)
+ path.touch()
+ git.run("add", filename)
+
+ git.run("commit", "-m", f"message {i}")
+
+ if name is None:
+ ref = "HEAD"
+ if len(changed_files) > 1:
+ ref = f"HEAD~{len(changed_files) - 1}"
+ proc = git.run("rev-parse", ref, stdout=subprocess.PIPE)
+ last_hash = proc.stdout.strip()
+ name = f"123-123-{last_hash}"
+
+ docker_data = {
+ "repositories/tlcpack": {
+ "results": [
+ {
+ "name": "ci-something",
+ },
+ {
+ "name": "something-else",
+ },
+ ],
+ },
+ "repositories/tlcpack/ci-something/tags": {
+ "results": [{"name": name}, {"name": name + "old"}],
+ },
+ }
+
+ proc = subprocess.run(
+ [
+ str(tag_script),
+ "--testing-docker-data",
+ json.dumps(docker_data),
+ ],
+ stdout=subprocess.PIPE,
+ stderr=subprocess.STDOUT,
+ encoding="utf-8",
+ cwd=git.cwd,
+ )
+
+ assert_in(check, proc.stdout)
+ assert proc.returncode == expected_code
+
+
if __name__ == "__main__":
tvm.testing.main()
diff --git a/tests/scripts/cmd_utils.py b/tests/scripts/cmd_utils.py
index 272086796e..771c3ee52d 100644
--- a/tests/scripts/cmd_utils.py
+++ b/tests/scripts/cmd_utils.py
@@ -44,18 +44,21 @@ def init_log():
class Sh:
- def __init__(self, env=None):
+ def __init__(self, env=None, cwd=None):
self.env = os.environ.copy()
if env is not None:
self.env.update(env)
+ self.cwd = cwd
def run(self, cmd: str, **kwargs):
logging.info(f"+ {cmd}")
- if "check" not in kwargs:
- kwargs["check"] = True
- if "shell" not in kwargs:
- kwargs["shell"] = True
- if "env" not in kwargs:
- kwargs["env"] = self.env
-
- subprocess.run(cmd, **kwargs)
+ defaults = {
+ "check": True,
+ "shell": True,
+ "env": self.env,
+ "encoding": "utf-8",
+ "cwd": self.cwd,
+ }
+ defaults.update(kwargs)
+
+ return subprocess.run(cmd, **defaults)
diff --git a/tests/scripts/git_utils.py b/tests/scripts/git_utils.py
index 267756d859..c5ea8d85e0 100644
--- a/tests/scripts/git_utils.py
+++ b/tests/scripts/git_utils.py
@@ -20,6 +20,7 @@ import json
import subprocess
import re
import base64
+import logging
from urllib import request
from typing import Dict, Tuple, Any, Optional, List
diff --git a/tests/scripts/http_utils.py b/tests/scripts/http_utils.py
new file mode 100644
index 0000000000..c14259479d
--- /dev/null
+++ b/tests/scripts/http_utils.py
@@ -0,0 +1,34 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+
+import json
+import logging
+from urllib import request
+from typing import Dict, Any, Optional
+
+
+def get(url: str, headers: Optional[Dict[str, str]] = None) -> Dict[str, Any]:
+ logging.info(f"Requesting GET to {url}")
+ if headers is None:
+ headers = {}
+ req = request.Request(url, headers=headers)
+ with request.urlopen(req) as response:
+ response_headers = {k: v for k, v in response.getheaders()}
+ response = json.loads(response.read())
+
+ return response, response_headers
diff --git a/tests/scripts/should_rebuild_docker.py b/tests/scripts/should_rebuild_docker.py
new file mode 100755
index 0000000000..dc12c38de8
--- /dev/null
+++ b/tests/scripts/should_rebuild_docker.py
@@ -0,0 +1,154 @@
+#!/usr/bin/env python3
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing,
+# software distributed under the License is distributed on an
+# "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+# KIND, either express or implied. See the License for the
+# specific language governing permissions and limitations
+# under the License.
+import argparse
+import datetime
+import json
+import logging
+import subprocess
+
+from typing import Dict, Any, List
+
+
+from http_utils import get
+from cmd_utils import Sh, init_log
+
+
+DOCKER_API_BASE = "https://hub.docker.com/v2/"
+PAGE_SIZE = 25
+TEST_DATA = None
+
+
+def docker_api(url: str) -> Dict[str, Any]:
+ """
+ Run a paginated fetch from the public Docker Hub API
+ """
+ if TEST_DATA is not None:
+ return TEST_DATA[url]
+ pagination = f"?page_size={PAGE_SIZE}&page=1"
+ url = DOCKER_API_BASE + url + pagination
+ r, headers = get(url)
+ reset = headers.get("x-ratelimit-reset")
+ if reset is not None:
+ reset = datetime.datetime.fromtimestamp(int(reset))
+ reset = reset.isoformat()
+ logging.info(
+ f"Docker API Rate Limit: {headers.get('x-ratelimit-remaining')} / {headers.get('x-ratelimit-limit')} (reset at {reset})"
+ )
+ if "results" not in r:
+ raise RuntimeError(f"Error fetching data, no results found in: {r}")
+ return r
+
+
+def any_docker_changes_since(hash: str) -> bool:
+ """
+ Check the docker/ directory, return True if there have been any code changes
+ since the specified hash
+ """
+ sh = Sh()
+ cmd = f"git diff {hash} -- docker/"
+ proc = sh.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT)
+ stdout = proc.stdout.strip()
+ return stdout != "", stdout
+
+
+def does_commit_exist(hash: str) -> bool:
+ """
+ Returns True if the hash exists in the repo
+ """
+ sh = Sh()
+ cmd = f"git rev-parse -q {hash}"
+ proc = sh.run(cmd, stdout=subprocess.PIPE, stderr=subprocess.STDOUT, check=False)
+ print(proc.stdout)
+ if proc.returncode == 0:
+ return True
+
+ if "unknown revision or path not in the working tree" in proc.stdout:
+ return False
+
+ raise RuntimeError(f"Unexpected failure when running: {cmd}")
+
+
+def find_hash_for_tag(tag: Dict[str, Any]) -> str:
+ """
+ Split the hash off of a name like <date>-<time>-<hash>
+ """
+ name = tag["name"]
+ name_parts = name.split("-")
+ if len(name_parts) != 3:
+ raise RuntimeError(f"Image {name} is not using new naming scheme")
+ shorthash = name_parts[2]
+ return shorthash
+
+
+def find_commit_in_repo(tags: List[Dict[str, Any]]):
+ """
+ Look through all the docker tags, find the most recent one which references
+ a commit that is present in the repo
+ """
+ for tag in tags["results"]:
+ shorthash = find_hash_for_tag(tag)
+ logging.info(f"Hash '{shorthash}' does not exist in repo")
+ if does_commit_exist(shorthash):
+ return shorthash, tag
+
+ raise RuntimeError(f"No extant hash found in tags:\n{tags}")
+
+
+def main():
+ # Fetch all tlcpack images
+ images = docker_api("repositories/tlcpack")
+
+ # Ignore all non-ci images
+ relevant_images = [image for image in images["results"] if image["name"].startswith("ci-")]
+ image_names = [image["name"] for image in relevant_images]
+ logging.info(f"Found {len(relevant_images)} images to check: {', '.join(image_names)}")
+
+ for image in relevant_images:
+ # Check the tags for the image
+ tags = docker_api(f"repositories/tlcpack/{image['name']}/tags")
+
+ # Find the hash of the most recent tag
+ shorthash, tag = find_commit_in_repo(tags)
+ name = tag["name"]
+ logging.info(f"Looking for docker/ changes since {shorthash}")
+
+ any_docker_changes, diff = any_docker_changes_since(shorthash)
+ if any_docker_changes:
+ logging.info(f"Found docker changes from {shorthash} when checking {name}")
+ logging.info(diff)
+ exit(2)
+
+ logging.info("Did not find changes, no rebuild necessary")
+ exit(0)
+
+
+if __name__ == "__main__":
+ init_log()
+ parser = argparse.ArgumentParser(
+ description="Exits 0 if Docker images don't need to be rebuilt, 1 otherwise"
+ )
+ parser.add_argument(
+ "--testing-docker-data",
+ help="(testing only) JSON data to mock response from Docker Hub API",
+ )
+ args = parser.parse_args()
+
+ if args.testing_docker_data is not None:
+ TEST_DATA = json.loads(args.testing_docker_data)
+
+ main()