You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mxnet.apache.org by la...@apache.org on 2020/04/28 20:20:14 UTC
[incubator-mxnet] branch master updated: CI: Add test for docker
cache job run on master branch commits (#18183)
This is an automated email from the ASF dual-hosted git repository.
lausen pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/incubator-mxnet.git
The following commit(s) were added to refs/heads/master by this push:
new bfd6b41 CI: Add test for docker cache job run on master branch commits (#18183)
bfd6b41 is described below
commit bfd6b411f0c1ce82b5b779ac87324df6808278c5
Author: Leonard Lausen <la...@amazon.com>
AuthorDate: Tue Apr 28 13:19:05 2020 -0700
CI: Add test for docker cache job run on master branch commits (#18183)
Test the job on PRs, to prevent PRs from breaking the production job
---
ci/Jenkinsfile_docker_cache | 2 +-
ci/docker/Dockerfile.publish.test.centos7 | 3 ---
ci/docker_cache.py | 24 ++++++++++--------------
ci/jenkins/Jenkins_steps.groovy | 12 ++++++++++++
ci/jenkins/Jenkinsfile_miscellaneous | 3 ++-
5 files changed, 25 insertions(+), 19 deletions(-)
diff --git a/ci/Jenkinsfile_docker_cache b/ci/Jenkinsfile_docker_cache
index 9d5022a..532b18a 100644
--- a/ci/Jenkinsfile_docker_cache
+++ b/ci/Jenkinsfile_docker_cache
@@ -38,7 +38,7 @@ core_logic: {
timeout(time: total_timeout, unit: 'MINUTES') {
utils.init_git()
sh "python3 ./ci/docker_cache.py --docker-registry ${env.DOCKER_CACHE_REGISTRY}"
- sh "cd ci && docker-compose -f docker/docker-compose.yml build --parallel && docker-compose -f docker/docker-compose.yml push "
+ sh "cd ci && python3 ./docker_login.py && docker-compose -f docker/docker-compose.yml build --parallel && docker-compose -f docker/docker-compose.yml push && docker logout"
}
}
}
diff --git a/ci/docker/Dockerfile.publish.test.centos7 b/ci/docker/Dockerfile.publish.test.centos7
index d9e8c38..f8586eb 100644
--- a/ci/docker/Dockerfile.publish.test.centos7
+++ b/ci/docker/Dockerfile.publish.test.centos7
@@ -31,9 +31,6 @@ FROM $BASE_IMAGE
WORKDIR /work/deps
-COPY install/centos7_scala.sh /work/
-RUN /work/centos7_scala.sh
-
# Install runtime dependencies for publish tests
# - make is used to run tests ci/publish/scala/test.sh
# - unzip is used to run org.apache.mxnetexamples.neuralstyle.NeuralStyleSuite
diff --git a/ci/docker_cache.py b/ci/docker_cache.py
index 14a0f46..7fb946b 100644
--- a/ci/docker_cache.py
+++ b/ci/docker_cache.py
@@ -41,7 +41,7 @@ PARALLEL_BUILDS = 10
DOCKER_CACHE_RETRY_SECONDS = 5
-def build_save_containers(platforms, registry, load_cache) -> int:
+def build_save_containers(platforms, registry, load_cache, no_publish) -> int:
"""
Entry point to build and upload all built dockerimages in parallel
:param platforms: List of platforms
@@ -54,7 +54,7 @@ def build_save_containers(platforms, registry, load_cache) -> int:
return 0
platform_results = Parallel(n_jobs=PARALLEL_BUILDS, backend="multiprocessing")(
- delayed(_build_save_container)(platform, registry, load_cache)
+ delayed(_build_save_container)(platform, registry, load_cache, no_publish)
for platform in platforms)
is_error = False
@@ -66,7 +66,7 @@ def build_save_containers(platforms, registry, load_cache) -> int:
return 1 if is_error else 0
-def _build_save_container(platform, registry, load_cache) -> Optional[str]:
+def _build_save_container(platform, registry, load_cache, no_publish) -> Optional[str]:
"""
Build image for passed platform and upload the cache to the specified S3 bucket
:param platform: Platform
@@ -74,14 +74,6 @@ def _build_save_container(platform, registry, load_cache) -> Optional[str]:
:param load_cache: Load cache before building
:return: Platform if failed, None otherwise
"""
- # Case 1: docker-compose
- if platform in build_util.DOCKER_COMPOSE_WHITELIST:
- build_util.build_docker(platform=platform, registry=registry, num_retries=10, no_cache=False)
- push_cmd = ['docker-compose', 'push', platform]
- subprocess.check_call(push_cmd)
- return None
-
- # Case 2: Deprecated way, will be removed
docker_tag = build_util.get_docker_tag(platform=platform, registry=registry)
# Preload cache
if load_cache:
@@ -95,7 +87,8 @@ def _build_save_container(platform, registry, load_cache) -> Optional[str]:
logging.info('Built %s as %s', docker_tag, image_id)
# Push cache to registry
- _upload_image(registry=registry, docker_tag=docker_tag, image_id=image_id)
+ if not no_publish:
+ _upload_image(registry=registry, docker_tag=docker_tag, image_id=image_id)
return None
except Exception:
logging.exception('Unexpected exception during build of %s', docker_tag)
@@ -187,6 +180,8 @@ def main() -> int:
help="Docker hub registry name",
type=str,
required=True)
+ parser.add_argument("--no-publish", help="Only build but don't publish. Used for testing.",
+ action='store_true')
args = parser.parse_args()
@@ -197,8 +192,9 @@ def main() -> int:
region_name = os.environ['DOCKERHUB_SECRET_ENDPOINT_REGION']
try:
- login_dockerhub(secret_name, endpoint_url, region_name)
- return build_save_containers(platforms=platforms, registry=args.docker_registry, load_cache=True)
+ if not args.no_publish:
+ login_dockerhub(secret_name, endpoint_url, region_name)
+ return build_save_containers(platforms=platforms, registry=args.docker_registry, load_cache=True, no_publish=args.no_publish)
finally:
logout_dockerhub()
diff --git a/ci/jenkins/Jenkins_steps.groovy b/ci/jenkins/Jenkins_steps.groovy
index 91b8e6e..28d76be 100644
--- a/ci/jenkins/Jenkins_steps.groovy
+++ b/ci/jenkins/Jenkins_steps.groovy
@@ -1758,4 +1758,16 @@ def test_artifact_repository() {
}]
}
+def misc_test_docker_cache_build() {
+ return ['Test Docker cache build': {
+ node(NODE_LINUX_CPU) {
+ ws('workspace/docker_cache') {
+ utils.init_git()
+ sh "python3 ./ci/docker_cache.py --docker-registry ${env.DOCKER_CACHE_REGISTRY} --no-publish"
+ sh "cd ci && docker-compose -f docker/docker-compose.yml build --parallel"
+ }
+ }
+ }]
+}
+
return this
diff --git a/ci/jenkins/Jenkinsfile_miscellaneous b/ci/jenkins/Jenkinsfile_miscellaneous
index a47e4c8..2900e37 100644
--- a/ci/jenkins/Jenkinsfile_miscellaneous
+++ b/ci/jenkins/Jenkinsfile_miscellaneous
@@ -44,7 +44,8 @@ core_logic: {
])
utils.parallel_stage('Tests', [
- custom_steps.misc_asan_cpu()
+ custom_steps.misc_asan_cpu(),
+ custom_steps.misc_test_docker_cache_build()
])
}
,