You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by cc...@apache.org on 2019/03/07 23:08:49 UTC
[beam] branch master updated: Update ValidatesContainer test run
scripts.
This is an automated email from the ASF dual-hosted git repository.
ccy pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new 058dfc7 Update ValidatesContainer test run scripts.
new e835a05 Merge pull request #7905 from markflyhigh/py3-merge-valcon-script
058dfc7 is described below
commit 058dfc765a25ceb3afa3a0da65a9690a614bbd07
Author: Mark Liu <ma...@google.com>
AuthorDate: Wed Feb 20 12:09:17 2019 -0800
Update ValidatesContainer test run scripts.
---
...ommit_Python_ValidatesContainer_Dataflow.groovy | 4 +-
sdks/python/container/run_validatescontainer.sh | 38 +++++++-
.../python/container/run_validatescontainer_py3.sh | 101 ---------------------
3 files changed, 35 insertions(+), 108 deletions(-)
diff --git a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
index 0b21c2d..da5094e 100644
--- a/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
+++ b/.test-infra/jenkins/job_PostCommit_Python_ValidatesContainer_Dataflow.groovy
@@ -31,7 +31,7 @@ PostcommitJobBuilder.postCommitJob('beam_PostCommit_Py_ValCont',
// Execute shell command to test Python SDK.
// TODO: Parallel the script run with Jenkins DSL or Gradle.
steps {
- shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh')
- shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer_py3.sh')
+ shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh python2')
+ shell('cd ' + commonJobProperties.checkoutDir + ' && bash sdks/python/container/run_validatescontainer.sh python3')
}
}
diff --git a/sdks/python/container/run_validatescontainer.sh b/sdks/python/container/run_validatescontainer.sh
index 87ee7bb..47f5b8d 100755
--- a/sdks/python/container/run_validatescontainer.sh
+++ b/sdks/python/container/run_validatescontainer.sh
@@ -23,10 +23,19 @@
# GCS_LOCATION -> Temporary location to use for service tests.
# PROJECT -> Project name to use for dataflow and docker images.
#
-# Execute from the root of the repository: sdks/python/container/run_validatescontainer.sh
+# Execute from the root of the repository:
+# test Python2 container: ./sdks/python/container/run_validatescontainer.sh python2
+# test Python3 container: ./sdks/python/container/run_validatescontainer.sh python3
echo "This script must be executed in the root of beam project. Please set LOCAL_PATH, GCS_LOCATION, and PROJECT as desired."
+if [[ $# != 1 ]]; then
+ printf "Usage: \n$> ./sdks/python/container/run_validatescontainer.sh <python_version>"
+ printf "\n\tpython_version: [required] Python version used for container build and run tests."
+ printf " Use 'python2' for Python2, 'python3' for Python3."
+ exit 1
+fi
+
set -e
set -v
@@ -36,6 +45,22 @@ GCS_LOCATION=${GCS_LOCATION:-gs://temp-storage-for-end-to-end-tests}
# Project for the container and integration test
PROJECT=${PROJECT:-apache-beam-testing}
+# Other variables branched by Python version.
+if [[ $1 == "python2" ]]; then
+ IMAGE_NAME="python" # Use this to create CONTAINER_IMAGE variable.
+ CONTAINER_PROJECT="beam-sdks-python-container" # Use this to build container by Gradle.
+ GRADLE_PY3_FLAG="" # Use this in Gradle command.
+ PY_INTERPRETER="python" # Use this in virtualenv command.
+elif [[ $1 == "python3" ]]; then
+ IMAGE_NAME="python3" # Use this to create CONTAINER_IMAGE variable.
+ CONTAINER_PROJECT="beam-sdks-python-container-py3" # Use this to build container by Gradle.
+ GRADLE_PY3_FLAG="-Ppython3" # Use this in Gradle command.
+ PY_INTERPRETER="python3.5" # Use this in virtualenv command.
+else
+ echo "Must set Python version with 'python2' or 'python3' from commandline."
+ exit 1
+fi
+
# Verify in the root of the repository
test -d sdks/python/container
@@ -47,9 +72,9 @@ gcloud -v
# Build the container
TAG=$(date +%Y%m%d-%H%M%S)
-CONTAINER=us.gcr.io/$PROJECT/$USER/python
+CONTAINER=us.gcr.io/$PROJECT/$USER/$IMAGE_NAME
echo "Using container $CONTAINER"
-./gradlew :beam-sdks-python-container:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG --info
+./gradlew :$CONTAINER_PROJECT:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG $GRADLE_PY3_FLAG --info
# Verify it exists
docker images | grep $TAG
@@ -65,9 +90,12 @@ function cleanup_container {
}
trap cleanup_container EXIT
+echo ">>> Successfully built and push container $CONTAINER"
+
# Virtualenv for the rest of the script to run setup & e2e test
-virtualenv sdks/python/container
-. sdks/python/container/bin/activate
+VENV_PATH=sdks/python/container/venv/$PY_INTERPRETER
+virtualenv $VENV_PATH -p $PY_INTERPRETER
+. $VENV_PATH/bin/activate
cd sdks/python
pip install -e .[gcp,test]
diff --git a/sdks/python/container/run_validatescontainer_py3.sh b/sdks/python/container/run_validatescontainer_py3.sh
deleted file mode 100755
index bff133c..0000000
--- a/sdks/python/container/run_validatescontainer_py3.sh
+++ /dev/null
@@ -1,101 +0,0 @@
-#!/bin/bash
-#
-# Licensed to the Apache Software Foundation (ASF) under one or more
-# contributor license agreements. See the NOTICE file distributed with
-# this work for additional information regarding copyright ownership.
-# The ASF licenses this file to You under the Apache License, Version 2.0
-# (the "License"); you may not use this file except in compliance with
-# the License. You may obtain a copy of the License at
-#
-# http://www.apache.org/licenses/LICENSE-2.0
-#
-# Unless required by applicable law or agreed to in writing, software
-# distributed under the License is distributed on an "AS IS" BASIS,
-# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
-# See the License for the specific language governing permissions and
-# limitations under the License.
-#
-
-# This script validates python 3 container. It will be run by Jenkins as a post commit test.
-# In order to run locally make the following changes:
-#
-# LOCAL_PATH -> Path of tox and virtualenv if you have them already installed.
-# GCS_LOCATION -> Temporary location to use for service tests.
-# PROJECT -> Project name to use for dataflow and docker images.
-#
-# Execute from the root of the repository: sdks/python/container/run_validatescontainer.sh
-
-echo "This script must be executed in the root of beam project. Please set LOCAL_PATH, GCS_LOCATION, and PROJECT as desired."
-
-set -e
-set -v
-
-# Where to store integration test outputs.
-GCS_LOCATION=${GCS_LOCATION:-gs://temp-storage-for-end-to-end-tests}
-
-# Project for the container and integration test
-PROJECT=${PROJECT:-apache-beam-testing}
-
-# Verify in the root of the repository
-test -d sdks/python/container
-
-# Verify docker and gcloud commands exist
-command -v docker
-command -v gcloud
-docker -v
-gcloud -v
-
-# Build the container image
-TAG=$(date +%Y%m%d-%H%M%S)
-CONTAINER=us.gcr.io/$PROJECT/$USER/python3
-echo "Building Python 3 container $CONTAINER"
-./gradlew :beam-sdks-python-container-py3:docker -Pdocker-repository-root=us.gcr.io/$PROJECT/$USER -Pdocker-tag=$TAG -Ppython3 --info
-
-# Verify it exists
-docker images | grep $TAG
-
-# Push the container
-gcloud docker -- push $CONTAINER
-
-function cleanup_container {
- # Delete the container locally and remotely
- docker rmi $CONTAINER:$TAG || echo "Failed to remove container"
- gcloud --quiet container images delete $CONTAINER:$TAG || echo "Failed to delete container"
- echo "Removed the container"
-}
-trap cleanup_container EXIT
-
-echo ">>> Successfully built Python 3 container $CONTAINER"
-
-# Python 3 virtualenv for the rest of the script to run setup & e2e test
-virtualenv sdks/python/container/py3/venv -p python3.5
-. sdks/python/container/py3/venv/bin/activate
-cd sdks/python
-pip install -e .[gcp,test]
-
-# Install test dependencies for ValidatesContainer tests.
-echo "pyhamcrest" > postcommit_requirements.txt
-
-# Create a tarball
-python setup.py sdist
-SDK_LOCATION=$(find dist/apache-beam-*.tar.gz)
-
-# Run ValidatesRunner tests on Google Cloud Dataflow service
-echo ">>> RUNNING DATAFLOW RUNNER VALIDATESCONTAINER TEST"
-python setup.py nosetests \
- --attr ValidatesContainer \
- --nologcapture \
- --processes=1 \
- --process-timeout=900 \
- --test-pipeline-options=" \
- --runner=TestDataflowRunner \
- --project=$PROJECT \
- --worker_harness_container_image=$CONTAINER:$TAG \
- --staging_location=$GCS_LOCATION/staging-validatesrunner-test \
- --temp_location=$GCS_LOCATION/temp-validatesrunner-test \
- --output=$GCS_LOCATION/output \
- --sdk_location=$SDK_LOCATION \
- --requirements_file=postcommit_requirements.txt \
- --num_workers=1"
-
-echo ">>> SUCCESS DATAFLOW RUNNER VALIDATESCONTAINER TEST"