You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by dh...@apache.org on 2016/12/28 19:12:14 UTC
[1/2] beam git commit: [BEAM-1221] Run Wordcount IT in Postcommit
Using Test Framework
Repository: beam
Updated Branches:
refs/heads/python-sdk 5107dfad8 -> eff50118b
[BEAM-1221] Run Wordcount IT in Postcommit Using Test Framework
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/d58b6495
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/d58b6495
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/d58b6495
Branch: refs/heads/python-sdk
Commit: d58b6495a929f4b86617214dab11ab467ebb53c1
Parents: 5107dfa
Author: Mark Liu <ma...@google.com>
Authored: Tue Dec 27 16:33:37 2016 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Wed Dec 28 11:11:52 2016 -0800
----------------------------------------------------------------------
sdks/python/run_postcommit.sh | 35 ++++++++++++++---------------------
1 file changed, 14 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/d58b6495/sdks/python/run_postcommit.sh
----------------------------------------------------------------------
diff --git a/sdks/python/run_postcommit.sh b/sdks/python/run_postcommit.sh
index 968d4f9..67a257e 100755
--- a/sdks/python/run_postcommit.sh
+++ b/sdks/python/run_postcommit.sh
@@ -60,7 +60,7 @@ python -m apache_beam.examples.wordcount --output /tmp/py-wordcount-direct
GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
# Job name needs to be unique
-JOBNAME_E2E=py-wordcount-`date +%s`
+JOBNAME_E2E_WC=py-wordcount-`date +%s`
JOBNAME_VR_TEST=py-validatesrunner-`date +%s`
PROJECT=apache-beam-testing
@@ -71,6 +71,7 @@ python setup.py sdist
SDK_LOCATION=$(find dist/apache-beam-sdk-*.tar.gz)
# Run ValidatesRunner tests on Google Cloud Dataflow service
+echo ">>> RUNNING DATAFLOW RUNNER VALIDATESRUNNER TESTS"
python setup.py nosetests \
-a ValidatesRunner --test-pipeline-options=" \
--runner=BlockingDataflowRunner \
@@ -82,23 +83,15 @@ python setup.py nosetests \
--num_workers=1"
# Run wordcount on the Google Cloud Dataflow service
-python -m apache_beam.examples.wordcount \
- --output $GCS_LOCATION/py-wordcount-cloud \
- --staging_location $GCS_LOCATION/staging-wordcount \
- --temp_location $GCS_LOCATION/temp-wordcount \
- --runner BlockingDataflowRunner \
- --job_name $JOBNAME_E2E \
- --project $PROJECT \
- --sdk_location $SDK_LOCATION \
- --num_workers 1 >> job_output 2>&1 || true;
-
-# Print full job output, validate correct, then remove it.
-echo ">>> JOB OUTPUT FOLLOWS"
-cat job_output
-
-# Grep will exit with status 1 if success message was not found.
-echo ">>> CHECKING JOB SUCCESS"
-grep JOB_STATE_DONE job_output
-
-# Remove the job output.
-rm job_output
+# and validate job that finishes successfully.
+echo ">>> RUNNING TEST DATAFLOW RUNNER py-wordcount"
+python setup.py nosetests \
+ -a IT --test-pipeline-options=" \
+ --runner=TestDataflowRunner \
+ --project=$PROJECT \
+ --staging_location=$GCS_LOCATION/staging-wordcount \
+ --temp_location=$GCS_LOCATION/temp-wordcount \
+ --output=$GCS_LOCATION/py-wordcount-cloud/output \
+ --sdk_location=$SDK_LOCATION \
+ --job_name=$JOBNAME_E2E_WC \
+ --num_workers=1"
[2/2] beam git commit: This closes #1705
Posted by dh...@apache.org.
This closes #1705
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/eff50118
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/eff50118
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/eff50118
Branch: refs/heads/python-sdk
Commit: eff50118bbfd4da14c2d5d9cb72fea8933e07fde
Parents: 5107dfa d58b649
Author: Dan Halperin <dh...@google.com>
Authored: Wed Dec 28 11:12:01 2016 -0800
Committer: Dan Halperin <dh...@google.com>
Committed: Wed Dec 28 11:12:01 2016 -0800
----------------------------------------------------------------------
sdks/python/run_postcommit.sh | 35 ++++++++++++++---------------------
1 file changed, 14 insertions(+), 21 deletions(-)
----------------------------------------------------------------------