You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by al...@apache.org on 2017/04/03 22:15:21 UTC
[1/2] beam git commit: Update integration tests in post commits
Repository: beam
Updated Branches:
refs/heads/master 570d0e2a1 -> 3711c0caf
Update integration tests in post commits
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/ffe8355b
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/ffe8355b
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/ffe8355b
Branch: refs/heads/master
Commit: ffe8355b212e2d7f98d2dd152bbf7bc37448ad58
Parents: 570d0e2
Author: Ahmet Altay <al...@google.com>
Authored: Fri Mar 31 15:11:43 2017 -0700
Committer: Ahmet Altay <al...@google.com>
Committed: Mon Apr 3 15:14:28 2017 -0700
----------------------------------------------------------------------
.../cookbook/bigquery_tornadoes_it_test.py | 3 +++
.../apache_beam/examples/wordcount_it_test.py | 6 +++++-
sdks/python/run_postcommit.sh | 20 +++++++++-----------
3 files changed, 17 insertions(+), 12 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/ffe8355b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py
index 306a09e..709e3d5 100644
--- a/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py
+++ b/sdks/python/apache_beam/examples/cookbook/bigquery_tornadoes_it_test.py
@@ -32,6 +32,9 @@ from apache_beam.tests.pipeline_verifiers import PipelineStateMatcher
class BigqueryTornadoesIT(unittest.TestCase):
+ # Enable nose tests running in parallel
+ _multiprocess_can_split_ = True
+
# The default checksum is a SHA-1 hash generated from sorted rows reading
# from expected Bigquery table.
DEFAULT_CHECKSUM = '83789a7c1bca7959dcf23d3bc37e9204e594330f'
http://git-wip-us.apache.org/repos/asf/beam/blob/ffe8355b/sdks/python/apache_beam/examples/wordcount_it_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/examples/wordcount_it_test.py b/sdks/python/apache_beam/examples/wordcount_it_test.py
index 1c700b6..54e54e8 100644
--- a/sdks/python/apache_beam/examples/wordcount_it_test.py
+++ b/sdks/python/apache_beam/examples/wordcount_it_test.py
@@ -18,6 +18,7 @@
"""End-to-end test for the wordcount example."""
import logging
+import time
import unittest
from hamcrest.core.core.allof import all_of
@@ -31,6 +32,9 @@ from apache_beam.tests.pipeline_verifiers import FileChecksumMatcher
class WordCountIT(unittest.TestCase):
+ # Enable nose tests running in parallel
+ _multiprocess_can_split_ = True
+
# The default checksum is a SHA-1 hash generated from a sorted list of
# lines read from expected output.
DEFAULT_CHECKSUM = '33535a832b7db6d78389759577d4ff495980b9c0'
@@ -41,7 +45,7 @@ class WordCountIT(unittest.TestCase):
# Set extra options to the pipeline for test purpose
output = '/'.join([test_pipeline.get_option('output'),
- test_pipeline.get_option('job_name'),
+ str(int(time.time())),
'results'])
arg_sleep_secs = test_pipeline.get_option('sleep_secs')
sleep_secs = int(arg_sleep_secs) if arg_sleep_secs is not None else None
http://git-wip-us.apache.org/repos/asf/beam/blob/ffe8355b/sdks/python/run_postcommit.sh
----------------------------------------------------------------------
diff --git a/sdks/python/run_postcommit.sh b/sdks/python/run_postcommit.sh
index 50338e2..dd3182a 100755
--- a/sdks/python/run_postcommit.sh
+++ b/sdks/python/run_postcommit.sh
@@ -56,12 +56,9 @@ python -m apache_beam.examples.wordcount --output /tmp/py-wordcount-direct
# Run tests on the service.
-# Where to store wordcount output.
+# Where to store integration test outputs.
GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
-# Job name needs to be unique
-JOBNAME_E2E_WC=py-wordcount-`date +%s`
-
PROJECT=apache-beam-testing
# Create a tarball
@@ -88,18 +85,19 @@ python setup.py nosetests \
--requirements_file=postcommit_requirements.txt \
--num_workers=1"
-# Run wordcount on the Google Cloud Dataflow service
-# and validate job that finishes successfully.
-echo ">>> RUNNING TEST DATAFLOW RUNNER py-wordcount"
+# Run integration tests on the Google Cloud Dataflow service
+# and validate that jobs finish successfully.
+echo ">>> RUNNING TEST DATAFLOW RUNNER it tests"
python setup.py nosetests \
-a IT \
+ --processes=4 \
+ --process-timeout=600 \
--test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
- --staging_location=$GCS_LOCATION/staging-wordcount \
- --temp_location=$GCS_LOCATION/temp-wordcount \
- --output=$GCS_LOCATION/py-wordcount-cloud/output \
+ --staging_location=$GCS_LOCATION/staging-it \
+ --temp_location=$GCS_LOCATION/temp-it \
+ --output=$GCS_LOCATION/py-it-cloud/output \
--sdk_location=$SDK_LOCATION \
- --job_name=$JOBNAME_E2E_WC \
--num_workers=1 \
--sleep_secs=20"
[2/2] beam git commit: This closes #2393
Posted by al...@apache.org.
This closes #2393
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/3711c0ca
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/3711c0ca
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/3711c0ca
Branch: refs/heads/master
Commit: 3711c0caf91e1c4d32c055bdff098f81f56b49c1
Parents: 570d0e2 ffe8355
Author: Ahmet Altay <al...@google.com>
Authored: Mon Apr 3 15:15:11 2017 -0700
Committer: Ahmet Altay <al...@google.com>
Committed: Mon Apr 3 15:15:11 2017 -0700
----------------------------------------------------------------------
.../cookbook/bigquery_tornadoes_it_test.py | 3 +++
.../apache_beam/examples/wordcount_it_test.py | 6 +++++-
sdks/python/run_postcommit.sh | 20 +++++++++-----------
3 files changed, 17 insertions(+), 12 deletions(-)
----------------------------------------------------------------------