You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by al...@apache.org on 2017/03/16 18:45:12 UTC
[1/2] beam git commit: [BEAM-1687] Parallel ValidatesRunner Tests
Repository: beam
Updated Branches:
refs/heads/master 444edae75 -> a0b8a854d
[BEAM-1687] Parallel ValidatesRunner Tests
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/2352c291
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/2352c291
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/2352c291
Branch: refs/heads/master
Commit: 2352c291288957b78dce5bfa3cfbef128fcd56b4
Parents: 444edae
Author: Mark Liu <ma...@google.com>
Authored: Fri Mar 10 11:06:27 2017 -0800
Committer: Ahmet Altay <al...@google.com>
Committed: Thu Mar 16 11:44:49 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/transforms/ptransform_test.py | 2 ++
sdks/python/apache_beam/transforms/sideinputs_test.py | 2 ++
sdks/python/run_postcommit.sh | 10 ++++++----
3 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/beam/blob/2352c291/sdks/python/apache_beam/transforms/ptransform_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/ptransform_test.py b/sdks/python/apache_beam/transforms/ptransform_test.py
index f7f157f..37ff2a8 100644
--- a/sdks/python/apache_beam/transforms/ptransform_test.py
+++ b/sdks/python/apache_beam/transforms/ptransform_test.py
@@ -45,6 +45,8 @@ from apache_beam.utils.pipeline_options import TypeOptions
class PTransformTest(unittest.TestCase):
+ # Enable nose tests running in parallel
+ _multiprocess_can_split_ = True
def assertStartswith(self, msg, prefix):
self.assertTrue(msg.startswith(prefix),
http://git-wip-us.apache.org/repos/asf/beam/blob/2352c291/sdks/python/apache_beam/transforms/sideinputs_test.py
----------------------------------------------------------------------
diff --git a/sdks/python/apache_beam/transforms/sideinputs_test.py b/sdks/python/apache_beam/transforms/sideinputs_test.py
index 7df870f..9278f4b 100644
--- a/sdks/python/apache_beam/transforms/sideinputs_test.py
+++ b/sdks/python/apache_beam/transforms/sideinputs_test.py
@@ -29,6 +29,8 @@ from apache_beam.transforms.util import assert_that, equal_to
class SideInputsTest(unittest.TestCase):
+ # Enable nose tests running in parallel
+ _multiprocess_can_split_ = True
def create_pipeline(self):
return TestPipeline()
http://git-wip-us.apache.org/repos/asf/beam/blob/2352c291/sdks/python/run_postcommit.sh
----------------------------------------------------------------------
diff --git a/sdks/python/run_postcommit.sh b/sdks/python/run_postcommit.sh
index 1c0bbb2..4d17942 100755
--- a/sdks/python/run_postcommit.sh
+++ b/sdks/python/run_postcommit.sh
@@ -61,7 +61,6 @@ GCS_LOCATION=gs://temp-storage-for-end-to-end-tests
# Job name needs to be unique
JOBNAME_E2E_WC=py-wordcount-`date +%s`
-JOBNAME_VR_TEST=py-validatesrunner-`date +%s`
PROJECT=apache-beam-testing
@@ -77,13 +76,15 @@ echo "mock" >> postcommit_requirements.txt
# Run ValidatesRunner tests on Google Cloud Dataflow service
echo ">>> RUNNING DATAFLOW RUNNER VALIDATESRUNNER TESTS"
python setup.py nosetests \
- -a ValidatesRunner --test-pipeline-options=" \
+ -a ValidatesRunner \
+ --processes=4 \
+ --process-timeout=600 \
+ --test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--staging_location=$GCS_LOCATION/staging-validatesrunner-test \
--temp_location=$GCS_LOCATION/temp-validatesrunner-test \
--sdk_location=$SDK_LOCATION \
- --job_name=$JOBNAME_VR_TEST \
--requirements_file=postcommit_requirements.txt \
--num_workers=1"
@@ -91,7 +92,8 @@ python setup.py nosetests \
# and validate job that finishes successfully.
echo ">>> RUNNING TEST DATAFLOW RUNNER py-wordcount"
python setup.py nosetests \
- -a IT --test-pipeline-options=" \
+ -a IT \
+ --test-pipeline-options=" \
--runner=TestDataflowRunner \
--project=$PROJECT \
--staging_location=$GCS_LOCATION/staging-wordcount \
[2/2] beam git commit: This closes #2220
Posted by al...@apache.org.
This closes #2220
Project: http://git-wip-us.apache.org/repos/asf/beam/repo
Commit: http://git-wip-us.apache.org/repos/asf/beam/commit/a0b8a854
Tree: http://git-wip-us.apache.org/repos/asf/beam/tree/a0b8a854
Diff: http://git-wip-us.apache.org/repos/asf/beam/diff/a0b8a854
Branch: refs/heads/master
Commit: a0b8a854df6129e2dbbbd4bad683e53d975f3146
Parents: 444edae 2352c29
Author: Ahmet Altay <al...@google.com>
Authored: Thu Mar 16 11:45:03 2017 -0700
Committer: Ahmet Altay <al...@google.com>
Committed: Thu Mar 16 11:45:03 2017 -0700
----------------------------------------------------------------------
sdks/python/apache_beam/transforms/ptransform_test.py | 2 ++
sdks/python/apache_beam/transforms/sideinputs_test.py | 2 ++
sdks/python/run_postcommit.sh | 10 ++++++----
3 files changed, 10 insertions(+), 4 deletions(-)
----------------------------------------------------------------------