You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by ma...@apache.org on 2019/05/17 22:36:22 UTC
[beam] branch master updated: [BEAM-7339] Enable 1Gb input for
Python wordcount benchmark
This is an automated email from the ASF dual-hosted git repository.
markliu pushed a commit to branch master
in repository https://gitbox.apache.org/repos/asf/beam.git
The following commit(s) were added to refs/heads/master by this push:
new ddbdcd1 [BEAM-7339] Enable 1Gb input for Python wordcount benchmark
new 21fece9 Merge pull request #8608 from markflyhigh/py-large-input
ddbdcd1 is described below
commit ddbdcd11ba37d2dd1767a7c1f303393947b7dc86
Author: Mark Liu <ma...@google.com>
AuthorDate: Fri May 17 13:00:10 2019 -0700
[BEAM-7339] Enable 1Gb input for Python wordcount benchmark
---
.test-infra/jenkins/job_PerformanceTests_Python.groovy | 16 ++++++++++++----
1 file changed, 12 insertions(+), 4 deletions(-)
diff --git a/.test-infra/jenkins/job_PerformanceTests_Python.groovy b/.test-infra/jenkins/job_PerformanceTests_Python.groovy
index 732272f..0ccd4ea 100644
--- a/.test-infra/jenkins/job_PerformanceTests_Python.groovy
+++ b/.test-infra/jenkins/job_PerformanceTests_Python.groovy
@@ -62,24 +62,32 @@ def dataflowPipelineArgs = [
def testConfigurations = [
new PerformanceTestConfigurations(
jobName : 'beam_PerformanceTests_WordCountIT_Py27',
- jobDescription : 'Python SDK Performance Test - Run WordCountIT in Py27',
+ jobDescription : 'Python SDK Performance Test - Run WordCountIT in Py27 with 1Gb files',
jobTriggerPhrase : 'Run Python27 WordCountIT Performance Test',
resultTable : 'beam_performance.wordcount_py27_pkb_results',
itClass : 'apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it',
itModule : 'sdks/python',
extraPipelineArgs : dataflowPipelineArgs + [
- output: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output'
+ input: 'gs://apache-beam-samples/input_small_files/ascii_sort_1MB_input.0000*', // 1Gb
+ output: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output',
+ expect_checksum: 'ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710',
+ num_workers: '10',
+ autoscaling_algorithm: 'NONE', // Disable autoscale the worker pool.
],
),
new PerformanceTestConfigurations(
jobName : 'beam_PerformanceTests_WordCountIT_Py35',
- jobDescription : 'Python SDK Performance Test - Run WordCountIT in Py35',
+ jobDescription : 'Python SDK Performance Test - Run WordCountIT in Py35 with 1Gb files',
jobTriggerPhrase : 'Run Python35 WordCountIT Performance Test',
resultTable : 'beam_performance.wordcount_py35_pkb_results',
itClass : 'apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it',
itModule : 'sdks/python/test-suites/dataflow/py35',
extraPipelineArgs : dataflowPipelineArgs + [
- output: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output'
+ input: 'gs://apache-beam-samples/input_small_files/ascii_sort_1MB_input.0000*', // 1Gb
+ output: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output',
+ expect_checksum: 'ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710',
+ num_workers: '10',
+ autoscaling_algorithm: 'NONE', // Disable autoscale the worker pool.
],
)
]