You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2019/09/28 18:38:19 UTC
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py36
#517
See <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/517/display/redirect>
------------------------------------------
[...truncated 157.24 KB...]
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "eNrNV2l728YRJkj5COK6TtI4cdyDdeMWSkPSSizVVtPELm0nCi1ahZQIPRxkCSy5sAAsZnchWqmYy6WspvfdpHfT+07v+/gd/TOdXZJymUipv7nPIwHg7M7szjvvvAs8W3ICkpGAUb9FSVJVgqSyzUUiqwEX1K6TOCatmC4LkmVUnOHnUhsKk8+B1YOi45UKhYLfTqE0FiTi+Nciktp+O0pJHD1F/a6IFLVhwttnXHIMKzoyowHsanp70JYJHlApYTeb8G7Wc9RaRn0WpUrCnvE94oCxV0OKmySKC2nPXVhC8yPabMNe3OBNzR7YjgkVpVmuTDwJNzfNDniurtn2NfMNeEMrb12E/c6rM1H0soq47S/hfTFKV2x4I0Y/0INbHO8ARmpHMfUzopifCdqOLsOtYxF4ht6prK6SOMd5gq9GIRX2oiIqCh7XxoWRDW7DwG/qwe2OZ2PggYfeIRwMwiiOq76+2n7MSWjsNtxhKiCVgDv7cMiFu7xdI0847F3A546crdUUTbKKRJxIh1awuhWahhXFBzcqlaxla5VIVYKY52FtgE1tanrm5MzJ6ePHp6ZPnKgJKvMYS/HmvHUt7ZQk1Jd5W6f9Fqa3/9YevM1hB9ld7LCHswpQxvk6mzRPfMmICCW8vVEw2ww4Zg1HxuAyNjm82Ut8UYko7dS5wecduMDdPTja8m7RWetogy1owhIF7/TuRnvlqDMcypPJY9NhhbfRdG19bYN3eQfHQ3Ri3hrFccxg5Z5tPSdN+gFPsNxSYm0HJbrHm0AzyRWHd3s34WMSJcPq3WsA0DyqZTGJUqh4hzTBdUelWElBohiT9FPaxTvysbppOsJnlGiEas28Nc7KVFGBfVXNVRTbp0UnT2iqFmISUMZjA9UxhGqK6et9Dcvc76c9OH4Rph1vUmeOjDtWMU1ZW752nUuyuHZu2LPGAjNjS2eGW7b/WJpFwUpMw0UMNKf7y4b39OCEY2AIiSJwcjvHrelncIoNs7ix9/bgAcfbr9ELtALoZtL5wfu8w2jU3rN6u75p41nck9CKNLs6BQ8aInSjNORdP0E8NYwoRw/tpGk6jgkjbVNxEvuv8bbhlIF/NaJdHez0WPMFghKFjMvTQDe2De932B3eXpyvSao1COp9OOPCWadhNQr4X2rcVrc3C+uFTetKcbEA55p9eHjSuIxygUf6MOcJtNQYT2jtEk1XolSO7hUZk1Va63KxIjEvWtNp+QtUGL6mAV3SPewvcxHWeZ6quSV/Ye3+mZoUQU2GK7q7FeNp7b9AqQ3qUc3W4FGzlQdikrRC8iA05i9Y9QKc927XXS544gsMqcm8tdt5I6EGn6G8QXMDLkwqWHDhA2NwdajysZzISNcs08qjWGFKsGgwxmE9Cksb8JgLj4+5RknGhfITHuYxSt2yd6umyGsoBV4fPujCh0x4H30D5fvw4Q34iAsXWaO5XfUCij/gCYa1wzIVsUylxp5Gvf6kKijrUnG9EGKprli94npRHlm3LpXCopq4UtDlU7vWcSQsrRTFzHoxnDhUQNvucNfArvYMbMOnkn5qlw7i9XkrxJngO81G0aQe0jZBRYUnNU28p9GyUOdxTA2tyrxdltiC5aNhuRspVk7wMC4rRnAkpWUaU93x5UHD0LBMZJmgQ9qJqUJvXZpq+VwkpCqrLh/Nl2WaBpogVGgfjHjkqDxyr7lWAYWvNWjfOJIKAqNiuuaK81hC6O3Wv2UcBRSoOXewetA2nXs2ydTaVm9DxwzHNAVmtM+cc2eF4AIidqeCS17RxIYVA8WIRLFZXaseJFchZY8aHfa3JyGff8WqHygUD1gT1n5rn7XXKlrFImSTSENwQbCLjDfxHUA2FSgX8j6sutBlT/Tg8g6qscZOyT485cJHN2C9Bz10fNqFZ/IWe4idyq/Cs1vKOXVdyvkcQ0V83mEnmZa5Kz34mMP+t6b19Wrs9JCbAwmxGov1iXXNnw0Uj6uTDPVi80bpxcfH9eKF+X9b7LxG/RMufBJRf0Gj/ikE79MufGYLvM9ugXffdYH3OQ3e50fgfaEHX7we8L60DXiv1t8vI4RfMRB+9UZB+OI4hC+h5LLzbJ6hin4Ngfy6C99AIF9qshshUd/UEsX+b2TpWwq+7bCAhYyyNuswxoyEfIetsJihTHyXvfh6MvHy9jLxPU3Y77vwA8T5ZU3YHyJhf+TCj/vwExd+qmXiZzvIxM+NTPzChV9uwK968Ao6/tqF32wx/bf0db+olk1AjGXD75DZv+/BHxxDh8FKuMAfd/IfzLAfNi8ugzj4sfMnjPJnI9b4rtzpIEVT+MtOIYZT7DOD02dp+BP+ikH+ZhLG4uVJHhNdb334Uvh7wzJnrwZWKpJkPr79tvBNVcA/cMh8rEXSHx1o/9zMWwr+Vf0PXw7S9Q==",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: '2019-09-28T18:30:46.395973Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2019-09-28_11_30_45-17060369084266354367'
location: 'us-central1'
name: 'beamapp-jenkins-0928183042-692085'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2019-09-28T18:30:46.395973Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2019-09-28_11_30_45-17060369084266354367]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-09-28_11_30_45-17060369084266354367?project=apache-beam-testing
root: INFO: Job 2019-09-28_11_30_45-17060369084266354367 is in state JOB_STATE_RUNNING
root: INFO: 2019-09-28T18:30:48.402Z: JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service Account.
root: INFO: 2019-09-28T18:30:48.875Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
root: INFO: 2019-09-28T18:30:49.473Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2019-09-28T18:30:49.510Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2019-09-28T18:30:49.540Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2019-09-28T18:30:49.568Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
root: INFO: 2019-09-28T18:30:49.601Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2019-09-28T18:30:49.688Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
root: INFO: 2019-09-28T18:30:49.735Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2019-09-28T18:30:49.758Z: JOB_MESSAGE_DETAILED: Fusing consumer split into read/Read
root: INFO: 2019-09-28T18:30:49.792Z: JOB_MESSAGE_DETAILED: Fusing consumer pair_with_one into split
root: INFO: 2019-09-28T18:30:49.816Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Reify into pair_with_one
root: INFO: 2019-09-28T18:30:49.845Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Write into group/Reify
root: INFO: 2019-09-28T18:30:49.873Z: JOB_MESSAGE_DETAILED: Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2019-09-28T18:30:49.901Z: JOB_MESSAGE_DETAILED: Fusing consumer count into group/GroupByWindow
root: INFO: 2019-09-28T18:30:49.932Z: JOB_MESSAGE_DETAILED: Fusing consumer format into count
root: INFO: 2019-09-28T18:30:49.958Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2019-09-28T18:30:49.981Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2019-09-28T18:30:50.015Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2019-09-28T18:30:50.051Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2019-09-28T18:30:50.087Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2019-09-28T18:30:50.123Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2019-09-28T18:30:50.150Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2019-09-28T18:30:50.182Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2019-09-28T18:30:50.218Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
root: INFO: 2019-09-28T18:30:50.252Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
root: INFO: 2019-09-28T18:30:50.289Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
root: INFO: 2019-09-28T18:30:50.315Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
root: INFO: 2019-09-28T18:30:50.465Z: JOB_MESSAGE_DEBUG: Executing wait step start26
root: INFO: 2019-09-28T18:30:50.522Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-28T18:30:50.557Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-28T18:30:50.570Z: JOB_MESSAGE_DEBUG: Starting worker pool setup.
root: INFO: 2019-09-28T18:30:50.593Z: JOB_MESSAGE_BASIC: Executing operation group/Create
root: INFO: 2019-09-28T18:30:50.604Z: JOB_MESSAGE_BASIC: Starting 10 workers in us-central1-f...
root: INFO: 2019-09-28T18:30:50.663Z: JOB_MESSAGE_BASIC: Finished operation group/Create
root: INFO: 2019-09-28T18:30:50.663Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-28T18:30:50.733Z: JOB_MESSAGE_DEBUG: Value "group/Session" materialized.
root: INFO: 2019-09-28T18:30:50.757Z: JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/GroupByKey/Session" materialized.
root: INFO: 2019-09-28T18:30:50.794Z: JOB_MESSAGE_BASIC: Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-28T18:31:18.051Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 10 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-28T18:32:45.552Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-28T18:32:45.587Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-28T18:35:21.475Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-28T18:35:23.536Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-28T18:35:25.615Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-28T18:35:27.853Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-28T18:35:29.922Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-28T18:35:31.981Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-28T18:35:34.033Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-28T18:35:34.060Z: JOB_MESSAGE_BASIC: Finished operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-28T18:35:34.141Z: JOB_MESSAGE_DEBUG: Executing failure step failure25
root: INFO: 2019-09-28T18:35:34.176Z: JOB_MESSAGE_ERROR: Workflow failed. Causes: S06:read/Read+split+pair_with_one+group/Reify+group/Write failed., Internal Issue (957059d153e68f19): 63963027:24514
root: INFO: 2019-09-28T18:35:34.542Z: JOB_MESSAGE_WARNING: S01:write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite failed.
root: INFO: 2019-09-28T18:35:34.580Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-28T18:35:34.696Z: JOB_MESSAGE_DETAILED: Cleaning up.
root: INFO: 2019-09-28T18:35:34.781Z: JOB_MESSAGE_DEBUG: Starting worker pool teardown.
root: INFO: 2019-09-28T18:35:34.819Z: JOB_MESSAGE_BASIC: Stopping worker pool...
root: INFO: 2019-09-28T18:38:09.380Z: JOB_MESSAGE_DETAILED: Autoscaling: Reduced the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-28T18:38:09.430Z: JOB_MESSAGE_BASIC: Worker pool stopped.
root: INFO: 2019-09-28T18:38:09.470Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
root: INFO: Job 2019-09-28_11_30_45-17060369084266354367 is in state JOB_STATE_FAILED
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569695441588/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569695441588/results*' -> 'gs\\:\\/\\/temp\\-storage\\-for\\-end\\-to\\-end\\-tests\\/py\\-it\\-cloud\\/output\\/1569695441588\\/results[^/\\\\]*'
root: INFO: Starting the size estimation of the input
root: INFO: Finished listing 0 files in 0.0467531681060791 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-integrationTest-perf.xml
----------------------------------------------------------------------
XML: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 455.672s
FAILED (errors=1)
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py36:integrationTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
BUILD FAILED in 8m 18s
2019-09-28 18:38:18,489 629a2cc8 MainThread beam_integration_benchmark(1/1) ERROR Error during benchmark beam_integration_benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-28 18:38:18,490 629a2cc8 MainThread beam_integration_benchmark(1/1) INFO Cleaning up benchmark beam_integration_benchmark
2019-09-28 18:38:18,492 629a2cc8 MainThread beam_integration_benchmark(1/1) ERROR Exception running benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 984, in RunBenchmarkTask
RunBenchmark(spec, collector)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-28 18:38:18,492 629a2cc8 MainThread beam_integration_benchmark(1/1) ERROR Benchmark 1/1 beam_integration_benchmark (UID: beam_integration_benchmark0) failed. Execution will continue.
2019-09-28 18:38:18,492 629a2cc8 MainThread beam_integration_benchmark(1/1) INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2019-09-28 18:38:18,493 629a2cc8 MainThread beam_integration_benchmark(1/1) INFO Complete logs can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/629a2cc8/pkb.log>
2019-09-28 18:38:18,493 629a2cc8 MainThread beam_integration_benchmark(1/1) INFO Completion statuses can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/629a2cc8/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Jenkins build is back to normal :
beam_PerformanceTests_WordCountIT_Py36 #529
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/529/display/redirect?page=changes>
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py36
#528
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/528/display/redirect?page=changes>
Changes:
[mxm] [BEAM-5428] Implement cross-bundle user state caching in the Python SDK
[mxm] [BEAM-5428] Add pipeline option to enable caching / Disable caching by
[lcwik] [BEAM-8021] Swap build-tools to be compile only so it isn't a "required"
------------------------------------------
[...truncated 179.39 KB...]
Successfully started process 'command 'sh''
Collecting future==0.16.0
Installing collected packages: future
Successfully installed future-0.16.0
Create distribution tar file apache-beam.tar.gz in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/build>
:sdks:python:sdist (Thread[Execution worker for ':',5,main]) completed. Took 6.238 secs.
:sdks:python:test-suites:dataflow:py36:installGcpTest (Thread[Execution worker for ':',5,main]) started.
> Task :sdks:python:test-suites:dataflow:py36:installGcpTest
Caching disabled for task ':sdks:python:test-suites:dataflow:py36:installGcpTest': Caching has not been enabled for the task
Task ':sdks:python:test-suites:dataflow:py36:installGcpTest' is not up-to-date because:
Task has not declared any outputs despite executing actions.
Custom actions are attached to task ':sdks:python:test-suites:dataflow:py36:installGcpTest'.
Starting process 'command 'sh''. Working directory: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/test-suites/dataflow/py36> Command: sh -c . <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/bin/activate> && pip install --retries 10 -e <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/[gcp,test]>
Successfully started process 'command 'sh''
Obtaining file://<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python>
Collecting crcmod<2.0,>=1.7 (from apache-beam==2.17.0.dev0)
Collecting dill<0.3.1,>=0.3.0 (from apache-beam==2.17.0.dev0)
Collecting fastavro<0.22,>=0.21.4 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/e1/28/0206330c0002b1e28e21473117d0dc813defbd5891562d27af5c68c93899/fastavro-0.21.24-cp36-cp36m-manylinux1_x86_64.whl
Collecting future<1.0.0,>=0.16.0 (from apache-beam==2.17.0.dev0)
Requirement already satisfied: grpcio<2,>=1.12.1 in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages> (from apache-beam==2.17.0.dev0) (1.24.0)
Collecting hdfs<3.0.0,>=2.1.0 (from apache-beam==2.17.0.dev0)
Collecting httplib2<=0.12.0,>=0.8 (from apache-beam==2.17.0.dev0)
Collecting mock<3.0.0,>=1.0.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/e6/35/f187bdf23be87092bd0f1200d43d23076cee4d0dec109f195173fd3ebc79/mock-2.0.0-py2.py3-none-any.whl
Collecting pymongo<4.0.0,>=3.8.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/6d/27/f30b90f40054948b32df04a8e6355946874d084ac73755986b28d3003578/pymongo-3.9.0-cp36-cp36m-manylinux1_x86_64.whl
Collecting oauth2client<4,>=2.0.1 (from apache-beam==2.17.0.dev0)
Requirement already satisfied: protobuf<4,>=3.5.0.post1 in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages> (from apache-beam==2.17.0.dev0) (3.9.2)
Collecting pydot<2,>=1.2.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/33/d1/b1479a770f66d962f545c2101630ce1d5592d90cb4f083d38862e93d16d2/pydot-1.4.1-py2.py3-none-any.whl
Collecting python-dateutil<3,>=2.8.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/41/17/c62faccbfbd163c7f57f3844689e3a78bae1f403648a6afb1d0866d87fbb/python_dateutil-2.8.0-py2.py3-none-any.whl
Collecting pytz>=2018.3 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/87/76/46d697698a143e05f77bec5a526bf4e56a0be61d63425b68f4ba553b51f2/pytz-2019.2-py2.py3-none-any.whl
Collecting pyyaml<4.0.0,>=3.12 (from apache-beam==2.17.0.dev0)
Collecting avro-python3<2.0.0,>=1.8.1 (from apache-beam==2.17.0.dev0)
Collecting pyarrow<0.15.0,>=0.11.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/6f/df/33d8d6b682750d16baa9f45db825f33e0e0feb479fac1da9758f7ac8fd4b/pyarrow-0.14.1-cp36-cp36m-manylinux2010_x86_64.whl
Collecting cachetools<4,>=3.1.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/2f/a6/30b0a0bef12283e83e58c1d6e7b5aabc7acfc4110df81a4471655d33e704/cachetools-3.1.1-py2.py3-none-any.whl
Collecting google-apitools<0.5.29,>=0.5.28 (from apache-beam==2.17.0.dev0)
Collecting google-cloud-datastore<1.8.0,>=1.7.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/d0/aa/29cbcf8cf7d08ce2d55b9dce858f7c632b434cb6451bed17cb4275804217/google_cloud_datastore-1.7.4-py2.py3-none-any.whl
Collecting google-cloud-pubsub<1.1.0,>=0.39.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/d3/91/07a82945a7396ea34debafd476724bb5fc267c292790fdf2138c693f95c5/google_cloud_pubsub-1.0.2-py2.py3-none-any.whl
Collecting google-cloud-bigquery<1.18.0,>=1.6.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/a4/96/1b9cf1d43869c47a205aad411dac7c3040df6093d63c39273fa4d4c45da7/google_cloud_bigquery-1.17.1-py2.py3-none-any.whl
Collecting google-cloud-core<2,>=0.28.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/ee/f0/084f598629db8e6ec3627688723875cdb03637acb6d86999bb105a71df64/google_cloud_core-1.0.3-py2.py3-none-any.whl
Collecting google-cloud-bigtable<1.1.0,>=0.31.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/95/af/0ef7d097a1d5ad0c843867600e86de915e8ab8864740f49a4636cfb51af6/google_cloud_bigtable-1.0.0-py2.py3-none-any.whl
Collecting nose>=1.3.7 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl
Collecting nose_xunitmp>=0.4.1 (from apache-beam==2.17.0.dev0)
Collecting numpy<2,>=1.14.3 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/e5/e6/c3fdc53aed9fa19d6ff3abf97dfad768ae3afce1b7431f7500000816bda5/numpy-1.17.2-cp36-cp36m-manylinux1_x86_64.whl
Collecting pandas<0.25,>=0.23.4 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/19/74/e50234bc82c553fecdbd566d8650801e3fe2d6d8c8d940638e3d8a7c5522/pandas-0.24.2-cp36-cp36m-manylinux1_x86_64.whl
Collecting parameterized<0.7.0,>=0.6.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/3a/49/75f6dadb09e2f8ace3cdffe0c99a04f1b98dff41fbf9e768665d8b469e29/parameterized-0.6.3-py2.py3-none-any.whl
Collecting pyhamcrest<2.0,>=1.9 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/9a/d5/d37fd731b7d0e91afcc84577edeccf4638b4f9b82f5ffe2f8b62e2ddc609/PyHamcrest-1.9.0-py2.py3-none-any.whl
Collecting tenacity<6.0,>=5.0.2 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/1e/a1/be8c8610f4620c56790965ba2b564dd76d13cbcd7c2ff8f6053ce63027fb/tenacity-5.1.1-py2.py3-none-any.whl
Requirement already satisfied: six>=1.5.2 in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages> (from grpcio<2,>=1.12.1->apache-beam==2.17.0.dev0) (1.12.0)
Collecting requests>=2.7.0 (from hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/51/bd/23c926cd341ea6b7dd0b2a00aba99ae0f828be89d72b2190f27c11d4b7fb/requests-2.22.0-py2.py3-none-any.whl
Collecting docopt (from hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Collecting pbr>=0.11 (from mock<3.0.0,>=1.0.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/46/a4/d5c83831a3452713e4b4f126149bc4fbda170f7cb16a86a00ce57ce0e9ad/pbr-5.4.3-py2.py3-none-any.whl
Collecting pyasn1>=0.1.7 (from oauth2client<4,>=2.0.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/a1/71/8f0d444e3a74e5640a3d5d967c1c6b015da9c655f35b2d308a55d907a517/pyasn1-0.4.7-py2.py3-none-any.whl
Collecting pyasn1-modules>=0.0.5 (from oauth2client<4,>=2.0.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/be/70/e5ea8afd6d08a4b99ebfc77bd1845248d56cfcf43d11f9dc324b9580a35c/pyasn1_modules-0.2.6-py2.py3-none-any.whl
Collecting rsa>=3.1.4 (from oauth2client<4,>=2.0.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/02/e5/38518af393f7c214357079ce67a317307936896e961e35450b70fad2a9cf/rsa-4.0-py2.py3-none-any.whl
Requirement already satisfied: setuptools in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages> (from protobuf<4,>=3.5.0.post1->apache-beam==2.17.0.dev0) (41.2.0)
Collecting pyparsing>=2.1.4 (from pydot<2,>=1.2.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/11/fa/0160cd525c62d7abd076a070ff02b2b94de589f1a9789774f17d7c54058e/pyparsing-2.4.2-py2.py3-none-any.whl
Collecting fasteners>=0.14 (from google-apitools<0.5.29,>=0.5.28->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/18/bd/55eb2d6397b9c0e263af9d091ebdb756b15756029b3cededf6461481bc63/fasteners-0.15-py2.py3-none-any.whl
Collecting google-api-core[grpc]<2.0.0dev,>=1.6.0 (from google-cloud-datastore<1.8.0,>=1.7.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/71/e5/7059475b3013a3c75abe35015c5761735ab224eb1b129fee7c8e376e7805/google_api_core-1.14.2-py2.py3-none-any.whl
Collecting grpc-google-iam-v1<0.13dev,>=0.12.3 (from google-cloud-pubsub<1.1.0,>=0.39.0->apache-beam==2.17.0.dev0)
Collecting google-resumable-media<0.5.0dev,>=0.3.1 (from google-cloud-bigquery<1.18.0,>=1.6.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/96/d7/b29a41b01b854480891dfc408211ffb0cc7a2a3d5f15a3b6740ec18c845b/google_resumable_media-0.4.1-py2.py3-none-any.whl
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/e0/da/55f51ea951e1b7c63a579c09dd7db825bb730ec1fe9c0180fc77bfb31448/urllib3-1.25.6-py2.py3-none-any.whl
Collecting certifi>=2017.4.17 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/18/b0/8146a4f8dd402f60744fa380bc73ca47303cccf8b9190fd16a827281eac2/certifi-2019.9.11-py2.py3-none-any.whl
Collecting chardet<3.1.0,>=3.0.2 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl
Collecting idna<2.9,>=2.5 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/14/2c/cd551d81dbe15200be1cf41cd03869a46fe7226e7450af7a6545bfc474c9/idna-2.8-py2.py3-none-any.whl
Collecting monotonic>=0.1 (from fasteners>=0.14->google-apitools<0.5.29,>=0.5.28->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/ac/aa/063eca6a416f397bd99552c534c6d11d57f58f2e94c14780f3bbf818c4cf/monotonic-1.5-py2.py3-none-any.whl
Collecting googleapis-common-protos<2.0dev,>=1.6.0 (from google-api-core[grpc]<2.0.0dev,>=1.6.0->google-cloud-datastore<1.8.0,>=1.7.1->apache-beam==2.17.0.dev0)
Collecting google-auth<2.0dev,>=0.4.0 (from google-api-core[grpc]<2.0.0dev,>=1.6.0->google-cloud-datastore<1.8.0,>=1.7.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/c5/9b/ed0516cc1f7609fb0217e3057ff4f0f9f3e3ce79a369c6af4a6c5ca25664/google_auth-1.6.3-py2.py3-none-any.whl
Installing collected packages: crcmod, dill, fastavro, future, urllib3, certifi, chardet, idna, requests, docopt, hdfs, httplib2, pbr, mock, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, pyparsing, pydot, python-dateutil, pytz, pyyaml, avro-python3, numpy, pyarrow, cachetools, monotonic, fasteners, google-apitools, googleapis-common-protos, google-auth, google-api-core, google-cloud-core, google-cloud-datastore, grpc-google-iam-v1, google-cloud-pubsub, google-resumable-media, google-cloud-bigquery, google-cloud-bigtable, nose, nose-xunitmp, pandas, parameterized, pyhamcrest, tenacity, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.1 cachetools-3.1.1 certifi-2019.9.11 chardet-3.0.4 crcmod-1.7 dill-0.3.0 docopt-0.6.2 fastavro-0.21.24 fasteners-0.15 future-0.17.1 google-api-core-1.14.2 google-apitools-0.5.28 google-auth-1.6.3 google-cloud-bigquery-1.17.1 google-cloud-bigtable-1.0.0 google-cloud-core-1.0.3 google-cloud-datastore-1.7.4 google-cloud-pubsub-1.0.2 google-resumable-media-0.4.1 googleapis-common-protos-1.6.0 grpc-google-iam-v1-0.12.3 hdfs-2.5.8 httplib2-0.12.0 idna-2.8 mock-2.0.0 monotonic-1.5 nose-1.3.7 nose-xunitmp-0.4.1 numpy-1.17.2 oauth2client-3.0.0 pandas-0.24.2 parameterized-0.6.3 pbr-5.4.3 pyarrow-0.14.1 pyasn1-0.4.7 pyasn1-modules-0.2.6 pydot-1.4.1 pyhamcrest-1.9.0 pymongo-3.9.0 pyparsing-2.4.2 python-dateutil-2.8.0 pytz-2019.2 pyyaml-3.13 requests-2.22.0 rsa-4.0 tenacity-5.1.1 urllib3-1.25.6
:sdks:python:test-suites:dataflow:py36:installGcpTest (Thread[Execution worker for ':',5,main]) completed. Took 16.034 secs.
:sdks:python:test-suites:dataflow:py36:integrationTest (Thread[Execution worker for ':',5,main]) started.
> Task :sdks:python:test-suites:dataflow:py36:integrationTest
Caching disabled for task ':sdks:python:test-suites:dataflow:py36:integrationTest': Caching has not been enabled for the task
Task ':sdks:python:test-suites:dataflow:py36:integrationTest' is not up-to-date because:
Task has not declared any outputs despite executing actions.
Custom actions are attached to task ':sdks:python:test-suites:dataflow:py36:integrationTest'.
Starting process 'command 'sh''. Working directory: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/test-suites/dataflow/py36> Command: sh -c . <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/bin/activate> && <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/scripts/run_integration_test.sh> --test_opts "--tests=apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it --attr=IT --nocapture" --pipeline_opts "--project=apache-beam-testing --staging_location=gs://temp-storage-for-end-to-end-tests/staging-it --temp_location=gs://temp-storage-for-end-to-end-tests/temp-it --input=gs://apache-beam-samples/input_small_files/ascii_sort_1MB_input.0000* --output=gs://temp-storage-for-end-to-end-tests/py-it-cloud/output --expect_checksum=ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710 --num_workers=10 --autoscaling_algorithm=NONE --runner=TestDataflowRunner --sdk_location=build/apache-beam.tar.gz" --suite integrationTest-perf
Successfully started process 'command 'sh''
>>> RUNNING integration tests with pipeline options: --project=apache-beam-testing --staging_location=gs://temp-storage-for-end-to-end-tests/staging-it --temp_location=gs://temp-storage-for-end-to-end-tests/temp-it --input=gs://apache-beam-samples/input_small_files/ascii_sort_1MB_input.0000* --output=gs://temp-storage-for-end-to-end-tests/py-it-cloud/output --expect_checksum=ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710 --num_workers=10 --autoscaling_algorithm=NONE --runner=TestDataflowRunner --sdk_location=build/apache-beam.tar.gz
>>> test options: --tests=apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it --attr=IT --nocapture
running nosetests
running egg_info
writing apache_beam.egg-info/PKG-INFO
writing dependency_links to apache_beam.egg-info/dependency_links.txt
writing entry points to apache_beam.egg-info/entry_points.txt
writing requirements to apache_beam.egg-info/requires.txt
writing top-level names to apache_beam.egg-info/top_level.txt
reading manifest file 'apache_beam.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
writing manifest file 'apache_beam.egg-info/SOURCES.txt'
STDERR: DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support
setup.py:186: UserWarning: You are using Apache Beam with Python 2. New releases of Apache Beam will soon support Python 3 only.
'You are using Apache Beam with Python 2. '
<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/1922375555/local/lib/python2.7/site-packages/setuptools/dist.py>:474: UserWarning: Normalizing '2.17.0.dev' to '2.17.0.dev0'
normalized_version,
beam_fn_api.proto: warning: Import google/protobuf/descriptor.proto but not used.
beam_fn_api.proto: warning: Import google/protobuf/wrappers.proto but not used.
DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
RefactoringTool: Skipping optional fixer: idioms
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
RefactoringTool: Skipping optional fixer: ws_comma
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2_grpc.py>
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_runner_api_pb2.py>
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/endpoints_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/external_transforms_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/metrics_pb2.py>
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/schema_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/standard_window_fns_pb2.py>
RefactoringTool: Files that were modified:
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_runner_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/endpoints_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/external_transforms_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/metrics_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/schema_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/standard_window_fns_pb2.py>
warning: no files found matching 'README.md'
warning: no files found matching 'NOTICE'
warning: no files found matching 'LICENSE'
warning: sdist: standard file not found: should have one of README, README.rst, README.txt, README.md
<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages/setuptools/dist.py>:474: UserWarning: Normalizing '2.17.0.dev' to '2.17.0.dev0'
normalized_version,
warning: no files found matching 'README.md'
warning: no files found matching 'NOTICE'
warning: no files found matching 'LICENSE'
test_wordcount_it (apache_beam.examples.wordcount_it_test.WordCountIT) ...
IssueCommand timed out after 1200 seconds. Process was killed by perfkitbenchmarker.
2019-10-01 13:02:38,433 912686a3 MainThread beam_integration_benchmark(1/1) ERROR Benchmark 1/1 beam_integration_benchmark (UID: beam_integration_benchmark0) failed. Execution will continue.
2019-10-01 13:02:38,434 912686a3 MainThread beam_integration_benchmark(1/1) INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2019-10-01 13:02:38,434 912686a3 MainThread beam_integration_benchmark(1/1) INFO Complete logs can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/912686a3/pkb.log>
2019-10-01 13:02:38,434 912686a3 MainThread beam_integration_benchmark(1/1) INFO Completion statuses can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/912686a3/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py36
#527
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/527/display/redirect>
Changes:
------------------------------------------
[...truncated 179.63 KB...]
Successfully started process 'command 'sh''
Collecting future==0.16.0
Installing collected packages: future
Successfully installed future-0.16.0
Create distribution tar file apache-beam.tar.gz in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/build>
:sdks:python:sdist (Thread[Execution worker for ':',5,main]) completed. Took 6.628 secs.
:sdks:python:test-suites:dataflow:py36:installGcpTest (Thread[Execution worker for ':',5,main]) started.
> Task :sdks:python:test-suites:dataflow:py36:installGcpTest
Caching disabled for task ':sdks:python:test-suites:dataflow:py36:installGcpTest': Caching has not been enabled for the task
Task ':sdks:python:test-suites:dataflow:py36:installGcpTest' is not up-to-date because:
Task has not declared any outputs despite executing actions.
Custom actions are attached to task ':sdks:python:test-suites:dataflow:py36:installGcpTest'.
Starting process 'command 'sh''. Working directory: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/test-suites/dataflow/py36> Command: sh -c . <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/bin/activate> && pip install --retries 10 -e <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/[gcp,test]>
Successfully started process 'command 'sh''
Obtaining file://<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python>
Collecting crcmod<2.0,>=1.7 (from apache-beam==2.17.0.dev0)
Collecting dill<0.3.1,>=0.3.0 (from apache-beam==2.17.0.dev0)
Collecting fastavro<0.22,>=0.21.4 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/e1/28/0206330c0002b1e28e21473117d0dc813defbd5891562d27af5c68c93899/fastavro-0.21.24-cp36-cp36m-manylinux1_x86_64.whl
Collecting future<1.0.0,>=0.16.0 (from apache-beam==2.17.0.dev0)
Requirement already satisfied: grpcio<2,>=1.12.1 in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages> (from apache-beam==2.17.0.dev0) (1.24.0)
Collecting hdfs<3.0.0,>=2.1.0 (from apache-beam==2.17.0.dev0)
Collecting httplib2<=0.12.0,>=0.8 (from apache-beam==2.17.0.dev0)
Collecting mock<3.0.0,>=1.0.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/e6/35/f187bdf23be87092bd0f1200d43d23076cee4d0dec109f195173fd3ebc79/mock-2.0.0-py2.py3-none-any.whl
Collecting pymongo<4.0.0,>=3.8.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/6d/27/f30b90f40054948b32df04a8e6355946874d084ac73755986b28d3003578/pymongo-3.9.0-cp36-cp36m-manylinux1_x86_64.whl
Collecting oauth2client<4,>=2.0.1 (from apache-beam==2.17.0.dev0)
Requirement already satisfied: protobuf<4,>=3.5.0.post1 in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages> (from apache-beam==2.17.0.dev0) (3.9.2)
Collecting pydot<2,>=1.2.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/33/d1/b1479a770f66d962f545c2101630ce1d5592d90cb4f083d38862e93d16d2/pydot-1.4.1-py2.py3-none-any.whl
Collecting python-dateutil<3,>=2.8.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/41/17/c62faccbfbd163c7f57f3844689e3a78bae1f403648a6afb1d0866d87fbb/python_dateutil-2.8.0-py2.py3-none-any.whl
Collecting pytz>=2018.3 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/87/76/46d697698a143e05f77bec5a526bf4e56a0be61d63425b68f4ba553b51f2/pytz-2019.2-py2.py3-none-any.whl
Collecting pyyaml<4.0.0,>=3.12 (from apache-beam==2.17.0.dev0)
Collecting avro-python3<2.0.0,>=1.8.1 (from apache-beam==2.17.0.dev0)
Collecting pyarrow<0.15.0,>=0.11.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/6f/df/33d8d6b682750d16baa9f45db825f33e0e0feb479fac1da9758f7ac8fd4b/pyarrow-0.14.1-cp36-cp36m-manylinux2010_x86_64.whl
Collecting cachetools<4,>=3.1.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/2f/a6/30b0a0bef12283e83e58c1d6e7b5aabc7acfc4110df81a4471655d33e704/cachetools-3.1.1-py2.py3-none-any.whl
Collecting google-apitools<0.5.29,>=0.5.28 (from apache-beam==2.17.0.dev0)
Collecting google-cloud-datastore<1.8.0,>=1.7.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/d0/aa/29cbcf8cf7d08ce2d55b9dce858f7c632b434cb6451bed17cb4275804217/google_cloud_datastore-1.7.4-py2.py3-none-any.whl
Collecting google-cloud-pubsub<1.1.0,>=0.39.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/d3/91/07a82945a7396ea34debafd476724bb5fc267c292790fdf2138c693f95c5/google_cloud_pubsub-1.0.2-py2.py3-none-any.whl
Collecting google-cloud-bigquery<1.18.0,>=1.6.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/a4/96/1b9cf1d43869c47a205aad411dac7c3040df6093d63c39273fa4d4c45da7/google_cloud_bigquery-1.17.1-py2.py3-none-any.whl
Collecting google-cloud-core<2,>=0.28.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/ee/f0/084f598629db8e6ec3627688723875cdb03637acb6d86999bb105a71df64/google_cloud_core-1.0.3-py2.py3-none-any.whl
Collecting google-cloud-bigtable<1.1.0,>=0.31.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/95/af/0ef7d097a1d5ad0c843867600e86de915e8ab8864740f49a4636cfb51af6/google_cloud_bigtable-1.0.0-py2.py3-none-any.whl
Collecting nose>=1.3.7 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl
Collecting nose_xunitmp>=0.4.1 (from apache-beam==2.17.0.dev0)
Collecting numpy<2,>=1.14.3 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/e5/e6/c3fdc53aed9fa19d6ff3abf97dfad768ae3afce1b7431f7500000816bda5/numpy-1.17.2-cp36-cp36m-manylinux1_x86_64.whl
Collecting pandas<0.25,>=0.23.4 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/19/74/e50234bc82c553fecdbd566d8650801e3fe2d6d8c8d940638e3d8a7c5522/pandas-0.24.2-cp36-cp36m-manylinux1_x86_64.whl
Collecting parameterized<0.7.0,>=0.6.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/3a/49/75f6dadb09e2f8ace3cdffe0c99a04f1b98dff41fbf9e768665d8b469e29/parameterized-0.6.3-py2.py3-none-any.whl
Collecting pyhamcrest<2.0,>=1.9 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/9a/d5/d37fd731b7d0e91afcc84577edeccf4638b4f9b82f5ffe2f8b62e2ddc609/PyHamcrest-1.9.0-py2.py3-none-any.whl
Collecting tenacity<6.0,>=5.0.2 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/1e/a1/be8c8610f4620c56790965ba2b564dd76d13cbcd7c2ff8f6053ce63027fb/tenacity-5.1.1-py2.py3-none-any.whl
Requirement already satisfied: six>=1.5.2 in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages> (from grpcio<2,>=1.12.1->apache-beam==2.17.0.dev0) (1.12.0)
Collecting docopt (from hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Collecting requests>=2.7.0 (from hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/51/bd/23c926cd341ea6b7dd0b2a00aba99ae0f828be89d72b2190f27c11d4b7fb/requests-2.22.0-py2.py3-none-any.whl
Collecting pbr>=0.11 (from mock<3.0.0,>=1.0.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/46/a4/d5c83831a3452713e4b4f126149bc4fbda170f7cb16a86a00ce57ce0e9ad/pbr-5.4.3-py2.py3-none-any.whl
Collecting pyasn1>=0.1.7 (from oauth2client<4,>=2.0.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/a1/71/8f0d444e3a74e5640a3d5d967c1c6b015da9c655f35b2d308a55d907a517/pyasn1-0.4.7-py2.py3-none-any.whl
Collecting pyasn1-modules>=0.0.5 (from oauth2client<4,>=2.0.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/be/70/e5ea8afd6d08a4b99ebfc77bd1845248d56cfcf43d11f9dc324b9580a35c/pyasn1_modules-0.2.6-py2.py3-none-any.whl
Collecting rsa>=3.1.4 (from oauth2client<4,>=2.0.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/02/e5/38518af393f7c214357079ce67a317307936896e961e35450b70fad2a9cf/rsa-4.0-py2.py3-none-any.whl
Requirement already satisfied: setuptools in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages> (from protobuf<4,>=3.5.0.post1->apache-beam==2.17.0.dev0) (41.2.0)
Collecting pyparsing>=2.1.4 (from pydot<2,>=1.2.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/11/fa/0160cd525c62d7abd076a070ff02b2b94de589f1a9789774f17d7c54058e/pyparsing-2.4.2-py2.py3-none-any.whl
Collecting fasteners>=0.14 (from google-apitools<0.5.29,>=0.5.28->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/18/bd/55eb2d6397b9c0e263af9d091ebdb756b15756029b3cededf6461481bc63/fasteners-0.15-py2.py3-none-any.whl
Collecting google-api-core[grpc]<2.0.0dev,>=1.6.0 (from google-cloud-datastore<1.8.0,>=1.7.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/71/e5/7059475b3013a3c75abe35015c5761735ab224eb1b129fee7c8e376e7805/google_api_core-1.14.2-py2.py3-none-any.whl
Collecting grpc-google-iam-v1<0.13dev,>=0.12.3 (from google-cloud-pubsub<1.1.0,>=0.39.0->apache-beam==2.17.0.dev0)
Collecting google-resumable-media<0.5.0dev,>=0.3.1 (from google-cloud-bigquery<1.18.0,>=1.6.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/96/d7/b29a41b01b854480891dfc408211ffb0cc7a2a3d5f15a3b6740ec18c845b/google_resumable_media-0.4.1-py2.py3-none-any.whl
Collecting chardet<3.1.0,>=3.0.2 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl
Collecting certifi>=2017.4.17 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/18/b0/8146a4f8dd402f60744fa380bc73ca47303cccf8b9190fd16a827281eac2/certifi-2019.9.11-py2.py3-none-any.whl
Collecting idna<2.9,>=2.5 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/14/2c/cd551d81dbe15200be1cf41cd03869a46fe7226e7450af7a6545bfc474c9/idna-2.8-py2.py3-none-any.whl
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/e0/da/55f51ea951e1b7c63a579c09dd7db825bb730ec1fe9c0180fc77bfb31448/urllib3-1.25.6-py2.py3-none-any.whl
Collecting monotonic>=0.1 (from fasteners>=0.14->google-apitools<0.5.29,>=0.5.28->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/ac/aa/063eca6a416f397bd99552c534c6d11d57f58f2e94c14780f3bbf818c4cf/monotonic-1.5-py2.py3-none-any.whl
Collecting googleapis-common-protos<2.0dev,>=1.6.0 (from google-api-core[grpc]<2.0.0dev,>=1.6.0->google-cloud-datastore<1.8.0,>=1.7.1->apache-beam==2.17.0.dev0)
Collecting google-auth<2.0dev,>=0.4.0 (from google-api-core[grpc]<2.0.0dev,>=1.6.0->google-cloud-datastore<1.8.0,>=1.7.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/c5/9b/ed0516cc1f7609fb0217e3057ff4f0f9f3e3ce79a369c6af4a6c5ca25664/google_auth-1.6.3-py2.py3-none-any.whl
Installing collected packages: crcmod, dill, fastavro, future, docopt, chardet, certifi, idna, urllib3, requests, hdfs, httplib2, pbr, mock, pymongo, pyasn1, pyasn1-modules, rsa, oauth2client, pyparsing, pydot, python-dateutil, pytz, pyyaml, avro-python3, numpy, pyarrow, cachetools, monotonic, fasteners, google-apitools, googleapis-common-protos, google-auth, google-api-core, google-cloud-core, google-cloud-datastore, grpc-google-iam-v1, google-cloud-pubsub, google-resumable-media, google-cloud-bigquery, google-cloud-bigtable, nose, nose-xunitmp, pandas, parameterized, pyhamcrest, tenacity, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.1 cachetools-3.1.1 certifi-2019.9.11 chardet-3.0.4 crcmod-1.7 dill-0.3.0 docopt-0.6.2 fastavro-0.21.24 fasteners-0.15 future-0.17.1 google-api-core-1.14.2 google-apitools-0.5.28 google-auth-1.6.3 google-cloud-bigquery-1.17.1 google-cloud-bigtable-1.0.0 google-cloud-core-1.0.3 google-cloud-datastore-1.7.4 google-cloud-pubsub-1.0.2 google-resumable-media-0.4.1 googleapis-common-protos-1.6.0 grpc-google-iam-v1-0.12.3 hdfs-2.5.8 httplib2-0.12.0 idna-2.8 mock-2.0.0 monotonic-1.5 nose-1.3.7 nose-xunitmp-0.4.1 numpy-1.17.2 oauth2client-3.0.0 pandas-0.24.2 parameterized-0.6.3 pbr-5.4.3 pyarrow-0.14.1 pyasn1-0.4.7 pyasn1-modules-0.2.6 pydot-1.4.1 pyhamcrest-1.9.0 pymongo-3.9.0 pyparsing-2.4.2 python-dateutil-2.8.0 pytz-2019.2 pyyaml-3.13 requests-2.22.0 rsa-4.0 tenacity-5.1.1 urllib3-1.25.6
:sdks:python:test-suites:dataflow:py36:installGcpTest (Thread[Execution worker for ':',5,main]) completed. Took 16.901 secs.
:sdks:python:test-suites:dataflow:py36:integrationTest (Thread[Execution worker for ':',5,main]) started.
> Task :sdks:python:test-suites:dataflow:py36:integrationTest
Caching disabled for task ':sdks:python:test-suites:dataflow:py36:integrationTest': Caching has not been enabled for the task
Task ':sdks:python:test-suites:dataflow:py36:integrationTest' is not up-to-date because:
Task has not declared any outputs despite executing actions.
Custom actions are attached to task ':sdks:python:test-suites:dataflow:py36:integrationTest'.
Starting process 'command 'sh''. Working directory: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/test-suites/dataflow/py36> Command: sh -c . <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/bin/activate> && <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/scripts/run_integration_test.sh> --test_opts "--tests=apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it --attr=IT --nocapture" --pipeline_opts "--project=apache-beam-testing --staging_location=gs://temp-storage-for-end-to-end-tests/staging-it --temp_location=gs://temp-storage-for-end-to-end-tests/temp-it --input=gs://apache-beam-samples/input_small_files/ascii_sort_1MB_input.0000* --output=gs://temp-storage-for-end-to-end-tests/py-it-cloud/output --expect_checksum=ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710 --num_workers=10 --autoscaling_algorithm=NONE --runner=TestDataflowRunner --sdk_location=build/apache-beam.tar.gz" --suite integrationTest-perf
Successfully started process 'command 'sh''
>>> RUNNING integration tests with pipeline options: --project=apache-beam-testing --staging_location=gs://temp-storage-for-end-to-end-tests/staging-it --temp_location=gs://temp-storage-for-end-to-end-tests/temp-it --input=gs://apache-beam-samples/input_small_files/ascii_sort_1MB_input.0000* --output=gs://temp-storage-for-end-to-end-tests/py-it-cloud/output --expect_checksum=ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710 --num_workers=10 --autoscaling_algorithm=NONE --runner=TestDataflowRunner --sdk_location=build/apache-beam.tar.gz
>>> test options: --tests=apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it --attr=IT --nocapture
running nosetests
running egg_info
writing apache_beam.egg-info/PKG-INFO
writing dependency_links to apache_beam.egg-info/dependency_links.txt
writing entry points to apache_beam.egg-info/entry_points.txt
writing requirements to apache_beam.egg-info/requires.txt
writing top-level names to apache_beam.egg-info/top_level.txt
reading manifest file 'apache_beam.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
writing manifest file 'apache_beam.egg-info/SOURCES.txt'
STDERR: DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support
setup.py:186: UserWarning: You are using Apache Beam with Python 2. New releases of Apache Beam will soon support Python 3 only.
'You are using Apache Beam with Python 2. '
<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/1922375555/local/lib/python2.7/site-packages/setuptools/dist.py>:474: UserWarning: Normalizing '2.17.0.dev' to '2.17.0.dev0'
normalized_version,
beam_fn_api.proto: warning: Import google/protobuf/descriptor.proto but not used.
beam_fn_api.proto: warning: Import google/protobuf/wrappers.proto but not used.
DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
RefactoringTool: Skipping optional fixer: idioms
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
RefactoringTool: Skipping optional fixer: ws_comma
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2_grpc.py>
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_runner_api_pb2.py>
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/endpoints_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/external_transforms_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/metrics_pb2.py>
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/schema_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/standard_window_fns_pb2.py>
RefactoringTool: Files that were modified:
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_runner_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/endpoints_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/external_transforms_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/metrics_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/schema_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/standard_window_fns_pb2.py>
warning: no files found matching 'README.md'
warning: no files found matching 'NOTICE'
warning: no files found matching 'LICENSE'
warning: sdist: standard file not found: should have one of README, README.rst, README.txt, README.md
<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages/setuptools/dist.py>:474: UserWarning: Normalizing '2.17.0.dev' to '2.17.0.dev0'
normalized_version,
warning: no files found matching 'README.md'
warning: no files found matching 'NOTICE'
warning: no files found matching 'LICENSE'
test_wordcount_it (apache_beam.examples.wordcount_it_test.WordCountIT) ...
IssueCommand timed out after 1200 seconds. Process was killed by perfkitbenchmarker.
2019-10-01 06:57:37,515 443f303a MainThread beam_integration_benchmark(1/1) ERROR Benchmark 1/1 beam_integration_benchmark (UID: beam_integration_benchmark0) failed. Execution will continue.
2019-10-01 06:57:37,516 443f303a MainThread beam_integration_benchmark(1/1) INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2019-10-01 06:57:37,516 443f303a MainThread beam_integration_benchmark(1/1) INFO Complete logs can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/443f303a/pkb.log>
2019-10-01 06:57:37,516 443f303a MainThread beam_integration_benchmark(1/1) INFO Completion statuses can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/443f303a/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py36
#526
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/526/display/redirect?page=changes>
Changes:
[kcweaver] [BEAM-8321] fix Flink portable jar test
[valentyn] Restrict dill's upper bound.
[lostluck] Helper to get the value of a KV type
------------------------------------------
[...truncated 179.28 KB...]
Successfully started process 'command 'sh''
Collecting future==0.16.0
Installing collected packages: future
Successfully installed future-0.16.0
Create distribution tar file apache-beam.tar.gz in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/build>
:sdks:python:sdist (Thread[Execution worker for ':',5,main]) completed. Took 8.606 secs.
:sdks:python:test-suites:dataflow:py36:installGcpTest (Thread[Execution worker for ':',5,main]) started.
> Task :sdks:python:test-suites:dataflow:py36:installGcpTest
Caching disabled for task ':sdks:python:test-suites:dataflow:py36:installGcpTest': Caching has not been enabled for the task
Task ':sdks:python:test-suites:dataflow:py36:installGcpTest' is not up-to-date because:
Task has not declared any outputs despite executing actions.
Custom actions are attached to task ':sdks:python:test-suites:dataflow:py36:installGcpTest'.
Starting process 'command 'sh''. Working directory: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/test-suites/dataflow/py36> Command: sh -c . <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/bin/activate> && pip install --retries 10 -e <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/[gcp,test]>
Successfully started process 'command 'sh''
Obtaining file://<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python>
Collecting crcmod<2.0,>=1.7 (from apache-beam==2.17.0.dev0)
Collecting dill<0.3.1,>=0.3.0 (from apache-beam==2.17.0.dev0)
Collecting fastavro<0.22,>=0.21.4 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/e1/28/0206330c0002b1e28e21473117d0dc813defbd5891562d27af5c68c93899/fastavro-0.21.24-cp36-cp36m-manylinux1_x86_64.whl
Collecting future<1.0.0,>=0.16.0 (from apache-beam==2.17.0.dev0)
Requirement already satisfied: grpcio<2,>=1.12.1 in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages> (from apache-beam==2.17.0.dev0) (1.24.0)
Collecting hdfs<3.0.0,>=2.1.0 (from apache-beam==2.17.0.dev0)
Collecting httplib2<=0.12.0,>=0.8 (from apache-beam==2.17.0.dev0)
Collecting mock<3.0.0,>=1.0.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/e6/35/f187bdf23be87092bd0f1200d43d23076cee4d0dec109f195173fd3ebc79/mock-2.0.0-py2.py3-none-any.whl
Collecting pymongo<4.0.0,>=3.8.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/6d/27/f30b90f40054948b32df04a8e6355946874d084ac73755986b28d3003578/pymongo-3.9.0-cp36-cp36m-manylinux1_x86_64.whl
Collecting oauth2client<4,>=2.0.1 (from apache-beam==2.17.0.dev0)
Requirement already satisfied: protobuf<4,>=3.5.0.post1 in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages> (from apache-beam==2.17.0.dev0) (3.9.2)
Collecting pydot<2,>=1.2.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/33/d1/b1479a770f66d962f545c2101630ce1d5592d90cb4f083d38862e93d16d2/pydot-1.4.1-py2.py3-none-any.whl
Collecting python-dateutil<3,>=2.8.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/41/17/c62faccbfbd163c7f57f3844689e3a78bae1f403648a6afb1d0866d87fbb/python_dateutil-2.8.0-py2.py3-none-any.whl
Collecting pytz>=2018.3 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/87/76/46d697698a143e05f77bec5a526bf4e56a0be61d63425b68f4ba553b51f2/pytz-2019.2-py2.py3-none-any.whl
Collecting pyyaml<4.0.0,>=3.12 (from apache-beam==2.17.0.dev0)
Collecting avro-python3<2.0.0,>=1.8.1 (from apache-beam==2.17.0.dev0)
Collecting pyarrow<0.15.0,>=0.11.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/6f/df/33d8d6b682750d16baa9f45db825f33e0e0feb479fac1da9758f7ac8fd4b/pyarrow-0.14.1-cp36-cp36m-manylinux2010_x86_64.whl
Collecting cachetools<4,>=3.1.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/2f/a6/30b0a0bef12283e83e58c1d6e7b5aabc7acfc4110df81a4471655d33e704/cachetools-3.1.1-py2.py3-none-any.whl
Collecting google-apitools<0.5.29,>=0.5.28 (from apache-beam==2.17.0.dev0)
Collecting google-cloud-datastore<1.8.0,>=1.7.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/d0/aa/29cbcf8cf7d08ce2d55b9dce858f7c632b434cb6451bed17cb4275804217/google_cloud_datastore-1.7.4-py2.py3-none-any.whl
Collecting google-cloud-pubsub<1.1.0,>=0.39.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/d3/91/07a82945a7396ea34debafd476724bb5fc267c292790fdf2138c693f95c5/google_cloud_pubsub-1.0.2-py2.py3-none-any.whl
Collecting google-cloud-bigquery<1.18.0,>=1.6.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/a4/96/1b9cf1d43869c47a205aad411dac7c3040df6093d63c39273fa4d4c45da7/google_cloud_bigquery-1.17.1-py2.py3-none-any.whl
Collecting google-cloud-core<2,>=0.28.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/ee/f0/084f598629db8e6ec3627688723875cdb03637acb6d86999bb105a71df64/google_cloud_core-1.0.3-py2.py3-none-any.whl
Collecting google-cloud-bigtable<1.1.0,>=0.31.1 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/95/af/0ef7d097a1d5ad0c843867600e86de915e8ab8864740f49a4636cfb51af6/google_cloud_bigtable-1.0.0-py2.py3-none-any.whl
Collecting nose>=1.3.7 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/15/d8/dd071918c040f50fa1cf80da16423af51ff8ce4a0f2399b7bf8de45ac3d9/nose-1.3.7-py3-none-any.whl
Collecting nose_xunitmp>=0.4.1 (from apache-beam==2.17.0.dev0)
Collecting numpy<2,>=1.14.3 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/e5/e6/c3fdc53aed9fa19d6ff3abf97dfad768ae3afce1b7431f7500000816bda5/numpy-1.17.2-cp36-cp36m-manylinux1_x86_64.whl
Collecting pandas<0.25,>=0.23.4 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/19/74/e50234bc82c553fecdbd566d8650801e3fe2d6d8c8d940638e3d8a7c5522/pandas-0.24.2-cp36-cp36m-manylinux1_x86_64.whl
Collecting parameterized<0.7.0,>=0.6.0 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/3a/49/75f6dadb09e2f8ace3cdffe0c99a04f1b98dff41fbf9e768665d8b469e29/parameterized-0.6.3-py2.py3-none-any.whl
Collecting pyhamcrest<2.0,>=1.9 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/9a/d5/d37fd731b7d0e91afcc84577edeccf4638b4f9b82f5ffe2f8b62e2ddc609/PyHamcrest-1.9.0-py2.py3-none-any.whl
Collecting tenacity<6.0,>=5.0.2 (from apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/1e/a1/be8c8610f4620c56790965ba2b564dd76d13cbcd7c2ff8f6053ce63027fb/tenacity-5.1.1-py2.py3-none-any.whl
Requirement already satisfied: six>=1.5.2 in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages> (from grpcio<2,>=1.12.1->apache-beam==2.17.0.dev0) (1.12.0)
Collecting docopt (from hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Collecting requests>=2.7.0 (from hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/51/bd/23c926cd341ea6b7dd0b2a00aba99ae0f828be89d72b2190f27c11d4b7fb/requests-2.22.0-py2.py3-none-any.whl
Collecting pbr>=0.11 (from mock<3.0.0,>=1.0.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/46/a4/d5c83831a3452713e4b4f126149bc4fbda170f7cb16a86a00ce57ce0e9ad/pbr-5.4.3-py2.py3-none-any.whl
Collecting pyasn1>=0.1.7 (from oauth2client<4,>=2.0.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/a1/71/8f0d444e3a74e5640a3d5d967c1c6b015da9c655f35b2d308a55d907a517/pyasn1-0.4.7-py2.py3-none-any.whl
Collecting rsa>=3.1.4 (from oauth2client<4,>=2.0.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/02/e5/38518af393f7c214357079ce67a317307936896e961e35450b70fad2a9cf/rsa-4.0-py2.py3-none-any.whl
Collecting pyasn1-modules>=0.0.5 (from oauth2client<4,>=2.0.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/be/70/e5ea8afd6d08a4b99ebfc77bd1845248d56cfcf43d11f9dc324b9580a35c/pyasn1_modules-0.2.6-py2.py3-none-any.whl
Requirement already satisfied: setuptools in <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages> (from protobuf<4,>=3.5.0.post1->apache-beam==2.17.0.dev0) (41.2.0)
Collecting pyparsing>=2.1.4 (from pydot<2,>=1.2.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/11/fa/0160cd525c62d7abd076a070ff02b2b94de589f1a9789774f17d7c54058e/pyparsing-2.4.2-py2.py3-none-any.whl
Collecting fasteners>=0.14 (from google-apitools<0.5.29,>=0.5.28->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/18/bd/55eb2d6397b9c0e263af9d091ebdb756b15756029b3cededf6461481bc63/fasteners-0.15-py2.py3-none-any.whl
Collecting google-api-core[grpc]<2.0.0dev,>=1.6.0 (from google-cloud-datastore<1.8.0,>=1.7.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/71/e5/7059475b3013a3c75abe35015c5761735ab224eb1b129fee7c8e376e7805/google_api_core-1.14.2-py2.py3-none-any.whl
Collecting grpc-google-iam-v1<0.13dev,>=0.12.3 (from google-cloud-pubsub<1.1.0,>=0.39.0->apache-beam==2.17.0.dev0)
Collecting google-resumable-media<0.5.0dev,>=0.3.1 (from google-cloud-bigquery<1.18.0,>=1.6.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/96/d7/b29a41b01b854480891dfc408211ffb0cc7a2a3d5f15a3b6740ec18c845b/google_resumable_media-0.4.1-py2.py3-none-any.whl
Collecting chardet<3.1.0,>=3.0.2 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/bc/a9/01ffebfb562e4274b6487b4bb1ddec7ca55ec7510b22e4c51f14098443b8/chardet-3.0.4-py2.py3-none-any.whl
Collecting urllib3!=1.25.0,!=1.25.1,<1.26,>=1.21.1 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/e0/da/55f51ea951e1b7c63a579c09dd7db825bb730ec1fe9c0180fc77bfb31448/urllib3-1.25.6-py2.py3-none-any.whl
Collecting certifi>=2017.4.17 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/18/b0/8146a4f8dd402f60744fa380bc73ca47303cccf8b9190fd16a827281eac2/certifi-2019.9.11-py2.py3-none-any.whl
Collecting idna<2.9,>=2.5 (from requests>=2.7.0->hdfs<3.0.0,>=2.1.0->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/14/2c/cd551d81dbe15200be1cf41cd03869a46fe7226e7450af7a6545bfc474c9/idna-2.8-py2.py3-none-any.whl
Collecting monotonic>=0.1 (from fasteners>=0.14->google-apitools<0.5.29,>=0.5.28->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/ac/aa/063eca6a416f397bd99552c534c6d11d57f58f2e94c14780f3bbf818c4cf/monotonic-1.5-py2.py3-none-any.whl
Collecting googleapis-common-protos<2.0dev,>=1.6.0 (from google-api-core[grpc]<2.0.0dev,>=1.6.0->google-cloud-datastore<1.8.0,>=1.7.1->apache-beam==2.17.0.dev0)
Collecting google-auth<2.0dev,>=0.4.0 (from google-api-core[grpc]<2.0.0dev,>=1.6.0->google-cloud-datastore<1.8.0,>=1.7.1->apache-beam==2.17.0.dev0)
Using cached https://files.pythonhosted.org/packages/c5/9b/ed0516cc1f7609fb0217e3057ff4f0f9f3e3ce79a369c6af4a6c5ca25664/google_auth-1.6.3-py2.py3-none-any.whl
Installing collected packages: crcmod, dill, fastavro, future, docopt, chardet, urllib3, certifi, idna, requests, hdfs, httplib2, pbr, mock, pymongo, pyasn1, rsa, pyasn1-modules, oauth2client, pyparsing, pydot, python-dateutil, pytz, pyyaml, avro-python3, numpy, pyarrow, cachetools, monotonic, fasteners, google-apitools, googleapis-common-protos, google-auth, google-api-core, google-cloud-core, google-cloud-datastore, grpc-google-iam-v1, google-cloud-pubsub, google-resumable-media, google-cloud-bigquery, google-cloud-bigtable, nose, nose-xunitmp, pandas, parameterized, pyhamcrest, tenacity, apache-beam
Running setup.py develop for apache-beam
Successfully installed apache-beam avro-python3-1.9.1 cachetools-3.1.1 certifi-2019.9.11 chardet-3.0.4 crcmod-1.7 dill-0.3.0 docopt-0.6.2 fastavro-0.21.24 fasteners-0.15 future-0.17.1 google-api-core-1.14.2 google-apitools-0.5.28 google-auth-1.6.3 google-cloud-bigquery-1.17.1 google-cloud-bigtable-1.0.0 google-cloud-core-1.0.3 google-cloud-datastore-1.7.4 google-cloud-pubsub-1.0.2 google-resumable-media-0.4.1 googleapis-common-protos-1.6.0 grpc-google-iam-v1-0.12.3 hdfs-2.5.8 httplib2-0.12.0 idna-2.8 mock-2.0.0 monotonic-1.5 nose-1.3.7 nose-xunitmp-0.4.1 numpy-1.17.2 oauth2client-3.0.0 pandas-0.24.2 parameterized-0.6.3 pbr-5.4.3 pyarrow-0.14.1 pyasn1-0.4.7 pyasn1-modules-0.2.6 pydot-1.4.1 pyhamcrest-1.9.0 pymongo-3.9.0 pyparsing-2.4.2 python-dateutil-2.8.0 pytz-2019.2 pyyaml-3.13 requests-2.22.0 rsa-4.0 tenacity-5.1.1 urllib3-1.25.6
:sdks:python:test-suites:dataflow:py36:installGcpTest (Thread[Execution worker for ':',5,main]) completed. Took 23.654 secs.
:sdks:python:test-suites:dataflow:py36:integrationTest (Thread[Execution worker for ':',5,main]) started.
> Task :sdks:python:test-suites:dataflow:py36:integrationTest
Caching disabled for task ':sdks:python:test-suites:dataflow:py36:integrationTest': Caching has not been enabled for the task
Task ':sdks:python:test-suites:dataflow:py36:integrationTest' is not up-to-date because:
Task has not declared any outputs despite executing actions.
Custom actions are attached to task ':sdks:python:test-suites:dataflow:py36:integrationTest'.
Starting process 'command 'sh''. Working directory: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/test-suites/dataflow/py36> Command: sh -c . <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/bin/activate> && <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/scripts/run_integration_test.sh> --test_opts "--tests=apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it --attr=IT --nocapture" --pipeline_opts "--project=apache-beam-testing --staging_location=gs://temp-storage-for-end-to-end-tests/staging-it --temp_location=gs://temp-storage-for-end-to-end-tests/temp-it --input=gs://apache-beam-samples/input_small_files/ascii_sort_1MB_input.0000* --output=gs://temp-storage-for-end-to-end-tests/py-it-cloud/output --expect_checksum=ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710 --num_workers=10 --autoscaling_algorithm=NONE --runner=TestDataflowRunner --sdk_location=build/apache-beam.tar.gz" --suite integrationTest-perf
Successfully started process 'command 'sh''
>>> RUNNING integration tests with pipeline options: --project=apache-beam-testing --staging_location=gs://temp-storage-for-end-to-end-tests/staging-it --temp_location=gs://temp-storage-for-end-to-end-tests/temp-it --input=gs://apache-beam-samples/input_small_files/ascii_sort_1MB_input.0000* --output=gs://temp-storage-for-end-to-end-tests/py-it-cloud/output --expect_checksum=ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710 --num_workers=10 --autoscaling_algorithm=NONE --runner=TestDataflowRunner --sdk_location=build/apache-beam.tar.gz
>>> test options: --tests=apache_beam.examples.wordcount_it_test:WordCountIT.test_wordcount_it --attr=IT --nocapture
running nosetests
running egg_info
writing apache_beam.egg-info/PKG-INFO
writing dependency_links to apache_beam.egg-info/dependency_links.txt
writing entry points to apache_beam.egg-info/entry_points.txt
writing requirements to apache_beam.egg-info/requires.txt
writing top-level names to apache_beam.egg-info/top_level.txt
reading manifest file 'apache_beam.egg-info/SOURCES.txt'
reading manifest template 'MANIFEST.in'
writing manifest file 'apache_beam.egg-info/SOURCES.txt'
STDERR: DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support
setup.py:186: UserWarning: You are using Apache Beam with Python 2. New releases of Apache Beam will soon support Python 3 only.
'You are using Apache Beam with Python 2. '
<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/1922375555/local/lib/python2.7/site-packages/setuptools/dist.py>:474: UserWarning: Normalizing '2.17.0.dev' to '2.17.0.dev0'
normalized_version,
beam_fn_api.proto: warning: Import google/protobuf/descriptor.proto but not used.
beam_fn_api.proto: warning: Import google/protobuf/wrappers.proto but not used.
DEPRECATION: Python 2.7 will reach the end of its life on January 1st, 2020. Please upgrade your Python as Python 2.7 won't be maintained after that date. A future version of pip will drop support for Python 2.7. More details about Python 2 support in pip, can be found at https://pip.pypa.io/en/latest/development/release-process/#python-2-support
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
RefactoringTool: Skipping optional fixer: idioms
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
RefactoringTool: Skipping optional fixer: ws_comma
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
root: Generating grammar tables from /usr/lib/python2.7/lib2to3/PatternGrammar.txt
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2_grpc.py>
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2_grpc.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_runner_api_pb2.py>
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/endpoints_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/external_transforms_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/metrics_pb2.py>
RefactoringTool: No changes to <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/schema_pb2.py>
RefactoringTool: Refactored <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/standard_window_fns_pb2.py>
RefactoringTool: Files that were modified:
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_artifact_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_expansion_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_fn_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_job_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_provision_api_pb2_grpc.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/beam_runner_api_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/endpoints_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/external_transforms_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/metrics_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/schema_pb2.py>
RefactoringTool: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/apache_beam/portability/api/standard_window_fns_pb2.py>
warning: no files found matching 'README.md'
warning: no files found matching 'NOTICE'
warning: no files found matching 'LICENSE'
warning: sdist: standard file not found: should have one of README, README.rst, README.txt, README.md
<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/build/gradleenv/-1734967053/lib/python3.6/site-packages/setuptools/dist.py>:474: UserWarning: Normalizing '2.17.0.dev' to '2.17.0.dev0'
normalized_version,
warning: no files found matching 'README.md'
warning: no files found matching 'NOTICE'
warning: no files found matching 'LICENSE'
test_wordcount_it (apache_beam.examples.wordcount_it_test.WordCountIT) ...
IssueCommand timed out after 1200 seconds. Process was killed by perfkitbenchmarker.
2019-10-01 00:54:11,135 41e91bd7 MainThread beam_integration_benchmark(1/1) ERROR Benchmark 1/1 beam_integration_benchmark (UID: beam_integration_benchmark0) failed. Execution will continue.
2019-10-01 00:54:11,136 41e91bd7 MainThread beam_integration_benchmark(1/1) INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2019-10-01 00:54:11,136 41e91bd7 MainThread beam_integration_benchmark(1/1) INFO Complete logs can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/41e91bd7/pkb.log>
2019-10-01 00:54:11,136 41e91bd7 MainThread beam_integration_benchmark(1/1) INFO Completion statuses can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/41e91bd7/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py36
#525
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/525/display/redirect?page=changes>
Changes:
[ihr] Add clarification about authorized views
[kirillkozlov] [BEAM-8275] Beam SQL should support BigQuery in DIRECT_READ mode
[github] Addressed review comments
[github] Added a test for BigQuery SQL read in EXPORT mode
[lukecwik] [BEAM-6923] limit gcs buffer size to 1MB for artifact upload (#9647)
------------------------------------------
[...truncated 160.42 KB...]
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "eNrNV/l/G8UV10rOwZKmAUogpIeaknZNkZSjcRKXQlIlAaNYcdcGb4+wjHZHmo13d/bNzFoxtbhSOS69Lyj0Lr1vel/Q9u/oP9M3I8mpwKb5Lf187N3Vm3lv5n3f931n9+mSE5CMBIz6LUqSqhIklW0uElkNuKB2ncQxacV0UZAso+IMP5faUJh8BqweFB2vVCgU/HYKpbEgEce/FpHU9ttRSuLoCep3RaSoDRPeLuOSY1jRkRkNYFvT24G2TPCASgnb2YR3s56jVjLqsyhVEnaM7xEHjL0aUtwkUVxIe+bCApof0mYbduIGb2r2wHZMqCjNcmXiSbi5aXbAc3XNtquZr8FbWnnrIux2Xp+JopdVxG1/Ae/zUbpkw1sx+p4e3OJ4ezBSO4qpnxHF/EzQdnQZbh2LwDP0TmV1mcQ5zhN8OQqpsOcVUVHwqDbOjWxwGwZ+Ww9udzwbAw889A5hbxBGcVz19dX2Y05CY7fhDlMBqQTc2Yd9LtzlbRt5wn7vAj535HStpmiSVSTiRDq0gtWt0DSsKD64UalkLVupRKoSxDwPawNsaoePTZ08MXXi+LHjR46eqAkq8xhL8fa8dS3tlCTUl3lbp/0Oprf/zh68y2F72V1sv4ezClDG+TqbNE98yYgIJby7UTDbDDhmDQfG4DI2ObzZC3xeiSjt1LnB5z24wN09ONjybtFZ62iDLWjCEgXv9e5Ge+WgMxzKk8lDx8IKb6Pp2vraBu/z9o6H6MS8NYrjmMHKPZt6Tpr0A55guaXE2g5KdI83gWaSKw7v927CxyRKhtW71wCgeVTLYhKlUPH2aYLrjkqxkoJEMSbpp7SLd+Rjdd10hM8o0QjVmnlrnJWpogL7qpqrKLZPi06e0FTNxSSgjMcGqkMI1WGmr0calrkfpT34wEU45niTOnNk3KGKacra4rXrTJLFtXPDnjUWmBpbOjPcsv1H0iwKlmIazmOgGd1fNhzvwQnHwBASReDkZo4b08/gFBumcWMf7MF9jrdboxdoBdDNpPODD3n70ai9p/V2fdPG07gnoRVpevkw3G+I0I3SkHf9BPHUMKIcPbCVpuk4Joy0TcVJ7L/B24ZTBv7liHZ1sNNjzRcIShQyLk8D3dg2fNhhd3g7cb4mqdYgqPfhjAtnnYbVKOB/qXFb3V4vrBbWrSvF+QKca/bhwUnjMsoFHurDjCfQUmM8obVLNF2KUjm6V2RMlmmty8WSxLxoTaflz1Fh+JoGdEH3sL/IRVjneapmFvy5laNTNSmCmgyXdHcrxtPaf4FSG9Sjmq3Aw2Yr98UkaYXkfmjMXrDqBTjv3a67XPDEFxhSk3ljt7NGQg0+Q3mD5hpcmFQw58JHxuDqUOVjOZGRrlmmlUexwpRg3mCMw3oUFtbgERceHXONkowL5Sc8zGOUukXvVk2RN1AKvD581IWPmfA++gbK9+Hja/AJFy6yRnOz6gUUf8BjDGuHZSpimUqNHY16/XFVUNal4mohxFJdsXrF1aI8sGpdKoVFNXGloMuntq3iSFhaKoqp1WI4sa+Atu3htoFd7RjYhk8l/dQu7cXrs1aIM8F3mo2iST2kbYKKCo9rmnhPomWuzuOYGlqVebsssQXLB8NyN1KsnOBhXFaM4EhKyzSmuuPLg4ahYZnIMkGHtBNThd66NNXyuUhIVVZdPpovyzQNNEGo0D4Y8cBBeeBec60CCl9r0L5xJBUERsV0zRXnsYTQ265/yzgKKFBz7mD1oG0692ySqZWN3oaOGY5pCsxonznnzgrBBUTsTgWXvKKJDUsGihGJYrO6Vj1IrkLKHjY67G9OQj77ilXfUyjusSas3dYua6dVtIpFyCaRhuCCYBcZb+I7gGwqUC7kfVh2ocse68HlLVRjhZ2SfXjChU+uwWoPeuj4pAtP5S32ADuVX4WnN5Tz8HUp5zMMFfFZh51kWuau9OBTDvvfmtbXq7HTQ24OJMRqzNcnVjV/1lA8rk4y1Iv1G6UXnx7Xi+dm/22x8xr1z7jwWUT9OY365xC8z7vwhQ3wvrgB3pHrAu9LGrwvj8D7Sg++ej3gfW0T8F6vv88jhC8YCL9+oyB8cRzCl1By2Xk2y1BFv4FAftOFbyGQLzXZjZCob2uJYv83svQdBd91WMBCRlmbdRhjRkK+x5ZYzFAmvs9efDOZeHlzmfiBJuwPXfgR4vyyJuyPkbA/ceGnffiZCz/XMvGLLWTil0YmfuXCr9fgNz14BR1/68LvNpj+e/qmX1SLJiDGsuEPyOw/9uBPjqHDYCVc4M9b+Q9m2A+aF5dBHPzY+QtG+asRa3xX7nSQoin8basQwyn2mcHpszD8CX/HIK+ahLF4eZLHRNdbH74UXmtY5uzVwEpFkszHt98WvqkK+AcOmY+1SPqjA+2f63lLwb+q/wGQBdL5",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: '2019-09-30T18:39:21.957052Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2019-09-30_11_39_20-6148750149774115934'
location: 'us-central1'
name: 'beamapp-jenkins-0930183918-461374'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2019-09-30T18:39:21.957052Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2019-09-30_11_39_20-6148750149774115934]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-09-30_11_39_20-6148750149774115934?project=apache-beam-testing
root: INFO: Job 2019-09-30_11_39_20-6148750149774115934 is in state JOB_STATE_RUNNING
root: INFO: 2019-09-30T18:39:24.419Z: JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service Account.
root: INFO: 2019-09-30T18:39:24.983Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
root: INFO: 2019-09-30T18:39:25.664Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2019-09-30T18:39:25.708Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2019-09-30T18:39:25.744Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2019-09-30T18:39:25.785Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
root: INFO: 2019-09-30T18:39:25.811Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2019-09-30T18:39:25.900Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
root: INFO: 2019-09-30T18:39:25.956Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2019-09-30T18:39:25.992Z: JOB_MESSAGE_DETAILED: Fusing consumer split into read/Read
root: INFO: 2019-09-30T18:39:26.028Z: JOB_MESSAGE_DETAILED: Fusing consumer pair_with_one into split
root: INFO: 2019-09-30T18:39:26.063Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Reify into pair_with_one
root: INFO: 2019-09-30T18:39:26.100Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Write into group/Reify
root: INFO: 2019-09-30T18:39:26.137Z: JOB_MESSAGE_DETAILED: Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2019-09-30T18:39:26.171Z: JOB_MESSAGE_DETAILED: Fusing consumer count into group/GroupByWindow
root: INFO: 2019-09-30T18:39:26.211Z: JOB_MESSAGE_DETAILED: Fusing consumer format into count
root: INFO: 2019-09-30T18:39:26.246Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2019-09-30T18:39:26.280Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2019-09-30T18:39:26.317Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2019-09-30T18:39:26.352Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2019-09-30T18:39:26.386Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2019-09-30T18:39:26.409Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2019-09-30T18:39:26.442Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2019-09-30T18:39:26.476Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2019-09-30T18:39:26.516Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
root: INFO: 2019-09-30T18:39:26.551Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
root: INFO: 2019-09-30T18:39:26.584Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
root: INFO: 2019-09-30T18:39:26.622Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
root: INFO: 2019-09-30T18:39:26.789Z: JOB_MESSAGE_DEBUG: Executing wait step start26
root: INFO: 2019-09-30T18:39:26.871Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-30T18:39:26.907Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-30T18:39:26.921Z: JOB_MESSAGE_DEBUG: Starting worker pool setup.
root: INFO: 2019-09-30T18:39:26.959Z: JOB_MESSAGE_BASIC: Executing operation group/Create
root: INFO: 2019-09-30T18:39:26.962Z: JOB_MESSAGE_BASIC: Starting 10 workers in us-central1-f...
root: INFO: 2019-09-30T18:39:27.028Z: JOB_MESSAGE_BASIC: Finished operation group/Create
root: INFO: 2019-09-30T18:39:27.028Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-30T18:39:27.081Z: JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/GroupByKey/Session" materialized.
root: INFO: 2019-09-30T18:39:27.113Z: JOB_MESSAGE_DEBUG: Value "group/Session" materialized.
root: INFO: 2019-09-30T18:39:27.178Z: JOB_MESSAGE_BASIC: Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-30T18:39:47.892Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 1 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-30T18:39:47.930Z: JOB_MESSAGE_DETAILED: Resized worker pool to 1, though goal was 10. This could be a quota issue.
root: INFO: 2019-09-30T18:39:53.326Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 10 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-30T18:40:14.922Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-30T18:40:14.956Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-30T18:44:06.787Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T18:44:08.862Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T18:44:10.928Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T18:44:13.244Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T18:44:15.066Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T18:44:15.312Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T18:44:17.133Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T18:44:17.363Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T18:44:17.392Z: JOB_MESSAGE_BASIC: Finished operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-30T18:44:17.456Z: JOB_MESSAGE_DEBUG: Executing failure step failure25
root: INFO: 2019-09-30T18:44:17.466Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-30T18:44:17.491Z: JOB_MESSAGE_ERROR: Workflow failed. Causes: S06:read/Read+split+pair_with_one+group/Reify+group/Write failed., Internal Issue (200167a00f9605ad): 63963027:24514
root: INFO: 2019-09-30T18:44:17.551Z: JOB_MESSAGE_DEBUG: Executing failure step failure25
root: INFO: 2019-09-30T18:44:17.593Z: JOB_MESSAGE_ERROR: Workflow failed. Causes: S01:write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite failed., Internal Issue (200167a00f960a58): 63963027:24514
root: INFO: 2019-09-30T18:44:17.734Z: JOB_MESSAGE_DETAILED: Cleaning up.
root: INFO: 2019-09-30T18:44:17.838Z: JOB_MESSAGE_DEBUG: Starting worker pool teardown.
root: INFO: 2019-09-30T18:44:17.871Z: JOB_MESSAGE_BASIC: Stopping worker pool...
root: INFO: 2019-09-30T18:46:16.029Z: JOB_MESSAGE_DETAILED: Autoscaling: Reduced the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-30T18:46:16.077Z: JOB_MESSAGE_BASIC: Worker pool stopped.
root: INFO: 2019-09-30T18:46:16.121Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
root: INFO: Job 2019-09-30_11_39_20-6148750149774115934 is in state JOB_STATE_FAILED
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569868757238/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569868757238/results*' -> 'gs\\:\\/\\/temp\\-storage\\-for\\-end\\-to\\-end\\-tests\\/py\\-it\\-cloud\\/output\\/1569868757238\\/results[^/\\\\]*'
root: INFO: Starting the size estimation of the input
root: INFO: Finished listing 0 files in 0.03777909278869629 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-integrationTest-perf.xml
----------------------------------------------------------------------
XML: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 430.680s
FAILED (errors=1)
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py36:integrationTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
BUILD FAILED in 8m 6s
2019-09-30 18:46:29,589 8eb1dee0 MainThread beam_integration_benchmark(1/1) ERROR Error during benchmark beam_integration_benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-30 18:46:29,590 8eb1dee0 MainThread beam_integration_benchmark(1/1) INFO Cleaning up benchmark beam_integration_benchmark
2019-09-30 18:46:29,592 8eb1dee0 MainThread beam_integration_benchmark(1/1) ERROR Exception running benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 984, in RunBenchmarkTask
RunBenchmark(spec, collector)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-30 18:46:29,593 8eb1dee0 MainThread beam_integration_benchmark(1/1) ERROR Benchmark 1/1 beam_integration_benchmark (UID: beam_integration_benchmark0) failed. Execution will continue.
2019-09-30 18:46:29,593 8eb1dee0 MainThread beam_integration_benchmark(1/1) INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2019-09-30 18:46:29,594 8eb1dee0 MainThread beam_integration_benchmark(1/1) INFO Complete logs can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/8eb1dee0/pkb.log>
2019-09-30 18:46:29,594 8eb1dee0 MainThread beam_integration_benchmark(1/1) INFO Completion statuses can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/8eb1dee0/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py36
#524
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/524/display/redirect>
Changes:
------------------------------------------
[...truncated 156.23 KB...]
"value": "_finalize_write"
}
],
"non_parallel_inputs": {
"side0-write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s18"
},
"side1-write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s19"
},
"side2-write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "eNrNV/l/G8UV10rOwZKmAUogpIdISbumSEpIHBKXQlIlAaNYcdcGb4+wjHZHmo13d/bNzFoxtSiQynGhd8vZu/S+6X0ff0f/mb4ZSU4FNs1v6edj767ezHsz7/u+7zu7T5ecgGQkYNRvUZJUlSCpbHORyGrABbXrJI5JK6aLgmQZFaf52dSGwuQzYPWg6HilQqHgt1MojQWJOP61iKS2345SEkdPUr8rIkVtmPB2GZccw4qOzGgA25reDrRlggdUStjOJrwb9Ry1klGfRamSsGN8jzhg7NWQ4iaJ4kLaM+cX0PywNtuwEzd4Q7MHtmNCRWmWKxNPwo1NswOeq6u2Xc18Dd7WylsXYLfzxkwUvaQibvsLeJ+P0iUb3o7R9/TgJsfbg5HaUUz9jCjmZ4K2o0tw81gEnqF3KqvLJM5xnuDLUUiFPa+IioLHtHFuZINbMPA7enCr49kYeOChdwh7gzCK46qvr7YfcxIauw23mQpIJeD2Puxz4Q5v28gT9nvn8bkjp2s1RZOsIhEn0qEVrG6FpmFF8cGNSiVr2UolUpUg5nlYG2BTOzx17MTxo/cdPzE1dfRITVCZx1iKd+atq2mnJKG+zNs67Xcxvf139+A9DtvL7mD7PZxVgDLO19mkeeJLRkQo4c5GwWwz4Jg1HBiDy9jk8GYv8HklorRT5waf9+ICd/XgYMu7SWetow22oAlLFLzPuwvtlYPOcChPJg9NhRXeRtPV9bUN3u/tHQ/RiXlrFMcxg5W7N/WcNOkHPMFyS4m1HZTobm8CzSRXHD7g3YCPSZQMq3ePAUDzqJbFJEqh4u3TBNcdlWIlBYliTNJPaRfvyMfquukIn1GiEao189Y4K1NFBfZVNVdRbJ8SnTyhqZqLSUAZjw1UhxCqw0xf721Y5n6E9uDoBZhyvEmdOTLuUMU0ZW3x6nUmyeLa2WHPGgscG1s6M9yy/UfTLAqWYhrOY6AZ3V823NeD446BISSKwInNHDemn8YpNkzjxj7Yg/sdb7dGL9AKoJtJ5wcf8vajUXtP6+36po2ncU9CK9L08mF4wBChG6Uh7/oJ4qlhRDl6cCtN03FMGGmbipPYf5O3DScN/MsR7epgp8aaLxCUKGRcnga6sW34sMNu83bifE1SrUFQ78NpF844DatRwP9S45a6vV5YLaxbl4vzBTjb7MNDk8ZllAs83IcZT6ClxnhCaxdpuhSlcnSvyJgs01qXiyWJedGaTsufo8LwNQ3ogu5hf5GLsM7zVM0s+HMrR47VpAhqMlzS3a0YT2v/BUptUI9qtgKPmK3cH5OkFZIHoDF73qoX4Jx3q+5ywRNfYEhN5o3dzhoJNfgM5Q2aa3B+UsGcCx8Zg6tDlY/lREa6ZplWHsUKU4J5gzEO61FYWINHXXhszDVKMi6Un/Awj1HqFr2bNUXeRCnw+vBRFz5mwvvoGyjfh4+vwSdcuMAazc2qF1D8AY8zrB2WqYhlKjV2NOr1J1RBWReLq4UQS3XZ6hVXi/LAqnWxFBbVxOWCLp/atoojYWmpKI6tFsOJfQW0bQ+3Dexqx8A2fCrpp3ZpL16ftUKcCb7TbBRN6iFtE1RUeELTxHsKLXN1HsfU0KrM22WJLVg+GJa7kWLlBA/jsmIER1JapjHVHV8eNAwNy0SWCTqknZgq9NalqZbPRkKqsury0XxZpmmgCUKF9sGIBw7KA/eYaxVQ+FqD9o0jqSAwKqZrrjiPJYTedv1bxlFAgZpzB6sHbdO5Z5JMrWz0NnTMcExTYEb7zDl3RgguIGK3K7joFU1sWDJQjEgUm9W16kFyBVL2iNFhf3MS8tnXrfqeQnGPNWHttnZZO62iVSxCNok0BBcEu8B4E98BZFOBciHvw7ILXfZ4Dy5toRor7KTsw5MufHINVnvQQ8enXPhU3mIPspP5FXh6QzkPX5NyPsNQEZ912AmmZe5yDz7tsP+taX29Gjs15OZAQqzGfH1iVfNnDcXjyiRDvVi/XnrxmXG9eG723xY7p1F/3oXPIurPadQ/h+B93oUvbID3xQ3w7r0m8L6kwfvyCLyv9OCr1wLeC5uA90b9fREhfMlA+PL1gvCVcQhfRcll59gsQxX9GgL5dRe+gUC+2mTXQ6K+qSWK/d/I0rcUfNthAQsZZW3WYYwZCfkOW2IxQ5n4LnvlrWTitc1l4nuasN934QeI82uasD9Ewv7IhR/34Scu/FTLxM+2kImfG5n4hQu/XINf9eB1dPy1C7/ZYPpv6Vt+US2agBjLht8hs3/fgz84hg6DlXCBP27lP5hhP2ReXAZx8GPnTxjlz0as8V2500GKpvCXrUIMp9inB6fPwvAn/BWD/M0kjMXLkzwmut768KXw94Zlzl4NrFQkyXx8+23hm6qAf+CQ+ViLpD860P65nrcU/Kv6H4PT0vg=",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: '2019-09-30T12:51:39.615163Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2019-09-30_05_51_38-14277359654686163704'
location: 'us-central1'
name: 'beamapp-jenkins-0930125136-596838'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2019-09-30T12:51:39.615163Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2019-09-30_05_51_38-14277359654686163704]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-09-30_05_51_38-14277359654686163704?project=apache-beam-testing
root: INFO: Job 2019-09-30_05_51_38-14277359654686163704 is in state JOB_STATE_RUNNING
root: INFO: 2019-09-30T12:51:41.708Z: JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service Account.
root: INFO: 2019-09-30T12:51:42.226Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-a.
root: INFO: 2019-09-30T12:51:42.949Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2019-09-30T12:51:42.989Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2019-09-30T12:51:43.022Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2019-09-30T12:51:43.062Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
root: INFO: 2019-09-30T12:51:43.093Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2019-09-30T12:51:43.179Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
root: INFO: 2019-09-30T12:51:43.230Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2019-09-30T12:51:43.267Z: JOB_MESSAGE_DETAILED: Fusing consumer split into read/Read
root: INFO: 2019-09-30T12:51:43.303Z: JOB_MESSAGE_DETAILED: Fusing consumer pair_with_one into split
root: INFO: 2019-09-30T12:51:43.338Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Reify into pair_with_one
root: INFO: 2019-09-30T12:51:43.376Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Write into group/Reify
root: INFO: 2019-09-30T12:51:43.412Z: JOB_MESSAGE_DETAILED: Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2019-09-30T12:51:43.446Z: JOB_MESSAGE_DETAILED: Fusing consumer count into group/GroupByWindow
root: INFO: 2019-09-30T12:51:43.482Z: JOB_MESSAGE_DETAILED: Fusing consumer format into count
root: INFO: 2019-09-30T12:51:43.522Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2019-09-30T12:51:43.544Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2019-09-30T12:51:43.575Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2019-09-30T12:51:43.611Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2019-09-30T12:51:43.649Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2019-09-30T12:51:43.683Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2019-09-30T12:51:43.721Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2019-09-30T12:51:43.755Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2019-09-30T12:51:43.795Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
root: INFO: 2019-09-30T12:51:43.829Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
root: INFO: 2019-09-30T12:51:43.866Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
root: INFO: 2019-09-30T12:51:43.903Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
root: INFO: 2019-09-30T12:51:44.046Z: JOB_MESSAGE_DEBUG: Executing wait step start26
root: INFO: 2019-09-30T12:51:44.122Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-30T12:51:44.159Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-30T12:51:44.171Z: JOB_MESSAGE_DEBUG: Starting worker pool setup.
root: INFO: 2019-09-30T12:51:44.186Z: JOB_MESSAGE_BASIC: Executing operation group/Create
root: INFO: 2019-09-30T12:51:44.207Z: JOB_MESSAGE_BASIC: Starting 10 workers in us-central1-a...
root: INFO: 2019-09-30T12:51:44.279Z: JOB_MESSAGE_BASIC: Finished operation group/Create
root: INFO: 2019-09-30T12:51:44.279Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-30T12:51:44.357Z: JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/GroupByKey/Session" materialized.
root: INFO: 2019-09-30T12:51:44.388Z: JOB_MESSAGE_DEBUG: Value "group/Session" materialized.
root: INFO: 2019-09-30T12:51:44.454Z: JOB_MESSAGE_BASIC: Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-30T12:52:18.773Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 10 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-30T12:52:53.891Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-30T12:52:53.919Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-30T12:57:15.180Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T12:57:17.261Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T12:57:19.335Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T12:57:21.402Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T12:57:21.611Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T12:57:21.640Z: JOB_MESSAGE_BASIC: Finished operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-30T12:57:21.703Z: JOB_MESSAGE_DEBUG: Executing failure step failure25
root: INFO: 2019-09-30T12:57:21.732Z: JOB_MESSAGE_ERROR: Workflow failed. Causes: S06:read/Read+split+pair_with_one+group/Reify+group/Write failed., Internal Issue (92ba1b7412fbabf6): 63963027:24514
root: INFO: 2019-09-30T12:57:22.074Z: JOB_MESSAGE_WARNING: S01:write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite failed.
root: INFO: 2019-09-30T12:57:22.113Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-30T12:57:22.228Z: JOB_MESSAGE_DETAILED: Cleaning up.
root: INFO: 2019-09-30T12:57:22.294Z: JOB_MESSAGE_DEBUG: Starting worker pool teardown.
root: INFO: 2019-09-30T12:57:22.328Z: JOB_MESSAGE_BASIC: Stopping worker pool...
root: INFO: 2019-09-30T13:03:40.394Z: JOB_MESSAGE_DETAILED: Autoscaling: Reduced the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-30T13:03:40.448Z: JOB_MESSAGE_BASIC: Worker pool stopped.
root: INFO: 2019-09-30T13:03:40.485Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
root: INFO: Job 2019-09-30_05_51_38-14277359654686163704 is in state JOB_STATE_FAILED
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569847895543/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569847895543/results*' -> 'gs\\:\\/\\/temp\\-storage\\-for\\-end\\-to\\-end\\-tests\\/py\\-it\\-cloud\\/output\\/1569847895543\\/results[^/\\\\]*'
root: INFO: Starting the size estimation of the input
root: INFO: Finished listing 0 files in 0.06242084503173828 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-integrationTest-perf.xml
----------------------------------------------------------------------
XML: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 735.106s
FAILED (errors=1)
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py36:integrationTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
BUILD FAILED in 13m 2s
2019-09-30 13:03:55,553 afee3381 MainThread beam_integration_benchmark(1/1) ERROR Error during benchmark beam_integration_benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-30 13:03:55,566 afee3381 MainThread beam_integration_benchmark(1/1) INFO Cleaning up benchmark beam_integration_benchmark
2019-09-30 13:03:55,569 afee3381 MainThread beam_integration_benchmark(1/1) ERROR Exception running benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 984, in RunBenchmarkTask
RunBenchmark(spec, collector)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-30 13:03:55,574 afee3381 MainThread beam_integration_benchmark(1/1) ERROR Benchmark 1/1 beam_integration_benchmark (UID: beam_integration_benchmark0) failed. Execution will continue.
2019-09-30 13:03:55,575 afee3381 MainThread beam_integration_benchmark(1/1) INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2019-09-30 13:03:55,586 afee3381 MainThread beam_integration_benchmark(1/1) INFO Complete logs can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/afee3381/pkb.log>
2019-09-30 13:03:55,587 afee3381 MainThread beam_integration_benchmark(1/1) INFO Completion statuses can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/afee3381/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py36
#523
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/523/display/redirect>
Changes:
------------------------------------------
[...truncated 157.30 KB...]
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "eNrNV/l/G8UV10rOwZKmAUogpIeaknZNkRQH4hCXQlIlAaNYcdcGb4+wjHZHmo13d/bNzFoxtSiQynHpXWg5epfeN73v4+/oP9M3I8mpwKb5Lf187N3Vm3lv5n3f931n9+mSE5CMBIz6LUqSqhIklW0uElkNuKB2ncQxacV0SZAso+I0P5vaUJh8BqweFB2vVCgU/HYKpbEgEce/FpHU9ttRSuLoSep3RaSoDRPeHuOSY1jRkRkNYEfT24W2TPCASgk72YR3o56jVjPqsyhVEnaN7xEHjL0aUtwkUVxIe/b8Ipof1mYbduMGb2j2wHZMqCjNcmXiSbixaXbAc3XVtqeZr8NbWnnrAux1Xp+JopdUxG1/Ee8LUbpsw1sx+r4e3OR4+zBSO4qpnxHF/EzQdnQJbh6LwDP0TmV1hcQ5zhN8JQqpsBcUUVHwmDbOj2xwCwZ+Ww9udTwbAw889A5hfxBGcVz19dX2Y05CY7fhNlMBqQTc3ocDLtzh7Rh5wkHvPD535EytpmiSVSTiRDq0gtWt0DSsKD64UalkLVutRKoSxDwPawNsalPHpk/cd/TY8amj9x6frgkq8xhL8fa8dTXtlCTUl3lbp/0Oprf/zh68y2H72R3soIezClDG+TqbNE98yYgIJby7UTDbDDhmDYfG4DI2ObzZi3xBiSjt1LnB5z24wJ09ONzybtJZ62iDLWjCEgXv9e5Ee+WwMxzKk8kjx8IKb6Pp6vraBu/z9o+H6MS8NYrjmMHKXVt6Tpr0A55guaXE2g5KdJc3gWaSKw7v927AxyRKhtW72wCgeVTLYhKlUPEOaILrjkqxkoJEMSbpp7SLd+RjdcN0hM8o0QjVmnlrnJWpogL7qpqrKLZPiU6e0FTNxySgjMcGqiMI1RTT16MNy9zvoT249wIcc7xJnTky7kjFNGVt6ep1Nsni2tlhzxoLTI8tnRlu2f6jaRYFyzENFzDQrO4vG4734D7HwBASReDEVo6b00/jFBtmcGMf6MH9jrdXoxdoBdDNpPODD3oH0ai9Z/R2fdPGM7gnoRVpZmUKHjBE6EZpyLt+gnhqGFGOHtxO03QcE0bapuIk9t/gbcNJA/9KRLs62Kmx5gsEJQoZl6eBbmwbPuSw27zdOF+TVGsQ1Ptw2oUzTsNqFPC/1Lilbm8U1gob1uXiQgHONvvw0KRxGeUCD/dh1hNoqTGe0NpFmi5HqRzdKzImK7TW5WJZYl60ptPy56kwfE0Duqh72F/iIqzzPFWzi/786j3TNSmCmgyXdXcrxtPaf4FSG9Sjmq3CI2Yr98ckaYXkAWjMnbfqBTjn3aq7XPDEFxhSk3lzt3NGQg0+Q3mD5jqcn1Qw78KHx+DqUOVjOZGRrlmmlUexwpRgwWCMw3oUFtfhURceG3ONkowL5Sc8zGOUuiXvZk2RN1AKvD58xIWPmvA++gbK9+Fj6/BxFy6wRnOr6gUUf8DjDGuHZSpimUqNXY16/QlVUNbF4lohxFJdtnrFtaI8tGZdLIVFNXG5oMundqzhSFhaLorptWI4caCAtp3hjoFd7RrYhk8l/dQu7cfrs1aIM8F3mo2iST2kbYKKCk9omnhPoWW+zuOYGlqVebsssQXLh8NyN1KsnOBhXFaM4EhKyzSmuuPLg4ahYZnIMkGHtBNThd66NNXy2UhIVVZdPpovyzQNNEGo0D4Y8dBheehuc60CCl9r0L5xJBUERsV0zRXnsYTQ26l/yzgKKFBz7mD1oG0690ySqdXN3oaOGY5pCsxonznnzgjBBUTsdgUXvaKJDcsGihGJYrO6Vj1IrkDKHjE67G9NQj73mlXfVyjusyasvdYea7dVtIpFyCaRhuCCYBcYb+I7gGwqUC7kfVhxocse78GlbVRjlZ2UfXjShU+sw1oPeuj4lAufzFvsQXYyvwJPbyrn1DUp5zMMFfFZh51gWuYu9+BTDvvfmtbXq7FTQ24OJMRqLNQn1jR/1lE8rkwy1IuN66UXnx7Xi+fm/m2xcxr1z7jwWUT9OY365xC8z7vwhU3wvrgJ3tFrAu9LGrwvj8B7vgcvXAt4X9kCvNfr71cRwhcNhC9dLwhfHofwFZRcdo7NMVTRryGQX3fhGwjkK012PSTqm1qi2P+NLH1LwbcdFrCQUdZmHcaYkZDvsGUWM5SJ77KX30wmXt1aJr6nCft9F36AOL+qCftDJOyPXPhxH37iwk+1TPxsG5n4uZGJX7jwy3X4VQ9eQ8dfu/CbTab/lr7pF9WSCYixbPgdMvv3PfiDY+gwWAkX+ON2/oMZ9kPmxWUQBz92/oRR/mzEGt+VOx2kaAp/2S7EcIp9enD6LA5/wl8xyN9Mwli8PMljouutD18Kf29Y5uzVwEpFkszHt98WvqkK+AcOmY+1SPqjA+2fG3lLwb+q/wH9QdLt",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: '2019-09-30T06:41:56.584387Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2019-09-29_23_41_55-14722745897377016068'
location: 'us-central1'
name: 'beamapp-jenkins-0930064153-472760'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2019-09-30T06:41:56.584387Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2019-09-29_23_41_55-14722745897377016068]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-09-29_23_41_55-14722745897377016068?project=apache-beam-testing
root: INFO: Job 2019-09-29_23_41_55-14722745897377016068 is in state JOB_STATE_RUNNING
root: INFO: 2019-09-30T06:41:59.526Z: JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service Account.
root: INFO: 2019-09-30T06:42:00.010Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-a.
root: INFO: 2019-09-30T06:42:00.674Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2019-09-30T06:42:00.712Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2019-09-30T06:42:00.748Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2019-09-30T06:42:00.793Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
root: INFO: 2019-09-30T06:42:00.821Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2019-09-30T06:42:00.927Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
root: INFO: 2019-09-30T06:42:00.962Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2019-09-30T06:42:00.999Z: JOB_MESSAGE_DETAILED: Fusing consumer split into read/Read
root: INFO: 2019-09-30T06:42:01.033Z: JOB_MESSAGE_DETAILED: Fusing consumer pair_with_one into split
root: INFO: 2019-09-30T06:42:01.070Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Reify into pair_with_one
root: INFO: 2019-09-30T06:42:01.102Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Write into group/Reify
root: INFO: 2019-09-30T06:42:01.140Z: JOB_MESSAGE_DETAILED: Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2019-09-30T06:42:01.174Z: JOB_MESSAGE_DETAILED: Fusing consumer count into group/GroupByWindow
root: INFO: 2019-09-30T06:42:01.212Z: JOB_MESSAGE_DETAILED: Fusing consumer format into count
root: INFO: 2019-09-30T06:42:01.249Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2019-09-30T06:42:01.284Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2019-09-30T06:42:01.322Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2019-09-30T06:42:01.360Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2019-09-30T06:42:01.394Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2019-09-30T06:42:01.432Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2019-09-30T06:42:01.465Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2019-09-30T06:42:01.502Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2019-09-30T06:42:01.543Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
root: INFO: 2019-09-30T06:42:01.574Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
root: INFO: 2019-09-30T06:42:01.609Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
root: INFO: 2019-09-30T06:42:01.649Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
root: INFO: 2019-09-30T06:42:01.805Z: JOB_MESSAGE_DEBUG: Executing wait step start26
root: INFO: 2019-09-30T06:42:01.867Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-30T06:42:01.899Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-30T06:42:01.912Z: JOB_MESSAGE_DEBUG: Starting worker pool setup.
root: INFO: 2019-09-30T06:42:01.935Z: JOB_MESSAGE_BASIC: Executing operation group/Create
root: INFO: 2019-09-30T06:42:01.945Z: JOB_MESSAGE_BASIC: Starting 10 workers in us-central1-a...
root: INFO: 2019-09-30T06:42:02.001Z: JOB_MESSAGE_BASIC: Finished operation group/Create
root: INFO: 2019-09-30T06:42:02.001Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-30T06:42:02.072Z: JOB_MESSAGE_DEBUG: Value "group/Session" materialized.
root: INFO: 2019-09-30T06:42:02.097Z: JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/GroupByKey/Session" materialized.
root: INFO: 2019-09-30T06:42:02.131Z: JOB_MESSAGE_BASIC: Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-30T06:42:31.382Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 10 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-30T06:43:04.691Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-30T06:43:04.729Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-30T06:47:22.688Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T06:47:24.775Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T06:47:25.731Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T06:47:26.413Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T06:47:26.785Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T06:47:27.463Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T06:47:27.847Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T06:47:27.872Z: JOB_MESSAGE_BASIC: Finished operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-30T06:47:27.944Z: JOB_MESSAGE_DEBUG: Executing failure step failure25
root: INFO: 2019-09-30T06:47:27.983Z: JOB_MESSAGE_ERROR: Workflow failed. Causes: S06:read/Read+split+pair_with_one+group/Reify+group/Write failed., Internal Issue (3d0666b961365ec0): 63963027:24514
root: INFO: 2019-09-30T06:47:28.292Z: JOB_MESSAGE_WARNING: S01:write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite failed.
root: INFO: 2019-09-30T06:47:28.329Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-30T06:47:28.443Z: JOB_MESSAGE_DETAILED: Cleaning up.
root: INFO: 2019-09-30T06:47:28.522Z: JOB_MESSAGE_DEBUG: Starting worker pool teardown.
root: INFO: 2019-09-30T06:47:28.559Z: JOB_MESSAGE_BASIC: Stopping worker pool...
root: INFO: 2019-09-30T06:51:15.387Z: JOB_MESSAGE_DETAILED: Autoscaling: Reduced the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-30T06:51:15.434Z: JOB_MESSAGE_BASIC: Worker pool stopped.
root: INFO: 2019-09-30T06:51:15.470Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
root: INFO: Job 2019-09-29_23_41_55-14722745897377016068 is in state JOB_STATE_FAILED
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569825712476/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569825712476/results*' -> 'gs\\:\\/\\/temp\\-storage\\-for\\-end\\-to\\-end\\-tests\\/py\\-it\\-cloud\\/output\\/1569825712476\\/results[^/\\\\]*'
root: INFO: Starting the size estimation of the input
root: INFO: Finished listing 0 files in 0.04228329658508301 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-integrationTest-perf.xml
----------------------------------------------------------------------
XML: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 569.975s
FAILED (errors=1)
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py36:integrationTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
BUILD FAILED in 10m 16s
2019-09-30 06:51:23,920 e7986a9b MainThread beam_integration_benchmark(1/1) ERROR Error during benchmark beam_integration_benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-30 06:51:23,922 e7986a9b MainThread beam_integration_benchmark(1/1) INFO Cleaning up benchmark beam_integration_benchmark
2019-09-30 06:51:23,924 e7986a9b MainThread beam_integration_benchmark(1/1) ERROR Exception running benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 984, in RunBenchmarkTask
RunBenchmark(spec, collector)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-30 06:51:23,924 e7986a9b MainThread beam_integration_benchmark(1/1) ERROR Benchmark 1/1 beam_integration_benchmark (UID: beam_integration_benchmark0) failed. Execution will continue.
2019-09-30 06:51:23,925 e7986a9b MainThread beam_integration_benchmark(1/1) INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2019-09-30 06:51:23,925 e7986a9b MainThread beam_integration_benchmark(1/1) INFO Complete logs can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/e7986a9b/pkb.log>
2019-09-30 06:51:23,925 e7986a9b MainThread beam_integration_benchmark(1/1) INFO Completion statuses can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/e7986a9b/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py36
#522
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/522/display/redirect>
Changes:
------------------------------------------
[...truncated 156.31 KB...]
"type": "STRING",
"value": "_finalize_write"
}
],
"non_parallel_inputs": {
"side0-write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s18"
},
"side1-write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s19"
},
"side2-write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "eNrNV/l/3MQVX62dA5GmAUogpMeSklam7MohTZq4FJJuEjAbb1zZYPUIYlaa3VEsafRmRt6YeimQruNC7xt6l943ve/j7+g/0zezu04XbJrf0s/HlrRv5r2Z933f9x3pmQknJDkJGQ1alKQ1JUgm21ykshZyQe06SRLSSuiSIHlOxRl+LrOhNPUsWD0oO/5EqVQK2hlMjAWJOf61iKR20I4zksRP0aArYkVtmPT3GJcCw4qOzGkIO5r+LrTlgodUStjJJv2b9Ry1mtOAxZmSsGt8jzhg7LWI4iaJ4kLasxcW0fyINtuwGzd4U7MHtmNCxVleKBNPws1NswNeqGu2Pc1iHd7QKloXYa/z6kwUvaxibgeLeF+Is2Ub3ojR9/XgFsffh5HacUKDnCgW5IK248tw61gEnqN3JmsrJClwnuArcUSFvaCIisPHtXF+ZIPbMPCbenC749sYeOChdwj7wyhOklqgr3aQcBIZuw13mApIJeDOPhzw4C5/x8gTDvoX8LkjZ1xX0TSvSsSJdGgVq1ulWVRVfHCjUkk3X63GqhomvIjcATbukWPHT56YPnpy+uixE9OuoLJIsBRvLlrX0s5ISgNZtHXab2F6+2/twdsctp/dxQ76OKsEFZyvs8mKNJCMiEjC3Y2S2WbIMWs4NAaXscnhzV7kC0rEWafODT5vxwXu6cHhln+LzlpHG2xBE5YoeId/D9qrh53hUJFOTR+LqryNpmvraxu8098/HqKT8NYojmMGq/du6Tll0g95iuWWEms7KNG9/iSaSaE4vMu/CR/TOB1W7z4DgOaRmyckzqDqH9AE1x2VYSUFiRNMMshoF+/Ix9qG6YiAUaIRcptFa5yVmaIC+6pWqDixT4tOkdJMzSckpIwnBqpphOoI09f7G5a5H6U9ePdFOOb4UzpzZNx01TSlu3TtOpvmiXtu2LPGAsfHls4Nt+zgsSyPw+WERgsYaFb3lw3v6cEJx8AQEUXg5FaOm9PP4BQbZnBj7+3BA46/V6MXagXQzaTzg/f5B9GovWf0dgPTxjO4J6EVaWblCDxoiNCNs4h3gxTx1DCiHD20nabpOCaMtE3FSRK8xtuGUwb+lZh2dbDTY80XCkoUMq7IQt3YNrzfYXf4u3G+JqnWIKj34YwHZ52G1Sjh/0Tjtrq9UVorbVhXygslONfsw8NTxmWUCzzSh1lfoMVlPKXuJZotx5kc3asyISvU7XKxLDEv6uq0gnkqDF+zkC7qHg6WuIjqvMjU7GIwv3r0uCtF6MpoWXe3Yjxz/wsUd1CPWr4Kj5qtPJCQtBWRB6Exd8Gql+C8f7vucsHTQGBITebN3c4ZCTX4DOUNmutwYUrBvAcfGIOrQ1WA5URGemaZVhEnClOCBYMxDutRWFyHxzx4fMw1TnMuVJDyqEhQ6pb8WzVFXkMp8PvwQQ8+ZMIH6BuqIIAPr8NHPLjIGs2tqhdS/AFPMKwdlqmMZZpo7GrU60+qkrIulddKEZbqitUrr5XloTXr0kRUVpNXSrp8ascajkQTy2VxfK0cTR4ooW1ntGNgV7sGtuHThH5qT+zH63NWhDMhcJqNskk9om2CigpPapr4T6Nlvs6ThBpaVXi7IrEFK4ejSjdWrJLiYVxRjOBIRis0obrjK4OGoVGFyApBh6yTUIXeujS1yrlYSFVRXT6aLys0CzVBqNA+GPHQYXnoPnOtAQpfa9C+SSwVhEbFdM0V54mEyN+pf8skDilQc+5g9aBtOvdsmqvVzd6GjhlOaAbMaJ85584KwQXE7E4Fl/yyiQ3LBooRiRKzulY9SK9Cxh41OhxsTUI+94pV31cq77Mmrb3WHmu3VbbKZcinkIbggWAXGW/iO4BsKlAeFH1Y8aDLnujB5W1UY5Wdkn14yoOPrsNaD3ro+LQHHyta7CF2qrgKz2wq55HrUs5nGSricw47ybTMXenBxx32vzWtr1djp4fcHEiI1VioT65p/qyjeFydYqgXGzdKLz4xrhfPz/3bYuc16i948ElE/XmN+qcQvE978JlN8D67Cd791wXe5zR4nx+B94UefPF6wPvSFuC9Wn+/jBB+xUD41RsF4YvjEL6EksvOszmGKvo1BPLrHnwDgXypyW6ERH1TSxT7v5Glbyn4tsNCFjHK2qzDGDMS8h22zBKGMvFd9uLrycTLW8vE9zRhv+/BDxDnlzVhf4iE/ZEHP+7DTzz4qZaJn20jEz83MvELD365Dr/qwSvo+GsPfrPJ9N/S1/2iWjIBMZYNv0Nm/74Hf3AMHQYr4QJ/3M5/MMN+2Ly4DOLgx86fMMqfjVjju3KngxTN4C/bhRhOsc8MTp/F4U/4Kwb5m0kYi1ekRUJ0vfXhS+HvDcucvRpYqUiaB/j228I3VQH/wCHzsRbLYHSg/XOjaCn4V+0/s/vS5w==",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: '2019-09-30T00:38:27.936676Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2019-09-29_17_38_26-2960565174968177893'
location: 'us-central1'
name: 'beamapp-jenkins-0930003824-655924'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2019-09-30T00:38:27.936676Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2019-09-29_17_38_26-2960565174968177893]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-09-29_17_38_26-2960565174968177893?project=apache-beam-testing
root: INFO: Job 2019-09-29_17_38_26-2960565174968177893 is in state JOB_STATE_RUNNING
root: INFO: 2019-09-30T00:38:29.865Z: JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service Account.
root: INFO: 2019-09-30T00:38:30.350Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-a.
root: INFO: 2019-09-30T00:38:30.894Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2019-09-30T00:38:30.934Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2019-09-30T00:38:30.958Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2019-09-30T00:38:30.997Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
root: INFO: 2019-09-30T00:38:31.032Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2019-09-30T00:38:31.125Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
root: INFO: 2019-09-30T00:38:31.222Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2019-09-30T00:38:31.246Z: JOB_MESSAGE_DETAILED: Fusing consumer split into read/Read
root: INFO: 2019-09-30T00:38:31.270Z: JOB_MESSAGE_DETAILED: Fusing consumer pair_with_one into split
root: INFO: 2019-09-30T00:38:31.296Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Reify into pair_with_one
root: INFO: 2019-09-30T00:38:31.322Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Write into group/Reify
root: INFO: 2019-09-30T00:38:31.351Z: JOB_MESSAGE_DETAILED: Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2019-09-30T00:38:31.389Z: JOB_MESSAGE_DETAILED: Fusing consumer count into group/GroupByWindow
root: INFO: 2019-09-30T00:38:31.425Z: JOB_MESSAGE_DETAILED: Fusing consumer format into count
root: INFO: 2019-09-30T00:38:31.462Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2019-09-30T00:38:31.487Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2019-09-30T00:38:31.519Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2019-09-30T00:38:31.557Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2019-09-30T00:38:31.593Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2019-09-30T00:38:31.629Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2019-09-30T00:38:31.655Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2019-09-30T00:38:31.690Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2019-09-30T00:38:31.730Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
root: INFO: 2019-09-30T00:38:31.769Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
root: INFO: 2019-09-30T00:38:31.804Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
root: INFO: 2019-09-30T00:38:31.842Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
root: INFO: 2019-09-30T00:38:32.001Z: JOB_MESSAGE_DEBUG: Executing wait step start26
root: INFO: 2019-09-30T00:38:32.070Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-30T00:38:32.102Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-30T00:38:32.115Z: JOB_MESSAGE_DEBUG: Starting worker pool setup.
root: INFO: 2019-09-30T00:38:32.133Z: JOB_MESSAGE_BASIC: Starting 10 workers in us-central1-a...
root: INFO: 2019-09-30T00:38:32.134Z: JOB_MESSAGE_BASIC: Executing operation group/Create
root: INFO: 2019-09-30T00:38:32.187Z: JOB_MESSAGE_BASIC: Finished operation group/Create
root: INFO: 2019-09-30T00:38:32.188Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-30T00:38:32.256Z: JOB_MESSAGE_DEBUG: Value "group/Session" materialized.
root: INFO: 2019-09-30T00:38:32.291Z: JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/GroupByKey/Session" materialized.
root: INFO: 2019-09-30T00:38:32.325Z: JOB_MESSAGE_BASIC: Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-30T00:38:56.225Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 10 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-30T00:39:35.300Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-30T00:39:35.333Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-30T00:43:23.194Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T00:43:25.264Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T00:43:27.337Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T00:43:29.421Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T00:43:31.486Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-30T00:43:31.791Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-30T00:43:31.860Z: JOB_MESSAGE_DEBUG: Executing failure step failure25
root: INFO: 2019-09-30T00:43:31.890Z: JOB_MESSAGE_ERROR: Workflow failed. Causes: S01:write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite failed., Internal Issue (28374aaaa224ed7b): 63963027:24514
root: INFO: 2019-09-30T00:43:31.999Z: JOB_MESSAGE_BASIC: Finished operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-30T00:43:32.113Z: JOB_MESSAGE_DETAILED: Cleaning up.
root: INFO: 2019-09-30T00:43:32.170Z: JOB_MESSAGE_DEBUG: Starting worker pool teardown.
root: INFO: 2019-09-30T00:43:32.205Z: JOB_MESSAGE_BASIC: Stopping worker pool...
root: INFO: 2019-09-30T00:47:56.838Z: JOB_MESSAGE_DETAILED: Autoscaling: Reduced the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-30T00:47:56.868Z: JOB_MESSAGE_BASIC: Worker pool stopped.
root: INFO: 2019-09-30T00:47:56.890Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
root: INFO: Job 2019-09-29_17_38_26-2960565174968177893 is in state JOB_STATE_FAILED
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569803903580/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569803903580/results*' -> 'gs\\:\\/\\/temp\\-storage\\-for\\-end\\-to\\-end\\-tests\\/py\\-it\\-cloud\\/output\\/1569803903580\\/results[^/\\\\]*'
root: INFO: Starting the size estimation of the input
root: INFO: Finished listing 0 files in 0.051508426666259766 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-integrationTest-perf.xml
----------------------------------------------------------------------
XML: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 585.148s
FAILED (errors=1)
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py36:integrationTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
BUILD FAILED in 10m 30s
2019-09-30 00:48:10,338 0bc33330 MainThread beam_integration_benchmark(1/1) ERROR Error during benchmark beam_integration_benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-30 00:48:10,339 0bc33330 MainThread beam_integration_benchmark(1/1) INFO Cleaning up benchmark beam_integration_benchmark
2019-09-30 00:48:10,341 0bc33330 MainThread beam_integration_benchmark(1/1) ERROR Exception running benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 984, in RunBenchmarkTask
RunBenchmark(spec, collector)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-30 00:48:10,342 0bc33330 MainThread beam_integration_benchmark(1/1) ERROR Benchmark 1/1 beam_integration_benchmark (UID: beam_integration_benchmark0) failed. Execution will continue.
2019-09-30 00:48:10,342 0bc33330 MainThread beam_integration_benchmark(1/1) INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2019-09-30 00:48:10,342 0bc33330 MainThread beam_integration_benchmark(1/1) INFO Complete logs can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/0bc33330/pkb.log>
2019-09-30 00:48:10,343 0bc33330 MainThread beam_integration_benchmark(1/1) INFO Completion statuses can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/0bc33330/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py36
#521
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/521/display/redirect>
Changes:
------------------------------------------
[...truncated 157.31 KB...]
},
"side2-write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "eNrNV/l/G8UV10rOwZKmAUogpIdISbumSIqBOMSlkFRJwChW3LXB2yMso92RZuPdnX0zs1ZMLQqkclxoOXpD79L7pvd9/B39Z/pmJDkV2DS/pZ+Pvbt6M+/NvO/7vu/sPl1yApKRgFG/RUlSVYKkss1FIqsBF9SukzgmrZguCZJlVJziZ1IbCpPPgNWDouOVCoWC306hNBYk4vjXIpLafjtKSRw9Sf2uiBS1YcLbY1xyDCs6MqMB7Gh6u9CWCR5QKWEnm/Cu13PUakZ9FqVKwq7xPeKAsVdDipskigtpz55bRPPD2mzDbtzgdc0e2I4JFaVZrkw8Cdc3zQ54rq7Y9jTzdXhbK2+dh73OGzNR9KKKuO0v4n0hSpdteDtG39eDGxxvH0ZqRzH1M6KYnwnaji7CjWMReIbeqayukDjHeYKvRCEV9oIiKgoe08b5kQ1uwsDv6MHNjmdj4IGH3iHsD8Iojqu+vtp+zElo7DbcYioglYBb+3DAhdu8HSNPOOidw+eOnKnVFE2yikScSIdWsLoVmoYVxQc3KpWsZauVSFWCmOdhbYBNbero9PFj900dO3L83unpmqAyj7EU78xbV9JOSUJ9mbd12u9ievvv7sF7HLaf3cYOejirAGWcr7NJ88SXjIhQwu2NgtlmwDFrODQGl7HJ4c1e5AtKRGmnzg0+78UF7ujB4ZZ3g85aRxtsQROWKHifdwfaK4ed4VCeTB45GlZ4G01X1tc2eL+3fzxEJ+atURzHDFbu3NJz0qQf8ATLLSXWdlCiO70JNJNccfiAdx0+JlEyrN5dBgDNo1oWkyiFindAE1x3VIqVFCSKMUk/pV28Ix+rG6YjfEaJRqjWzFvjrEwVFdhX1VxFsX1SdPKEpmo+JgFlPDZQHUGoppi+3t2wzP0e2oN7z8NRx5vUmSPjjlRMU9aWrlxnkyyunRn2rLHA9NjSmeGW7T+aZlGwHNNwAQPN6v6y4VgP7nMMDCFRBI5v5bg5/RROsWEGN/bBHtzveHs1eoFWAN1MOj/4kHcQjdp7Rm/XN208g3sSWpFmVqbgAUOEbpSGvOsniKeGEeXowe00TccxYaRtKk5i/03eNpww8K9EtKuDnRxrvkBQopBxeRroxrbhww67xduN8zVJtQZBvQ+nXDjtNKxGAf9LjZvq9kZhrbBhXSouFOBMsw8PTRqXUS7wcB9mPYGWGuMJrV2g6XKUytG9ImOyQmtdLpYl5kVrOi1/ngrD1zSgi7qH/SUuwjrPUzW76M+v3jNdkyKoyXBZd7diPK39Fyi1QT2q2So8YrZyf0ySVkgegMbcOategLPezbrLBU98gSE1mTd3O2ck1OAzlDdorsO5SQXzLnxkDK4OVT6WExnpmmVaeRQrTAkWDMY4rEdhcR0edeGxMdcoybhQfsLDPEapW/Ju1BR5E6XA68NHXfiYCe+jb6B8Hz6+Dp9w4TxrNLeqXkDxBzzOsHZYpiKWqdTY1ajXn1AFZV0orhVCLNUlq1dcK8pDa9aFUlhUE5cKunxqxxqOhKXlopheK4YTBwpo2xnuGNjVroFt+FTST+3Sfrw+a4U4E3yn2Sia1EPaJqio8ISmifcUWubrPI6poVWZt8sSW7B8OCx3I8XKCR7GZcUIjqS0TGOqO748aBgaloksE3RIOzFV6K1LUy2fiYRUZdXlo/myTNNAE4QK7YMRDx2Wh+4y1yqg8LUG7RtHUkFgVEzXXHEeSwi9nfq3jKOAAjXnDlYP2qZzTyeZWt3sbeiY4ZimwIz2mXPutBBcQMRuVXDBK5rYsGygGJEoNqtr1YPkMqTsEaPD/tYk5HOvW/V9heI+a8Laa+2xdltFq1iEbBJpCC4Idp7xJr4DyKYC5ULehxUXuuzxHlzcRjVW2QnZhydd+OQ6rPWgh45PufCpvMUeZCfyy/D0pnJOXZVyPsNQEZ912HGmZe5SDz7tsP+taX29Gjs55OZAQqzGQn1iTfNnHcXj8iRDvdi4VnrxmXG9eG7u3xY7q1F/3oXPIurPadQ/h+C94MKLm+C9tAne3VcF3ssavM+PwPtCD754NeB9aQvw3qi/X0YIv2Ig/Oq1gvCVcQhfRcllZ9kcQxX9GgL5dRe+gUC+2mTXQqK+qSWK/d/I0rcUfNthAQsZZW3WYYwZCfkOW2YxQ5n4LnvlrWTita1l4nuasN934QeI82uasD9Ewv7IhR/34Scu/FTLxM+2kYmfG5n4hQu/XIdf9eB1dPy1C7/ZZPpv6Vt+US2ZgBjLht8hs3/fgz84hg6DlXCBP27nP5hhP2ReXAZx8GPnTxjlz0as8V2500GKpvCX7UIMp9inBqfP4vAn/BWD/M0kjMXLkzwmut768KXw94Zlzl4NrFQkyXx8+23hm6qAf+CQ+ViLpD860P65kbcU/Kv6H0ai0vM=",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: '2019-09-29T18:28:33.739761Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2019-09-29_11_28_32-698110663761782636'
location: 'us-central1'
name: 'beamapp-jenkins-0929182830-530145'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2019-09-29T18:28:33.739761Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2019-09-29_11_28_32-698110663761782636]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-09-29_11_28_32-698110663761782636?project=apache-beam-testing
root: INFO: Job 2019-09-29_11_28_32-698110663761782636 is in state JOB_STATE_RUNNING
root: INFO: 2019-09-29T18:28:36.587Z: JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service Account.
root: INFO: 2019-09-29T18:28:36.968Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
root: INFO: 2019-09-29T18:28:37.540Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2019-09-29T18:28:37.575Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2019-09-29T18:28:37.598Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2019-09-29T18:28:37.623Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
root: INFO: 2019-09-29T18:28:37.646Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2019-09-29T18:28:37.743Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
root: INFO: 2019-09-29T18:28:37.778Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2019-09-29T18:28:37.806Z: JOB_MESSAGE_DETAILED: Fusing consumer split into read/Read
root: INFO: 2019-09-29T18:28:37.826Z: JOB_MESSAGE_DETAILED: Fusing consumer pair_with_one into split
root: INFO: 2019-09-29T18:28:37.842Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Reify into pair_with_one
root: INFO: 2019-09-29T18:28:37.864Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Write into group/Reify
root: INFO: 2019-09-29T18:28:37.890Z: JOB_MESSAGE_DETAILED: Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2019-09-29T18:28:37.911Z: JOB_MESSAGE_DETAILED: Fusing consumer count into group/GroupByWindow
root: INFO: 2019-09-29T18:28:37.935Z: JOB_MESSAGE_DETAILED: Fusing consumer format into count
root: INFO: 2019-09-29T18:28:37.952Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2019-09-29T18:28:37.976Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2019-09-29T18:28:37.997Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2019-09-29T18:28:38.019Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2019-09-29T18:28:38.056Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2019-09-29T18:28:38.077Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2019-09-29T18:28:38.102Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2019-09-29T18:28:38.127Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2019-09-29T18:28:38.150Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
root: INFO: 2019-09-29T18:28:38.175Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
root: INFO: 2019-09-29T18:28:38.200Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
root: INFO: 2019-09-29T18:28:38.226Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
root: INFO: 2019-09-29T18:28:38.323Z: JOB_MESSAGE_DEBUG: Executing wait step start26
root: INFO: 2019-09-29T18:28:38.363Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-29T18:28:38.379Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-29T18:28:38.393Z: JOB_MESSAGE_BASIC: Executing operation group/Create
root: INFO: 2019-09-29T18:28:38.397Z: JOB_MESSAGE_DEBUG: Starting worker pool setup.
root: INFO: 2019-09-29T18:28:38.417Z: JOB_MESSAGE_BASIC: Starting 10 workers in us-central1-f...
root: INFO: 2019-09-29T18:28:38.461Z: JOB_MESSAGE_BASIC: Finished operation group/Create
root: INFO: 2019-09-29T18:28:38.462Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-29T18:28:38.503Z: JOB_MESSAGE_DEBUG: Value "group/Session" materialized.
root: INFO: 2019-09-29T18:28:38.518Z: JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/GroupByKey/Session" materialized.
root: INFO: 2019-09-29T18:28:38.545Z: JOB_MESSAGE_BASIC: Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-29T18:28:59.761Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 1 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-29T18:28:59.793Z: JOB_MESSAGE_DETAILED: Resized worker pool to 1, though goal was 10. This could be a quota issue.
root: INFO: 2019-09-29T18:29:05.218Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 10 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-29T18:29:28.441Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-29T18:29:28.480Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-29T18:33:20.174Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T18:33:21.423Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T18:33:22.253Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T18:33:23.506Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T18:33:24.320Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T18:33:24.564Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T18:33:24.588Z: JOB_MESSAGE_BASIC: Finished operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-29T18:33:24.660Z: JOB_MESSAGE_DEBUG: Executing failure step failure25
root: INFO: 2019-09-29T18:33:24.694Z: JOB_MESSAGE_ERROR: Workflow failed. Causes: S06:read/Read+split+pair_with_one+group/Reify+group/Write failed., Internal Issue (86ebed5c9a4d8269): 63963027:24514
root: INFO: 2019-09-29T18:33:25.099Z: JOB_MESSAGE_WARNING: S01:write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite failed.
root: INFO: 2019-09-29T18:33:25.134Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-29T18:33:25.236Z: JOB_MESSAGE_DETAILED: Cleaning up.
root: INFO: 2019-09-29T18:33:25.301Z: JOB_MESSAGE_DEBUG: Starting worker pool teardown.
root: INFO: 2019-09-29T18:33:25.326Z: JOB_MESSAGE_BASIC: Stopping worker pool...
root: INFO: 2019-09-29T18:35:34.600Z: JOB_MESSAGE_DETAILED: Autoscaling: Reduced the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-29T18:35:34.648Z: JOB_MESSAGE_BASIC: Worker pool stopped.
root: INFO: 2019-09-29T18:35:34.679Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
root: INFO: Job 2019-09-29_11_28_32-698110663761782636 is in state JOB_STATE_FAILED
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569781709466/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569781709466/results*' -> 'gs\\:\\/\\/temp\\-storage\\-for\\-end\\-to\\-end\\-tests\\/py\\-it\\-cloud\\/output\\/1569781709466\\/results[^/\\\\]*'
root: INFO: Starting the size estimation of the input
root: INFO: Finished listing 0 files in 0.06062579154968262 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-integrationTest-perf.xml
----------------------------------------------------------------------
XML: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 434.970s
FAILED (errors=1)
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py36:integrationTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
BUILD FAILED in 8m 4s
2019-09-29 18:35:45,994 bc9a0e75 MainThread beam_integration_benchmark(1/1) ERROR Error during benchmark beam_integration_benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-29 18:35:45,995 bc9a0e75 MainThread beam_integration_benchmark(1/1) INFO Cleaning up benchmark beam_integration_benchmark
2019-09-29 18:35:45,998 bc9a0e75 MainThread beam_integration_benchmark(1/1) ERROR Exception running benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 984, in RunBenchmarkTask
RunBenchmark(spec, collector)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-29 18:35:45,998 bc9a0e75 MainThread beam_integration_benchmark(1/1) ERROR Benchmark 1/1 beam_integration_benchmark (UID: beam_integration_benchmark0) failed. Execution will continue.
2019-09-29 18:35:45,999 bc9a0e75 MainThread beam_integration_benchmark(1/1) INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2019-09-29 18:35:45,999 bc9a0e75 MainThread beam_integration_benchmark(1/1) INFO Complete logs can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/bc9a0e75/pkb.log>
2019-09-29 18:35:45,999 bc9a0e75 MainThread beam_integration_benchmark(1/1) INFO Completion statuses can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/bc9a0e75/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py36
#520
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/520/display/redirect>
Changes:
------------------------------------------
[...truncated 157.95 KB...]
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "eNrNV2l728YRJkj5COK6TtI4cdyDceMWSkPSsms5VtPELm0nCi1ahZQIPRxkCSy5sAAsZnchWqmYy6WspPfdpHfT+07v+/gd/TOdXZJymUipv7nPIwHg7M7szjvvvAs8W3ICkpGAUb9FSVJVgqSyzUUiqwEX1K6TOCatmC4JkmVUnOHnUhsKk8+B1YOi45UKhYLfTqE0FiTi+Nciktp+O0pJHD1F/a6IFLVhwttjXHIMKzoyowHsaHq70JYJHlApYSeb8G7Wc9RqRn0WpUrCrvE94oCxV0OKmySKC2nPXlhE8yPabMNu3OBNzR7YjgkVpVmuTDwJNzfNDniurtn2NPN1eFMrb12Evc5rM1H0soq47S/ifSFKl214M0bf14NbHG8fRmpHMfUzopifCdqOLsOtYxF4ht6prK6QOMd5gq9EIRX2giIqCh7XxvmRDW7DwG/pwe2OZ2PggYfeIewPwiiOq76+2n7MSWjsNtxhKiCVgDv7cMCFu7wdI0846F3A546cqdUUTbKKRJxIh1awuhWahhXFBzcqlaxlq5VIVYKY52FtgE1t6vj0yRPTR09MTd1/5FhNUJnHWIq35q1raackob7M2zrttzG9/bf34B0O28/uYgc9nFWAMs7X2aR54ktGRCjh7kbBbDPgmDUcGoPL2OTwZi/yBSWitFPnBp934gL39OBwy7tFZ62jDbagCUsUvMu7B+2Vw85wKE8mjxwPK7yNpmvraxu829s/HqIT89YojmMGK/du6Tlp0g94guWWEms7KNG93gSaSa44vMe7CR+TKBlW7z4DgOZRLYtJlELFO6AJrjsqxUoKEsWYpJ/SLt6Rj9UN0xE+o0QjVGvmrXFWpooK7KtqrqLYPi06eUJTNR+TgDIeG6iOIFRTTF+PNixzP0Z78N6LcNzxJnXmyLgjFdOUtaVr19kki2vnhj1rLDA9tnRmuGX7j6VZFCzHNFzAQLO6v2w40YP7HQNDSBSBk1s5bk4/g1NsmMGNva8HDzjeXo1eoBVAN5POD97vHUSj9p7R2/VNG8/gnoRWpJmVKXjQEKEbpSHv+gniqWFEOXpoO03TcUwYaZuKk9h/nbcNpwz8KxHt6mCnx5ovEJQoZFyeBrqxbfiAw+7wduN8TVKtQVDvwxkXzjoNq1HA/1Ljtrq9UVgrbFhXigsFONfsw8OTxmWUCzzSh1lPoKXGeEJrl2i6HKVydK/ImKzQWpeLZYl50ZpOy5+nwvA1Deii7mF/iYuwzvNUzS7686vHpmtSBDUZLuvuVoyntf8CpTaoRzVbhUfNVh6ISdIKyYPQmLtg1Qtw3rtdd7ngiS8wpCbz5m7njIQafIbyBs11uDCpYN6FD47B1aHKx3IiI12zTCuPYoUpwYLBGIf1KCyuw2MuPD7mGiUZF8pPeJjHKHVL3q2aIq+jFHh9+JALHzbhffQNlO/DR9bhoy5cZI3mVtULKP6AJxjWDstUxDKVGrsa9fqTqqCsS8W1QoilumL1imtFeWjNulQKi2riSkGXT+1Yw5GwtFwU02vFcOJAAW07wx0Du9o1sA2fSvqpXdqP1+etEGeC7zQbRZN6SNsEFRWe1DTxnkbLfJ3HMTW0KvN2WWILlg+H5W6kWDnBw7isGMGRlJZpTHXHlwcNQ8MykWWCDmknpgq9dWmq5XORkKqsunw0X5ZpGmiCUKF9MOKhw/LQfeZaBRS+1qB940gqCIyK6ZorzmMJobdT/5ZxFFCg5tzB6kHbdO7ZJFOrm70NHTMc0xSY0T5zzp0VgguI2J0KLnlFExuWDRQjEsVmda16kFyFlD1qdNjfmoR87lWrvq9Q3GdNWHutPdZuq2gVi5BNIg3BBcEuMt7EdwDZVKBcyPuw4kKXPdGDy9uoxio7JfvwlAsfW4e1HvTQ8WkXnslb7CF2Kr8Kz24q59R1KedzDBXxeYedZFrmrvTg4w7735rW16ux00NuDiTEaizUJ9Y0f9ZRPK5OMtSLjRulFy+M68WLc/+22HmN+idc+CSi/qJG/VMI3qdd+MwmeJ/dBO/odYH3OQ3e50fgfaEHX7we8L60BXiv1d8vI4RfMRB+9UZB+NI4hC+j5LLzbI6hin4Ngfy6C99AIF9ushshUd/UEsX+b2TpWwq+7bCAhYyyNuswxoyEfIcts5ihTHyXvfRGMvHK1jLxPU3Y77vwA8T5FU3YHyJhf+TCj/vwExd+qmXiZ9vIxM+NTPzChV+uw6968Co6/tqF32wy/bf0Db+olkxAjGXD75DZv+/BHxxDh8FKuMAft/MfzLAfNi8ugzj4sfMnjPJnI9b4rtzpIEVT+Mt2IYZT7DOD02dx+BP+ikH+ZhLG4uVJHhNdb334Uvh7wzJnrwZWKpJkPr79tvBNVcA/cMh8rEXSHx1o/9zIWwr+Vf0Pp9XS5g==",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: '2019-09-29T13:11:56.309061Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2019-09-29_06_11_54-8672757647317743432'
location: 'us-central1'
name: 'beamapp-jenkins-0929131152-853360'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2019-09-29T13:11:56.309061Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2019-09-29_06_11_54-8672757647317743432]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-09-29_06_11_54-8672757647317743432?project=apache-beam-testing
root: INFO: Job 2019-09-29_06_11_54-8672757647317743432 is in state JOB_STATE_RUNNING
root: INFO: 2019-09-29T13:11:58.336Z: JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service Account.
root: INFO: 2019-09-29T13:11:58.917Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-a.
root: INFO: 2019-09-29T13:11:59.521Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2019-09-29T13:11:59.554Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2019-09-29T13:11:59.581Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2019-09-29T13:11:59.623Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
root: INFO: 2019-09-29T13:11:59.653Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2019-09-29T13:11:59.739Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
root: INFO: 2019-09-29T13:11:59.783Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2019-09-29T13:11:59.813Z: JOB_MESSAGE_DETAILED: Fusing consumer split into read/Read
root: INFO: 2019-09-29T13:11:59.833Z: JOB_MESSAGE_DETAILED: Fusing consumer pair_with_one into split
root: INFO: 2019-09-29T13:11:59.861Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Reify into pair_with_one
root: INFO: 2019-09-29T13:11:59.890Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Write into group/Reify
root: INFO: 2019-09-29T13:11:59.925Z: JOB_MESSAGE_DETAILED: Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2019-09-29T13:11:59.949Z: JOB_MESSAGE_DETAILED: Fusing consumer count into group/GroupByWindow
root: INFO: 2019-09-29T13:11:59.980Z: JOB_MESSAGE_DETAILED: Fusing consumer format into count
root: INFO: 2019-09-29T13:12:00.008Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2019-09-29T13:12:00.038Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2019-09-29T13:12:00.060Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2019-09-29T13:12:00.084Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2019-09-29T13:12:00.108Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2019-09-29T13:12:00.127Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2019-09-29T13:12:00.156Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2019-09-29T13:12:00.185Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2019-09-29T13:12:00.211Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
root: INFO: 2019-09-29T13:12:00.237Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
root: INFO: 2019-09-29T13:12:00.261Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
root: INFO: 2019-09-29T13:12:00.288Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
root: INFO: 2019-09-29T13:12:00.407Z: JOB_MESSAGE_DEBUG: Executing wait step start26
root: INFO: 2019-09-29T13:12:00.454Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-29T13:12:00.481Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-29T13:12:00.485Z: JOB_MESSAGE_DEBUG: Starting worker pool setup.
root: INFO: 2019-09-29T13:12:00.507Z: JOB_MESSAGE_BASIC: Starting 10 workers in us-central1-a...
root: INFO: 2019-09-29T13:12:00.508Z: JOB_MESSAGE_BASIC: Executing operation group/Create
root: INFO: 2019-09-29T13:12:00.557Z: JOB_MESSAGE_BASIC: Finished operation group/Create
root: INFO: 2019-09-29T13:12:00.557Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-29T13:12:00.619Z: JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/GroupByKey/Session" materialized.
root: INFO: 2019-09-29T13:12:00.650Z: JOB_MESSAGE_DEBUG: Value "group/Session" materialized.
root: INFO: 2019-09-29T13:12:00.711Z: JOB_MESSAGE_BASIC: Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-29T13:12:34.407Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 4 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-29T13:12:34.446Z: JOB_MESSAGE_DETAILED: Resized worker pool to 4, though goal was 10. This could be a quota issue.
root: INFO: 2019-09-29T13:12:39.781Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 9 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-29T13:12:39.811Z: JOB_MESSAGE_DETAILED: Resized worker pool to 9, though goal was 10. This could be a quota issue.
root: INFO: 2019-09-29T13:12:45.194Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 10 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-29T13:13:04.687Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-29T13:13:04.712Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-29T13:17:16.073Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T13:17:18.159Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T13:17:18.191Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T13:17:20.229Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T13:17:20.260Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T13:17:22.306Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T13:17:22.344Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T13:17:22.750Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-29T13:17:22.871Z: JOB_MESSAGE_DEBUG: Executing failure step failure25
root: INFO: 2019-09-29T13:17:22.951Z: JOB_MESSAGE_ERROR: Workflow failed. Causes: S01:write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite failed., Internal Issue (49f8fc27634f0a41): 63963027:24514
root: INFO: 2019-09-29T13:17:23.051Z: JOB_MESSAGE_BASIC: Finished operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-29T13:17:23.157Z: JOB_MESSAGE_DETAILED: Cleaning up.
root: INFO: 2019-09-29T13:17:23.234Z: JOB_MESSAGE_DEBUG: Starting worker pool teardown.
root: INFO: 2019-09-29T13:17:23.267Z: JOB_MESSAGE_BASIC: Stopping worker pool...
root: INFO: 2019-09-29T13:21:06.885Z: JOB_MESSAGE_DETAILED: Autoscaling: Reduced the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-29T13:21:06.931Z: JOB_MESSAGE_BASIC: Worker pool stopped.
root: INFO: 2019-09-29T13:21:06.967Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
root: INFO: Job 2019-09-29_06_11_54-8672757647317743432 is in state JOB_STATE_FAILED
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569762711803/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569762711803/results*' -> 'gs\\:\\/\\/temp\\-storage\\-for\\-end\\-to\\-end\\-tests\\/py\\-it\\-cloud\\/output\\/1569762711803\\/results[^/\\\\]*'
root: INFO: Starting the size estimation of the input
root: INFO: Finished listing 0 files in 0.0590972900390625 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-integrationTest-perf.xml
----------------------------------------------------------------------
XML: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 565.472s
FAILED (errors=1)
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py36:integrationTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
BUILD FAILED in 10m 15s
2019-09-29 13:21:18,858 32c2cd7a MainThread beam_integration_benchmark(1/1) ERROR Error during benchmark beam_integration_benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-29 13:21:18,859 32c2cd7a MainThread beam_integration_benchmark(1/1) INFO Cleaning up benchmark beam_integration_benchmark
2019-09-29 13:21:18,861 32c2cd7a MainThread beam_integration_benchmark(1/1) ERROR Exception running benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 984, in RunBenchmarkTask
RunBenchmark(spec, collector)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-29 13:21:18,861 32c2cd7a MainThread beam_integration_benchmark(1/1) ERROR Benchmark 1/1 beam_integration_benchmark (UID: beam_integration_benchmark0) failed. Execution will continue.
2019-09-29 13:21:18,862 32c2cd7a MainThread beam_integration_benchmark(1/1) INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2019-09-29 13:21:18,862 32c2cd7a MainThread beam_integration_benchmark(1/1) INFO Complete logs can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/32c2cd7a/pkb.log>
2019-09-29 13:21:18,862 32c2cd7a MainThread beam_integration_benchmark(1/1) INFO Completion statuses can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/32c2cd7a/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py36
#519
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/519/display/redirect>
------------------------------------------
[...truncated 156.73 KB...]
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "eNrNV2l728YRJij5COK6TtI4cdyDdeMWSkPClmOnVtPELm0nCi1ahZQIPRxkCSy5sAAsZnchWqmYy6WspvfdpHfT+07v+/gd/TOdXZJymUipv7nPIwHg7M7szjvvvAs8O+GEJCcho0GLkrSmBMlkm4tU1kIuqF0nSUJaCV0SJM+pOMPPZTaUpp4Dqwdlx58olUpBO4OJsSAxx78WkdQO2nFGkvgpGnRFrKgNk/4e41JgWNGROQ1hR9PfhbZc8JBKCTvZpH+znqNWcxqwOFMSdo3vEQeMvRZR3CRRXEh79sIimh/RZht24wZvavbAdkyoOMsLZeJJuLlpdsALdc22p1mswxtaResi7HVenYmil1XM7WAR7wtxtmzDGzH6vh7c4vj7MFI7TmiQE8WCXNB2fBluHYvAc/TOZG2FJAXOE3wljqiwFxRRcfi4Ns6PbHAbBn5TD253fBsDDzz0DmF/GMVJUgv01Q4STiJjt+EOUwGpBNzZhwMe3OXvGHnCQf8CPnfkjOsqmuZViTiRDq1idas0i6qKD25UKunmq9VYVcOEF5E7wMY9evzEyfuPnTxyfHp6+j5XUFkkWIo3F61raWckpYEs2jrttzC9/bf24G0O28/uYgd9nFWCCs7X2WRFGkhGRCTh7Y2S2WbIMWs4NAaXscnhzV7kC0rEWafODT7vwAXu7sHhln+LzlpHG2xBE5YoeKd/N9qrh53hUJFOHTkeVXkbTdfW1zZ4l79/PEQn4a1RHMcMVu/Z0nPKpB/yFMstJdZ2UKJ7/Ek0k0JxeLd/Ez6mcTqs3r0GAM0jN09InEHVP6AJrjsqw0oKEieYZJDRLt6Rj7UN0xEBo0Qj5DaL1jgrM0UF9lWtUHFinxadIqWZmk9ISBlPDFRHEKqjTF+nG5a5H6M9uO8iHHf8KZ05Mu5I1TSlu3TtOpvmiXtu2LPGAifGls4Nt+zgsSyPw+WERgsYaFb3lw339+A9joEhIorAya0cN6efwSk2zODG3tuDBxx/r0Yv1Aqgm0nnB+/zD6JRe8/o7QamjWdwT0Ir0szKUXjQEKEbZxHvBiniqWFEOXpoO03TcUwYaZuKkyR4jbcNpwz8KzHt6mCnx5ovFJQoZFyRhbqxbXi/w+7wd+N8TVKtQVDvwxkPzjoNq1HC/4nGbXV7o7RW2rCulBdKcK7Zh4enjMsoF3ikD7O+QIvLeErdSzRbjjM5uldlQlao2+ViWWJe1NVpBfNUGL5mIV3UPRwscRHVeZGp2cVgfvXYCVeK0JXRsu5uxXjm/hco7qAetXwVHjVbeSAhaSsiD0Jj7oJVL8F5/3bd5YKngcCQmsybu50zEmrwGcobNNfhwpSCeQ8+MAZXh6oAy4mM9MwyrSJOFKYECwZjHNajsLgOj3nw+JhrnOZcqCDlUZGg1C35t2qKvIZS4Pfhgx58yIQP0DdUQQAfXoePeHCRNZpbVS+k+AOeYFg7LFMZyzTR2NWo159UJWVdKq+VIizVFatXXivLQ2vWpYmorCavlHT51I41HIkmlsvixFo5mjxQQtvOaMfArnYNbMOnCf3UntiP1+etCGdC4DQbZZN6RNsEFRWe1DTxn0bLfJ0nCTW0qvB2RWILVg5HlW6sWCXFw7iiGMGRjFZoQnXHVwYNQ6MKkRWCDlknoQq9dWlqlXOxkKqiunw0X1ZoFmqCUKF9MOKhw/LQveZaAxS+1qB9k1gqCI2K6ZorzhMJkb9T/5ZJHFKg5tzB6kHbdO7ZNFerm70NHTOc0AyY0T5zzp0VgguI2Z0KLvllExuWDRQjEiVmda16kF6FjD1qdDjYmoR87hWrvq9U3mdNWnutPdZuq2yVy5BPIQ3BA8EuMt7EdwDZVKA8KPqw4kGXPdGDy9uoxio7JfvwlAcfXYe1HvTQ8WkPnila7CF2qrgKz24q59HrUs7nGCri8w47ybTMXenBxxz2vzWtr1djp4fcHEiI1VioT65p/qyjeFydYqgXGzdKLz4+rhcvzP3bYuc16p/w4JOI+gsa9U8heJ/24DOb4H12E7zp6wLvcxq8z4/A+0IPvng94H1pC/Berb9fRgi/YiD86o2C8MVxCF9CyWXn2RxDFf0aAvl1D76BQL7UZDdCor6pJYr938jStxR822EhixhlbdZhjBkJ+Q5bZglDmfgue/H1ZOLlrWXie5qw3/fgB4jzy5qwP0TC/siDH/fhJx78VMvEz7aRiZ8bmfiFB79ch1/14BV0/LUHv9lk+m/p635RLZmAGMuG3yGzf9+DPziGDoOVcIE/buc/mGE/bF5cBnHwY+dPGOXPRqzxXbnTQYpm8JftQgyn2GcGp8/i8Cf8FYP8zSSMxSvSIiG63vrwpfD3hmXOXg2sVCTNA3z7beGbqoB/4JD5WItlMDrQ/rlRtBT8q/Yfm5zS5Q==",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: '2019-09-29T06:37:36.442082Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2019-09-28_23_37_35-8960032847715983566'
location: 'us-central1'
name: 'beamapp-jenkins-0929063733-263898'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2019-09-29T06:37:36.442082Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2019-09-28_23_37_35-8960032847715983566]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-09-28_23_37_35-8960032847715983566?project=apache-beam-testing
root: INFO: Job 2019-09-28_23_37_35-8960032847715983566 is in state JOB_STATE_RUNNING
root: INFO: 2019-09-29T06:37:38.638Z: JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service Account.
root: INFO: 2019-09-29T06:37:39.159Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-a.
root: INFO: 2019-09-29T06:37:39.721Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2019-09-29T06:37:39.756Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2019-09-29T06:37:39.791Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2019-09-29T06:37:39.823Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
root: INFO: 2019-09-29T06:37:39.854Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2019-09-29T06:37:39.965Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
root: INFO: 2019-09-29T06:37:40.015Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2019-09-29T06:37:40.046Z: JOB_MESSAGE_DETAILED: Fusing consumer split into read/Read
root: INFO: 2019-09-29T06:37:40.075Z: JOB_MESSAGE_DETAILED: Fusing consumer pair_with_one into split
root: INFO: 2019-09-29T06:37:40.104Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Reify into pair_with_one
root: INFO: 2019-09-29T06:37:40.133Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Write into group/Reify
root: INFO: 2019-09-29T06:37:40.155Z: JOB_MESSAGE_DETAILED: Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2019-09-29T06:37:40.193Z: JOB_MESSAGE_DETAILED: Fusing consumer count into group/GroupByWindow
root: INFO: 2019-09-29T06:37:40.227Z: JOB_MESSAGE_DETAILED: Fusing consumer format into count
root: INFO: 2019-09-29T06:37:40.258Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2019-09-29T06:37:40.289Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2019-09-29T06:37:40.324Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2019-09-29T06:37:40.352Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2019-09-29T06:37:40.377Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2019-09-29T06:37:40.414Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2019-09-29T06:37:40.450Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2019-09-29T06:37:40.485Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2019-09-29T06:37:40.523Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
root: INFO: 2019-09-29T06:37:40.551Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
root: INFO: 2019-09-29T06:37:40.584Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
root: INFO: 2019-09-29T06:37:40.609Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
root: INFO: 2019-09-29T06:37:40.766Z: JOB_MESSAGE_DEBUG: Executing wait step start26
root: INFO: 2019-09-29T06:37:40.837Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-29T06:37:40.872Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-29T06:37:40.885Z: JOB_MESSAGE_DEBUG: Starting worker pool setup.
root: INFO: 2019-09-29T06:37:40.910Z: JOB_MESSAGE_BASIC: Starting 10 workers in us-central1-a...
root: INFO: 2019-09-29T06:37:40.911Z: JOB_MESSAGE_BASIC: Executing operation group/Create
root: INFO: 2019-09-29T06:37:40.970Z: JOB_MESSAGE_BASIC: Finished operation group/Create
root: INFO: 2019-09-29T06:37:40.970Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-29T06:37:41.044Z: JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/GroupByKey/Session" materialized.
root: INFO: 2019-09-29T06:37:41.076Z: JOB_MESSAGE_DEBUG: Value "group/Session" materialized.
root: INFO: 2019-09-29T06:37:41.149Z: JOB_MESSAGE_BASIC: Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-29T06:38:10.894Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 10 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-29T06:38:37.600Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-29T06:38:37.627Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-29T06:43:02.078Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T06:43:02.691Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T06:43:03.733Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T06:43:04.216Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T06:43:05.795Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T06:43:06.278Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T06:43:07.851Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T06:43:08.202Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-29T06:43:08.241Z: JOB_MESSAGE_DEBUG: Executing failure step failure25
root: INFO: 2019-09-29T06:43:08.264Z: JOB_MESSAGE_ERROR: Workflow failed. Causes: S01:write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite failed., Internal Issue (d0c56e499aebe3b2): 63963027:24514
root: INFO: 2019-09-29T06:43:08.334Z: JOB_MESSAGE_BASIC: Finished operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-29T06:43:08.415Z: JOB_MESSAGE_DETAILED: Cleaning up.
root: INFO: 2019-09-29T06:43:08.455Z: JOB_MESSAGE_DEBUG: Starting worker pool teardown.
root: INFO: 2019-09-29T06:43:08.473Z: JOB_MESSAGE_BASIC: Stopping worker pool...
root: INFO: 2019-09-29T06:47:46.186Z: JOB_MESSAGE_DETAILED: Autoscaling: Reduced the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-29T06:47:46.261Z: JOB_MESSAGE_BASIC: Worker pool stopped.
root: INFO: 2019-09-29T06:47:46.283Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
root: INFO: Job 2019-09-28_23_37_35-8960032847715983566 is in state JOB_STATE_FAILED
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569739052224/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569739052224/results*' -> 'gs\\:\\/\\/temp\\-storage\\-for\\-end\\-to\\-end\\-tests\\/py\\-it\\-cloud\\/output\\/1569739052224\\/results[^/\\\\]*'
root: INFO: Starting the size estimation of the input
root: INFO: Finished listing 0 files in 0.04228639602661133 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-integrationTest-perf.xml
----------------------------------------------------------------------
XML: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 625.049s
FAILED (errors=1)
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py36:integrationTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
BUILD FAILED in 11m 11s
2019-09-29 06:47:58,783 66cafb28 MainThread beam_integration_benchmark(1/1) ERROR Error during benchmark beam_integration_benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-29 06:47:58,784 66cafb28 MainThread beam_integration_benchmark(1/1) INFO Cleaning up benchmark beam_integration_benchmark
2019-09-29 06:47:58,786 66cafb28 MainThread beam_integration_benchmark(1/1) ERROR Exception running benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 984, in RunBenchmarkTask
RunBenchmark(spec, collector)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-29 06:47:58,787 66cafb28 MainThread beam_integration_benchmark(1/1) ERROR Benchmark 1/1 beam_integration_benchmark (UID: beam_integration_benchmark0) failed. Execution will continue.
2019-09-29 06:47:58,788 66cafb28 MainThread beam_integration_benchmark(1/1) INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2019-09-29 06:47:58,788 66cafb28 MainThread beam_integration_benchmark(1/1) INFO Complete logs can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/66cafb28/pkb.log>
2019-09-29 06:47:58,788 66cafb28 MainThread beam_integration_benchmark(1/1) INFO Completion statuses can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/66cafb28/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py36
#518
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/518/display/redirect>
------------------------------------------
[...truncated 156.69 KB...]
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYE5OLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqYIfgYGhvi0xJycpMTk7HiwlkJ8pgVkJmfnpEJNYQGawlpbyJZUnKQHACYlLgM=",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "eNrNV/l/G8UV10rOwZKmAUogpIdISbumSIqhcbBLIamSgFGsuGuDt0dYRrsjzca7O/tmZq2YWlypHEPvu9C79L7pfR9/R/+ZvhlJTgU2zW/p52Pvrt7MezPv+77vO7vPlpyAZCRg1G9RklSVIKlsc5HIasAFteskjkkrpsuCZBkVp/nZ1IbC5HNg9aDoeKVCoeC3UyiNBYk4/rWIpLbfjlISR09RvysiRW2Y8PYZlxzDio7MaAC7mt4etGWCB1RK2M0mvBv1HLWWUZ9FqZKwZ3yPOGDs1ZDiJoniQtpz55fQ/Ig227AXN3hDswe2Y0JFaZYrE0/CjU2zA56rq7Z9zXwD3tTKWxdgv/PaTBS9pCJu+0t4X4zSFRvejNEP9OAmxzuAkdpRTP2MKOZngrajS3DzWASeoXcqq6skznGe4KtRSIW9qIiKgse1cWFkg1sw8Ft6cKvj2Rh44KF3CAeDMIrjqq+vth9zEhq7DbeZCkgl4PY+HHLhDm/XyBMOe+fxuSNnazVFk6wiESfSoRWsboWmYUXxwY1KJWvZWiVSlSDmeVgbYFObOj49c2LqxLGZ+2emj9cElXmMpXhr3rqadkoS6su8rdN+G9Pbf3sP3uGwg+wOdtjDWQUo43ydTZonvmREhBLubBTMNgOOWcORMbiMTQ5v9hJfVCJKO3Vu8HknLnBXD462vJt01jraYAuasETBu7y70F456gyH8mTy2PGwwttourq+tsG7vYPjIToxb43iOGawcve2npMm/YAnWG4psbaDEt3tTaCZ5IrDe7wb8DGJkmH17jEAaB7VsphEKVS8Q5rguqNSrKQgUYxJ+int4h35WN00HeEzSjRCtWbeGmdlqqjAvqrmKortU6KTJzRVCzEJKOOxgeoYQjXF9PXehmXu99EevPcCHHe8SZ05Mu5YxTRlbfnqdS7J4trZYc8aC0yPLZ0Zbtn+Y2kWBSsxDRcx0JzuLxtO9OB+x8AQEkVgZjvHremncYoNs7ix9/XgAcfbr9ELtALoZtL5wfu9w2jU3rN6u75p41nck9CKNLs6BQ8aInSjNORdP0E8NYwoRw/tpGk6jgkjbVNxEvuv87bhpIF/NaJdHezUWPMFghKFjMvTQDe2DR9w2G3eXpyvSao1COp9OO3CGadhNQr4X2rcUrc3C+uFTetycbEAZ5t9eHjSuIxygUf6MOcJtNQYT2jtIk1XolSO7hUZk1Va63KxIjEvWtNp+QtUGL6mAV3SPewvcxHWeZ6quSV/Ye2+6ZoUQU2GK7q7FeNp7b9AqQ3qUc3W4FGzlQdikrRC8iA05s9b9QKc827VXS544gsMqcm8tdt5I6EGn6G8QXMDzk8qWHDhg2NwdajysZzISNcs08qjWGFKsGgwxmE9Cksb8JgLj4+5RknGhfITHuYxSt2yd7OmyOsoBV4fPuTCh014H30D5fvwkQ34qAsXWKO5XfUCij/gCYa1wzIVsUylxp5Gvf6kKijrYnG9EGKpLlu94npRHlm3LpbCopq4XNDlU7vWcSQsrRTF9HoxnDhUQNvucNfArvYMbMOnkn5qlw7i9XkrxJngO81G0aQe0jZBRYUnNU28p9GyUOdxTA2tyrxdltiC5aNhuRspVk7wMC4rRnAkpWUaU93x5UHD0LBMZJmgQ9qJqUJvXZpq+WwkpCqrLh/Nl2WaBpogVGgfjHjkqDxyj7lWAYWvNWjfOJIKAqNiuuaK81hC6O3Wv2UcBRSoOXewetA2nXsmydTaVm9DxwzHNAVmtM+cc2eE4AIidruCi17RxIYVA8WIRLFZXaseJFcgZY8aHfa3JyGff9WqHygUD1gT1n5rn7XXKlrFImSTSENwQbALjDfxHUA2FSgX8j6sutBlT/Tg0g6qscZOyj485cLHNmC9Bz10fNqFZ/IWe4idzK/As1vKOXVNyvkcQ0V83mEzTMvc5R583GH/W9P6ejV2asjNgYRYjcX6xLrmzwaKx5VJhnqxeb304oVxvXhx/t8WO6dR/4QLn0TUX9SofwrB+7QLn9kC77Nb4N17TeB9ToP3+RF4X+jBF68FvC9tA95r9ffLCOFXDIRfvV4QvjQO4csouewcm2eool9DIL/uwjcQyJeb7HpI1De1RLH/G1n6loJvOyxgIaOszTqMMSMh32ErLGYoE99lL72RTLyyvUx8TxP2+y78AHF+RRP2h0jYH7nw4z78xIWfapn42Q4y8XMjE79w4Zcb8KsevIqOv3bhN1tM/y19wy+qZRMQY9nwO2T273vwB8fQYbASLvDHnfwHM+yHzYvLIA5+7PwJo/zZiDW+K3c6SNEU/rJTiOEU+/Tg9Fka/oS/YpC/mYSxeHmSx0TXWx++FP7esMzZq4GViiSZj2+/LXxTFfAPHDIfa5H0RwfaPzfzloJ/Vf8Dd23S9w==",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: '2019-09-29T00:31:43.016526Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2019-09-28_17_31_41-15694561143181347133'
location: 'us-central1'
name: 'beamapp-jenkins-0929003139-979459'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2019-09-29T00:31:43.016526Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2019-09-28_17_31_41-15694561143181347133]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2019-09-28_17_31_41-15694561143181347133?project=apache-beam-testing
root: INFO: Job 2019-09-28_17_31_41-15694561143181347133 is in state JOB_STATE_RUNNING
root: INFO: 2019-09-29T00:31:45.054Z: JOB_MESSAGE_DETAILED: Checking permissions granted to controller Service Account.
root: INFO: 2019-09-29T00:31:45.476Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
root: INFO: 2019-09-29T00:31:46.023Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2019-09-29T00:31:46.059Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2019-09-29T00:31:46.094Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2019-09-29T00:31:46.121Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
root: INFO: 2019-09-29T00:31:46.141Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2019-09-29T00:31:46.222Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
root: INFO: 2019-09-29T00:31:46.268Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2019-09-29T00:31:46.303Z: JOB_MESSAGE_DETAILED: Fusing consumer split into read/Read
root: INFO: 2019-09-29T00:31:46.331Z: JOB_MESSAGE_DETAILED: Fusing consumer pair_with_one into split
root: INFO: 2019-09-29T00:31:46.365Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Reify into pair_with_one
root: INFO: 2019-09-29T00:31:46.401Z: JOB_MESSAGE_DETAILED: Fusing consumer group/Write into group/Reify
root: INFO: 2019-09-29T00:31:46.434Z: JOB_MESSAGE_DETAILED: Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2019-09-29T00:31:46.469Z: JOB_MESSAGE_DETAILED: Fusing consumer count into group/GroupByWindow
root: INFO: 2019-09-29T00:31:46.506Z: JOB_MESSAGE_DETAILED: Fusing consumer format into count
root: INFO: 2019-09-29T00:31:46.538Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2019-09-29T00:31:46.573Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2019-09-29T00:31:46.608Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2019-09-29T00:31:46.632Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2019-09-29T00:31:46.666Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2019-09-29T00:31:46.700Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2019-09-29T00:31:46.731Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2019-09-29T00:31:46.762Z: JOB_MESSAGE_DETAILED: Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2019-09-29T00:31:46.796Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
root: INFO: 2019-09-29T00:31:46.832Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
root: INFO: 2019-09-29T00:31:46.857Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
root: INFO: 2019-09-29T00:31:46.893Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
root: INFO: 2019-09-29T00:31:47.011Z: JOB_MESSAGE_DEBUG: Executing wait step start26
root: INFO: 2019-09-29T00:31:47.075Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-29T00:31:47.103Z: JOB_MESSAGE_BASIC: Executing operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-29T00:31:47.115Z: JOB_MESSAGE_DEBUG: Starting worker pool setup.
root: INFO: 2019-09-29T00:31:47.131Z: JOB_MESSAGE_BASIC: Executing operation group/Create
root: INFO: 2019-09-29T00:31:47.146Z: JOB_MESSAGE_BASIC: Starting 10 workers in us-central1-f...
root: INFO: 2019-09-29T00:31:47.195Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/GroupByKey/Create
root: INFO: 2019-09-29T00:31:47.195Z: JOB_MESSAGE_BASIC: Finished operation group/Create
root: INFO: 2019-09-29T00:31:47.252Z: JOB_MESSAGE_DEBUG: Value "group/Session" materialized.
root: INFO: 2019-09-29T00:31:47.279Z: JOB_MESSAGE_DEBUG: Value "write/Write/WriteImpl/GroupByKey/Session" materialized.
root: INFO: 2019-09-29T00:31:47.314Z: JOB_MESSAGE_BASIC: Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-29T00:32:13.090Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 9 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-29T00:32:13.125Z: JOB_MESSAGE_DETAILED: Resized worker pool to 9, though goal was 10. This could be a quota issue.
root: INFO: 2019-09-29T00:32:18.499Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of workers to 10 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-29T00:32:37.786Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-29T00:32:37.824Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
root: INFO: 2019-09-29T00:36:49.613Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T00:36:51.673Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T00:36:53.745Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T00:36:54.984Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T00:36:55.810Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T00:36:56.048Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T00:36:56.868Z: JOB_MESSAGE_ERROR: Traceback (most recent call last):
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 773, in run
self._load_main_session(self.local_staging_directory)
File "/usr/local/lib/python3.6/site-packages/dataflow_worker/batchworker.py", line 489, in _load_main_session
pickler.load_session(session_file)
File "/usr/local/lib/python3.6/site-packages/apache_beam/internal/pickler.py", line 287, in load_session
return dill.load_session(file_path)
File "/usr/local/lib/python3.6/site-packages/dill/_dill.py", line 410, in load_session
module = unpickler.load()
TypeError: _create_function() takes from 2 to 6 positional arguments but 7 were given
root: INFO: 2019-09-29T00:36:57.167Z: JOB_MESSAGE_BASIC: Finished operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2019-09-29T00:36:57.224Z: JOB_MESSAGE_DEBUG: Executing failure step failure25
root: INFO: 2019-09-29T00:36:57.256Z: JOB_MESSAGE_ERROR: Workflow failed. Causes: S01:write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite failed., Internal Issue (93eb8727af2a6290): 63963027:24514
root: INFO: 2019-09-29T00:36:57.360Z: JOB_MESSAGE_BASIC: Finished operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2019-09-29T00:36:57.463Z: JOB_MESSAGE_DETAILED: Cleaning up.
root: INFO: 2019-09-29T00:36:57.510Z: JOB_MESSAGE_DEBUG: Starting worker pool teardown.
root: INFO: 2019-09-29T00:36:57.536Z: JOB_MESSAGE_BASIC: Stopping worker pool...
root: INFO: 2019-09-29T00:38:51.257Z: JOB_MESSAGE_DETAILED: Autoscaling: Reduced the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2019-09-29T00:38:51.301Z: JOB_MESSAGE_BASIC: Worker pool stopped.
root: INFO: 2019-09-29T00:38:51.321Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
root: INFO: Job 2019-09-28_17_31_41-15694561143181347133 is in state JOB_STATE_FAILED
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569717098965/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1569717098965/results*' -> 'gs\\:\\/\\/temp\\-storage\\-for\\-end\\-to\\-end\\-tests\\/py\\-it\\-cloud\\/output\\/1569717098965\\/results[^/\\\\]*'
root: INFO: Starting the size estimation of the input
root: INFO: Finished listing 0 files in 0.035767555236816406 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-integrationTest-perf.xml
----------------------------------------------------------------------
XML: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 444.740s
FAILED (errors=1)
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py36:integrationTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
BUILD FAILED in 8m 5s
2019-09-29 00:39:04,962 b35fd78d MainThread beam_integration_benchmark(1/1) ERROR Error during benchmark beam_integration_benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-29 00:39:04,964 b35fd78d MainThread beam_integration_benchmark(1/1) INFO Cleaning up benchmark beam_integration_benchmark
2019-09-29 00:39:04,967 b35fd78d MainThread beam_integration_benchmark(1/1) ERROR Exception running benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 984, in RunBenchmarkTask
RunBenchmark(spec, collector)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 841, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 687, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 160, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 91, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2019-09-29 00:39:04,967 b35fd78d MainThread beam_integration_benchmark(1/1) ERROR Benchmark 1/1 beam_integration_benchmark (UID: beam_integration_benchmark0) failed. Execution will continue.
2019-09-29 00:39:04,968 b35fd78d MainThread beam_integration_benchmark(1/1) INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2019-09-29 00:39:04,968 b35fd78d MainThread beam_integration_benchmark(1/1) INFO Complete logs can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/b35fd78d/pkb.log>
2019-09-29 00:39:04,968 b35fd78d MainThread beam_integration_benchmark(1/1) INFO Completion statuses can be found at: <https://builds.apache.org/job/beam_PerformanceTests_WordCountIT_Py36/ws/runs/b35fd78d/completion_statuses.json>
Build step 'Execute shell' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org