You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/01/08 15:17:59 UTC
Build failed in Jenkins: beam_PostCommit_Python_Verify #3903
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/3903/display/redirect>
------------------------------------------
[...truncated 1.19 MB...]
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s2"
},
"serialized_fn": "<string of 952 bytes>",
"user_name": "pair_with_one"
}
},
{
"kind": "GroupByKey",
"name": "s4",
"properties": {
"display_data": [],
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:pair",
"component_encodings": [
{
"@type": "StrUtf8Coder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlzBJUWhJWkWziAeVyGDZmMhY20hU5IeAAajEkY=",
"component_encodings": []
},
{
"@type": "kind:stream",
"component_encodings": [
{
"@type": "VarIntCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxhiUWeeSXOIA5XIYNmYyFjbSFTkh4A89cR+g==",
"component_encodings": []
}
],
"is_stream_like": true
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "group.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s3"
},
"serialized_fn": "%0AJ%22H%0A%1Dref_Coder_GlobalWindowCoder_1%12%27%0A%25%0A%23%0A%21urn%3Abeam%3Acoders%3Aglobal_window%3A0.1jT%0A%25%0A%23%0A%21beam%3Awindowfn%3Aglobal_windows%3Av0.1%10%01%1A%1Dref_Coder_GlobalWindowCoder_1%22%02%3A%00%28%010%018%01H%01",
"user_name": "group"
}
},
{
"kind": "ParallelDo",
"name": "s5",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "count_ones"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:pair",
"component_encodings": [
{
"@type": "StrUtf8Coder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlzBJUWhJWkWziAeVyGDZmMhY20hU5IeAAajEkY=",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "count.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s4"
},
"serialized_fn": "<string of 1032 bytes>",
"user_name": "count"
}
},
{
"kind": "ParallelDo",
"name": "s6",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "format_result"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "format.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s5"
},
"serialized_fn": "<string of 1028 bytes>",
"user_name": "format"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-08T15:15:45.225850Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_07_15_44-11092434259277105742'
location: u'us-central1'
name: u'beamapp-jenkins-0108151543-797920'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_07_15_44-11092434259277105742]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_07_15_44-11092434259277105742?project=apache-beam-testing
root: INFO: Job 2018-01-08_07_15_44-11092434259277105742 is in state JOB_STATE_PENDING
root: INFO: 2018-01-08T15:15:44.668Z: JOB_MESSAGE_WARNING: (99f03e304695c3fa): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-08T15:15:46.792Z: JOB_MESSAGE_DETAILED: (f2a2ed88ea2a1e71): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-08T15:15:47.490Z: JOB_MESSAGE_DETAILED: (f2a2ed88ea2a1faa): Expanding CollectionToSingleton operations into optimizable parts.
root: INFO: 2018-01-08T15:15:47.507Z: JOB_MESSAGE_DETAILED: (f2a2ed88ea2a1c41): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T15:15:47.534Z: JOB_MESSAGE_DEBUG: (f2a2ed88ea2a18d8): Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2018-01-08T15:15:47.560Z: JOB_MESSAGE_DETAILED: (f2a2ed88ea2a1692): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T15:15:47.587Z: JOB_MESSAGE_DEBUG: (f2a2ed88ea2a144c): Annotating graph with Autotuner information.
root: INFO: 2018-01-08T15:15:47.625Z: JOB_MESSAGE_DETAILED: (f2a2ed88ea2a1fc0): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-08T15:15:47.641Z: JOB_MESSAGE_DETAILED: (f2a2ed88ea2a1d7a): Fusing consumer split into read/Read
root: INFO: 2018-01-08T15:15:47.670Z: JOB_MESSAGE_DETAILED: (f2a2ed88ea2a1b34): Fusing consumer group/Reify into pair_with_one
root: INFO: 2018-01-08T15:15:47.700Z: JOB_MESSAGE_DETAILED: (f2a2ed88ea2a18ee): Fusing consumer format into count
root: INFO: 2018-01-08T15:15:47.728Z: JOB_MESSAGE_DETAILED: (f2a2ed88ea2a16a8): Fusing consumer count into group/GroupByWindow
root: INFO: 2018-01-08T15:15:47.755Z: JOB_MESSAGE_DETAILED: (f2a2ed88ea2a1462): Fusing consumer pair_with_one into split
root: INFO: 2018-01-08T15:15:47.783Z: JOB_MESSAGE_DETAILED: (f2a2ed88ea2a121c): Fusing consumer group/Write into group/Reify
root: INFO: 2018-01-08T15:15:47.812Z: JOB_MESSAGE_DETAILED: (f2a2ed88ea2a1fd6): Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2018-01-08T15:15:47.833Z: JOB_MESSAGE_DEBUG: (f2a2ed88ea2a1d90): Workflow config is missing a default resource spec.
root: INFO: 2018-01-08T15:15:47.860Z: JOB_MESSAGE_DEBUG: (f2a2ed88ea2a1b4a): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-08T15:15:47.889Z: JOB_MESSAGE_DEBUG: (f2a2ed88ea2a1904): Adding workflow start and stop steps.
root: INFO: 2018-01-08T15:15:47.917Z: JOB_MESSAGE_DEBUG: (f2a2ed88ea2a16be): Assigning stage ids.
root: INFO: 2018-01-08T15:15:48.043Z: JOB_MESSAGE_DEBUG: (b479c4beeb3bdd2a): Executing wait step start13
root: INFO: 2018-01-08T15:15:48.098Z: JOB_MESSAGE_BASIC: (b479c4beeb3bd9cb): Executing operation group/Create
root: INFO: 2018-01-08T15:15:48.136Z: JOB_MESSAGE_DEBUG: (f957c10e833aed02): Starting worker pool setup.
root: INFO: 2018-01-08T15:15:48.165Z: JOB_MESSAGE_BASIC: (f957c10e833aef4c): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-08T15:15:48.230Z: JOB_MESSAGE_DEBUG: (b479c4beeb3bd30d): Value "group/Session" materialized.
root: INFO: 2018-01-08T15:15:48.288Z: JOB_MESSAGE_BASIC: (ed3015f4003848f0): Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: Job 2018-01-08_07_15_44-11092434259277105742 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-08T15:15:54.495Z: JOB_MESSAGE_DETAILED: (287c79bc133a0d23): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-08T15:16:34.258Z: JOB_MESSAGE_ERROR: (287c79bc133a0552): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-08T15:16:34.287Z: JOB_MESSAGE_ERROR: (287c79bc133a0554): Workflow failed.
root: INFO: 2018-01-08T15:16:34.549Z: JOB_MESSAGE_DETAILED: (f2a2ed88ea2a1c83): Cleaning up.
root: INFO: 2018-01-08T15:16:34.592Z: JOB_MESSAGE_DEBUG: (f2a2ed88ea2a17f7): Starting worker pool teardown.
root: INFO: 2018-01-08T15:16:34.622Z: JOB_MESSAGE_BASIC: (f2a2ed88ea2a15b1): Stopping worker pool...
root: INFO: 2018-01-08T15:17:45.483Z: JOB_MESSAGE_BASIC: (f2a2ed88ea2a1c99): Worker pool stopped.
root: INFO: 2018-01-08T15:17:45.516Z: JOB_MESSAGE_DEBUG: (f2a2ed88ea2a180d): Tearing down pending resources...
root: INFO: Job 2018-01-08_07_15_44-11092434259277105742 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 3 tests in 132.722s
FAILED (errors=4)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user ehudm@google.com
Jenkins build is back to normal : beam_PostCommit_Python_Verify
#3913
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/3913/display/redirect?page=changes>
Build failed in Jenkins: beam_PostCommit_Python_Verify #3912
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/3912/display/redirect>
------------------------------------------
[...truncated 1.19 MB...]
"component_encodings": [
{
"@type": "VarIntCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxhiUWeeSXOIA5XIYNmYyFjbSFTkh4A89cR+g==",
"component_encodings": []
}
],
"is_stream_like": true
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "monthly count/GroupByKey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s2"
},
"serialized_fn": "%0AJ%22H%0A%1Dref_Coder_GlobalWindowCoder_1%12%27%0A%25%0A%23%0A%21urn%3Abeam%3Acoders%3Aglobal_window%3A0.1jT%0A%25%0A%23%0A%21beam%3Awindowfn%3Aglobal_windows%3Av0.1%10%01%1A%1Dref_Coder_GlobalWindowCoder_1%22%02%3A%00%28%010%018%01H%01",
"user_name": "monthly count/GroupByKey"
}
},
{
"kind": "CombineValues",
"name": "s4",
"properties": {
"display_data": [],
"encoding": {
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "monthly count/Combine.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s3"
},
"serialized_fn": "<string of 236 bytes>",
"user_name": "monthly count/Combine"
}
},
{
"kind": "ParallelDo",
"name": "s5",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "format.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s4"
},
"serialized_fn": "<string of 1012 bytes>",
"user_name": "format"
}
},
{
"kind": "ParallelWrite",
"name": "s6",
"properties": {
"create_disposition": "CREATE_IF_NEEDED",
"dataset": "BigQueryTornadoesIT",
"display_data": [],
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "RowAsDictJsonCoder$eNprYEpOLEhMzkiNT0pNzNXLzNdLTy7QS8pMLyxNLarkCsovdyx2yUwu8SrOz3POT0kt4ipk0GwsZKwtZErSAwCu1BVY",
"component_encodings": []
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"format": "bigquery",
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s5"
},
"schema": "{\"fields\": [{\"type\": \"INTEGER\", \"name\": \"month\", \"mode\": \"NULLABLE\"}, {\"type\": \"INTEGER\", \"name\": \"tornado_count\", \"mode\": \"NULLABLE\"}]}",
"table": "monthly_tornadoes_1515510936428",
"user_name": "Write/WriteToBigQuery/NativeWrite",
"write_disposition": "WRITE_TRUNCATE"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-09T15:15:37.745309Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-09_07_15_37-16647480242682086812'
location: u'us-central1'
name: u'beamapp-jenkins-0109151536-471332'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-09_07_15_37-16647480242682086812]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-09_07_15_37-16647480242682086812?project=apache-beam-testing
root: INFO: Job 2018-01-09_07_15_37-16647480242682086812 is in state JOB_STATE_PENDING
root: INFO: 2018-01-09T15:15:37.187Z: JOB_MESSAGE_WARNING: (e707bb141e664cf8): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-09T15:15:39.493Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b5790ca): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-09T15:15:40.327Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b579943): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T15:15:40.362Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b57929a): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T15:15:40.386Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b5798d4): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-09T15:15:40.415Z: JOB_MESSAGE_DEBUG: (6d55ae7f5b579548): Annotating graph with Autotuner information.
root: INFO: 2018-01-09T15:15:40.506Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b5794d9): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-09T15:15:40.531Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b579b13): Fusing consumer months with tornadoes into read
root: INFO: 2018-01-09T15:15:40.562Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b57914d): Fusing consumer monthly count/GroupByKey/Reify into monthly count/GroupByKey+monthly count/Combine/Partial
root: INFO: 2018-01-09T15:15:40.594Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b579787): Fusing consumer format into monthly count/Combine/Extract
root: INFO: 2018-01-09T15:15:40.625Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b579dc1): Fusing consumer monthly count/Combine/Extract into monthly count/Combine
root: INFO: 2018-01-09T15:15:40.648Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b5793fb): Fusing consumer Write/WriteToBigQuery/NativeWrite into format
root: INFO: 2018-01-09T15:15:40.671Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b579a35): Fusing consumer monthly count/Combine into monthly count/GroupByKey/Read
root: INFO: 2018-01-09T15:15:40.697Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b57906f): Fusing consumer monthly count/GroupByKey+monthly count/Combine/Partial into months with tornadoes
root: INFO: 2018-01-09T15:15:40.723Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b5796a9): Fusing consumer monthly count/GroupByKey/Write into monthly count/GroupByKey/Reify
root: INFO: 2018-01-09T15:15:40.755Z: JOB_MESSAGE_DEBUG: (6d55ae7f5b579ce3): Workflow config is missing a default resource spec.
root: INFO: 2018-01-09T15:15:40.786Z: JOB_MESSAGE_DEBUG: (6d55ae7f5b57931d): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-09T15:15:40.818Z: JOB_MESSAGE_DEBUG: (6d55ae7f5b579957): Adding workflow start and stop steps.
root: INFO: 2018-01-09T15:15:40.848Z: JOB_MESSAGE_DEBUG: (6d55ae7f5b579f91): Assigning stage ids.
root: INFO: 2018-01-09T15:15:40.983Z: JOB_MESSAGE_DEBUG: (744af15e948f7c3e): Executing wait step start22
root: INFO: Job 2018-01-09_07_15_37-16647480242682086812 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-09T15:15:41.041Z: JOB_MESSAGE_BASIC: (744af15e948f7d2b): Executing operation monthly count/GroupByKey/Create
root: INFO: 2018-01-09T15:15:41.083Z: JOB_MESSAGE_DEBUG: (35c2c1f94838bd80): Starting worker pool setup.
root: INFO: 2018-01-09T15:15:41.111Z: JOB_MESSAGE_BASIC: (35c2c1f94838b4ca): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-09T15:15:41.180Z: JOB_MESSAGE_DEBUG: (6130267680178e76): Value "monthly count/GroupByKey/Session" materialized.
root: INFO: 2018-01-09T15:15:41.235Z: JOB_MESSAGE_BASIC: (6130267680178bde): Executing operation read+months with tornadoes+monthly count/GroupByKey+monthly count/Combine/Partial+monthly count/GroupByKey/Reify+monthly count/GroupByKey/Write
root: INFO: 2018-01-09T15:15:41.833Z: JOB_MESSAGE_BASIC: (119be7a56ad58031): BigQuery export job "dataflow_job_1268862417682925551" started. You can check its status with the bq tool: "bq show -j --project_id=clouddataflow-readonly dataflow_job_1268862417682925551".
root: INFO: 2018-01-09T15:15:48.056Z: JOB_MESSAGE_DETAILED: (c20043ba94cf240a): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-09T15:16:12.200Z: JOB_MESSAGE_DETAILED: (c2f7dc0cf8a2ef91): BigQuery export job progress: "dataflow_job_1268862417682925551" observed total of 1 exported files thus far.
root: INFO: 2018-01-09T15:16:12.223Z: JOB_MESSAGE_BASIC: (c2f7dc0cf8a2e317): BigQuery export job finished: "dataflow_job_1268862417682925551"
root: INFO: 2018-01-09T15:16:16.975Z: JOB_MESSAGE_ERROR: (c20043ba94cf2ded): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-09T15:16:17.002Z: JOB_MESSAGE_ERROR: (c20043ba94cf246f): Workflow failed.
root: INFO: 2018-01-09T15:16:17.163Z: JOB_MESSAGE_DETAILED: (6d55ae7f5b579b96): Cleaning up.
root: INFO: 2018-01-09T15:16:17.242Z: JOB_MESSAGE_DEBUG: (6d55ae7f5b57980a): Starting worker pool teardown.
root: INFO: 2018-01-09T15:16:17.274Z: JOB_MESSAGE_BASIC: (6d55ae7f5b579e44): Stopping worker pool...
root: INFO: 2018-01-09T15:17:33.686Z: JOB_MESSAGE_BASIC: (6d55ae7f5b57972c): Worker pool stopped.
root: INFO: 2018-01-09T15:17:33.716Z: JOB_MESSAGE_DEBUG: (6d55ae7f5b5793a0): Tearing down pending resources...
root: INFO: Job 2018-01-09_07_15_37-16647480242682086812 is in state JOB_STATE_FAILED
root: INFO: Clean up a Bigquery table with project: apache-beam-testing, dataset: BigQueryTornadoesIT, table: monthly_tornadoes_1515510936428.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
google_auth_httplib2: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
google_auth_httplib2: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 3 tests in 127.634s
FAILED (errors=4)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user ehudm@google.com
Not sending mail to unregistered user jb@nanthrax.net
Build failed in Jenkins: beam_PostCommit_Python_Verify #3911
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/3911/display/redirect?page=changes>
Changes:
[jbonofre] [BEAM-3282] MqttIO reader now use receive for timeout in order to return
------------------------------------------
[...truncated 1.19 MB...]
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/Extract.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 760 bytes>",
"user_name": "write/Write/WriteImpl/Extract"
}
},
{
"kind": "CollectionToSingleton",
"name": "SideInput-s15",
"properties": {
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s8"
},
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)"
}
},
{
"kind": "CollectionToSingleton",
"name": "SideInput-s16",
"properties": {
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s14"
},
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)"
}
},
{
"kind": "ParallelDo",
"name": "s17",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_finalize_write"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {
"SideInput-s15": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s15"
},
"SideInput-s16": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s16"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "<string of 2444 bytes>",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-09T11:12:52.997344Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-09_03_12_52-10390288235067446738'
location: u'us-central1'
name: u'beamapp-jenkins-0109111251-222203'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-09_03_12_52-10390288235067446738]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-09_03_12_52-10390288235067446738?project=apache-beam-testing
root: INFO: Job 2018-01-09_03_12_52-10390288235067446738 is in state JOB_STATE_PENDING
root: INFO: 2018-01-09T11:12:52.469Z: JOB_MESSAGE_WARNING: (9031b819851d9dae): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-09T11:12:54.639Z: JOB_MESSAGE_DETAILED: (cbe42ae64e3975dc): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-09T11:12:55.519Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397035): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T11:12:55.552Z: JOB_MESSAGE_DEBUG: (cbe42ae64e397dac): Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-09T11:12:55.579Z: JOB_MESSAGE_DEBUG: (cbe42ae64e3976a6): Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2018-01-09T11:12:55.602Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397fa0): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T11:12:55.633Z: JOB_MESSAGE_DETAILED: (cbe42ae64e39789a): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-09T11:12:55.668Z: JOB_MESSAGE_DEBUG: (cbe42ae64e397388): Annotating graph with Autotuner information.
root: INFO: 2018-01-09T11:12:55.708Z: JOB_MESSAGE_DETAILED: (cbe42ae64e39757c): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-09T11:12:55.735Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397e76): Fusing consumer split into read/Read
root: INFO: 2018-01-09T11:12:55.764Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397770): Fusing consumer group/Write into group/Reify
root: INFO: 2018-01-09T11:12:55.793Z: JOB_MESSAGE_DETAILED: (cbe42ae64e39706a): Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2018-01-09T11:12:55.824Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397964): Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2018-01-09T11:12:55.852Z: JOB_MESSAGE_DETAILED: (cbe42ae64e39725e): Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2018-01-09T11:12:55.882Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397b58): Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2018-01-09T11:12:55.914Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397452): Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2018-01-09T11:12:55.942Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397d4c): Fusing consumer pair_with_one into split
root: INFO: 2018-01-09T11:12:55.972Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397646): Fusing consumer group/Reify into pair_with_one
root: INFO: 2018-01-09T11:12:55.998Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397f40): Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2018-01-09T11:12:56.027Z: JOB_MESSAGE_DETAILED: (cbe42ae64e39783a): Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2018-01-09T11:12:56.052Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397134): Fusing consumer format into count
root: INFO: 2018-01-09T11:12:56.077Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397a2e): Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2018-01-09T11:12:56.100Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397328): Fusing consumer count into group/GroupByWindow
root: INFO: 2018-01-09T11:12:56.121Z: JOB_MESSAGE_DETAILED: (cbe42ae64e397c22): Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2018-01-09T11:12:56.145Z: JOB_MESSAGE_DEBUG: (cbe42ae64e39751c): Workflow config is missing a default resource spec.
root: INFO: 2018-01-09T11:12:56.173Z: JOB_MESSAGE_DEBUG: (cbe42ae64e397e16): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-09T11:12:56.203Z: JOB_MESSAGE_DEBUG: (cbe42ae64e397710): Adding workflow start and stop steps.
root: INFO: 2018-01-09T11:12:56.234Z: JOB_MESSAGE_DEBUG: (cbe42ae64e39700a): Assigning stage ids.
root: INFO: 2018-01-09T11:12:56.354Z: JOB_MESSAGE_DEBUG: (8b15f8a678b5b9d6): Executing wait step start25
root: INFO: 2018-01-09T11:12:56.415Z: JOB_MESSAGE_BASIC: (d682dfe56947b4da): Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2018-01-09T11:12:56.442Z: JOB_MESSAGE_BASIC: (8b15f8a678b5b650): Executing operation group/Create
root: INFO: 2018-01-09T11:12:56.454Z: JOB_MESSAGE_DEBUG: (1f0091ade8590be): Starting worker pool setup.
root: INFO: 2018-01-09T11:12:56.485Z: JOB_MESSAGE_BASIC: (1f0091ade859e00): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-09T11:12:56.562Z: JOB_MESSAGE_DEBUG: (231a566cc01d858f): Value "group/Session" materialized.
root: INFO: 2018-01-09T11:12:56.623Z: JOB_MESSAGE_BASIC: (8b15f8a678b5b5c6): Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: Job 2018-01-09_03_12_52-10390288235067446738 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-09T11:13:04.632Z: JOB_MESSAGE_DETAILED: (d4b2e694798fd523): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-09T11:13:28.484Z: JOB_MESSAGE_ERROR: (d4b2e694798fd6d6): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-09T11:13:28.507Z: JOB_MESSAGE_ERROR: (d4b2e694798fd6f4): Workflow failed.
root: INFO: 2018-01-09T11:13:28.646Z: JOB_MESSAGE_DETAILED: (cbe42ae64e39786f): Cleaning up.
root: INFO: 2018-01-09T11:13:28.699Z: JOB_MESSAGE_DEBUG: (cbe42ae64e397a63): Starting worker pool teardown.
root: INFO: 2018-01-09T11:13:28.726Z: JOB_MESSAGE_BASIC: (cbe42ae64e39735d): Stopping worker pool...
root: INFO: 2018-01-09T11:14:44.663Z: JOB_MESSAGE_DEBUG: (cbe42ae64e39703f): Tearing down pending resources...
root: INFO: Job 2018-01-09_03_12_52-10390288235067446738 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 3 tests in 128.483s
FAILED (errors=4)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user ehudm@google.com
Not sending mail to unregistered user jb@nanthrax.net
Build failed in Jenkins: beam_PostCommit_Python_Verify #3910
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/3910/display/redirect>
------------------------------------------
[...truncated 1.14 MB...]
"@type": "OutputReference",
"output_name": "out",
"step_name": "s2"
},
"serialized_fn": "<string of 952 bytes>",
"user_name": "pair_with_one"
}
},
{
"kind": "GroupByKey",
"name": "s4",
"properties": {
"display_data": [],
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:pair",
"component_encodings": [
{
"@type": "StrUtf8Coder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlzBJUWhJWkWziAeVyGDZmMhY20hU5IeAAajEkY=",
"component_encodings": []
},
{
"@type": "kind:stream",
"component_encodings": [
{
"@type": "VarIntCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxhiUWeeSXOIA5XIYNmYyFjbSFTkh4A89cR+g==",
"component_encodings": []
}
],
"is_stream_like": true
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "group.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s3"
},
"serialized_fn": "%0AJ%22H%0A%1Dref_Coder_GlobalWindowCoder_1%12%27%0A%25%0A%23%0A%21urn%3Abeam%3Acoders%3Aglobal_window%3A0.1jT%0A%25%0A%23%0A%21beam%3Awindowfn%3Aglobal_windows%3Av0.1%10%01%1A%1Dref_Coder_GlobalWindowCoder_1%22%02%3A%00%28%010%018%01H%01",
"user_name": "group"
}
},
{
"kind": "ParallelDo",
"name": "s5",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "count_ones"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:pair",
"component_encodings": [
{
"@type": "StrUtf8Coder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlzBJUWhJWkWziAeVyGDZmMhY20hU5IeAAajEkY=",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "count.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s4"
},
"serialized_fn": "<string of 1032 bytes>",
"user_name": "count"
}
},
{
"kind": "ParallelDo",
"name": "s6",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "format_result"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "format.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s5"
},
"serialized_fn": "<string of 1028 bytes>",
"user_name": "format"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-09T09:27:00.868220Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-09_01_27_00-3796338122094653772'
location: u'us-central1'
name: u'beamapp-jenkins-0109092658-618743'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-09_01_27_00-3796338122094653772]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-09_01_27_00-3796338122094653772?project=apache-beam-testing
root: INFO: Job 2018-01-09_01_27_00-3796338122094653772 is in state JOB_STATE_PENDING
root: INFO: 2018-01-09T09:27:00.296Z: JOB_MESSAGE_WARNING: (34af4d687eb9bfb0): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-09T09:27:02.752Z: JOB_MESSAGE_DETAILED: (279d55b9eedae6d0): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-09T09:27:03.602Z: JOB_MESSAGE_DETAILED: (279d55b9eedae38d): Expanding CollectionToSingleton operations into optimizable parts.
root: INFO: 2018-01-09T09:27:03.634Z: JOB_MESSAGE_DETAILED: (279d55b9eedaede0): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T09:27:03.660Z: JOB_MESSAGE_DEBUG: (279d55b9eedae833): Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2018-01-09T09:27:03.691Z: JOB_MESSAGE_DETAILED: (279d55b9eedaef15): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T09:27:03.721Z: JOB_MESSAGE_DEBUG: (279d55b9eedae5f7): Annotating graph with Autotuner information.
root: INFO: 2018-01-09T09:27:03.756Z: JOB_MESSAGE_DETAILED: (279d55b9eedae3bb): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-09T09:27:03.788Z: JOB_MESSAGE_DETAILED: (279d55b9eedaea9d): Fusing consumer split into read/Read
root: INFO: 2018-01-09T09:27:03.818Z: JOB_MESSAGE_DETAILED: (279d55b9eedae17f): Fusing consumer group/Reify into pair_with_one
root: INFO: 2018-01-09T09:27:03.849Z: JOB_MESSAGE_DETAILED: (279d55b9eedae861): Fusing consumer format into count
root: INFO: 2018-01-09T09:27:03.880Z: JOB_MESSAGE_DETAILED: (279d55b9eedaef43): Fusing consumer count into group/GroupByWindow
root: INFO: 2018-01-09T09:27:03.912Z: JOB_MESSAGE_DETAILED: (279d55b9eedae625): Fusing consumer pair_with_one into split
root: INFO: 2018-01-09T09:27:03.935Z: JOB_MESSAGE_DETAILED: (279d55b9eedaed07): Fusing consumer group/Write into group/Reify
root: INFO: 2018-01-09T09:27:03.956Z: JOB_MESSAGE_DETAILED: (279d55b9eedae3e9): Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2018-01-09T09:27:03.977Z: JOB_MESSAGE_DEBUG: (279d55b9eedaeacb): Workflow config is missing a default resource spec.
root: INFO: 2018-01-09T09:27:03.988Z: JOB_MESSAGE_DEBUG: (279d55b9eedae1ad): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-09T09:27:04Z: JOB_MESSAGE_DEBUG: (279d55b9eedae88f): Adding workflow start and stop steps.
root: INFO: 2018-01-09T09:27:04.020Z: JOB_MESSAGE_DEBUG: (279d55b9eedaef71): Assigning stage ids.
root: INFO: 2018-01-09T09:27:04.110Z: JOB_MESSAGE_DEBUG: (ea4572ef6b66f1a6): Executing wait step start13
root: INFO: 2018-01-09T09:27:04.145Z: JOB_MESSAGE_BASIC: (f30bcf178c2256e6): Executing operation group/Create
root: INFO: Job 2018-01-09_01_27_00-3796338122094653772 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-09T09:27:04.173Z: JOB_MESSAGE_DEBUG: (a9abec243c008ca8): Starting worker pool setup.
root: INFO: 2018-01-09T09:27:04.193Z: JOB_MESSAGE_BASIC: (a9abec243c008faa): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-09T09:27:04.246Z: JOB_MESSAGE_DEBUG: (f30bcf178c2250b0): Value "group/Session" materialized.
root: INFO: 2018-01-09T09:27:04.285Z: JOB_MESSAGE_BASIC: (f30bcf178c225bb8): Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2018-01-09T09:27:12.638Z: JOB_MESSAGE_DETAILED: (fe34485267edcd16): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-09T09:27:41.531Z: JOB_MESSAGE_ERROR: (fe34485267edc3e9): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-09T09:27:41.545Z: JOB_MESSAGE_ERROR: (fe34485267edc70b): Workflow failed.
root: INFO: 2018-01-09T09:27:41.801Z: JOB_MESSAGE_DETAILED: (279d55b9eedaee6a): Cleaning up.
root: INFO: 2018-01-09T09:27:41.845Z: JOB_MESSAGE_DEBUG: (279d55b9eedaec2e): Starting worker pool teardown.
root: INFO: 2018-01-09T09:27:41.859Z: JOB_MESSAGE_BASIC: (279d55b9eedae310): Stopping worker pool...
root: INFO: 2018-01-09T09:28:58.581Z: JOB_MESSAGE_BASIC: (279d55b9eedaee98): Worker pool stopped.
root: INFO: 2018-01-09T09:28:58.604Z: JOB_MESSAGE_DEBUG: (279d55b9eedaec5c): Tearing down pending resources...
root: INFO: Job 2018-01-09_01_27_00-3796338122094653772 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 3 tests in 425.641s
FAILED (errors=2)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user ehudm@google.com
Not sending mail to unregistered user jb@nanthrax.net
Build failed in Jenkins: beam_PostCommit_Python_Verify #3909
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/3909/display/redirect?page=changes>
Changes:
[jb] Exclude go from rat configuration, use .gitignore for excludes and align
[iemejia] [BEAM-3422] Update reproducible-build-maven-plugin to version 0.4
------------------------------------------
[...truncated 1.19 MB...]
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/Extract.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 760 bytes>",
"user_name": "write/Write/WriteImpl/Extract"
}
},
{
"kind": "CollectionToSingleton",
"name": "SideInput-s15",
"properties": {
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s8"
},
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)"
}
},
{
"kind": "CollectionToSingleton",
"name": "SideInput-s16",
"properties": {
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s14"
},
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)"
}
},
{
"kind": "ParallelDo",
"name": "s17",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_finalize_write"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {
"SideInput-s15": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s15"
},
"SideInput-s16": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s16"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "<string of 2444 bytes>",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-09T07:11:49.222503Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_23_11_48-7103978468489907865'
location: u'us-central1'
name: u'beamapp-jenkins-0109071147-263698'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_23_11_48-7103978468489907865]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_23_11_48-7103978468489907865?project=apache-beam-testing
root: INFO: Job 2018-01-08_23_11_48-7103978468489907865 is in state JOB_STATE_PENDING
root: INFO: 2018-01-09T07:11:48.502Z: JOB_MESSAGE_WARNING: (629666c7525d40ed): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-09T07:11:50.994Z: JOB_MESSAGE_DETAILED: (ac978764490cd46): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-09T07:11:51.834Z: JOB_MESSAGE_DETAILED: (ac978764490c18c): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T07:11:51.860Z: JOB_MESSAGE_DEBUG: (ac978764490c39b): Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-09T07:11:51.875Z: JOB_MESSAGE_DEBUG: (ac978764490cfa5): Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2018-01-09T07:11:51.897Z: JOB_MESSAGE_DETAILED: (ac978764490cbaf): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T07:11:51.919Z: JOB_MESSAGE_DETAILED: (ac978764490c7b9): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-09T07:11:51.936Z: JOB_MESSAGE_DEBUG: (ac978764490cbd7): Annotating graph with Autotuner information.
root: INFO: 2018-01-09T07:11:51.966Z: JOB_MESSAGE_DETAILED: (ac978764490c3eb): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-09T07:11:51.979Z: JOB_MESSAGE_DETAILED: (ac978764490cff5): Fusing consumer split into read/Read
root: INFO: 2018-01-09T07:11:52.001Z: JOB_MESSAGE_DETAILED: (ac978764490cbff): Fusing consumer group/Write into group/Reify
root: INFO: 2018-01-09T07:11:52.023Z: JOB_MESSAGE_DETAILED: (ac978764490c809): Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2018-01-09T07:11:52.045Z: JOB_MESSAGE_DETAILED: (ac978764490c413): Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2018-01-09T07:11:52.067Z: JOB_MESSAGE_DETAILED: (ac978764490c01d): Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2018-01-09T07:11:52.082Z: JOB_MESSAGE_DETAILED: (ac978764490cc27): Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2018-01-09T07:11:52.099Z: JOB_MESSAGE_DETAILED: (ac978764490c831): Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2018-01-09T07:11:52.122Z: JOB_MESSAGE_DETAILED: (ac978764490c43b): Fusing consumer pair_with_one into split
root: INFO: 2018-01-09T07:11:52.135Z: JOB_MESSAGE_DETAILED: (ac978764490c045): Fusing consumer group/Reify into pair_with_one
root: INFO: 2018-01-09T07:11:52.151Z: JOB_MESSAGE_DETAILED: (ac978764490cc4f): Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2018-01-09T07:11:52.169Z: JOB_MESSAGE_DETAILED: (ac978764490c859): Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2018-01-09T07:11:52.187Z: JOB_MESSAGE_DETAILED: (ac978764490c463): Fusing consumer format into count
root: INFO: 2018-01-09T07:11:52.211Z: JOB_MESSAGE_DETAILED: (ac978764490c06d): Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2018-01-09T07:11:52.233Z: JOB_MESSAGE_DETAILED: (ac978764490cc77): Fusing consumer count into group/GroupByWindow
root: INFO: 2018-01-09T07:11:52.255Z: JOB_MESSAGE_DETAILED: (ac978764490c881): Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2018-01-09T07:11:52.281Z: JOB_MESSAGE_DEBUG: (ac978764490c48b): Workflow config is missing a default resource spec.
root: INFO: 2018-01-09T07:11:52.299Z: JOB_MESSAGE_DEBUG: (ac978764490c095): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-09T07:11:52.311Z: JOB_MESSAGE_DEBUG: (ac978764490cc9f): Adding workflow start and stop steps.
root: INFO: 2018-01-09T07:11:52.329Z: JOB_MESSAGE_DEBUG: (ac978764490c8a9): Assigning stage ids.
root: INFO: Job 2018-01-08_23_11_48-7103978468489907865 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-09T07:11:52.420Z: JOB_MESSAGE_DEBUG: (b88937fdb77126d1): Executing wait step start25
root: INFO: 2018-01-09T07:11:52.462Z: JOB_MESSAGE_BASIC: (8d16606f048e39c4): Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2018-01-09T07:11:52.474Z: JOB_MESSAGE_BASIC: (b88937fdb7712087): Executing operation group/Create
root: INFO: 2018-01-09T07:11:52.486Z: JOB_MESSAGE_DEBUG: (d7d386978af4af87): Starting worker pool setup.
root: INFO: 2018-01-09T07:11:52.504Z: JOB_MESSAGE_BASIC: (d7d386978af4a3b5): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-09T07:11:52.562Z: JOB_MESSAGE_DEBUG: (cbf4485d2011d136): Value "group/Session" materialized.
root: INFO: 2018-01-09T07:11:52.604Z: JOB_MESSAGE_BASIC: (cbf4485d2011dcf9): Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2018-01-09T07:11:59.372Z: JOB_MESSAGE_DETAILED: (5793a6762af55aea): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-09T07:12:39.182Z: JOB_MESSAGE_ERROR: (5793a6762af5563d): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-09T07:12:39.208Z: JOB_MESSAGE_ERROR: (5793a6762af55a77): Workflow failed.
root: INFO: 2018-01-09T07:12:39.511Z: JOB_MESSAGE_DETAILED: (ac978764490ced6): Cleaning up.
root: INFO: 2018-01-09T07:12:39.557Z: JOB_MESSAGE_DEBUG: (ac978764490c6ea): Starting worker pool teardown.
root: INFO: 2018-01-09T07:12:39.580Z: JOB_MESSAGE_BASIC: (ac978764490c2f4): Stopping worker pool...
root: INFO: 2018-01-09T07:13:48.255Z: JOB_MESSAGE_DEBUG: (ac978764490cf26): Tearing down pending resources...
root: INFO: Job 2018-01-08_23_11_48-7103978468489907865 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 3 tests in 133.689s
FAILED (errors=4)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user ehudm@google.com
Not sending mail to unregistered user jb@nanthrax.net
Build failed in Jenkins: beam_PostCommit_Python_Verify #3908
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/3908/display/redirect>
------------------------------------------
[...truncated 1.19 MB...]
{
"@type": "kind:stream",
"component_encodings": [
{
"@type": "VarIntCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxhiUWeeSXOIA5XIYNmYyFjbSFTkh4A89cR+g==",
"component_encodings": []
}
],
"is_stream_like": true
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "monthly count/GroupByKey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s2"
},
"serialized_fn": "%0AJ%22H%0A%1Dref_Coder_GlobalWindowCoder_1%12%27%0A%25%0A%23%0A%21urn%3Abeam%3Acoders%3Aglobal_window%3A0.1jT%0A%25%0A%23%0A%21beam%3Awindowfn%3Aglobal_windows%3Av0.1%10%01%1A%1Dref_Coder_GlobalWindowCoder_1%22%02%3A%00%28%010%018%01H%01",
"user_name": "monthly count/GroupByKey"
}
},
{
"kind": "CombineValues",
"name": "s4",
"properties": {
"display_data": [],
"encoding": {
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "monthly count/Combine.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s3"
},
"serialized_fn": "<string of 236 bytes>",
"user_name": "monthly count/Combine"
}
},
{
"kind": "ParallelDo",
"name": "s5",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "format.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s4"
},
"serialized_fn": "<string of 1012 bytes>",
"user_name": "format"
}
},
{
"kind": "ParallelWrite",
"name": "s6",
"properties": {
"create_disposition": "CREATE_IF_NEEDED",
"dataset": "BigQueryTornadoesIT",
"display_data": [],
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "RowAsDictJsonCoder$eNprYEpOLEhMzkiNT0pNzNXLzNdLTy7QS8pMLyxNLarkCsovdyx2yUwu8SrOz3POT0kt4ipk0GwsZKwtZErSAwCu1BVY",
"component_encodings": []
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"format": "bigquery",
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s5"
},
"schema": "{\"fields\": [{\"type\": \"INTEGER\", \"name\": \"month\", \"mode\": \"NULLABLE\"}, {\"type\": \"INTEGER\", \"name\": \"tornado_count\", \"mode\": \"NULLABLE\"}]}",
"table": "monthly_tornadoes_1515467759724",
"user_name": "Write/WriteToBigQuery/NativeWrite",
"write_disposition": "WRITE_TRUNCATE"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-09T03:16:01.511898Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_19_16_00-3503180796637202280'
location: u'us-central1'
name: u'beamapp-jenkins-0109031559-769004'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_19_16_00-3503180796637202280]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_19_16_00-3503180796637202280?project=apache-beam-testing
root: INFO: Job 2018-01-08_19_16_00-3503180796637202280 is in state JOB_STATE_PENDING
root: INFO: 2018-01-09T03:16:00.753Z: JOB_MESSAGE_WARNING: (309dcc5fb787c274): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-09T03:16:03.080Z: JOB_MESSAGE_DETAILED: (108995781510c26f): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-09T03:16:04.069Z: JOB_MESSAGE_DETAILED: (108995781510c744): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T03:16:04.102Z: JOB_MESSAGE_DETAILED: (108995781510c0ff): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T03:16:04.125Z: JOB_MESSAGE_DETAILED: (108995781510ccd1): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-09T03:16:04.155Z: JOB_MESSAGE_DEBUG: (108995781510c475): Annotating graph with Autotuner information.
root: INFO: 2018-01-09T03:16:04.267Z: JOB_MESSAGE_DETAILED: (108995781510ca02): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-09T03:16:04.297Z: JOB_MESSAGE_DETAILED: (108995781510c5d4): Fusing consumer months with tornadoes into read
root: INFO: 2018-01-09T03:16:04.317Z: JOB_MESSAGE_DETAILED: (108995781510c1a6): Fusing consumer monthly count/GroupByKey/Reify into monthly count/GroupByKey+monthly count/Combine/Partial
root: INFO: 2018-01-09T03:16:04.336Z: JOB_MESSAGE_DETAILED: (108995781510cd78): Fusing consumer format into monthly count/Combine/Extract
root: INFO: 2018-01-09T03:16:04.352Z: JOB_MESSAGE_DETAILED: (108995781510c94a): Fusing consumer monthly count/Combine/Extract into monthly count/Combine
root: INFO: 2018-01-09T03:16:04.382Z: JOB_MESSAGE_DETAILED: (108995781510c51c): Fusing consumer Write/WriteToBigQuery/NativeWrite into format
root: INFO: 2018-01-09T03:16:04.414Z: JOB_MESSAGE_DETAILED: (108995781510c0ee): Fusing consumer monthly count/Combine into monthly count/GroupByKey/Read
root: INFO: 2018-01-09T03:16:04.441Z: JOB_MESSAGE_DETAILED: (108995781510ccc0): Fusing consumer monthly count/GroupByKey+monthly count/Combine/Partial into months with tornadoes
root: INFO: 2018-01-09T03:16:04.471Z: JOB_MESSAGE_DETAILED: (108995781510c892): Fusing consumer monthly count/GroupByKey/Write into monthly count/GroupByKey/Reify
root: INFO: 2018-01-09T03:16:04.498Z: JOB_MESSAGE_DEBUG: (108995781510c464): Workflow config is missing a default resource spec.
root: INFO: 2018-01-09T03:16:04.521Z: JOB_MESSAGE_DEBUG: (108995781510c036): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-09T03:16:04.552Z: JOB_MESSAGE_DEBUG: (108995781510cc08): Adding workflow start and stop steps.
root: INFO: 2018-01-09T03:16:04.574Z: JOB_MESSAGE_DEBUG: (108995781510c7da): Assigning stage ids.
root: INFO: Job 2018-01-08_19_16_00-3503180796637202280 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-09T03:16:04.712Z: JOB_MESSAGE_DEBUG: (2eaed10f6bac88d9): Executing wait step start22
root: INFO: 2018-01-09T03:16:04.772Z: JOB_MESSAGE_BASIC: (29e5550981b66149): Executing operation monthly count/GroupByKey/Create
root: INFO: 2018-01-09T03:16:04.806Z: JOB_MESSAGE_DEBUG: (48ee540ca972377b): Starting worker pool setup.
root: INFO: 2018-01-09T03:16:04.844Z: JOB_MESSAGE_BASIC: (48ee540ca9723c35): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-09T03:16:04.901Z: JOB_MESSAGE_DEBUG: (116961d32427811b): Value "monthly count/GroupByKey/Session" materialized.
root: INFO: 2018-01-09T03:16:04.954Z: JOB_MESSAGE_BASIC: (29e5550981b6600f): Executing operation read+months with tornadoes+monthly count/GroupByKey+monthly count/Combine/Partial+monthly count/GroupByKey/Reify+monthly count/GroupByKey/Write
root: INFO: 2018-01-09T03:16:05.529Z: JOB_MESSAGE_BASIC: (87afd3eda227c8f0): BigQuery export job "dataflow_job_9777266333625861106" started. You can check its status with the bq tool: "bq show -j --project_id=clouddataflow-readonly dataflow_job_9777266333625861106".
root: INFO: 2018-01-09T03:16:13.504Z: JOB_MESSAGE_DETAILED: (b944639b2962e234): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-09T03:16:35.825Z: JOB_MESSAGE_DETAILED: (eb9707af2a7d4895): BigQuery export job progress: "dataflow_job_9777266333625861106" observed total of 1 exported files thus far.
root: INFO: 2018-01-09T03:16:35.849Z: JOB_MESSAGE_BASIC: (eb9707af2a7d429b): BigQuery export job finished: "dataflow_job_9777266333625861106"
root: INFO: 2018-01-09T03:16:55.614Z: JOB_MESSAGE_ERROR: (b944639b2962e0a7): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-09T03:16:55.643Z: JOB_MESSAGE_ERROR: (b944639b2962e2a1): Workflow failed.
root: INFO: 2018-01-09T03:16:55.911Z: JOB_MESSAGE_DETAILED: (108995781510c50b): Cleaning up.
root: INFO: 2018-01-09T03:16:55.950Z: JOB_MESSAGE_DEBUG: (108995781510ccaf): Starting worker pool teardown.
root: INFO: 2018-01-09T03:16:55.980Z: JOB_MESSAGE_BASIC: (108995781510c881): Stopping worker pool...
root: INFO: 2018-01-09T03:18:05.455Z: JOB_MESSAGE_DEBUG: (108995781510c39b): Tearing down pending resources...
root: INFO: Job 2018-01-08_19_16_00-3503180796637202280 is in state JOB_STATE_FAILED
root: INFO: Clean up a Bigquery table with project: apache-beam-testing, dataset: BigQueryTornadoesIT, table: monthly_tornadoes_1515467759724.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
google_auth_httplib2: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
google_auth_httplib2: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 3 tests in 138.176s
FAILED (errors=4)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user ehudm@google.com
Build failed in Jenkins: beam_PostCommit_Python_Verify #3907
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/3907/display/redirect?page=changes>
Changes:
[chamikara] Reduce the size of file-based I/O tests
[chamikara] Reduce the size of file-based I/O tests
[ehudm] Split up hdfs and sdk into separate containers.
------------------------------------------
[...truncated 1.19 MB...]
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/Extract.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 760 bytes>",
"user_name": "write/Write/WriteImpl/Extract"
}
},
{
"kind": "CollectionToSingleton",
"name": "SideInput-s15",
"properties": {
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s8"
},
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)"
}
},
{
"kind": "CollectionToSingleton",
"name": "SideInput-s16",
"properties": {
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s14"
},
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)"
}
},
{
"kind": "ParallelDo",
"name": "s17",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_finalize_write"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {
"SideInput-s15": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s15"
},
"SideInput-s16": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s16"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "<string of 2444 bytes>",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-09T02:35:11.156034Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_18_35_10-13037444169894927327'
location: u'us-central1'
name: u'beamapp-jenkins-0109023509-628835'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_18_35_10-13037444169894927327]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_18_35_10-13037444169894927327?project=apache-beam-testing
root: INFO: Job 2018-01-08_18_35_10-13037444169894927327 is in state JOB_STATE_PENDING
root: INFO: 2018-01-09T02:35:10.673Z: JOB_MESSAGE_WARNING: (b4ee4e65ad931783): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-09T02:35:12.749Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a1089b): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-09T02:35:13.542Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a104d9): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T02:35:13.565Z: JOB_MESSAGE_DEBUG: (ebd16e7b95a10e54): Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-09T02:35:13.587Z: JOB_MESSAGE_DEBUG: (ebd16e7b95a104a6): Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2018-01-09T02:35:13.615Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a10af8): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T02:35:13.642Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a1014a): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-09T02:35:13.679Z: JOB_MESSAGE_DEBUG: (ebd16e7b95a10440): Annotating graph with Autotuner information.
root: INFO: 2018-01-09T02:35:13.713Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a100e4): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-09T02:35:13.727Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a10736): Fusing consumer split into read/Read
root: INFO: 2018-01-09T02:35:13.748Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a10d88): Fusing consumer group/Write into group/Reify
root: INFO: 2018-01-09T02:35:13.770Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a103da): Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2018-01-09T02:35:13.788Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a10a2c): Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2018-01-09T02:35:13.810Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a1007e): Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2018-01-09T02:35:13.839Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a106d0): Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2018-01-09T02:35:13.851Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a10d22): Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2018-01-09T02:35:13.875Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a10374): Fusing consumer pair_with_one into split
root: INFO: 2018-01-09T02:35:13.897Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a109c6): Fusing consumer group/Reify into pair_with_one
root: INFO: 2018-01-09T02:35:13.907Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a10018): Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2018-01-09T02:35:13.928Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a1066a): Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2018-01-09T02:35:13.964Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a10cbc): Fusing consumer format into count
root: INFO: 2018-01-09T02:35:13.988Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a1030e): Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2018-01-09T02:35:14.015Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a10960): Fusing consumer count into group/GroupByWindow
root: INFO: 2018-01-09T02:35:14.045Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a10fb2): Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2018-01-09T02:35:14.057Z: JOB_MESSAGE_DEBUG: (ebd16e7b95a10604): Workflow config is missing a default resource spec.
root: INFO: 2018-01-09T02:35:14.086Z: JOB_MESSAGE_DEBUG: (ebd16e7b95a10c56): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-09T02:35:14.108Z: JOB_MESSAGE_DEBUG: (ebd16e7b95a102a8): Adding workflow start and stop steps.
root: INFO: 2018-01-09T02:35:14.129Z: JOB_MESSAGE_DEBUG: (ebd16e7b95a108fa): Assigning stage ids.
root: INFO: 2018-01-09T02:35:14.252Z: JOB_MESSAGE_DEBUG: (5829fb3c5236b0b4): Executing wait step start25
root: INFO: 2018-01-09T02:35:14.312Z: JOB_MESSAGE_BASIC: (b8622818a6bf69c4): Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2018-01-09T02:35:14.332Z: JOB_MESSAGE_BASIC: (5829fb3c5236b922): Executing operation group/Create
root: INFO: 2018-01-09T02:35:14.343Z: JOB_MESSAGE_DEBUG: (d12cbd835151dfef): Starting worker pool setup.
root: INFO: Job 2018-01-08_18_35_10-13037444169894927327 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-09T02:35:14.364Z: JOB_MESSAGE_BASIC: (d12cbd835151dde5): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-09T02:35:14.418Z: JOB_MESSAGE_DEBUG: (7c7dccf16a07c440): Value "group/Session" materialized.
root: INFO: 2018-01-09T02:35:14.465Z: JOB_MESSAGE_BASIC: (5829fb3c5236bc84): Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2018-01-09T02:35:22.950Z: JOB_MESSAGE_DETAILED: (69830f3f6354bbf5): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-09T02:35:46.945Z: JOB_MESSAGE_ERROR: (69830f3f6354b2ca): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-09T02:35:46.970Z: JOB_MESSAGE_ERROR: (69830f3f6354bafc): Workflow failed.
root: INFO: 2018-01-09T02:35:47.248Z: JOB_MESSAGE_DETAILED: (ebd16e7b95a1056b): Cleaning up.
root: INFO: 2018-01-09T02:35:47.285Z: JOB_MESSAGE_DEBUG: (ebd16e7b95a1020f): Starting worker pool teardown.
root: INFO: 2018-01-09T02:35:47.312Z: JOB_MESSAGE_BASIC: (ebd16e7b95a10861): Stopping worker pool...
root: INFO: 2018-01-09T02:37:03.162Z: JOB_MESSAGE_DEBUG: (ebd16e7b95a107fb): Tearing down pending resources...
root: INFO: Job 2018-01-08_18_35_10-13037444169894927327 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 3 tests in 128.036s
FAILED (errors=4)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user ehudm@google.com
Build failed in Jenkins: beam_PostCommit_Python_Verify #3906
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/3906/display/redirect?page=changes>
Changes:
[chamikara] Reduce the size of file-based I/O tests
[chamikara] Reduce the size of file-based I/O tests
------------------------------------------
[...truncated 1.19 MB...]
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/Extract.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 760 bytes>",
"user_name": "write/Write/WriteImpl/Extract"
}
},
{
"kind": "CollectionToSingleton",
"name": "SideInput-s15",
"properties": {
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s8"
},
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)"
}
},
{
"kind": "CollectionToSingleton",
"name": "SideInput-s16",
"properties": {
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s14"
},
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)"
}
},
{
"kind": "ParallelDo",
"name": "s17",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_finalize_write"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {
"SideInput-s15": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s15"
},
"SideInput-s16": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s16"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "<string of 2444 bytes>",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-08T22:31:43.209063Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_14_31_42-18104110613483799127'
location: u'us-central1'
name: u'beamapp-jenkins-0108223141-306421'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_14_31_42-18104110613483799127]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_14_31_42-18104110613483799127?project=apache-beam-testing
root: INFO: Job 2018-01-08_14_31_42-18104110613483799127 is in state JOB_STATE_PENDING
root: INFO: 2018-01-08T22:31:42.540Z: JOB_MESSAGE_WARNING: (fb3eb8aeeb14c8bb): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-08T22:31:44.844Z: JOB_MESSAGE_DETAILED: (53cfaa268470417b): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-08T22:31:45.587Z: JOB_MESSAGE_DETAILED: (53cfaa2684704b69): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T22:31:45.616Z: JOB_MESSAGE_DEBUG: (53cfaa268470421c): Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-08T22:31:45.648Z: JOB_MESSAGE_DEBUG: (53cfaa268470413e): Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2018-01-08T22:31:45.674Z: JOB_MESSAGE_DETAILED: (53cfaa2684704060): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T22:31:45.704Z: JOB_MESSAGE_DETAILED: (53cfaa2684704f82): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-08T22:31:45.733Z: JOB_MESSAGE_DEBUG: (53cfaa2684704ce8): Annotating graph with Autotuner information.
root: INFO: 2018-01-08T22:31:45.769Z: JOB_MESSAGE_DETAILED: (53cfaa2684704b2c): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-08T22:31:45.794Z: JOB_MESSAGE_DETAILED: (53cfaa2684704a4e): Fusing consumer split into read/Read
root: INFO: 2018-01-08T22:31:45.816Z: JOB_MESSAGE_DETAILED: (53cfaa2684704970): Fusing consumer group/Write into group/Reify
root: INFO: 2018-01-08T22:31:45.826Z: JOB_MESSAGE_DETAILED: (53cfaa2684704892): Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2018-01-08T22:31:45.845Z: JOB_MESSAGE_DETAILED: (53cfaa26847047b4): Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2018-01-08T22:31:45.870Z: JOB_MESSAGE_DETAILED: (53cfaa26847046d6): Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2018-01-08T22:31:45.900Z: JOB_MESSAGE_DETAILED: (53cfaa26847045f8): Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2018-01-08T22:31:45.926Z: JOB_MESSAGE_DETAILED: (53cfaa268470451a): Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2018-01-08T22:31:45.961Z: JOB_MESSAGE_DETAILED: (53cfaa268470443c): Fusing consumer pair_with_one into split
root: INFO: 2018-01-08T22:31:46.006Z: JOB_MESSAGE_DETAILED: (53cfaa268470435e): Fusing consumer group/Reify into pair_with_one
root: INFO: 2018-01-08T22:31:46.037Z: JOB_MESSAGE_DETAILED: (53cfaa2684704280): Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2018-01-08T22:31:46.064Z: JOB_MESSAGE_DETAILED: (53cfaa26847041a2): Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2018-01-08T22:31:46.090Z: JOB_MESSAGE_DETAILED: (53cfaa26847040c4): Fusing consumer format into count
root: INFO: 2018-01-08T22:31:46.120Z: JOB_MESSAGE_DETAILED: (53cfaa2684704fe6): Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2018-01-08T22:31:46.150Z: JOB_MESSAGE_DETAILED: (53cfaa2684704f08): Fusing consumer count into group/GroupByWindow
root: INFO: 2018-01-08T22:31:46.180Z: JOB_MESSAGE_DETAILED: (53cfaa2684704e2a): Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2018-01-08T22:31:46.205Z: JOB_MESSAGE_DEBUG: (53cfaa2684704d4c): Workflow config is missing a default resource spec.
root: INFO: 2018-01-08T22:31:46.234Z: JOB_MESSAGE_DEBUG: (53cfaa2684704c6e): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-08T22:31:46.263Z: JOB_MESSAGE_DEBUG: (53cfaa2684704b90): Adding workflow start and stop steps.
root: INFO: 2018-01-08T22:31:46.291Z: JOB_MESSAGE_DEBUG: (53cfaa2684704ab2): Assigning stage ids.
root: INFO: Job 2018-01-08_14_31_42-18104110613483799127 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-08T22:31:46.420Z: JOB_MESSAGE_DEBUG: (91dd82064f8c084a): Executing wait step start25
root: INFO: 2018-01-08T22:31:46.482Z: JOB_MESSAGE_BASIC: (621b544429a00a88): Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2018-01-08T22:31:46.514Z: JOB_MESSAGE_BASIC: (91dd82064f8c0b40): Executing operation group/Create
root: INFO: 2018-01-08T22:31:46.525Z: JOB_MESSAGE_DEBUG: (3a772a06e12c0cc4): Starting worker pool setup.
root: INFO: 2018-01-08T22:31:46.556Z: JOB_MESSAGE_BASIC: (3a772a06e12c09e2): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-08T22:31:46.622Z: JOB_MESSAGE_DEBUG: (a765e5815b73a364): Value "group/Session" materialized.
root: INFO: 2018-01-08T22:31:46.679Z: JOB_MESSAGE_BASIC: (a765e5815b73a2b3): Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2018-01-08T22:31:55.681Z: JOB_MESSAGE_DETAILED: (b1a178361420ac01): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-08T22:32:35.087Z: JOB_MESSAGE_ERROR: (b1a178361420a3b4): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-08T22:32:35.122Z: JOB_MESSAGE_ERROR: (b1a178361420ab2e): Workflow failed.
root: INFO: 2018-01-08T22:32:35.368Z: JOB_MESSAGE_DETAILED: (53cfaa2684704ecb): Cleaning up.
root: INFO: 2018-01-08T22:32:35.407Z: JOB_MESSAGE_DEBUG: (53cfaa2684704d0f): Starting worker pool teardown.
root: INFO: 2018-01-08T22:32:35.427Z: JOB_MESSAGE_BASIC: (53cfaa2684704c31): Stopping worker pool...
root: INFO: 2018-01-08T22:33:50.209Z: JOB_MESSAGE_DEBUG: (53cfaa26847047db): Tearing down pending resources...
root: INFO: Job 2018-01-08_14_31_42-18104110613483799127 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 3 tests in 138.468s
FAILED (errors=4)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user ehudm@google.com
Build failed in Jenkins: beam_PostCommit_Python_Verify #3905
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/3905/display/redirect>
------------------------------------------
[...truncated 1.19 MB...]
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/Extract.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 760 bytes>",
"user_name": "write/Write/WriteImpl/Extract"
}
},
{
"kind": "CollectionToSingleton",
"name": "SideInput-s15",
"properties": {
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s8"
},
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)"
}
},
{
"kind": "CollectionToSingleton",
"name": "SideInput-s16",
"properties": {
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s14"
},
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)"
}
},
{
"kind": "ParallelDo",
"name": "s17",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_finalize_write"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {
"SideInput-s15": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s15"
},
"SideInput-s16": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s16"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "<string of 2444 bytes>",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-08T21:23:40.437415Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_13_23_39-1432012221936230517'
location: u'us-central1'
name: u'beamapp-jenkins-0108212338-936047'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_13_23_39-1432012221936230517]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_13_23_39-1432012221936230517?project=apache-beam-testing
root: INFO: Job 2018-01-08_13_23_39-1432012221936230517 is in state JOB_STATE_PENDING
root: INFO: 2018-01-08T21:23:39.894Z: JOB_MESSAGE_WARNING: (13df87875a86ea61): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-08T21:23:42.487Z: JOB_MESSAGE_DETAILED: (f5e99badd97dcabd): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-08T21:23:43.213Z: JOB_MESSAGE_DETAILED: (f5e99badd97dcec7): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T21:23:43.245Z: JOB_MESSAGE_DEBUG: (f5e99badd97dc680): Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-08T21:23:43.280Z: JOB_MESSAGE_DEBUG: (f5e99badd97dcba6): Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2018-01-08T21:23:43.310Z: JOB_MESSAGE_DETAILED: (f5e99badd97dc0cc): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T21:23:43.341Z: JOB_MESSAGE_DETAILED: (f5e99badd97dc5f2): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-08T21:23:43.384Z: JOB_MESSAGE_DEBUG: (f5e99badd97dc564): Annotating graph with Autotuner information.
root: INFO: 2018-01-08T21:23:43.430Z: JOB_MESSAGE_DETAILED: (f5e99badd97dcfb0): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-08T21:23:43.459Z: JOB_MESSAGE_DETAILED: (f5e99badd97dc4d6): Fusing consumer split into read/Read
root: INFO: 2018-01-08T21:23:43.489Z: JOB_MESSAGE_DETAILED: (f5e99badd97dc9fc): Fusing consumer group/Write into group/Reify
root: INFO: 2018-01-08T21:23:43.518Z: JOB_MESSAGE_DETAILED: (f5e99badd97dcf22): Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2018-01-08T21:23:43.550Z: JOB_MESSAGE_DETAILED: (f5e99badd97dc448): Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2018-01-08T21:23:43.582Z: JOB_MESSAGE_DETAILED: (f5e99badd97dc96e): Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2018-01-08T21:23:43.607Z: JOB_MESSAGE_DETAILED: (f5e99badd97dce94): Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2018-01-08T21:23:43.636Z: JOB_MESSAGE_DETAILED: (f5e99badd97dc3ba): Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2018-01-08T21:23:43.666Z: JOB_MESSAGE_DETAILED: (f5e99badd97dc8e0): Fusing consumer pair_with_one into split
root: INFO: 2018-01-08T21:23:43.692Z: JOB_MESSAGE_DETAILED: (f5e99badd97dce06): Fusing consumer group/Reify into pair_with_one
root: INFO: 2018-01-08T21:23:43.717Z: JOB_MESSAGE_DETAILED: (f5e99badd97dc32c): Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2018-01-08T21:23:43.742Z: JOB_MESSAGE_DETAILED: (f5e99badd97dc852): Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2018-01-08T21:23:43.770Z: JOB_MESSAGE_DETAILED: (f5e99badd97dcd78): Fusing consumer format into count
root: INFO: 2018-01-08T21:23:43.800Z: JOB_MESSAGE_DETAILED: (f5e99badd97dc29e): Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2018-01-08T21:23:43.833Z: JOB_MESSAGE_DETAILED: (f5e99badd97dc7c4): Fusing consumer count into group/GroupByWindow
root: INFO: 2018-01-08T21:23:43.855Z: JOB_MESSAGE_DETAILED: (f5e99badd97dccea): Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2018-01-08T21:23:43.889Z: JOB_MESSAGE_DEBUG: (f5e99badd97dc210): Workflow config is missing a default resource spec.
root: INFO: 2018-01-08T21:23:43.916Z: JOB_MESSAGE_DEBUG: (f5e99badd97dc736): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-08T21:23:43.947Z: JOB_MESSAGE_DEBUG: (f5e99badd97dcc5c): Adding workflow start and stop steps.
root: INFO: 2018-01-08T21:23:43.978Z: JOB_MESSAGE_DEBUG: (f5e99badd97dc182): Assigning stage ids.
root: INFO: 2018-01-08T21:23:44.128Z: JOB_MESSAGE_DEBUG: (366ce270ff44f896): Executing wait step start25
root: INFO: 2018-01-08T21:23:44.184Z: JOB_MESSAGE_BASIC: (34e0da8637762205): Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2018-01-08T21:23:44.215Z: JOB_MESSAGE_BASIC: (366ce270ff44f92c): Executing operation group/Create
root: INFO: 2018-01-08T21:23:44.227Z: JOB_MESSAGE_DEBUG: (67339a5557597fa9): Starting worker pool setup.
root: INFO: 2018-01-08T21:23:44.254Z: JOB_MESSAGE_BASIC: (67339a555759758f): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-08T21:23:44.334Z: JOB_MESSAGE_DEBUG: (3bf66ee32c01bbe2): Value "group/Session" materialized.
root: INFO: 2018-01-08T21:23:44.378Z: JOB_MESSAGE_BASIC: (366ce270ff44fa26): Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: Job 2018-01-08_13_23_39-1432012221936230517 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-08T21:23:52.523Z: JOB_MESSAGE_DETAILED: (c337f16139f0143c): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-08T21:24:21.533Z: JOB_MESSAGE_ERROR: (c337f16139f01f69): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-08T21:24:21.559Z: JOB_MESSAGE_ERROR: (c337f16139f01847): Workflow failed.
root: INFO: 2018-01-08T21:24:21.706Z: JOB_MESSAGE_DETAILED: (f5e99badd97dc8ad): Cleaning up.
root: INFO: 2018-01-08T21:24:21.751Z: JOB_MESSAGE_DEBUG: (f5e99badd97dc2f9): Starting worker pool teardown.
root: INFO: 2018-01-08T21:24:21.768Z: JOB_MESSAGE_BASIC: (f5e99badd97dc81f): Stopping worker pool...
root: INFO: 2018-01-08T21:25:33.553Z: JOB_MESSAGE_BASIC: (f5e99badd97dccb7): Worker pool stopped.
root: INFO: 2018-01-08T21:25:33.593Z: JOB_MESSAGE_DEBUG: (f5e99badd97dc703): Tearing down pending resources...
root: INFO: Job 2018-01-08_13_23_39-1432012221936230517 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 3 tests in 127.945s
FAILED (errors=4)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user ehudm@google.com
Build failed in Jenkins: beam_PostCommit_Python_Verify #3904
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_Verify/3904/display/redirect?page=changes>
Changes:
[iemejia] [BEAM-3404] Update KinesisIO to use AWS SDK 1.11.255 and KCL 1.8.8
[iemejia] [BEAM-3404] Fix and enable KinesisReaderIT
[iemejia] [BEAM-3187] Ensure that teardown is called in case of Exception on
[iemejia] [BEAM-3187] Enable PardoLifecycleTest for the Spark runner
[ankurgoenka] [BEAM-3411] Fixing KeyError when on progress update request
[iemejia] [BEAM-3373] Add serviceEndpoint parameter to KinesisIO
------------------------------------------
[...truncated 1.19 MB...]
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/Extract.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 760 bytes>",
"user_name": "write/Write/WriteImpl/Extract"
}
},
{
"kind": "CollectionToSingleton",
"name": "SideInput-s15",
"properties": {
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s8"
},
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)"
}
},
{
"kind": "CollectionToSingleton",
"name": "SideInput-s16",
"properties": {
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
}
]
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s14"
},
"user_name": "write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)"
}
},
{
"kind": "ParallelDo",
"name": "s17",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_finalize_write"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {
"SideInput-s15": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s15"
},
"SideInput-s16": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s16"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "<string of 2444 bytes>",
"user_name": "write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-08T19:12:09.133862Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_11_12_08-14746680311414383184'
location: u'us-central1'
name: u'beamapp-jenkins-0108191207-413161'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_11_12_08-14746680311414383184]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_11_12_08-14746680311414383184?project=apache-beam-testing
root: INFO: Job 2018-01-08_11_12_08-14746680311414383184 is in state JOB_STATE_PENDING
root: INFO: 2018-01-08T19:12:08.555Z: JOB_MESSAGE_WARNING: (cca6bb9caa19e1d4): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-08T19:12:10.808Z: JOB_MESSAGE_DETAILED: (669c5e754138797e): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-08T19:12:11.518Z: JOB_MESSAGE_DETAILED: (669c5e75413875a4): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T19:12:11.554Z: JOB_MESSAGE_DEBUG: (669c5e7541387963): Combiner lifting skipped for step write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-08T19:12:11.578Z: JOB_MESSAGE_DEBUG: (669c5e754138768d): Combiner lifting skipped for step group: GroupByKey not followed by a combiner.
root: INFO: 2018-01-08T19:12:11.609Z: JOB_MESSAGE_DETAILED: (669c5e75413873b7): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T19:12:11.639Z: JOB_MESSAGE_DETAILED: (669c5e75413870e1): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-08T19:12:11.674Z: JOB_MESSAGE_DEBUG: (669c5e754138785f): Annotating graph with Autotuner information.
root: INFO: 2018-01-08T19:12:11.715Z: JOB_MESSAGE_DETAILED: (669c5e75413872b3): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-08T19:12:11.747Z: JOB_MESSAGE_DETAILED: (669c5e7541387fdd): Fusing consumer split into read/Read
root: INFO: 2018-01-08T19:12:11.771Z: JOB_MESSAGE_DETAILED: (669c5e7541387d07): Fusing consumer group/Write into group/Reify
root: INFO: 2018-01-08T19:12:11.802Z: JOB_MESSAGE_DETAILED: (669c5e7541387a31): Fusing consumer group/GroupByWindow into group/Read
root: INFO: 2018-01-08T19:12:11.824Z: JOB_MESSAGE_DETAILED: (669c5e754138775b): Fusing consumer write/Write/WriteImpl/GroupByKey/GroupByWindow into write/Write/WriteImpl/GroupByKey/Read
root: INFO: 2018-01-08T19:12:11.854Z: JOB_MESSAGE_DETAILED: (669c5e7541387485): Fusing consumer write/Write/WriteImpl/GroupByKey/Write into write/Write/WriteImpl/GroupByKey/Reify
root: INFO: 2018-01-08T19:12:11.878Z: JOB_MESSAGE_DETAILED: (669c5e75413871af): Fusing consumer write/Write/WriteImpl/WindowInto(WindowIntoFn) into write/Write/WriteImpl/Pair
root: INFO: 2018-01-08T19:12:11.908Z: JOB_MESSAGE_DETAILED: (669c5e7541387ed9): Fusing consumer write/Write/WriteImpl/GroupByKey/Reify into write/Write/WriteImpl/WindowInto(WindowIntoFn)
root: INFO: 2018-01-08T19:12:11.934Z: JOB_MESSAGE_DETAILED: (669c5e7541387c03): Fusing consumer pair_with_one into split
root: INFO: 2018-01-08T19:12:11.963Z: JOB_MESSAGE_DETAILED: (669c5e754138792d): Fusing consumer group/Reify into pair_with_one
root: INFO: 2018-01-08T19:12:11.994Z: JOB_MESSAGE_DETAILED: (669c5e7541387657): Fusing consumer write/Write/WriteImpl/WriteBundles/WriteBundles into format
root: INFO: 2018-01-08T19:12:12.023Z: JOB_MESSAGE_DETAILED: (669c5e7541387381): Fusing consumer write/Write/WriteImpl/Pair into write/Write/WriteImpl/WriteBundles/WriteBundles
root: INFO: 2018-01-08T19:12:12.047Z: JOB_MESSAGE_DETAILED: (669c5e75413870ab): Fusing consumer format into count
root: INFO: 2018-01-08T19:12:12.079Z: JOB_MESSAGE_DETAILED: (669c5e7541387dd5): Fusing consumer write/Write/WriteImpl/Extract into write/Write/WriteImpl/GroupByKey/GroupByWindow
root: INFO: 2018-01-08T19:12:12.104Z: JOB_MESSAGE_DETAILED: (669c5e7541387aff): Fusing consumer count into group/GroupByWindow
root: INFO: 2018-01-08T19:12:12.135Z: JOB_MESSAGE_DETAILED: (669c5e7541387829): Fusing consumer write/Write/WriteImpl/InitializeWrite into write/Write/WriteImpl/DoOnce/Read
root: INFO: 2018-01-08T19:12:12.163Z: JOB_MESSAGE_DEBUG: (669c5e7541387553): Workflow config is missing a default resource spec.
root: INFO: 2018-01-08T19:12:12.190Z: JOB_MESSAGE_DEBUG: (669c5e754138727d): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-08T19:12:12.215Z: JOB_MESSAGE_DEBUG: (669c5e7541387fa7): Adding workflow start and stop steps.
root: INFO: 2018-01-08T19:12:12.247Z: JOB_MESSAGE_DEBUG: (669c5e7541387cd1): Assigning stage ids.
root: INFO: Job 2018-01-08_11_12_08-14746680311414383184 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-08T19:12:12.361Z: JOB_MESSAGE_DEBUG: (d019c0de29dc543): Executing wait step start25
root: INFO: 2018-01-08T19:12:12.417Z: JOB_MESSAGE_BASIC: (e3e8b1b840ec8c56): Executing operation write/Write/WriteImpl/DoOnce/Read+write/Write/WriteImpl/InitializeWrite
root: INFO: 2018-01-08T19:12:12.440Z: JOB_MESSAGE_BASIC: (d019c0de29dc105): Executing operation group/Create
root: INFO: 2018-01-08T19:12:12.451Z: JOB_MESSAGE_DEBUG: (f77cd40cf38fd5fb): Starting worker pool setup.
root: INFO: 2018-01-08T19:12:12.481Z: JOB_MESSAGE_BASIC: (f77cd40cf38fd2f9): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-08T19:12:12.551Z: JOB_MESSAGE_DEBUG: (3995dad64a3f609a): Value "group/Session" materialized.
root: INFO: 2018-01-08T19:12:12.612Z: JOB_MESSAGE_BASIC: (3995dad64a3f6677): Executing operation read/Read+split+pair_with_one+group/Reify+group/Write
root: INFO: 2018-01-08T19:12:19.863Z: JOB_MESSAGE_DETAILED: (42e7f9f4c4f78b17): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-08T19:12:58.672Z: JOB_MESSAGE_ERROR: (42e7f9f4c4f786e6): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-08T19:12:58.689Z: JOB_MESSAGE_ERROR: (42e7f9f4c4f783a8): Workflow failed.
root: INFO: 2018-01-08T19:12:59.418Z: JOB_MESSAGE_DETAILED: (669c5e754138780e): Cleaning up.
root: INFO: 2018-01-08T19:12:59.453Z: JOB_MESSAGE_DEBUG: (669c5e7541387262): Starting worker pool teardown.
root: INFO: 2018-01-08T19:12:59.473Z: JOB_MESSAGE_BASIC: (669c5e7541387f8c): Stopping worker pool...
root: INFO: 2018-01-08T19:14:13.047Z: JOB_MESSAGE_DEBUG: (669c5e754138715e): Tearing down pending resources...
root: INFO: Job 2018-01-08_11_12_08-14746680311414383184 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 3 tests in 138.219s
FAILED (errors=4)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user ehudm@google.com