You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/01/07 15:13:37 UTC
Build failed in Jenkins:
beam_PostCommit_Python_ValidatesRunner_Dataflow #598
See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/598/display/redirect>
------------------------------------------
[...truncated 1003.01 KB...]
"name": "s12",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_merge_tagged_vals_under_key"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s11"
},
"serialized_fn": "<string of 1380 bytes>",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key)"
}
},
{
"kind": "ParallelDo",
"name": "s13",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Unkey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s12"
},
"serialized_fn": "<string of 980 bytes>",
"user_name": "assert_that/Unkey"
}
},
{
"kind": "ParallelDo",
"name": "s14",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_equal"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Match.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1172 bytes>",
"user_name": "assert_that/Match"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-07T15:06:50.852102Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-07_07_06_50-11033586593107725253'
location: u'us-central1'
name: u'beamapp-jenkins-0107150644-241455'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-07_07_06_50-11033586593107725253]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-07_07_06_50-11033586593107725253?project=apache-beam-testing
root: INFO: Job 2018-01-07_07_06_50-11033586593107725253 is in state JOB_STATE_PENDING
root: INFO: 2018-01-07T15:06:50.320Z: JOB_MESSAGE_WARNING: (991f2c8c48b2c2a9): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-07T15:06:52.512Z: JOB_MESSAGE_DETAILED: (1bd0407a05177340): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-07T15:06:53.793Z: JOB_MESSAGE_DETAILED: (1bd0407a05177fa9): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-07T15:06:53.818Z: JOB_MESSAGE_DEBUG: (1bd0407a051773b8): Combiner lifting skipped for step assert_that/Group/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-07T15:06:53.839Z: JOB_MESSAGE_DETAILED: (1bd0407a05177bc2): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-07T15:06:53.860Z: JOB_MESSAGE_DETAILED: (1bd0407a051773cc): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-07T15:06:53.885Z: JOB_MESSAGE_DEBUG: (1bd0407a051773e0): Annotating graph with Autotuner information.
root: INFO: 2018-01-07T15:06:53.917Z: JOB_MESSAGE_DETAILED: (1bd0407a051773f4): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-07T15:06:53.940Z: JOB_MESSAGE_DETAILED: (1bd0407a05177bfe): Unzipping flatten s10 for input s8.out
root: INFO: 2018-01-07T15:06:53.953Z: JOB_MESSAGE_DETAILED: (1bd0407a05177408): Fusing unzipped copy of assert_that/Group/GroupByKey/Reify, through flatten assert_that/Group/Flatten, into producer assert_that/Group/pair_with_0
root: INFO: 2018-01-07T15:06:53.973Z: JOB_MESSAGE_DETAILED: (1bd0407a05177c12): Fusing consumer assert_that/Group/Map(_merge_tagged_vals_under_key) into assert_that/Group/GroupByKey/GroupByWindow
root: INFO: 2018-01-07T15:06:53.992Z: JOB_MESSAGE_DETAILED: (1bd0407a0517741c): Fusing consumer assert_that/Match into assert_that/Unkey
root: INFO: 2018-01-07T15:06:54.011Z: JOB_MESSAGE_DETAILED: (1bd0407a05177c26): Fusing consumer assert_that/Unkey into assert_that/Group/Map(_merge_tagged_vals_under_key)
root: INFO: 2018-01-07T15:06:54.032Z: JOB_MESSAGE_DETAILED: (1bd0407a05177430): Fusing consumer assert_that/Group/GroupByKey/GroupByWindow into assert_that/Group/GroupByKey/Read
root: INFO: 2018-01-07T15:06:54.052Z: JOB_MESSAGE_DETAILED: (1bd0407a05177c3a): Unzipping flatten s10-u13 for input s11-reify-value0-c11
root: INFO: 2018-01-07T15:06:54.073Z: JOB_MESSAGE_DETAILED: (1bd0407a05177444): Fusing unzipped copy of assert_that/Group/GroupByKey/Write, through flatten s10-u13, into producer assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-07T15:06:54.092Z: JOB_MESSAGE_DETAILED: (1bd0407a05177c4e): Fusing consumer assert_that/Group/GroupByKey/Reify into assert_that/Group/pair_with_1
root: INFO: 2018-01-07T15:06:54.112Z: JOB_MESSAGE_DETAILED: (1bd0407a05177458): Fusing consumer assert_that/Group/GroupByKey/Write into assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-07T15:06:54.131Z: JOB_MESSAGE_DETAILED: (1bd0407a05177c62): Fusing consumer assert_that/Group/pair_with_0 into assert_that/Create/Read
root: INFO: 2018-01-07T15:06:54.145Z: JOB_MESSAGE_DETAILED: (1bd0407a0517746c): Fusing consumer assert_that/ToVoidKey into assert_that/WindowInto(WindowIntoFn)
root: INFO: 2018-01-07T15:06:54.164Z: JOB_MESSAGE_DETAILED: (1bd0407a05177c76): Fusing consumer assert_that/Group/pair_with_1 into assert_that/ToVoidKey
root: INFO: 2018-01-07T15:06:54.186Z: JOB_MESSAGE_DETAILED: (1bd0407a05177480): Fusing consumer compute/compute into start/Read
root: INFO: 2018-01-07T15:06:54.205Z: JOB_MESSAGE_DETAILED: (1bd0407a05177c8a): Fusing consumer assert_that/WindowInto(WindowIntoFn) into compute/compute
root: INFO: 2018-01-07T15:06:54.228Z: JOB_MESSAGE_DEBUG: (1bd0407a05177494): Workflow config is missing a default resource spec.
root: INFO: 2018-01-07T15:06:54.246Z: JOB_MESSAGE_DEBUG: (1bd0407a05177c9e): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-07T15:06:54.258Z: JOB_MESSAGE_DEBUG: (1bd0407a051774a8): Adding workflow start and stop steps.
root: INFO: 2018-01-07T15:06:54.270Z: JOB_MESSAGE_DEBUG: (1bd0407a05177cb2): Assigning stage ids.
root: INFO: 2018-01-07T15:06:54.374Z: JOB_MESSAGE_DEBUG: (43e8e6bd957dad29): Executing wait step start21
root: INFO: 2018-01-07T15:06:54.413Z: JOB_MESSAGE_BASIC: (3b2a2ffa078bd00): Executing operation side/Read
root: INFO: 2018-01-07T15:06:54.457Z: JOB_MESSAGE_DEBUG: (3b2a2ffa078bbb2): Value "side/Read.out" materialized.
root: INFO: 2018-01-07T15:06:54.492Z: JOB_MESSAGE_BASIC: (43e8e6bd957da190): Executing operation compute/_UnpickledSideInput(Read.out.0)
root: INFO: 2018-01-07T15:06:54.534Z: JOB_MESSAGE_DEBUG: (3b2a2ffa078bff1): Value "compute/_UnpickledSideInput(Read.out.0).output" materialized.
root: INFO: 2018-01-07T15:06:54.559Z: JOB_MESSAGE_BASIC: (43e8e6bd957da072): Executing operation assert_that/Group/GroupByKey/Create
root: INFO: 2018-01-07T15:06:54.591Z: JOB_MESSAGE_DEBUG: (a05dccdb0d149dd2): Starting worker pool setup.
root: INFO: 2018-01-07T15:06:54.610Z: JOB_MESSAGE_BASIC: (a05dccdb0d149550): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-07T15:06:54.646Z: JOB_MESSAGE_DEBUG: (43e8e6bd957da940): Value "assert_that/Group/GroupByKey/Session" materialized.
root: INFO: 2018-01-07T15:06:54.686Z: JOB_MESSAGE_BASIC: (3558df663ca5ace2): Executing operation start/Read+compute/compute+assert_that/WindowInto(WindowIntoFn)+assert_that/ToVoidKey+assert_that/Group/pair_with_1+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-07T15:06:54.708Z: JOB_MESSAGE_BASIC: (43e8e6bd957da793): Executing operation assert_that/Create/Read+assert_that/Group/pair_with_0+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: Job 2018-01-07_07_06_50-11033586593107725253 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-07T15:07:01.028Z: JOB_MESSAGE_DETAILED: (fa43b55ff726509c): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-07T15:07:29.820Z: JOB_MESSAGE_ERROR: (fa43b55ff7265dc7): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-07T15:07:29.843Z: JOB_MESSAGE_ERROR: (fa43b55ff7265479): Workflow failed.
root: INFO: 2018-01-07T15:07:30.085Z: JOB_MESSAGE_DETAILED: (1bd0407a051778df): Cleaning up.
root: INFO: 2018-01-07T15:07:30.132Z: JOB_MESSAGE_DEBUG: (1bd0407a051778f3): Starting worker pool teardown.
root: INFO: 2018-01-07T15:07:30.152Z: JOB_MESSAGE_BASIC: (1bd0407a051770fd): Stopping worker pool...
root: INFO: 2018-01-07T15:08:46.822Z: JOB_MESSAGE_DEBUG: (1bd0407a0517792f): Tearing down pending resources...
root: INFO: Job 2018-01-07_07_06_50-11033586593107725253 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 15 tests in 774.350s
FAILED (errors=12)
Build step 'Execute shell' marked build as failure
Jenkins build is back to normal :
beam_PostCommit_Python_ValidatesRunner_Dataflow #610
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/610/display/redirect?page=changes>
Build failed in Jenkins:
beam_PostCommit_Python_ValidatesRunner_Dataflow #609
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/609/display/redirect>
------------------------------------------
[...truncated 1.06 MB...]
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_merge_tagged_vals_under_key"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s11"
},
"serialized_fn": "<string of 1380 bytes>",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key)"
}
},
{
"kind": "ParallelDo",
"name": "s13",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Unkey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s12"
},
"serialized_fn": "<string of 980 bytes>",
"user_name": "assert_that/Unkey"
}
},
{
"kind": "ParallelDo",
"name": "s14",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_equal"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Match.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1160 bytes>",
"user_name": "assert_that/Match"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-09T15:07:15.482419Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-09_07_07_14-1700221786659886619'
location: u'us-central1'
name: u'beamapp-jenkins-0109150708-229635'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-09_07_07_14-1700221786659886619]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-09_07_07_14-1700221786659886619?project=apache-beam-testing
root: INFO: Job 2018-01-09_07_07_14-1700221786659886619 is in state JOB_STATE_PENDING
root: INFO: 2018-01-09T15:07:14.980Z: JOB_MESSAGE_WARNING: (179866b4f13e1bcf): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-09T15:07:17.253Z: JOB_MESSAGE_DETAILED: (e013a3886e0637e1): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-09T15:07:18.581Z: JOB_MESSAGE_DETAILED: (e013a3886e06303f): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T15:07:18.604Z: JOB_MESSAGE_DEBUG: (e013a3886e063506): Combiner lifting skipped for step assert_that/Group/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-09T15:07:18.629Z: JOB_MESSAGE_DETAILED: (e013a3886e0632e0): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T15:07:18.659Z: JOB_MESSAGE_DETAILED: (e013a3886e0630ba): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-09T15:07:18.689Z: JOB_MESSAGE_DEBUG: (e013a3886e063c6e): Annotating graph with Autotuner information.
root: INFO: 2018-01-09T15:07:18.729Z: JOB_MESSAGE_DETAILED: (e013a3886e063822): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-09T15:07:18.760Z: JOB_MESSAGE_DETAILED: (e013a3886e0635fc): Unzipping flatten s10 for input s8.out
root: INFO: 2018-01-09T15:07:18.777Z: JOB_MESSAGE_DETAILED: (e013a3886e0633d6): Fusing unzipped copy of assert_that/Group/GroupByKey/Reify, through flatten assert_that/Group/Flatten, into producer assert_that/Group/pair_with_0
root: INFO: 2018-01-09T15:07:18.808Z: JOB_MESSAGE_DETAILED: (e013a3886e0631b0): Fusing consumer assert_that/Group/Map(_merge_tagged_vals_under_key) into assert_that/Group/GroupByKey/GroupByWindow
root: INFO: 2018-01-09T15:07:18.826Z: JOB_MESSAGE_DETAILED: (e013a3886e063f8a): Fusing consumer assert_that/Match into assert_that/Unkey
root: INFO: 2018-01-09T15:07:18.857Z: JOB_MESSAGE_DETAILED: (e013a3886e063d64): Fusing consumer assert_that/Unkey into assert_that/Group/Map(_merge_tagged_vals_under_key)
root: INFO: 2018-01-09T15:07:18.885Z: JOB_MESSAGE_DETAILED: (e013a3886e063b3e): Fusing consumer assert_that/Group/GroupByKey/GroupByWindow into assert_that/Group/GroupByKey/Read
root: INFO: 2018-01-09T15:07:18.916Z: JOB_MESSAGE_DETAILED: (e013a3886e063918): Unzipping flatten s10-u13 for input s11-reify-value0-c11
root: INFO: 2018-01-09T15:07:18.940Z: JOB_MESSAGE_DETAILED: (e013a3886e0636f2): Fusing unzipped copy of assert_that/Group/GroupByKey/Write, through flatten s10-u13, into producer assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-09T15:07:18.961Z: JOB_MESSAGE_DETAILED: (e013a3886e0634cc): Fusing consumer assert_that/Group/GroupByKey/Reify into assert_that/Group/pair_with_1
root: INFO: 2018-01-09T15:07:18.991Z: JOB_MESSAGE_DETAILED: (e013a3886e0632a6): Fusing consumer assert_that/Group/GroupByKey/Write into assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-09T15:07:19.022Z: JOB_MESSAGE_DETAILED: (e013a3886e063080): Fusing consumer assert_that/Group/pair_with_0 into assert_that/Create/Read
root: INFO: 2018-01-09T15:07:19.053Z: JOB_MESSAGE_DETAILED: (e013a3886e063e5a): Fusing consumer assert_that/WindowInto(WindowIntoFn) into compute/compute
root: INFO: 2018-01-09T15:07:19.077Z: JOB_MESSAGE_DETAILED: (e013a3886e063c34): Fusing consumer assert_that/Group/pair_with_1 into assert_that/ToVoidKey
root: INFO: 2018-01-09T15:07:19.101Z: JOB_MESSAGE_DETAILED: (e013a3886e063a0e): Fusing consumer assert_that/ToVoidKey into assert_that/WindowInto(WindowIntoFn)
root: INFO: 2018-01-09T15:07:19.132Z: JOB_MESSAGE_DETAILED: (e013a3886e0637e8): Fusing consumer compute/compute into start/Read
root: INFO: 2018-01-09T15:07:19.165Z: JOB_MESSAGE_DEBUG: (e013a3886e0635c2): Workflow config is missing a default resource spec.
root: INFO: 2018-01-09T15:07:19.188Z: JOB_MESSAGE_DEBUG: (e013a3886e06339c): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-09T15:07:19.216Z: JOB_MESSAGE_DEBUG: (e013a3886e063176): Adding workflow start and stop steps.
root: INFO: 2018-01-09T15:07:19.241Z: JOB_MESSAGE_DEBUG: (e013a3886e063f50): Assigning stage ids.
root: INFO: 2018-01-09T15:07:19.380Z: JOB_MESSAGE_DEBUG: (2e3fdd27a082e49a): Executing wait step start21
root: INFO: 2018-01-09T15:07:19.433Z: JOB_MESSAGE_BASIC: (8a2f5c4c260b15eb): Executing operation side/Read
root: INFO: 2018-01-09T15:07:19.483Z: JOB_MESSAGE_DEBUG: (2e3fdd27a082e4c3): Value "side/Read.out" materialized.
root: INFO: 2018-01-09T15:07:19.541Z: JOB_MESSAGE_BASIC: (8a2f5c4c260b1169): Executing operation compute/_UnpickledSideInput(Read.out.0)
root: INFO: 2018-01-09T15:07:19.595Z: JOB_MESSAGE_DEBUG: (8a2f5c4c260b1ce7): Value "compute/_UnpickledSideInput(Read.out.0).output" materialized.
root: INFO: 2018-01-09T15:07:19.653Z: JOB_MESSAGE_BASIC: (2e3fdd27a082e4ec): Executing operation assert_that/Group/GroupByKey/Create
root: INFO: Job 2018-01-09_07_07_14-1700221786659886619 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-09T15:07:19.685Z: JOB_MESSAGE_DEBUG: (e172506258715fcb): Starting worker pool setup.
root: INFO: 2018-01-09T15:07:19.717Z: JOB_MESSAGE_BASIC: (e172506258715df9): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-09T15:07:19.772Z: JOB_MESSAGE_DEBUG: (2e3fdd27a082e53e): Value "assert_that/Group/GroupByKey/Session" materialized.
root: INFO: 2018-01-09T15:07:19.823Z: JOB_MESSAGE_BASIC: (2e3fdd27a082e8bb): Executing operation start/Read+compute/compute+assert_that/WindowInto(WindowIntoFn)+assert_that/ToVoidKey+assert_that/Group/pair_with_1+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-09T15:07:19.850Z: JOB_MESSAGE_BASIC: (d8ac6b5f304f96fe): Executing operation assert_that/Create/Read+assert_that/Group/pair_with_0+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-09T15:07:26.226Z: JOB_MESSAGE_DETAILED: (26e8810822b82710): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-09T15:08:00.759Z: JOB_MESSAGE_ERROR: (26e8810822b82083): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-09T15:08:00.786Z: JOB_MESSAGE_ERROR: (26e8810822b820f9): Workflow failed.
root: INFO: 2018-01-09T15:08:00.911Z: JOB_MESSAGE_DETAILED: (e013a3886e063da5): Cleaning up.
root: INFO: 2018-01-09T15:08:00.963Z: JOB_MESSAGE_DEBUG: (e013a3886e063959): Starting worker pool teardown.
root: INFO: 2018-01-09T15:08:00.989Z: JOB_MESSAGE_BASIC: (e013a3886e063733): Stopping worker pool...
root: INFO: 2018-01-09T15:09:15.254Z: JOB_MESSAGE_BASIC: (e013a3886e063e9b): Worker pool stopped.
root: INFO: 2018-01-09T15:09:15.297Z: JOB_MESSAGE_DEBUG: (e013a3886e063a4f): Tearing down pending resources...
root: INFO: Job 2018-01-09_07_07_14-1700221786659886619 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 15 tests in 510.367s
FAILED (errors=14)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user jb@nanthrax.net
Build failed in Jenkins:
beam_PostCommit_Python_ValidatesRunner_Dataflow #608
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/608/display/redirect?page=changes>
Changes:
[jbonofre] [BEAM-3282] MqttIO reader now use receive for timeout in order to return
------------------------------------------
[...truncated 1.06 MB...]
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_merge_tagged_vals_under_key"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s11"
},
"serialized_fn": "<string of 1380 bytes>",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key)"
}
},
{
"kind": "ParallelDo",
"name": "s13",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Unkey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s12"
},
"serialized_fn": "<string of 980 bytes>",
"user_name": "assert_that/Unkey"
}
},
{
"kind": "ParallelDo",
"name": "s14",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_equal"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Match.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1160 bytes>",
"user_name": "assert_that/Match"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-09T11:21:59.810899Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-09_03_21_59-13300801501679223962'
location: u'us-central1'
name: u'beamapp-jenkins-0109112153-284066'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-09_03_21_59-13300801501679223962]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-09_03_21_59-13300801501679223962?project=apache-beam-testing
root: INFO: Job 2018-01-09_03_21_59-13300801501679223962 is in state JOB_STATE_PENDING
root: INFO: 2018-01-09T11:21:59.223Z: JOB_MESSAGE_WARNING: (b895f07ebf92853e): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-09T11:22:01.483Z: JOB_MESSAGE_DETAILED: (989f12685e7bb643): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-09T11:22:02.666Z: JOB_MESSAGE_DETAILED: (989f12685e7bb6f5): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T11:22:02.692Z: JOB_MESSAGE_DEBUG: (989f12685e7bb1f6): Combiner lifting skipped for step assert_that/Group/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-09T11:22:02.722Z: JOB_MESSAGE_DETAILED: (989f12685e7bb94c): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T11:22:02.743Z: JOB_MESSAGE_DETAILED: (989f12685e7bb0a2): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-09T11:22:02.769Z: JOB_MESSAGE_DEBUG: (989f12685e7bbf4e): Annotating graph with Autotuner information.
root: INFO: 2018-01-09T11:22:02.789Z: JOB_MESSAGE_DETAILED: (989f12685e7bbdfa): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-09T11:22:02.814Z: JOB_MESSAGE_DETAILED: (989f12685e7bb550): Unzipping flatten s10 for input s8.out
root: INFO: 2018-01-09T11:22:02.839Z: JOB_MESSAGE_DETAILED: (989f12685e7bbca6): Fusing unzipped copy of assert_that/Group/GroupByKey/Reify, through flatten assert_that/Group/Flatten, into producer assert_that/Group/pair_with_0
root: INFO: 2018-01-09T11:22:02.862Z: JOB_MESSAGE_DETAILED: (989f12685e7bb3fc): Fusing consumer assert_that/Group/Map(_merge_tagged_vals_under_key) into assert_that/Group/GroupByKey/GroupByWindow
root: INFO: 2018-01-09T11:22:02.885Z: JOB_MESSAGE_DETAILED: (989f12685e7bbb52): Fusing consumer assert_that/Match into assert_that/Unkey
root: INFO: 2018-01-09T11:22:02.905Z: JOB_MESSAGE_DETAILED: (989f12685e7bb2a8): Fusing consumer assert_that/Unkey into assert_that/Group/Map(_merge_tagged_vals_under_key)
root: INFO: 2018-01-09T11:22:02.924Z: JOB_MESSAGE_DETAILED: (989f12685e7bb9fe): Fusing consumer assert_that/Group/GroupByKey/GroupByWindow into assert_that/Group/GroupByKey/Read
root: INFO: 2018-01-09T11:22:02.951Z: JOB_MESSAGE_DETAILED: (989f12685e7bb154): Unzipping flatten s10-u13 for input s11-reify-value0-c11
root: INFO: 2018-01-09T11:22:02.981Z: JOB_MESSAGE_DETAILED: (989f12685e7bb8aa): Fusing unzipped copy of assert_that/Group/GroupByKey/Write, through flatten s10-u13, into producer assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-09T11:22:03.011Z: JOB_MESSAGE_DETAILED: (989f12685e7bb000): Fusing consumer assert_that/Group/GroupByKey/Reify into assert_that/Group/pair_with_1
root: INFO: 2018-01-09T11:22:03.031Z: JOB_MESSAGE_DETAILED: (989f12685e7bb756): Fusing consumer assert_that/Group/GroupByKey/Write into assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-09T11:22:03.053Z: JOB_MESSAGE_DETAILED: (989f12685e7bbeac): Fusing consumer assert_that/Group/pair_with_0 into assert_that/Create/Read
root: INFO: 2018-01-09T11:22:03.072Z: JOB_MESSAGE_DETAILED: (989f12685e7bb602): Fusing consumer assert_that/WindowInto(WindowIntoFn) into compute/compute
root: INFO: 2018-01-09T11:22:03.101Z: JOB_MESSAGE_DETAILED: (989f12685e7bbd58): Fusing consumer assert_that/Group/pair_with_1 into assert_that/ToVoidKey
root: INFO: 2018-01-09T11:22:03.113Z: JOB_MESSAGE_DETAILED: (989f12685e7bb4ae): Fusing consumer assert_that/ToVoidKey into assert_that/WindowInto(WindowIntoFn)
root: INFO: 2018-01-09T11:22:03.131Z: JOB_MESSAGE_DETAILED: (989f12685e7bbc04): Fusing consumer compute/compute into start/Read
root: INFO: 2018-01-09T11:22:03.162Z: JOB_MESSAGE_DEBUG: (989f12685e7bb35a): Workflow config is missing a default resource spec.
root: INFO: 2018-01-09T11:22:03.192Z: JOB_MESSAGE_DEBUG: (989f12685e7bbab0): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-09T11:22:03.222Z: JOB_MESSAGE_DEBUG: (989f12685e7bb206): Adding workflow start and stop steps.
root: INFO: 2018-01-09T11:22:03.251Z: JOB_MESSAGE_DEBUG: (989f12685e7bb95c): Assigning stage ids.
root: INFO: 2018-01-09T11:22:03.389Z: JOB_MESSAGE_DEBUG: (59e608b64c5d2cb1): Executing wait step start21
root: INFO: 2018-01-09T11:22:03.442Z: JOB_MESSAGE_BASIC: (59e608b64c5d23d6): Executing operation side/Read
root: INFO: 2018-01-09T11:22:03.506Z: JOB_MESSAGE_DEBUG: (f445bbe2bf69871): Value "side/Read.out" materialized.
root: INFO: 2018-01-09T11:22:03.557Z: JOB_MESSAGE_BASIC: (59e608b64c5d2f9c): Executing operation compute/_UnpickledSideInput(Read.out.0)
root: INFO: 2018-01-09T11:22:03.611Z: JOB_MESSAGE_DEBUG: (f445bbe2bf69308): Value "compute/_UnpickledSideInput(Read.out.0).output" materialized.
root: INFO: 2018-01-09T11:22:03.666Z: JOB_MESSAGE_BASIC: (59e608b64c5d2b62): Executing operation assert_that/Group/GroupByKey/Create
root: INFO: 2018-01-09T11:22:03.705Z: JOB_MESSAGE_DEBUG: (d296d82af3e41143): Starting worker pool setup.
root: INFO: 2018-01-09T11:22:03.726Z: JOB_MESSAGE_BASIC: (d296d82af3e41ab5): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-09T11:22:03.779Z: JOB_MESSAGE_DEBUG: (97e1f26b8d03aa1e): Value "assert_that/Group/GroupByKey/Session" materialized.
root: INFO: 2018-01-09T11:22:03.838Z: JOB_MESSAGE_BASIC: (59e608b64c5d29ac): Executing operation start/Read+compute/compute+assert_that/WindowInto(WindowIntoFn)+assert_that/ToVoidKey+assert_that/Group/pair_with_1+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-09T11:22:03.867Z: JOB_MESSAGE_BASIC: (97e1f26b8d03a329): Executing operation assert_that/Create/Read+assert_that/Group/pair_with_0+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: Job 2018-01-09_03_21_59-13300801501679223962 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-09T11:22:10.868Z: JOB_MESSAGE_DETAILED: (78763677bb46cdd2): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-09T11:22:39.842Z: JOB_MESSAGE_ERROR: (78763677bb46c3a3): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-09T11:22:39.871Z: JOB_MESSAGE_ERROR: (78763677bb46c299): Workflow failed.
root: INFO: 2018-01-09T11:22:40.117Z: JOB_MESSAGE_DETAILED: (989f12685e7bba5f): Cleaning up.
root: INFO: 2018-01-09T11:22:40.179Z: JOB_MESSAGE_DEBUG: (989f12685e7bb90b): Starting worker pool teardown.
root: INFO: 2018-01-09T11:22:40.204Z: JOB_MESSAGE_BASIC: (989f12685e7bb061): Stopping worker pool...
root: INFO: 2018-01-09T11:23:50.444Z: JOB_MESSAGE_BASIC: (989f12685e7bbdb9): Worker pool stopped.
root: INFO: 2018-01-09T11:23:50.462Z: JOB_MESSAGE_DEBUG: (989f12685e7bbc65): Tearing down pending resources...
root: INFO: Job 2018-01-09_03_21_59-13300801501679223962 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 15 tests in 501.536s
FAILED (errors=14)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user jb@nanthrax.net
Build failed in Jenkins:
beam_PostCommit_Python_ValidatesRunner_Dataflow #607
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/607/display/redirect>
------------------------------------------
[...truncated 1.06 MB...]
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_merge_tagged_vals_under_key"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1380 bytes>",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key)"
}
},
{
"kind": "ParallelDo",
"name": "s15",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Unkey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s14"
},
"serialized_fn": "<string of 980 bytes>",
"user_name": "assert_that/Unkey"
}
},
{
"kind": "ParallelDo",
"name": "s16",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_equal"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Match.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s15"
},
"serialized_fn": "<string of 1160 bytes>",
"user_name": "assert_that/Match"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-09T09:06:12.187038Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-09_01_06_11-14000546594437023026'
location: u'us-central1'
name: u'beamapp-jenkins-0109090605-054053'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-09_01_06_11-14000546594437023026]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-09_01_06_11-14000546594437023026?project=apache-beam-testing
root: INFO: Job 2018-01-09_01_06_11-14000546594437023026 is in state JOB_STATE_PENDING
root: INFO: 2018-01-09T09:06:11.663Z: JOB_MESSAGE_WARNING: (c24beef316b94e36): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-09T09:06:14.156Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2de7): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-09T09:06:15.042Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2e55): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T09:06:15.076Z: JOB_MESSAGE_DEBUG: (e6ec93942c1b2e64): Combiner lifting skipped for step assert_that/Group/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-09T09:06:15.107Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2e6e): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T09:06:15.139Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2e78): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-09T09:06:15.173Z: JOB_MESSAGE_DEBUG: (e6ec93942c1b2e8c): Annotating graph with Autotuner information.
root: INFO: 2018-01-09T09:06:15.210Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2ea0): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-09T09:06:15.228Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2eaa): Unzipping flatten s12 for input s10.out
root: INFO: 2018-01-09T09:06:15.261Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2eb4): Fusing unzipped copy of assert_that/Group/GroupByKey/Reify, through flatten assert_that/Group/Flatten, into producer assert_that/Group/pair_with_0
root: INFO: 2018-01-09T09:06:15.285Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2ebe): Fusing consumer assert_that/Group/Map(_merge_tagged_vals_under_key) into assert_that/Group/GroupByKey/GroupByWindow
root: INFO: 2018-01-09T09:06:15.326Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2ec8): Fusing consumer assert_that/Unkey into assert_that/Group/Map(_merge_tagged_vals_under_key)
root: INFO: 2018-01-09T09:06:15.351Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2ed2): Fusing consumer assert_that/Group/GroupByKey/GroupByWindow into assert_that/Group/GroupByKey/Read
root: INFO: 2018-01-09T09:06:15.383Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2edc): Fusing consumer assert_that/Match into assert_that/Unkey
root: INFO: 2018-01-09T09:06:15.406Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2ee6): Unzipping flatten s12-u13 for input s13-reify-value0-c11
root: INFO: 2018-01-09T09:06:15.430Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2ef0): Fusing unzipped copy of assert_that/Group/GroupByKey/Write, through flatten s12-u13, into producer assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-09T09:06:15.453Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2efa): Fusing consumer assert_that/Group/pair_with_0 into assert_that/Create/Read
root: INFO: 2018-01-09T09:06:15.476Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2f04): Fusing consumer assert_that/Group/GroupByKey/Write into assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-09T09:06:15.510Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2f0e): Fusing consumer assert_that/Group/GroupByKey/Reify into assert_that/Group/pair_with_1
root: INFO: 2018-01-09T09:06:15.543Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2f18): Fusing consumer FlatMap(<lambda at sideinputs_test.py:302>)/FlatMap(<lambda at sideinputs_test.py:302>) into main input/Read
root: INFO: 2018-01-09T09:06:15.574Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2f22): Fusing consumer assert_that/Group/pair_with_1 into assert_that/ToVoidKey
root: INFO: 2018-01-09T09:06:15.598Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2f2c): Fusing consumer assert_that/WindowInto(WindowIntoFn) into FlatMap(<lambda at sideinputs_test.py:302>)/FlatMap(<lambda at sideinputs_test.py:302>)
root: INFO: 2018-01-09T09:06:15.657Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2f36): Fusing consumer assert_that/ToVoidKey into assert_that/WindowInto(WindowIntoFn)
root: INFO: 2018-01-09T09:06:15.691Z: JOB_MESSAGE_DEBUG: (e6ec93942c1b2f40): Workflow config is missing a default resource spec.
root: INFO: 2018-01-09T09:06:15.720Z: JOB_MESSAGE_DEBUG: (e6ec93942c1b2f4a): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-09T09:06:15.743Z: JOB_MESSAGE_DEBUG: (e6ec93942c1b2f54): Adding workflow start and stop steps.
root: INFO: 2018-01-09T09:06:15.771Z: JOB_MESSAGE_DEBUG: (e6ec93942c1b2f5e): Assigning stage ids.
root: INFO: 2018-01-09T09:06:15.928Z: JOB_MESSAGE_DEBUG: (b1670b2103de32d5): Executing wait step start22
root: INFO: 2018-01-09T09:06:15.983Z: JOB_MESSAGE_BASIC: (fdfdae5e7b7c8288): Executing operation side1/Read
root: INFO: 2018-01-09T09:06:16.008Z: JOB_MESSAGE_BASIC: (b1670b2103de326f): Executing operation side2/Read
root: INFO: 2018-01-09T09:06:16.048Z: JOB_MESSAGE_DEBUG: (fdfdae5e7b7c8ca6): Value "side1/Read.out" materialized.
root: INFO: 2018-01-09T09:06:16.081Z: JOB_MESSAGE_DEBUG: (b1670b2103de3209): Value "side2/Read.out" materialized.
root: INFO: 2018-01-09T09:06:16.148Z: JOB_MESSAGE_BASIC: (b1670b2103de39b4): Executing operation Flatten
root: INFO: 2018-01-09T09:06:16.205Z: JOB_MESSAGE_DEBUG: (fdfdae5e7b7c87bf): Value "Flatten.out" materialized.
root: INFO: 2018-01-09T09:06:16.264Z: JOB_MESSAGE_BASIC: (fdfdae5e7b7c8b00): Executing operation FlatMap(<lambda at sideinputs_test.py:302>)/_UnpickledSideInput(Flatten.out.0)
root: INFO: 2018-01-09T09:06:16.334Z: JOB_MESSAGE_DEBUG: (fdfdae5e7b7c8d46): Value "FlatMap(<lambda at sideinputs_test.py:302>)/_UnpickledSideInput(Flatten.out.0).output" materialized.
root: INFO: Job 2018-01-09_01_06_11-14000546594437023026 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-09T09:06:16.396Z: JOB_MESSAGE_BASIC: (fdfdae5e7b7c8f8c): Executing operation assert_that/Group/GroupByKey/Create
root: INFO: 2018-01-09T09:06:16.431Z: JOB_MESSAGE_DEBUG: (d2f7d9bd73dc7f1): Starting worker pool setup.
root: INFO: 2018-01-09T09:06:16.464Z: JOB_MESSAGE_BASIC: (d2f7d9bd73dc9ab): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-09T09:06:16.532Z: JOB_MESSAGE_DEBUG: (fdfdae5e7b7c85be): Value "assert_that/Group/GroupByKey/Session" materialized.
root: INFO: 2018-01-09T09:06:16.597Z: JOB_MESSAGE_BASIC: (fdfdae5e7b7c8ceb): Executing operation main input/Read+FlatMap(<lambda at sideinputs_test.py:302>)/FlatMap(<lambda at sideinputs_test.py:302>)+assert_that/WindowInto(WindowIntoFn)+assert_that/ToVoidKey+assert_that/Group/pair_with_1+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-09T09:06:16.624Z: JOB_MESSAGE_BASIC: (9b6c691ee5a35182): Executing operation assert_that/Create/Read+assert_that/Group/pair_with_0+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-09T09:06:24.557Z: JOB_MESSAGE_DETAILED: (f677d227ec2c32aa): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-09T09:07:15.095Z: JOB_MESSAGE_ERROR: (f677d227ec2c315d): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-09T09:07:15.122Z: JOB_MESSAGE_ERROR: (f677d227ec2c392f): Workflow failed.
root: INFO: 2018-01-09T09:07:15.405Z: JOB_MESSAGE_DETAILED: (e6ec93942c1b2f8b): Cleaning up.
root: INFO: 2018-01-09T09:07:15.453Z: JOB_MESSAGE_DEBUG: (e6ec93942c1b2f9f): Starting worker pool teardown.
root: INFO: 2018-01-09T09:07:15.487Z: JOB_MESSAGE_BASIC: (e6ec93942c1b2fa9): Stopping worker pool...
root: INFO: 2018-01-09T09:08:30.828Z: JOB_MESSAGE_DEBUG: (e6ec93942c1b2fdb): Tearing down pending resources...
root: INFO: Job 2018-01-09_01_06_11-14000546594437023026 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 15 tests in 490.884s
FAILED (errors=14)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user jb@nanthrax.net
Build failed in Jenkins:
beam_PostCommit_Python_ValidatesRunner_Dataflow #606
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/606/display/redirect?page=changes>
Changes:
[jb] Exclude go from rat configuration, use .gitignore for excludes and align
[iemejia] [BEAM-3422] Update reproducible-build-maven-plugin to version 0.4
------------------------------------------
[...truncated 1023.65 KB...]
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_merge_tagged_vals_under_key"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s11"
},
"serialized_fn": "<string of 1380 bytes>",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key)"
}
},
{
"kind": "ParallelDo",
"name": "s13",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Unkey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s12"
},
"serialized_fn": "<string of 980 bytes>",
"user_name": "assert_that/Unkey"
}
},
{
"kind": "ParallelDo",
"name": "s14",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_equal"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Match.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1160 bytes>",
"user_name": "assert_that/Match"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-09T07:15:28.332391Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_23_15_27-13639918581061008883'
location: u'us-central1'
name: u'beamapp-jenkins-0109071521-203670'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_23_15_27-13639918581061008883]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_23_15_27-13639918581061008883?project=apache-beam-testing
root: INFO: Job 2018-01-08_23_15_27-13639918581061008883 is in state JOB_STATE_PENDING
root: INFO: 2018-01-09T07:15:27.733Z: JOB_MESSAGE_WARNING: (bd4ab9ae9d64386f): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-09T07:15:30.023Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0dce8): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-09T07:15:31.110Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0d632): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T07:15:31.136Z: JOB_MESSAGE_DEBUG: (cf1052e39dd0da5f): Combiner lifting skipped for step assert_that/Group/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-09T07:15:31.158Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0d27d): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T07:15:31.181Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0da9b): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-09T07:15:31.205Z: JOB_MESSAGE_DEBUG: (cf1052e39dd0dad7): Annotating graph with Autotuner information.
root: INFO: 2018-01-09T07:15:31.250Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0db13): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-09T07:15:31.283Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0d331): Unzipping flatten s10 for input s8.out
root: INFO: 2018-01-09T07:15:31.314Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0db4f): Fusing unzipped copy of assert_that/Group/GroupByKey/Reify, through flatten assert_that/Group/Flatten, into producer assert_that/Group/pair_with_0
root: INFO: 2018-01-09T07:15:31.336Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0d36d): Fusing consumer assert_that/Group/Map(_merge_tagged_vals_under_key) into assert_that/Group/GroupByKey/GroupByWindow
root: INFO: 2018-01-09T07:15:31.368Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0db8b): Fusing consumer assert_that/Match into assert_that/Unkey
root: INFO: 2018-01-09T07:15:31.390Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0d3a9): Fusing consumer assert_that/Unkey into assert_that/Group/Map(_merge_tagged_vals_under_key)
root: INFO: 2018-01-09T07:15:31.414Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0dbc7): Fusing consumer assert_that/Group/GroupByKey/GroupByWindow into assert_that/Group/GroupByKey/Read
root: INFO: 2018-01-09T07:15:31.431Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0d3e5): Unzipping flatten s10-u13 for input s11-reify-value0-c11
root: INFO: 2018-01-09T07:15:31.457Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0dc03): Fusing unzipped copy of assert_that/Group/GroupByKey/Write, through flatten s10-u13, into producer assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-09T07:15:31.477Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0d421): Fusing consumer assert_that/Group/GroupByKey/Reify into assert_that/Group/pair_with_1
root: INFO: 2018-01-09T07:15:31.508Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0dc3f): Fusing consumer assert_that/Group/GroupByKey/Write into assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-09T07:15:31.522Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0d45d): Fusing consumer assert_that/Group/pair_with_0 into assert_that/Create/Read
root: INFO: 2018-01-09T07:15:31.554Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0dc7b): Fusing consumer assert_that/WindowInto(WindowIntoFn) into compute/compute
root: INFO: 2018-01-09T07:15:31.585Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0d499): Fusing consumer assert_that/Group/pair_with_1 into assert_that/ToVoidKey
root: INFO: 2018-01-09T07:15:31.614Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0dcb7): Fusing consumer assert_that/ToVoidKey into assert_that/WindowInto(WindowIntoFn)
root: INFO: 2018-01-09T07:15:31.638Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0d4d5): Fusing consumer compute/compute into start/Read
root: INFO: 2018-01-09T07:15:31.663Z: JOB_MESSAGE_DEBUG: (cf1052e39dd0dcf3): Workflow config is missing a default resource spec.
root: INFO: 2018-01-09T07:15:31.685Z: JOB_MESSAGE_DEBUG: (cf1052e39dd0d511): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-09T07:15:31.717Z: JOB_MESSAGE_DEBUG: (cf1052e39dd0dd2f): Adding workflow start and stop steps.
root: INFO: 2018-01-09T07:15:31.738Z: JOB_MESSAGE_DEBUG: (cf1052e39dd0d54d): Assigning stage ids.
root: INFO: 2018-01-09T07:15:31.875Z: JOB_MESSAGE_DEBUG: (2f6c97f5bda1a769): Executing wait step start21
root: INFO: 2018-01-09T07:15:31.935Z: JOB_MESSAGE_BASIC: (b397e940a3375d2): Executing operation side/Read
root: INFO: 2018-01-09T07:15:32.002Z: JOB_MESSAGE_DEBUG: (2f6c97f5bda1ad1e): Value "side/Read.out" materialized.
root: INFO: 2018-01-09T07:15:32.055Z: JOB_MESSAGE_BASIC: (b397e940a337094): Executing operation compute/_UnpickledSideInput(Read.out.0)
root: INFO: 2018-01-09T07:15:32.120Z: JOB_MESSAGE_DEBUG: (dcad8ad682c154d2): Value "compute/_UnpickledSideInput(Read.out.0).output" materialized.
root: INFO: 2018-01-09T07:15:32.179Z: JOB_MESSAGE_BASIC: (dcad8ad682c150ec): Executing operation assert_that/Group/GroupByKey/Create
root: INFO: 2018-01-09T07:15:32.221Z: JOB_MESSAGE_DEBUG: (5c4d01adcc526779): Starting worker pool setup.
root: INFO: 2018-01-09T07:15:32.243Z: JOB_MESSAGE_BASIC: (5c4d01adcc526fff): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-09T07:15:32.336Z: JOB_MESSAGE_DEBUG: (e2da739e3921c0a1): Value "assert_that/Group/GroupByKey/Session" materialized.
root: INFO: Job 2018-01-08_23_15_27-13639918581061008883 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-09T07:15:32.590Z: JOB_MESSAGE_BASIC: (dcad8ad682c15fc2): Executing operation start/Read+compute/compute+assert_that/WindowInto(WindowIntoFn)+assert_that/ToVoidKey+assert_that/Group/pair_with_1+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-09T07:15:32.621Z: JOB_MESSAGE_BASIC: (e2da739e3921ce8e): Executing operation assert_that/Create/Read+assert_that/Group/pair_with_0+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-09T07:15:38.193Z: JOB_MESSAGE_DETAILED: (2f6c97f5bda1a798): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-09T07:16:01.981Z: JOB_MESSAGE_ERROR: (2f6c97f5bda1abe5): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-09T07:16:01.998Z: JOB_MESSAGE_ERROR: (2f6c97f5bda1a7c7): Workflow failed.
root: INFO: 2018-01-09T07:16:02.774Z: JOB_MESSAGE_DETAILED: (cf1052e39dd0d1d4): Cleaning up.
root: INFO: 2018-01-09T07:16:02.813Z: JOB_MESSAGE_DEBUG: (cf1052e39dd0d210): Starting worker pool teardown.
root: INFO: 2018-01-09T07:16:02.837Z: JOB_MESSAGE_BASIC: (cf1052e39dd0da2e): Stopping worker pool...
root: INFO: 2018-01-09T07:17:11.904Z: JOB_MESSAGE_DEBUG: (cf1052e39dd0d2c4): Tearing down pending resources...
root: INFO: Job 2018-01-08_23_15_27-13639918581061008883 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 15 tests in 587.117s
FAILED (errors=13)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Not sending mail to unregistered user jb@nanthrax.net
Build failed in Jenkins:
beam_PostCommit_Python_ValidatesRunner_Dataflow #605
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/605/display/redirect>
------------------------------------------
[...truncated 1.06 MB...]
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_merge_tagged_vals_under_key"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s11"
},
"serialized_fn": "<string of 1380 bytes>",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key)"
}
},
{
"kind": "ParallelDo",
"name": "s13",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Unkey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s12"
},
"serialized_fn": "<string of 980 bytes>",
"user_name": "assert_that/Unkey"
}
},
{
"kind": "ParallelDo",
"name": "s14",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_equal"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Match.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1160 bytes>",
"user_name": "assert_that/Match"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-09T03:06:39.428914Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_19_06_38-16327728576386954215'
location: u'us-central1'
name: u'beamapp-jenkins-0109030632-641906'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_19_06_38-16327728576386954215]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_19_06_38-16327728576386954215?project=apache-beam-testing
root: INFO: Job 2018-01-08_19_06_38-16327728576386954215 is in state JOB_STATE_PENDING
root: INFO: 2018-01-09T03:06:38.969Z: JOB_MESSAGE_WARNING: (e297bea1d889f613): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-09T03:06:41.164Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2312): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-09T03:06:42.353Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2514): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T03:06:42.386Z: JOB_MESSAGE_DEBUG: (75eeccd10b8f20fd): Combiner lifting skipped for step assert_that/Group/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-09T03:06:42.418Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2e43): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-09T03:06:42.441Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2b89): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-09T03:06:42.478Z: JOB_MESSAGE_DEBUG: (75eeccd10b8f2615): Annotating graph with Autotuner information.
root: INFO: 2018-01-09T03:06:42.520Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f20a1): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-09T03:06:42.544Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2de7): Unzipping flatten s10 for input s8.out
root: INFO: 2018-01-09T03:06:42.567Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2b2d): Fusing unzipped copy of assert_that/Group/GroupByKey/Reify, through flatten assert_that/Group/Flatten, into producer assert_that/Group/pair_with_0
root: INFO: 2018-01-09T03:06:42.597Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2873): Fusing consumer assert_that/Group/Map(_merge_tagged_vals_under_key) into assert_that/Group/GroupByKey/GroupByWindow
root: INFO: 2018-01-09T03:06:42.627Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f25b9): Fusing consumer assert_that/Match into assert_that/Unkey
root: INFO: 2018-01-09T03:06:42.661Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f22ff): Fusing consumer assert_that/Unkey into assert_that/Group/Map(_merge_tagged_vals_under_key)
root: INFO: 2018-01-09T03:06:42.686Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2045): Fusing consumer assert_that/Group/GroupByKey/GroupByWindow into assert_that/Group/GroupByKey/Read
root: INFO: 2018-01-09T03:06:42.708Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2d8b): Unzipping flatten s10-u13 for input s11-reify-value0-c11
root: INFO: 2018-01-09T03:06:42.739Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2ad1): Fusing unzipped copy of assert_that/Group/GroupByKey/Write, through flatten s10-u13, into producer assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-09T03:06:42.770Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2817): Fusing consumer assert_that/Group/GroupByKey/Reify into assert_that/Group/pair_with_1
root: INFO: 2018-01-09T03:06:42.792Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f255d): Fusing consumer assert_that/Group/GroupByKey/Write into assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-09T03:06:42.810Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f22a3): Fusing consumer assert_that/Group/pair_with_0 into assert_that/Create/Read
root: INFO: 2018-01-09T03:06:42.842Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2fe9): Fusing consumer assert_that/WindowInto(WindowIntoFn) into compute/compute
root: INFO: 2018-01-09T03:06:42.868Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2d2f): Fusing consumer assert_that/Group/pair_with_1 into assert_that/ToVoidKey
root: INFO: 2018-01-09T03:06:42.898Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f2a75): Fusing consumer assert_that/ToVoidKey into assert_that/WindowInto(WindowIntoFn)
root: INFO: 2018-01-09T03:06:42.921Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f27bb): Fusing consumer compute/compute into start/Read
root: INFO: 2018-01-09T03:06:42.957Z: JOB_MESSAGE_DEBUG: (75eeccd10b8f2501): Workflow config is missing a default resource spec.
root: INFO: 2018-01-09T03:06:42.990Z: JOB_MESSAGE_DEBUG: (75eeccd10b8f2247): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-09T03:06:43.014Z: JOB_MESSAGE_DEBUG: (75eeccd10b8f2f8d): Adding workflow start and stop steps.
root: INFO: 2018-01-09T03:06:43.038Z: JOB_MESSAGE_DEBUG: (75eeccd10b8f2cd3): Assigning stage ids.
root: INFO: 2018-01-09T03:06:43.193Z: JOB_MESSAGE_DEBUG: (485b45dceb3a8584): Executing wait step start21
root: INFO: 2018-01-09T03:06:43.255Z: JOB_MESSAGE_BASIC: (108e9bbab0905707): Executing operation side/Read
root: INFO: 2018-01-09T03:06:43.323Z: JOB_MESSAGE_DEBUG: (108e9bbab0905ce1): Value "side/Read.out" materialized.
root: INFO: 2018-01-09T03:06:43.379Z: JOB_MESSAGE_BASIC: (108e9bbab090546c): Executing operation compute/_UnpickledSideInput(Read.out.0)
root: INFO: 2018-01-09T03:06:43.440Z: JOB_MESSAGE_DEBUG: (485b45dceb3a8a07): Value "compute/_UnpickledSideInput(Read.out.0).output" materialized.
root: INFO: 2018-01-09T03:06:43.499Z: JOB_MESSAGE_BASIC: (108e9bbab0905a46): Executing operation assert_that/Group/GroupByKey/Create
root: INFO: 2018-01-09T03:06:43.535Z: JOB_MESSAGE_DEBUG: (f2703e96358f4fa3): Starting worker pool setup.
root: INFO: 2018-01-09T03:06:43.567Z: JOB_MESSAGE_BASIC: (f2703e96358f4a29): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-09T03:06:43.623Z: JOB_MESSAGE_DEBUG: (8c95da77d47d1068): Value "assert_that/Group/GroupByKey/Session" materialized.
root: INFO: Job 2018-01-08_19_06_38-16327728576386954215 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-09T03:06:43.676Z: JOB_MESSAGE_BASIC: (108e9bbab090595c): Executing operation start/Read+compute/compute+assert_that/WindowInto(WindowIntoFn)+assert_that/ToVoidKey+assert_that/Group/pair_with_1+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-09T03:06:43.708Z: JOB_MESSAGE_BASIC: (8c95da77d47d1081): Executing operation assert_that/Create/Read+assert_that/Group/pair_with_0+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-09T03:06:52.076Z: JOB_MESSAGE_DETAILED: (c1899ea7056ba0d1): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-09T03:07:21.217Z: JOB_MESSAGE_ERROR: (c1899ea7056ba9de): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-09T03:07:21.243Z: JOB_MESSAGE_ERROR: (c1899ea7056ba3fc): Workflow failed.
root: INFO: 2018-01-09T03:07:21.494Z: JOB_MESSAGE_DETAILED: (75eeccd10b8f208e): Cleaning up.
root: INFO: 2018-01-09T03:07:21.536Z: JOB_MESSAGE_DEBUG: (75eeccd10b8f2b1a): Starting worker pool teardown.
root: INFO: 2018-01-09T03:07:21.561Z: JOB_MESSAGE_BASIC: (75eeccd10b8f2860): Stopping worker pool...
root: INFO: 2018-01-09T03:08:36.465Z: JOB_MESSAGE_BASIC: (75eeccd10b8f2d78): Worker pool stopped.
root: INFO: 2018-01-09T03:08:36.516Z: JOB_MESSAGE_DEBUG: (75eeccd10b8f2804): Tearing down pending resources...
root: INFO: Job 2018-01-08_19_06_38-16327728576386954215 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 15 tests in 534.909s
FAILED (errors=14)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Build failed in Jenkins:
beam_PostCommit_Python_ValidatesRunner_Dataflow #604
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/604/display/redirect?page=changes>
Changes:
[chamikara] Reduce the size of file-based I/O tests
[chamikara] Reduce the size of file-based I/O tests
------------------------------------------
[...truncated 1.06 MB...]
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_merge_tagged_vals_under_key"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1380 bytes>",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key)"
}
},
{
"kind": "ParallelDo",
"name": "s15",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Unkey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s14"
},
"serialized_fn": "<string of 980 bytes>",
"user_name": "assert_that/Unkey"
}
},
{
"kind": "ParallelDo",
"name": "s16",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_equal"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Match.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s15"
},
"serialized_fn": "<string of 1160 bytes>",
"user_name": "assert_that/Match"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-08T22:08:31.908358Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_14_08_31-11140097440825730708'
location: u'us-central1'
name: u'beamapp-jenkins-0108220812-390292'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_14_08_31-11140097440825730708]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_14_08_31-11140097440825730708?project=apache-beam-testing
root: INFO: Job 2018-01-08_14_08_31-11140097440825730708 is in state JOB_STATE_PENDING
root: INFO: 2018-01-08T22:08:31.403Z: JOB_MESSAGE_WARNING: (9a99939983a7e488): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-08T22:08:33.565Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3dcec): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-08T22:08:34.645Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3de62): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T22:08:34.657Z: JOB_MESSAGE_DEBUG: (a1afa8577ed3d695): Combiner lifting skipped for step assert_that/Group/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-08T22:08:34.665Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d6b7): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T22:08:34.675Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d6d9): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-08T22:08:34.686Z: JOB_MESSAGE_DEBUG: (a1afa8577ed3d71d): Annotating graph with Autotuner information.
root: INFO: 2018-01-08T22:08:34.706Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d761): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-08T22:08:34.717Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d783): Unzipping flatten s12 for input s10.out
root: INFO: 2018-01-08T22:08:34.727Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d7a5): Fusing unzipped copy of assert_that/Group/GroupByKey/Reify, through flatten assert_that/Group/Flatten, into producer assert_that/Group/pair_with_0
root: INFO: 2018-01-08T22:08:34.736Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d7c7): Fusing consumer assert_that/Group/Map(_merge_tagged_vals_under_key) into assert_that/Group/GroupByKey/GroupByWindow
root: INFO: 2018-01-08T22:08:34.745Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d7e9): Fusing consumer assert_that/Unkey into assert_that/Group/Map(_merge_tagged_vals_under_key)
root: INFO: 2018-01-08T22:08:34.756Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d80b): Fusing consumer assert_that/Group/GroupByKey/GroupByWindow into assert_that/Group/GroupByKey/Read
root: INFO: 2018-01-08T22:08:34.766Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d82d): Fusing consumer assert_that/Match into assert_that/Unkey
root: INFO: 2018-01-08T22:08:34.775Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d84f): Unzipping flatten s12-u13 for input s13-reify-value0-c11
root: INFO: 2018-01-08T22:08:34.785Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d871): Fusing unzipped copy of assert_that/Group/GroupByKey/Write, through flatten s12-u13, into producer assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-08T22:08:34.794Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d893): Fusing consumer assert_that/Group/pair_with_0 into assert_that/Create/Read
root: INFO: 2018-01-08T22:08:34.804Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d8b5): Fusing consumer assert_that/Group/GroupByKey/Write into assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-08T22:08:34.813Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d8d7): Fusing consumer assert_that/Group/GroupByKey/Reify into assert_that/Group/pair_with_1
root: INFO: 2018-01-08T22:08:34.823Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d8f9): Fusing consumer FlatMap(<lambda at sideinputs_test.py:302>)/FlatMap(<lambda at sideinputs_test.py:302>) into main input/Read
root: INFO: 2018-01-08T22:08:34.833Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d91b): Fusing consumer assert_that/Group/pair_with_1 into assert_that/ToVoidKey
root: INFO: 2018-01-08T22:08:34.843Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d93d): Fusing consumer assert_that/WindowInto(WindowIntoFn) into FlatMap(<lambda at sideinputs_test.py:302>)/FlatMap(<lambda at sideinputs_test.py:302>)
root: INFO: 2018-01-08T22:08:34.853Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d95f): Fusing consumer assert_that/ToVoidKey into assert_that/WindowInto(WindowIntoFn)
root: INFO: 2018-01-08T22:08:34.864Z: JOB_MESSAGE_DEBUG: (a1afa8577ed3d981): Workflow config is missing a default resource spec.
root: INFO: 2018-01-08T22:08:34.875Z: JOB_MESSAGE_DEBUG: (a1afa8577ed3d9a3): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-08T22:08:34.885Z: JOB_MESSAGE_DEBUG: (a1afa8577ed3d9c5): Adding workflow start and stop steps.
root: INFO: 2018-01-08T22:08:34.894Z: JOB_MESSAGE_DEBUG: (a1afa8577ed3d9e7): Assigning stage ids.
root: INFO: 2018-01-08T22:08:34.952Z: JOB_MESSAGE_DEBUG: (47947800a1cf097f): Executing wait step start22
root: INFO: 2018-01-08T22:08:34.975Z: JOB_MESSAGE_BASIC: (47947800a1cf0c6d): Executing operation side1/Read
root: INFO: 2018-01-08T22:08:34.984Z: JOB_MESSAGE_BASIC: (e69998e5c77eba30): Executing operation side2/Read
root: INFO: 2018-01-08T22:08:34.997Z: JOB_MESSAGE_DEBUG: (47947800a1cf0f5b): Value "side1/Read.out" materialized.
root: INFO: 2018-01-08T22:08:35.009Z: JOB_MESSAGE_DEBUG: (e69998e5c77eb46e): Value "side2/Read.out" materialized.
root: INFO: 2018-01-08T22:08:35.029Z: JOB_MESSAGE_BASIC: (e69998e5c77ebcf7): Executing operation Flatten
root: INFO: 2018-01-08T22:08:35.052Z: JOB_MESSAGE_DEBUG: (e69998e5c77ebe09): Value "Flatten.out" materialized.
root: INFO: 2018-01-08T22:08:35.073Z: JOB_MESSAGE_BASIC: (e69998e5c77eb43a): Executing operation FlatMap(<lambda at sideinputs_test.py:302>)/_UnpickledSideInput(Flatten.out.0)
root: INFO: Job 2018-01-08_14_08_31-11140097440825730708 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-08T22:08:35.095Z: JOB_MESSAGE_DEBUG: (e69998e5c77ebc20): Value "FlatMap(<lambda at sideinputs_test.py:302>)/_UnpickledSideInput(Flatten.out.0).output" materialized.
root: INFO: 2018-01-08T22:08:35.116Z: JOB_MESSAGE_BASIC: (e69998e5c77eb406): Executing operation assert_that/Group/GroupByKey/Create
root: INFO: 2018-01-08T22:08:35.136Z: JOB_MESSAGE_DEBUG: (fad1fc152a3a423f): Starting worker pool setup.
root: INFO: 2018-01-08T22:08:35.144Z: JOB_MESSAGE_BASIC: (fad1fc152a3a4e05): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-08T22:08:35.175Z: JOB_MESSAGE_DEBUG: (e69998e5c77eb518): Value "assert_that/Group/GroupByKey/Session" materialized.
root: INFO: 2018-01-08T22:08:35.199Z: JOB_MESSAGE_BASIC: (e69998e5c77eb475): Executing operation main input/Read+FlatMap(<lambda at sideinputs_test.py:302>)/FlatMap(<lambda at sideinputs_test.py:302>)+assert_that/WindowInto(WindowIntoFn)+assert_that/ToVoidKey+assert_that/Group/pair_with_1+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-08T22:08:35.208Z: JOB_MESSAGE_BASIC: (f3e70b4b25edbbe8): Executing operation assert_that/Create/Read+assert_that/Group/pair_with_0+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-08T22:08:43.518Z: JOB_MESSAGE_DETAILED: (e9bcfc76590d5830): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-08T22:09:12.410Z: JOB_MESSAGE_ERROR: (e9bcfc76590d5319): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-08T22:09:12.428Z: JOB_MESSAGE_ERROR: (e9bcfc76590d5a1f): Workflow failed.
root: INFO: 2018-01-08T22:09:12.542Z: JOB_MESSAGE_DETAILED: (a1afa8577ed3d280): Cleaning up.
root: INFO: 2018-01-08T22:09:12.576Z: JOB_MESSAGE_DEBUG: (a1afa8577ed3d2c4): Starting worker pool teardown.
root: INFO: 2018-01-08T22:09:12.593Z: JOB_MESSAGE_BASIC: (a1afa8577ed3d2e6): Stopping worker pool...
root: INFO: 2018-01-08T22:10:31.060Z: JOB_MESSAGE_BASIC: (a1afa8577ed3d36e): Worker pool stopped.
root: INFO: 2018-01-08T22:10:31.083Z: JOB_MESSAGE_DEBUG: (a1afa8577ed3d3b2): Tearing down pending resources...
root: INFO: Job 2018-01-08_14_08_31-11140097440825730708 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 15 tests in 606.712s
FAILED (errors=14)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Build failed in Jenkins:
beam_PostCommit_Python_ValidatesRunner_Dataflow #603
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/603/display/redirect>
------------------------------------------
[...truncated 1005.10 KB...]
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_merge_tagged_vals_under_key"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s11"
},
"serialized_fn": "<string of 1380 bytes>",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key)"
}
},
{
"kind": "ParallelDo",
"name": "s13",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Unkey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s12"
},
"serialized_fn": "<string of 980 bytes>",
"user_name": "assert_that/Unkey"
}
},
{
"kind": "ParallelDo",
"name": "s14",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_equal"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Match.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1160 bytes>",
"user_name": "assert_that/Match"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-08T21:10:25.794969Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_13_10_25-424042131538349009'
location: u'us-central1'
name: u'beamapp-jenkins-0108211009-263875'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_13_10_25-424042131538349009]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_13_10_25-424042131538349009?project=apache-beam-testing
root: INFO: Job 2018-01-08_13_10_25-424042131538349009 is in state JOB_STATE_PENDING
root: INFO: 2018-01-08T21:10:25.250Z: JOB_MESSAGE_WARNING: (5e280126c2489c5): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-08T21:10:27.305Z: JOB_MESSAGE_DETAILED: (91d5aab573cfb1a1): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-08T21:10:28.275Z: JOB_MESSAGE_DETAILED: (91d5aab573cfbcef): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T21:10:28.305Z: JOB_MESSAGE_DEBUG: (91d5aab573cfbfee): Combiner lifting skipped for step assert_that/Group/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-08T21:10:28.327Z: JOB_MESSAGE_DETAILED: (91d5aab573cfbc98): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T21:10:28.357Z: JOB_MESSAGE_DETAILED: (91d5aab573cfb942): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-08T21:10:28.389Z: JOB_MESSAGE_DEBUG: (91d5aab573cfb296): Annotating graph with Autotuner information.
root: INFO: 2018-01-08T21:10:28.426Z: JOB_MESSAGE_DETAILED: (91d5aab573cfbbea): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-08T21:10:28.451Z: JOB_MESSAGE_DETAILED: (91d5aab573cfb894): Unzipping flatten s10 for input s8.out
root: INFO: 2018-01-08T21:10:28.482Z: JOB_MESSAGE_DETAILED: (91d5aab573cfb53e): Fusing unzipped copy of assert_that/Group/GroupByKey/Reify, through flatten assert_that/Group/Flatten, into producer assert_that/Group/pair_with_0
root: INFO: 2018-01-08T21:10:28.512Z: JOB_MESSAGE_DETAILED: (91d5aab573cfb1e8): Fusing consumer assert_that/Group/Map(_merge_tagged_vals_under_key) into assert_that/Group/GroupByKey/GroupByWindow
root: INFO: 2018-01-08T21:10:28.537Z: JOB_MESSAGE_DETAILED: (91d5aab573cfbe92): Fusing consumer assert_that/Match into assert_that/Unkey
root: INFO: 2018-01-08T21:10:28.567Z: JOB_MESSAGE_DETAILED: (91d5aab573cfbb3c): Fusing consumer assert_that/Unkey into assert_that/Group/Map(_merge_tagged_vals_under_key)
root: INFO: 2018-01-08T21:10:28.597Z: JOB_MESSAGE_DETAILED: (91d5aab573cfb7e6): Fusing consumer assert_that/Group/GroupByKey/GroupByWindow into assert_that/Group/GroupByKey/Read
root: INFO: 2018-01-08T21:10:28.623Z: JOB_MESSAGE_DETAILED: (91d5aab573cfb490): Unzipping flatten s10-u13 for input s11-reify-value0-c11
root: INFO: 2018-01-08T21:10:28.649Z: JOB_MESSAGE_DETAILED: (91d5aab573cfb13a): Fusing unzipped copy of assert_that/Group/GroupByKey/Write, through flatten s10-u13, into producer assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-08T21:10:28.682Z: JOB_MESSAGE_DETAILED: (91d5aab573cfbde4): Fusing consumer assert_that/Group/GroupByKey/Reify into assert_that/Group/pair_with_1
root: INFO: 2018-01-08T21:10:28.711Z: JOB_MESSAGE_DETAILED: (91d5aab573cfba8e): Fusing consumer assert_that/Group/GroupByKey/Write into assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-08T21:10:28.742Z: JOB_MESSAGE_DETAILED: (91d5aab573cfb738): Fusing consumer assert_that/Group/pair_with_0 into assert_that/Create/Read
root: INFO: 2018-01-08T21:10:28.772Z: JOB_MESSAGE_DETAILED: (91d5aab573cfb3e2): Fusing consumer assert_that/WindowInto(WindowIntoFn) into compute/compute
root: INFO: 2018-01-08T21:10:28.804Z: JOB_MESSAGE_DETAILED: (91d5aab573cfb08c): Fusing consumer assert_that/Group/pair_with_1 into assert_that/ToVoidKey
root: INFO: 2018-01-08T21:10:28.827Z: JOB_MESSAGE_DETAILED: (91d5aab573cfbd36): Fusing consumer assert_that/ToVoidKey into assert_that/WindowInto(WindowIntoFn)
root: INFO: 2018-01-08T21:10:28.845Z: JOB_MESSAGE_DETAILED: (91d5aab573cfb9e0): Fusing consumer compute/compute into start/Read
root: INFO: 2018-01-08T21:10:28.868Z: JOB_MESSAGE_DEBUG: (91d5aab573cfb68a): Workflow config is missing a default resource spec.
root: INFO: 2018-01-08T21:10:28.900Z: JOB_MESSAGE_DEBUG: (91d5aab573cfb334): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-08T21:10:28.925Z: JOB_MESSAGE_DEBUG: (91d5aab573cfbfde): Adding workflow start and stop steps.
root: INFO: 2018-01-08T21:10:28.954Z: JOB_MESSAGE_DEBUG: (91d5aab573cfbc88): Assigning stage ids.
root: INFO: 2018-01-08T21:10:29.073Z: JOB_MESSAGE_DEBUG: (19d50f09693aff04): Executing wait step start21
root: INFO: 2018-01-08T21:10:29.120Z: JOB_MESSAGE_BASIC: (6bcb03754297fd24): Executing operation side/Read
root: INFO: 2018-01-08T21:10:29.180Z: JOB_MESSAGE_DEBUG: (19d50f09693af725): Value "side/Read.out" materialized.
root: INFO: 2018-01-08T21:10:29.236Z: JOB_MESSAGE_BASIC: (6bcb03754297fea2): Executing operation compute/_UnpickledSideInput(Read.out.0)
root: INFO: 2018-01-08T21:10:29.300Z: JOB_MESSAGE_DEBUG: (5aa939028304dcac): Value "compute/_UnpickledSideInput(Read.out.0).output" materialized.
root: INFO: 2018-01-08T21:10:29.359Z: JOB_MESSAGE_BASIC: (6bcb03754297f020): Executing operation assert_that/Group/GroupByKey/Create
root: INFO: 2018-01-08T21:10:29.398Z: JOB_MESSAGE_DEBUG: (6901911a92768729): Starting worker pool setup.
root: INFO: 2018-01-08T21:10:29.429Z: JOB_MESSAGE_BASIC: (6901911a92768903): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-08T21:10:29.495Z: JOB_MESSAGE_DEBUG: (8b9e57450d9daa23): Value "assert_that/Group/GroupByKey/Session" materialized.
root: INFO: 2018-01-08T21:10:29.546Z: JOB_MESSAGE_BASIC: (8b9e57450d9da2b8): Executing operation start/Read+compute/compute+assert_that/WindowInto(WindowIntoFn)+assert_that/ToVoidKey+assert_that/Group/pair_with_1+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-08T21:10:29.569Z: JOB_MESSAGE_BASIC: (6bcb03754297f7f2): Executing operation assert_that/Create/Read+assert_that/Group/pair_with_0+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: Job 2018-01-08_13_10_25-424042131538349009 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-08T21:10:37.831Z: JOB_MESSAGE_DETAILED: (19d50f09693afb11): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-08T21:11:07.534Z: JOB_MESSAGE_ERROR: (19d50f09693af3f8): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-08T21:11:07.548Z: JOB_MESSAGE_ERROR: (19d50f09693af0d2): Workflow failed.
root: INFO: 2018-01-08T21:11:07.665Z: JOB_MESSAGE_DETAILED: (91d5aab573cfb585): Cleaning up.
root: INFO: 2018-01-08T21:11:07.698Z: JOB_MESSAGE_DEBUG: (91d5aab573cfbed9): Starting worker pool teardown.
root: INFO: 2018-01-08T21:11:07.717Z: JOB_MESSAGE_BASIC: (91d5aab573cfbb83): Stopping worker pool...
root: INFO: 2018-01-08T21:12:21.134Z: JOB_MESSAGE_BASIC: (91d5aab573cfbe2b): Worker pool stopped.
root: INFO: 2018-01-08T21:12:21.167Z: JOB_MESSAGE_DEBUG: (91d5aab573cfb77f): Tearing down pending resources...
root: INFO: Job 2018-01-08_13_10_25-424042131538349009 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 15 tests in 642.282s
FAILED (errors=12)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Build failed in Jenkins:
beam_PostCommit_Python_ValidatesRunner_Dataflow #602
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/602/display/redirect>
------------------------------------------
[...truncated 1.04 MB...]
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_merge_tagged_vals_under_key"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s11"
},
"serialized_fn": "<string of 1380 bytes>",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key)"
}
},
{
"kind": "ParallelDo",
"name": "s13",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Unkey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s12"
},
"serialized_fn": "<string of 980 bytes>",
"user_name": "assert_that/Unkey"
}
},
{
"kind": "ParallelDo",
"name": "s14",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_equal"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Match.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1160 bytes>",
"user_name": "assert_that/Match"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-08T15:07:35.759911Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_07_07_35-12501536408384030644'
location: u'us-central1'
name: u'beamapp-jenkins-0108150727-243211'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_07_07_35-12501536408384030644]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_07_07_35-12501536408384030644?project=apache-beam-testing
root: INFO: Job 2018-01-08_07_07_35-12501536408384030644 is in state JOB_STATE_PENDING
root: INFO: 2018-01-08T15:07:35.261Z: JOB_MESSAGE_WARNING: (ad7e6120a057c738): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-08T15:07:37.389Z: JOB_MESSAGE_DETAILED: (6c894b17b3471f53): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-08T15:07:38.459Z: JOB_MESSAGE_DETAILED: (6c894b17b3471445): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T15:07:38.484Z: JOB_MESSAGE_DEBUG: (6c894b17b3471666): Combiner lifting skipped for step assert_that/Group/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-08T15:07:38.504Z: JOB_MESSAGE_DETAILED: (6c894b17b347127c): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T15:07:38.536Z: JOB_MESSAGE_DETAILED: (6c894b17b3471e92): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-08T15:07:38.563Z: JOB_MESSAGE_DEBUG: (6c894b17b34716be): Annotating graph with Autotuner information.
root: INFO: 2018-01-08T15:07:38.601Z: JOB_MESSAGE_DETAILED: (6c894b17b3471eea): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-08T15:07:38.622Z: JOB_MESSAGE_DETAILED: (6c894b17b3471b00): Unzipping flatten s10 for input s8.out
root: INFO: 2018-01-08T15:07:38.646Z: JOB_MESSAGE_DETAILED: (6c894b17b3471716): Fusing unzipped copy of assert_that/Group/GroupByKey/Reify, through flatten assert_that/Group/Flatten, into producer assert_that/Group/pair_with_0
root: INFO: 2018-01-08T15:07:38.670Z: JOB_MESSAGE_DETAILED: (6c894b17b347132c): Fusing consumer assert_that/Group/Map(_merge_tagged_vals_under_key) into assert_that/Group/GroupByKey/GroupByWindow
root: INFO: 2018-01-08T15:07:38.702Z: JOB_MESSAGE_DETAILED: (6c894b17b3471f42): Fusing consumer assert_that/Match into assert_that/Unkey
root: INFO: 2018-01-08T15:07:38.725Z: JOB_MESSAGE_DETAILED: (6c894b17b3471b58): Fusing consumer assert_that/Unkey into assert_that/Group/Map(_merge_tagged_vals_under_key)
root: INFO: 2018-01-08T15:07:38.755Z: JOB_MESSAGE_DETAILED: (6c894b17b347176e): Fusing consumer assert_that/Group/GroupByKey/GroupByWindow into assert_that/Group/GroupByKey/Read
root: INFO: 2018-01-08T15:07:38.787Z: JOB_MESSAGE_DETAILED: (6c894b17b3471384): Unzipping flatten s10-u13 for input s11-reify-value0-c11
root: INFO: 2018-01-08T15:07:38.813Z: JOB_MESSAGE_DETAILED: (6c894b17b3471f9a): Fusing unzipped copy of assert_that/Group/GroupByKey/Write, through flatten s10-u13, into producer assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-08T15:07:38.843Z: JOB_MESSAGE_DETAILED: (6c894b17b3471bb0): Fusing consumer assert_that/Group/GroupByKey/Reify into assert_that/Group/pair_with_1
root: INFO: 2018-01-08T15:07:38.875Z: JOB_MESSAGE_DETAILED: (6c894b17b34717c6): Fusing consumer assert_that/Group/GroupByKey/Write into assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-08T15:07:38.906Z: JOB_MESSAGE_DETAILED: (6c894b17b34713dc): Fusing consumer assert_that/Group/pair_with_0 into assert_that/Create/Read
root: INFO: 2018-01-08T15:07:38.931Z: JOB_MESSAGE_DETAILED: (6c894b17b3471ff2): Fusing consumer assert_that/WindowInto(WindowIntoFn) into compute/compute
root: INFO: 2018-01-08T15:07:38.957Z: JOB_MESSAGE_DETAILED: (6c894b17b3471c08): Fusing consumer assert_that/Group/pair_with_1 into assert_that/ToVoidKey
root: INFO: 2018-01-08T15:07:38.972Z: JOB_MESSAGE_DETAILED: (6c894b17b347181e): Fusing consumer assert_that/ToVoidKey into assert_that/WindowInto(WindowIntoFn)
root: INFO: 2018-01-08T15:07:39.001Z: JOB_MESSAGE_DETAILED: (6c894b17b3471434): Fusing consumer compute/compute into start/Read
root: INFO: 2018-01-08T15:07:39.024Z: JOB_MESSAGE_DEBUG: (6c894b17b347104a): Workflow config is missing a default resource spec.
root: INFO: 2018-01-08T15:07:39.057Z: JOB_MESSAGE_DEBUG: (6c894b17b3471c60): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-08T15:07:39.078Z: JOB_MESSAGE_DEBUG: (6c894b17b3471876): Adding workflow start and stop steps.
root: INFO: 2018-01-08T15:07:39.101Z: JOB_MESSAGE_DEBUG: (6c894b17b347148c): Assigning stage ids.
root: INFO: 2018-01-08T15:07:39.232Z: JOB_MESSAGE_DEBUG: (3a907acd7cc28f42): Executing wait step start21
root: INFO: 2018-01-08T15:07:39.281Z: JOB_MESSAGE_BASIC: (bdfb51be8ec14051): Executing operation side/Read
root: INFO: 2018-01-08T15:07:39.348Z: JOB_MESSAGE_DEBUG: (3a907acd7cc28ce3): Value "side/Read.out" materialized.
root: INFO: 2018-01-08T15:07:39.403Z: JOB_MESSAGE_BASIC: (bdfb51be8ec14faf): Executing operation compute/_UnpickledSideInput(Read.out.0)
root: INFO: 2018-01-08T15:07:39.453Z: JOB_MESSAGE_DEBUG: (3a907acd7cc28a84): Value "compute/_UnpickledSideInput(Read.out.0).output" materialized.
root: INFO: 2018-01-08T15:07:39.505Z: JOB_MESSAGE_BASIC: (3a907acd7cc28e12): Executing operation assert_that/Group/GroupByKey/Create
root: INFO: 2018-01-08T15:07:39.549Z: JOB_MESSAGE_DEBUG: (269ddaab21ecdb30): Starting worker pool setup.
root: INFO: 2018-01-08T15:07:39.572Z: JOB_MESSAGE_BASIC: (269ddaab21ecdd22): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-08T15:07:39.648Z: JOB_MESSAGE_DEBUG: (3a907acd7cc28954): Value "assert_that/Group/GroupByKey/Session" materialized.
root: INFO: 2018-01-08T15:07:39.707Z: JOB_MESSAGE_BASIC: (3a907acd7cc28ea9): Executing operation start/Read+compute/compute+assert_that/WindowInto(WindowIntoFn)+assert_that/ToVoidKey+assert_that/Group/pair_with_1+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-08T15:07:39.740Z: JOB_MESSAGE_BASIC: (fa4899e609a1bc3c): Executing operation assert_that/Create/Read+assert_that/Group/pair_with_0+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: Job 2018-01-08_07_07_35-12501536408384030644 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-08T15:07:46.113Z: JOB_MESSAGE_DETAILED: (de511b4a61c001b0): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-08T15:08:15.690Z: JOB_MESSAGE_ERROR: (de511b4a61c00c6d): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-08T15:08:15.716Z: JOB_MESSAGE_ERROR: (de511b4a61c00bab): Workflow failed.
root: INFO: 2018-01-08T15:08:15.971Z: JOB_MESSAGE_DETAILED: (6c894b17b3471aef): Cleaning up.
root: INFO: 2018-01-08T15:08:16.013Z: JOB_MESSAGE_DEBUG: (6c894b17b347131b): Starting worker pool teardown.
root: INFO: 2018-01-08T15:08:16.037Z: JOB_MESSAGE_BASIC: (6c894b17b3471f31): Stopping worker pool...
root: INFO: 2018-01-08T15:09:29.387Z: JOB_MESSAGE_DEBUG: (6c894b17b3471b9f): Tearing down pending resources...
root: INFO: Job 2018-01-08_07_07_35-12501536408384030644 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 15 tests in 832.084s
FAILED (errors=13)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Build failed in Jenkins:
beam_PostCommit_Python_ValidatesRunner_Dataflow #601
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/601/display/redirect?page=changes>
Changes:
[iemejia] [BEAM-3373] Add serviceEndpoint parameter to KinesisIO
------------------------------------------
[...truncated 1023.13 KB...]
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_merge_tagged_vals_under_key"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1380 bytes>",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key)"
}
},
{
"kind": "ParallelDo",
"name": "s15",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Unkey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s14"
},
"serialized_fn": "<string of 980 bytes>",
"user_name": "assert_that/Unkey"
}
},
{
"kind": "ParallelDo",
"name": "s16",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_equal"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Match.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s15"
},
"serialized_fn": "<string of 1160 bytes>",
"user_name": "assert_that/Match"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-08T14:53:42.864231Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_06_53_42-7960447886795721241'
location: u'us-central1'
name: u'beamapp-jenkins-0108145336-363498'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_06_53_42-7960447886795721241]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_06_53_42-7960447886795721241?project=apache-beam-testing
root: INFO: Job 2018-01-08_06_53_42-7960447886795721241 is in state JOB_STATE_PENDING
root: INFO: 2018-01-08T14:53:42.329Z: JOB_MESSAGE_WARNING: (6e79312d4177c3c5): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-08T14:53:44.605Z: JOB_MESSAGE_DETAILED: (37549e03c86ad362): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-08T14:53:45.674Z: JOB_MESSAGE_DETAILED: (37549e03c86ad7c8): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T14:53:45.689Z: JOB_MESSAGE_DEBUG: (37549e03c86ade33): Combiner lifting skipped for step assert_that/Group/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-08T14:53:45.700Z: JOB_MESSAGE_DETAILED: (37549e03c86add25): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T14:53:45.712Z: JOB_MESSAGE_DETAILED: (37549e03c86adc17): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-08T14:53:45.725Z: JOB_MESSAGE_DEBUG: (37549e03c86ad9fb): Annotating graph with Autotuner information.
root: INFO: 2018-01-08T14:53:45.746Z: JOB_MESSAGE_DETAILED: (37549e03c86ad7df): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-08T14:53:45.759Z: JOB_MESSAGE_DETAILED: (37549e03c86ad6d1): Unzipping flatten s12 for input s10.out
root: INFO: 2018-01-08T14:53:45.769Z: JOB_MESSAGE_DETAILED: (37549e03c86ad5c3): Fusing unzipped copy of assert_that/Group/GroupByKey/Reify, through flatten assert_that/Group/Flatten, into producer assert_that/Group/pair_with_0
root: INFO: 2018-01-08T14:53:45.779Z: JOB_MESSAGE_DETAILED: (37549e03c86ad4b5): Fusing consumer assert_that/Group/Map(_merge_tagged_vals_under_key) into assert_that/Group/GroupByKey/GroupByWindow
root: INFO: 2018-01-08T14:53:45.789Z: JOB_MESSAGE_DETAILED: (37549e03c86ad3a7): Fusing consumer assert_that/Unkey into assert_that/Group/Map(_merge_tagged_vals_under_key)
root: INFO: 2018-01-08T14:53:45.799Z: JOB_MESSAGE_DETAILED: (37549e03c86ad299): Fusing consumer assert_that/Group/GroupByKey/GroupByWindow into assert_that/Group/GroupByKey/Read
root: INFO: 2018-01-08T14:53:45.810Z: JOB_MESSAGE_DETAILED: (37549e03c86ad18b): Fusing consumer assert_that/Match into assert_that/Unkey
root: INFO: 2018-01-08T14:53:45.820Z: JOB_MESSAGE_DETAILED: (37549e03c86ad07d): Unzipping flatten s12-u13 for input s13-reify-value0-c11
root: INFO: 2018-01-08T14:53:45.832Z: JOB_MESSAGE_DETAILED: (37549e03c86adf6f): Fusing unzipped copy of assert_that/Group/GroupByKey/Write, through flatten s12-u13, into producer assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-08T14:53:45.843Z: JOB_MESSAGE_DETAILED: (37549e03c86ade61): Fusing consumer assert_that/Group/pair_with_0 into assert_that/Create/Read
root: INFO: 2018-01-08T14:53:45.853Z: JOB_MESSAGE_DETAILED: (37549e03c86add53): Fusing consumer assert_that/Group/GroupByKey/Write into assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-08T14:53:45.863Z: JOB_MESSAGE_DETAILED: (37549e03c86adc45): Fusing consumer assert_that/Group/GroupByKey/Reify into assert_that/Group/pair_with_1
root: INFO: 2018-01-08T14:53:45.874Z: JOB_MESSAGE_DETAILED: (37549e03c86adb37): Fusing consumer FlatMap(<lambda at sideinputs_test.py:302>)/FlatMap(<lambda at sideinputs_test.py:302>) into main input/Read
root: INFO: 2018-01-08T14:53:45.885Z: JOB_MESSAGE_DETAILED: (37549e03c86ada29): Fusing consumer assert_that/Group/pair_with_1 into assert_that/ToVoidKey
root: INFO: 2018-01-08T14:53:45.896Z: JOB_MESSAGE_DETAILED: (37549e03c86ad91b): Fusing consumer assert_that/WindowInto(WindowIntoFn) into FlatMap(<lambda at sideinputs_test.py:302>)/FlatMap(<lambda at sideinputs_test.py:302>)
root: INFO: 2018-01-08T14:53:45.907Z: JOB_MESSAGE_DETAILED: (37549e03c86ad80d): Fusing consumer assert_that/ToVoidKey into assert_that/WindowInto(WindowIntoFn)
root: INFO: 2018-01-08T14:53:45.919Z: JOB_MESSAGE_DEBUG: (37549e03c86ad6ff): Workflow config is missing a default resource spec.
root: INFO: 2018-01-08T14:53:45.931Z: JOB_MESSAGE_DEBUG: (37549e03c86ad5f1): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-08T14:53:45.943Z: JOB_MESSAGE_DEBUG: (37549e03c86ad4e3): Adding workflow start and stop steps.
root: INFO: 2018-01-08T14:53:45.953Z: JOB_MESSAGE_DEBUG: (37549e03c86ad3d5): Assigning stage ids.
root: INFO: Job 2018-01-08_06_53_42-7960447886795721241 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-08T14:53:46.026Z: JOB_MESSAGE_DEBUG: (2ea63c2130f2a0da): Executing wait step start22
root: INFO: 2018-01-08T14:53:46.053Z: JOB_MESSAGE_BASIC: (2ea63c2130f2af6c): Executing operation side1/Read
root: INFO: 2018-01-08T14:53:46.063Z: JOB_MESSAGE_BASIC: (31cceceef2cf16a3): Executing operation side2/Read
root: INFO: 2018-01-08T14:53:46.079Z: JOB_MESSAGE_DEBUG: (2ea63c2130f2adfe): Value "side1/Read.out" materialized.
root: INFO: 2018-01-08T14:53:46.089Z: JOB_MESSAGE_DEBUG: (31cceceef2cf13d1): Value "side2/Read.out" materialized.
root: INFO: 2018-01-08T14:53:46.113Z: JOB_MESSAGE_BASIC: (31cceceef2cf1c22): Executing operation Flatten
root: INFO: 2018-01-08T14:53:46.139Z: JOB_MESSAGE_DEBUG: (31cceceef2cf1cc4): Value "Flatten.out" materialized.
root: INFO: 2018-01-08T14:53:46.163Z: JOB_MESSAGE_BASIC: (31cceceef2cf1bfd): Executing operation FlatMap(<lambda at sideinputs_test.py:302>)/_UnpickledSideInput(Flatten.out.0)
root: INFO: 2018-01-08T14:53:46.194Z: JOB_MESSAGE_DEBUG: (31cceceef2cf1013): Value "FlatMap(<lambda at sideinputs_test.py:302>)/_UnpickledSideInput(Flatten.out.0).output" materialized.
root: INFO: 2018-01-08T14:53:46.219Z: JOB_MESSAGE_BASIC: (31cceceef2cf1429): Executing operation assert_that/Group/GroupByKey/Create
root: INFO: 2018-01-08T14:53:46.243Z: JOB_MESSAGE_DEBUG: (d2e1a1e4004763e3): Starting worker pool setup.
root: INFO: 2018-01-08T14:53:46.252Z: JOB_MESSAGE_BASIC: (d2e1a1e40047619d): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-08T14:53:46.284Z: JOB_MESSAGE_DEBUG: (31cceceef2cf14cb): Value "assert_that/Group/GroupByKey/Session" materialized.
root: INFO: 2018-01-08T14:53:46.308Z: JOB_MESSAGE_BASIC: (31cceceef2cf1090): Executing operation main input/Read+FlatMap(<lambda at sideinputs_test.py:302>)/FlatMap(<lambda at sideinputs_test.py:302>)+assert_that/WindowInto(WindowIntoFn)+assert_that/ToVoidKey+assert_that/Group/pair_with_1+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-08T14:53:46.318Z: JOB_MESSAGE_BASIC: (f2ea77b38c5af838): Executing operation assert_that/Create/Read+assert_that/Group/pair_with_0+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-08T14:53:52.539Z: JOB_MESSAGE_DETAILED: (874d3f5cdb953b7c): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-08T14:54:15.984Z: JOB_MESSAGE_ERROR: (874d3f5cdb953e31): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-08T14:54:16.016Z: JOB_MESSAGE_ERROR: (874d3f5cdb953123): Workflow failed.
root: INFO: 2018-01-08T14:54:16.297Z: JOB_MESSAGE_DETAILED: (37549e03c86ad716): Cleaning up.
root: INFO: 2018-01-08T14:54:16.350Z: JOB_MESSAGE_DEBUG: (37549e03c86ad4fa): Starting worker pool teardown.
root: INFO: 2018-01-08T14:54:16.381Z: JOB_MESSAGE_BASIC: (37549e03c86ad3ec): Stopping worker pool...
root: INFO: 2018-01-08T14:55:25.456Z: JOB_MESSAGE_BASIC: (37549e03c86adfb4): Worker pool stopped.
root: INFO: 2018-01-08T14:55:25.495Z: JOB_MESSAGE_DEBUG: (37549e03c86add98): Tearing down pending resources...
root: INFO: Job 2018-01-08_06_53_42-7960447886795721241 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 15 tests in 783.542s
FAILED (errors=13)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Build failed in Jenkins:
beam_PostCommit_Python_ValidatesRunner_Dataflow #600
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/600/display/redirect>
------------------------------------------
[...truncated 1.06 MB...]
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_merge_tagged_vals_under_key"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s11"
},
"serialized_fn": "<string of 1380 bytes>",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key)"
}
},
{
"kind": "ParallelDo",
"name": "s13",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Unkey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s12"
},
"serialized_fn": "<string of 980 bytes>",
"user_name": "assert_that/Unkey"
}
},
{
"kind": "ParallelDo",
"name": "s14",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_equal"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Match.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1160 bytes>",
"user_name": "assert_that/Match"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-08T09:18:38.903757Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_01_18_38-12898849157086774885'
location: u'us-central1'
name: u'beamapp-jenkins-0108091833-283318'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_01_18_38-12898849157086774885]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_01_18_38-12898849157086774885?project=apache-beam-testing
root: INFO: Job 2018-01-08_01_18_38-12898849157086774885 is in state JOB_STATE_PENDING
root: INFO: 2018-01-08T09:18:38.328Z: JOB_MESSAGE_WARNING: (b301eaf7a38103d9): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-08T09:18:40.671Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa78e): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-08T09:18:41.974Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa7e4): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T09:18:41.993Z: JOB_MESSAGE_DEBUG: (65f63b7932cfa507): Combiner lifting skipped for step assert_that/Group/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-08T09:18:42.010Z: JOB_MESSAGE_DETAILED: (65f63b7932cfadc9): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T09:18:42.033Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa68b): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-08T09:18:42.055Z: JOB_MESSAGE_DEBUG: (65f63b7932cfa80f): Annotating graph with Autotuner information.
root: INFO: 2018-01-08T09:18:42.090Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa993): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-08T09:18:42.103Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa255): Unzipping flatten s10 for input s8.out
root: INFO: 2018-01-08T09:18:42.121Z: JOB_MESSAGE_DETAILED: (65f63b7932cfab17): Fusing unzipped copy of assert_that/Group/GroupByKey/Reify, through flatten assert_that/Group/Flatten, into producer assert_that/Group/pair_with_0
root: INFO: 2018-01-08T09:18:42.134Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa3d9): Fusing consumer assert_that/Group/Map(_merge_tagged_vals_under_key) into assert_that/Group/GroupByKey/GroupByWindow
root: INFO: 2018-01-08T09:18:42.149Z: JOB_MESSAGE_DETAILED: (65f63b7932cfac9b): Fusing consumer assert_that/Match into assert_that/Unkey
root: INFO: 2018-01-08T09:18:42.162Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa55d): Fusing consumer assert_that/Unkey into assert_that/Group/Map(_merge_tagged_vals_under_key)
root: INFO: 2018-01-08T09:18:42.179Z: JOB_MESSAGE_DETAILED: (65f63b7932cfae1f): Fusing consumer assert_that/Group/GroupByKey/GroupByWindow into assert_that/Group/GroupByKey/Read
root: INFO: 2018-01-08T09:18:42.192Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa6e1): Unzipping flatten s10-u13 for input s11-reify-value0-c11
root: INFO: 2018-01-08T09:18:42.209Z: JOB_MESSAGE_DETAILED: (65f63b7932cfafa3): Fusing unzipped copy of assert_that/Group/GroupByKey/Write, through flatten s10-u13, into producer assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-08T09:18:42.223Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa865): Fusing consumer assert_that/Group/GroupByKey/Reify into assert_that/Group/pair_with_1
root: INFO: 2018-01-08T09:18:42.237Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa127): Fusing consumer assert_that/Group/GroupByKey/Write into assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-08T09:18:42.254Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa9e9): Fusing consumer assert_that/Group/pair_with_0 into assert_that/Create/Read
root: INFO: 2018-01-08T09:18:42.276Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa2ab): Fusing consumer assert_that/WindowInto(WindowIntoFn) into compute/compute
root: INFO: 2018-01-08T09:18:42.294Z: JOB_MESSAGE_DETAILED: (65f63b7932cfab6d): Fusing consumer assert_that/Group/pair_with_1 into assert_that/ToVoidKey
root: INFO: 2018-01-08T09:18:42.316Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa42f): Fusing consumer assert_that/ToVoidKey into assert_that/WindowInto(WindowIntoFn)
root: INFO: 2018-01-08T09:18:42.338Z: JOB_MESSAGE_DETAILED: (65f63b7932cfacf1): Fusing consumer compute/compute into start/Read
root: INFO: 2018-01-08T09:18:42.359Z: JOB_MESSAGE_DEBUG: (65f63b7932cfa5b3): Workflow config is missing a default resource spec.
root: INFO: 2018-01-08T09:18:42.376Z: JOB_MESSAGE_DEBUG: (65f63b7932cfae75): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-08T09:18:42.390Z: JOB_MESSAGE_DEBUG: (65f63b7932cfa737): Adding workflow start and stop steps.
root: INFO: 2018-01-08T09:18:42.403Z: JOB_MESSAGE_DEBUG: (65f63b7932cfaff9): Assigning stage ids.
root: INFO: 2018-01-08T09:18:42.500Z: JOB_MESSAGE_DEBUG: (141262fbb9dbb214): Executing wait step start21
root: INFO: 2018-01-08T09:18:42.542Z: JOB_MESSAGE_BASIC: (141262fbb9dbb8f9): Executing operation side/Read
root: INFO: 2018-01-08T09:18:42.596Z: JOB_MESSAGE_DEBUG: (141262fbb9dbb13f): Value "side/Read.out" materialized.
root: INFO: 2018-01-08T09:18:42.638Z: JOB_MESSAGE_BASIC: (141262fbb9dbb824): Executing operation compute/_UnpickledSideInput(Read.out.0)
root: INFO: 2018-01-08T09:18:42.688Z: JOB_MESSAGE_DEBUG: (930a2da5d7d00463): Value "compute/_UnpickledSideInput(Read.out.0).output" materialized.
root: INFO: 2018-01-08T09:18:42.736Z: JOB_MESSAGE_BASIC: (141262fbb9dbb06a): Executing operation assert_that/Group/GroupByKey/Create
root: INFO: 2018-01-08T09:18:42.770Z: JOB_MESSAGE_DEBUG: (8f1bae69a7659bac): Starting worker pool setup.
root: INFO: 2018-01-08T09:18:42.784Z: JOB_MESSAGE_BASIC: (8f1bae69a76598ea): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-08T09:18:42.831Z: JOB_MESSAGE_DEBUG: (77c742057664e089): Value "assert_that/Group/GroupByKey/Session" materialized.
root: INFO: 2018-01-08T09:18:42.871Z: JOB_MESSAGE_BASIC: (141262fbb9dbbe34): Executing operation start/Read+compute/compute+assert_that/WindowInto(WindowIntoFn)+assert_that/ToVoidKey+assert_that/Group/pair_with_1+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-08T09:18:42.895Z: JOB_MESSAGE_BASIC: (77c742057664e318): Executing operation assert_that/Create/Read+assert_that/Group/pair_with_0+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: Job 2018-01-08_01_18_38-12898849157086774885 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-08T09:18:49.587Z: JOB_MESSAGE_DETAILED: (883d437fc2841230): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-08T09:19:14.644Z: JOB_MESSAGE_ERROR: (883d437fc284184d): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-08T09:19:14.655Z: JOB_MESSAGE_ERROR: (883d437fc284144f): Workflow failed.
root: INFO: 2018-01-08T09:19:14.885Z: JOB_MESSAGE_DETAILED: (65f63b7932cfa762): Cleaning up.
root: INFO: 2018-01-08T09:19:14.914Z: JOB_MESSAGE_DEBUG: (65f63b7932cfa8e6): Starting worker pool teardown.
root: INFO: 2018-01-08T09:19:14.923Z: JOB_MESSAGE_BASIC: (65f63b7932cfa1a8): Stopping worker pool...
root: INFO: 2018-01-08T09:20:28.804Z: JOB_MESSAGE_BASIC: (65f63b7932cfa4b0): Worker pool stopped.
root: INFO: 2018-01-08T09:20:28.830Z: JOB_MESSAGE_DEBUG: (65f63b7932cfa634): Tearing down pending resources...
root: INFO: Job 2018-01-08_01_18_38-12898849157086774885 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 15 tests in 485.206s
FAILED (errors=14)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com
Build failed in Jenkins:
beam_PostCommit_Python_ValidatesRunner_Dataflow #599
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PostCommit_Python_ValidatesRunner_Dataflow/599/display/redirect?page=changes>
Changes:
[ankurgoenka] [BEAM-3411] Fixing KeyError when on progress update request
------------------------------------------
[...truncated 1.05 MB...]
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_merge_tagged_vals_under_key"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key).out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s11"
},
"serialized_fn": "<string of 1380 bytes>",
"user_name": "assert_that/Group/Map(_merge_tagged_vals_under_key)"
}
},
{
"kind": "ParallelDo",
"name": "s13",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "<lambda>"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Unkey.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s12"
},
"serialized_fn": "<string of 980 bytes>",
"user_name": "assert_that/Unkey"
}
},
{
"kind": "ParallelDo",
"name": "s14",
"properties": {
"display_data": [
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.CallableWrapperDoFn",
"type": "STRING",
"value": "_equal"
},
{
"key": "fn",
"label": "Transform Function",
"namespace": "apache_beam.transforms.core.ParDo",
"shortValue": "CallableWrapperDoFn",
"type": "STRING",
"value": "apache_beam.transforms.core.CallableWrapperDoFn"
}
],
"non_parallel_inputs": {},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
},
{
"@type": "FastPrimitivesCoder$eNprYEpOLEhMzkiNT0pNzNVLzk9JLSqGUlxuicUlAUWZuZklmWWpxc4gQa5CBs3GQsbaQqZQ/vi0xJycpMTk7Hiw+kJmPEYFZCZn56RCjWABGsFaW8iWVJykBwDlGS3/",
"component_encodings": []
}
],
"is_pair_like": true
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "out",
"user_name": "assert_that/Match.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s13"
},
"serialized_fn": "<string of 1160 bytes>",
"user_name": "assert_that/Match"
}
}
],
"type": "JOB_TYPE_BATCH"
}
root: INFO: Create job: <Job
createTime: u'2018-01-08T08:14:15.547011Z'
currentStateTime: u'1970-01-01T00:00:00Z'
id: u'2018-01-08_00_14_14-11501116380979676381'
location: u'us-central1'
name: u'beamapp-jenkins-0108081409-415938'
projectId: u'apache-beam-testing'
stageStates: []
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
root: INFO: Created job with id: [2018-01-08_00_14_14-11501116380979676381]
root: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobsDetail/locations/us-central1/jobs/2018-01-08_00_14_14-11501116380979676381?project=apache-beam-testing
root: INFO: Job 2018-01-08_00_14_14-11501116380979676381 is in state JOB_STATE_PENDING
root: INFO: 2018-01-08T08:14:14.826Z: JOB_MESSAGE_WARNING: (9f9c2c69b8b84861): Setting the number of workers (1) disables autoscaling for this job. If you are trying to cap autoscaling, consider only setting max_num_workers. If you want to disable autoscaling altogether, the documented way is to explicitly use autoscalingAlgorithm=NONE.
root: INFO: 2018-01-08T08:14:17.819Z: JOB_MESSAGE_DETAILED: (89da48800fedc8b7): Checking required Cloud APIs are enabled.
root: INFO: 2018-01-08T08:14:18.625Z: JOB_MESSAGE_DETAILED: (89da48800fedc845): Expanding CoGroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T08:14:18.654Z: JOB_MESSAGE_DEBUG: (89da48800fedc264): Combiner lifting skipped for step assert_that/Group/GroupByKey: GroupByKey not followed by a combiner.
root: INFO: 2018-01-08T08:14:18.680Z: JOB_MESSAGE_DETAILED: (89da48800fedc3ce): Expanding GroupByKey operations into optimizable parts.
root: INFO: 2018-01-08T08:14:18.708Z: JOB_MESSAGE_DETAILED: (89da48800fedc538): Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
root: INFO: 2018-01-08T08:14:18.741Z: JOB_MESSAGE_DEBUG: (89da48800fedc80c): Annotating graph with Autotuner information.
root: INFO: 2018-01-08T08:14:18.792Z: JOB_MESSAGE_DETAILED: (89da48800fedcae0): Fusing adjacent ParDo, Read, Write, and Flatten operations
root: INFO: 2018-01-08T08:14:18.822Z: JOB_MESSAGE_DETAILED: (89da48800fedcc4a): Unzipping flatten s10 for input s8.out
root: INFO: 2018-01-08T08:14:18.844Z: JOB_MESSAGE_DETAILED: (89da48800fedcdb4): Fusing unzipped copy of assert_that/Group/GroupByKey/Reify, through flatten assert_that/Group/Flatten, into producer assert_that/Group/pair_with_0
root: INFO: 2018-01-08T08:14:18.876Z: JOB_MESSAGE_DETAILED: (89da48800fedcf1e): Fusing consumer assert_that/Group/Map(_merge_tagged_vals_under_key) into assert_that/Group/GroupByKey/GroupByWindow
root: INFO: 2018-01-08T08:14:18.902Z: JOB_MESSAGE_DETAILED: (89da48800fedc088): Fusing consumer assert_that/Match into assert_that/Unkey
root: INFO: 2018-01-08T08:14:18.928Z: JOB_MESSAGE_DETAILED: (89da48800fedc1f2): Fusing consumer assert_that/Unkey into assert_that/Group/Map(_merge_tagged_vals_under_key)
root: INFO: 2018-01-08T08:14:18.960Z: JOB_MESSAGE_DETAILED: (89da48800fedc35c): Fusing consumer assert_that/Group/GroupByKey/GroupByWindow into assert_that/Group/GroupByKey/Read
root: INFO: 2018-01-08T08:14:18.982Z: JOB_MESSAGE_DETAILED: (89da48800fedc4c6): Unzipping flatten s10-u13 for input s11-reify-value0-c11
root: INFO: 2018-01-08T08:14:19.007Z: JOB_MESSAGE_DETAILED: (89da48800fedc630): Fusing unzipped copy of assert_that/Group/GroupByKey/Write, through flatten s10-u13, into producer assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-08T08:14:19.039Z: JOB_MESSAGE_DETAILED: (89da48800fedc79a): Fusing consumer assert_that/Group/GroupByKey/Reify into assert_that/Group/pair_with_1
root: INFO: 2018-01-08T08:14:19.071Z: JOB_MESSAGE_DETAILED: (89da48800fedc904): Fusing consumer assert_that/Group/GroupByKey/Write into assert_that/Group/GroupByKey/Reify
root: INFO: 2018-01-08T08:14:19.101Z: JOB_MESSAGE_DETAILED: (89da48800fedca6e): Fusing consumer assert_that/Group/pair_with_0 into assert_that/Create/Read
root: INFO: 2018-01-08T08:14:19.132Z: JOB_MESSAGE_DETAILED: (89da48800fedcbd8): Fusing consumer assert_that/WindowInto(WindowIntoFn) into compute/compute
root: INFO: 2018-01-08T08:14:19.157Z: JOB_MESSAGE_DETAILED: (89da48800fedcd42): Fusing consumer assert_that/Group/pair_with_1 into assert_that/ToVoidKey
root: INFO: 2018-01-08T08:14:19.188Z: JOB_MESSAGE_DETAILED: (89da48800fedceac): Fusing consumer assert_that/ToVoidKey into assert_that/WindowInto(WindowIntoFn)
root: INFO: 2018-01-08T08:14:19.219Z: JOB_MESSAGE_DETAILED: (89da48800fedc016): Fusing consumer compute/compute into start/Read
root: INFO: 2018-01-08T08:14:19.252Z: JOB_MESSAGE_DEBUG: (89da48800fedc180): Workflow config is missing a default resource spec.
root: INFO: 2018-01-08T08:14:19.273Z: JOB_MESSAGE_DEBUG: (89da48800fedc2ea): Adding StepResource setup and teardown to workflow graph.
root: INFO: 2018-01-08T08:14:19.297Z: JOB_MESSAGE_DEBUG: (89da48800fedc454): Adding workflow start and stop steps.
root: INFO: 2018-01-08T08:14:19.328Z: JOB_MESSAGE_DEBUG: (89da48800fedc5be): Assigning stage ids.
root: INFO: 2018-01-08T08:14:19.472Z: JOB_MESSAGE_DEBUG: (d473f33c01d764cd): Executing wait step start21
root: INFO: 2018-01-08T08:14:19.531Z: JOB_MESSAGE_BASIC: (9553413884fb3d88): Executing operation side/Read
root: INFO: 2018-01-08T08:14:19.592Z: JOB_MESSAGE_DEBUG: (d473f33c01d763ee): Value "side/Read.out" materialized.
root: INFO: 2018-01-08T08:14:19.651Z: JOB_MESSAGE_BASIC: (9553413884fb3dce): Executing operation compute/_UnpickledSideInput(Read.out.0)
root: INFO: 2018-01-08T08:14:19.697Z: JOB_MESSAGE_DEBUG: (d473f33c01d7630f): Value "compute/_UnpickledSideInput(Read.out.0).output" materialized.
root: INFO: Job 2018-01-08_00_14_14-11501116380979676381 is in state JOB_STATE_RUNNING
root: INFO: 2018-01-08T08:14:19.754Z: JOB_MESSAGE_BASIC: (9553413884fb3e14): Executing operation assert_that/Group/GroupByKey/Create
root: INFO: 2018-01-08T08:14:19.786Z: JOB_MESSAGE_DEBUG: (ddb5d62f419d3fdc): Starting worker pool setup.
root: INFO: 2018-01-08T08:14:19.809Z: JOB_MESSAGE_BASIC: (ddb5d62f419d3c3a): Starting 1 workers in us-central1-f...
root: INFO: 2018-01-08T08:14:19.876Z: JOB_MESSAGE_DEBUG: (69284e82b51f00ee): Value "assert_that/Group/GroupByKey/Session" materialized.
root: INFO: 2018-01-08T08:14:19.939Z: JOB_MESSAGE_BASIC: (9553413884fb33de): Executing operation start/Read+compute/compute+assert_that/WindowInto(WindowIntoFn)+assert_that/ToVoidKey+assert_that/Group/pair_with_1+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-08T08:14:19.965Z: JOB_MESSAGE_BASIC: (69284e82b51f0b5d): Executing operation assert_that/Create/Read+assert_that/Group/pair_with_0+assert_that/Group/GroupByKey/Reify+assert_that/Group/GroupByKey/Write
root: INFO: 2018-01-08T08:14:28.272Z: JOB_MESSAGE_DETAILED: (aa895484fc6e9aac): Autoscaling: Raised the number of workers to 0 based on the rate of progress in the currently running step(s).
root: INFO: 2018-01-08T08:14:57.140Z: JOB_MESSAGE_ERROR: (aa895484fc6e9815): Startup of the worker pool in zone us-central1-f failed to bring up any of the desired 1 workers. QUOTA_EXCEEDED: Quota 'DISKS_TOTAL_GB' exceeded. Limit: 21000.0 in region us-central1.
root: INFO: 2018-01-08T08:14:57.168Z: JOB_MESSAGE_ERROR: (aa895484fc6e961b): Workflow failed.
root: INFO: 2018-01-08T08:14:57.297Z: JOB_MESSAGE_DETAILED: (89da48800fedc41b): Cleaning up.
root: INFO: 2018-01-08T08:14:57.341Z: JOB_MESSAGE_DEBUG: (89da48800fedc6ef): Starting worker pool teardown.
root: INFO: 2018-01-08T08:14:57.370Z: JOB_MESSAGE_BASIC: (89da48800fedc859): Stopping worker pool...
root: INFO: 2018-01-08T08:16:06.029Z: JOB_MESSAGE_DEBUG: (89da48800fedcf6b): Tearing down pending resources...
root: INFO: Job 2018-01-08_00_14_14-11501116380979676381 is in state JOB_STATE_FAILED
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
Ran 15 tests in 426.802s
FAILED (errors=14)
Build step 'Execute shell' marked build as failure
Not sending mail to unregistered user ankurgoenka@gmail.com