You are viewing a plain text version of this content. The canonical link for it is here.
Posted to builds@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2020/08/23 06:59:52 UTC
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1835
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1835/display/redirect>
Changes:
------------------------------------------
[...truncated 290.96 KB...]
"python_side_input2-Write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWcmAn38ACKN/////////////////////////////////////////////4AjT6gN2OudMdm0LGyAdvXConiDRQhlJiemjSNNMR6j0m0nqek9TJk00NANNA09IAABoAAAAAAGgAAAA0DINAAAaABoHqHqaaDTQ0GQ02kGSIaT0TEwTSYmmpPSPUNGmm1AAAA0DRpkAAAAABoAAAAAAAAAAAAAAAAAAAAABjU8SeUjQQyPRGmINDTQ00yDaTTJo9R6QYhpkwmTIaYEaYEYgaaMQwmTR6mhibSDNEDQaaaAaaMgGmhgRggGjQ0ZMmhB6n+qQYhMAI0xNMAEwRpkwEaaZMATAEYAJgAjABGmmJgBMAACNMAAmAACaYAAgwho0DAAkwAEwABMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABIkQCNTCTI2k1T8jVP0InqNB4oHqZqPU2pptIxAADQNNNA0ZBpsiaAAAANDQAAAAAAAAAAAAAAAZZMbVuRhAlk30WHLsVEbRPM0TQ2bNWHDhGOovOQVli6uPJvRRNt22POuUVlyqz84E9DX6aYFztgFYx4N01wiEBChQICRpiTaQDM2gd9mkuVTGNd16B6hKbGNCUxCCdZySAOLO7BYCgTw4RgFQfC+DEVyxfnu3r1rBgwLdzMs2EsMxhLzOPGf4NJI8ObIInAtRmkaZtDUOMx5g0Sy4tOlIsFvYzzniZgOlPDEgrLwv78VHiLgvtJlSKsRIO/07jKVBVOkmEwnBOQpWEphMCdZPx81OUBHP+Aji9D0bbnOclKrooqTc2dFI0a4Zh5+glOmEXXcS2dWQvCV8BGIlDkokSkNhM0iUkNjeQghsONIUbckMloOaZDaZGI7AM5UneZ5swoNEmWLGGgZikrJlZTZ1kLAhnNxezllQdfCRlRg+nsu4YP/VuQjK97Wd2d42AbX+T7PF7SQEuuKS6fvnqIW9kci3XSMCIHm+HL3oS51242oI3A2JTTPGm0/KUHHHHHHxjEIQhDGMQghv0JuVWTGYwCMwRkLDQ42lqSI+k4HPay5ORwy3lBUC6aJpq5hashObJiboZCx8XOoVYrOEoVKqULBVYphxPNrirwdaq11r4ZUV8WfUFRGimVaUZlw1G+L6NM4iw019DTK95Xes9ZVI0E7x9hTgbOZg08pt23hQyYdYMqH2xQomHh8YL5bLWtIuEiBqgtxRF45QTOuFCiApqsUCmY12M4kHH8VbssU2dMmeOJEzlAw3Nd6CSPTPReXY+p1GERWJYh2d1FGl67Dj9U9MtO8dZf5jhtQqWg0tmTBgGCJKlQM8tls9qtZsE9rUdzjZXHS3b9LiYmfxOw76DMwYrL4+VZ+ZP6EAUAT4iSW/EUoDgFUtEFuMQGAONJpjWkw08amzeW3CXhU2jGkArpzM9S83szEku0CLMOkd4HDzJxVyeZS3L7pE3hXrG0jrjZXJODTQ5BpskEnZCSREpSEMfpX2kNJC7lsdOp3W8KxifE4XhIguX4NGplxCQfxfiTizOKIgVVio41hLKu/AlAl5yHUyzOmoobiLpQKTXISqDp1MygriiV3AjBjFYpVF0SdpOe5nj1CnflS60qmFcx5VNoFk5u4qKMFAMLuas5xHZExWJJVqoLxZDGsAuugrWJs3alnugDGAyhqCTQc3ojL1FaBaQByF2YyMNI2fwLQV1yYDyEKhMBAEoS8kiTA73p++5UxAA3HfnFA6GQdUCD2Vb6PVsfUh92wxaMnyeQjTOWt65js0YCqc7Jc/o8tWUKRsXyFKnU9UQBQoBqhGLQmAcUAURnyLiJqklIu4ZUaR2kkMPcFIdIeCsNjYVCalilrGGIzoDYXYW4gTIIC+gIFFDxM4OIa5BkD6mxmOBpLE3ZvL966DnC4gxchYqxYlxy36tVDd1K6upVKKMCvQKmmyHXUKFVN0SbqqNFJLCv+2W2EzERwCdu/qKp3y48IrDOkaxTpT5gVNwtOnKyl0ErYiZF2yaB0zQRyygmBs2Cg0aSdixOMAqeRVFpcPTCqml+L1zagXblGCZ6AtejqBt2LSFYuAsUBIDhTFbgmAdCwtkSU3jNQuDJ7FKTrpZ+jy/tdBEAxBF5/dLTdVIOoiBBhQC8Bcyhni4XNUTAFLN/fNRfPm3YgCYsaK8VBKCiLZDVNmss5LwU1BNFruPoZCygN5z4T7YMwGwBgR4E7MzbmzgQCgeJww9Pb7G2B+hGZMyKNDEwH8DhXcicGwFI4TVzOSGV275WdeOXGhtmpllkCMZksFVmemzY4GTU6R3i1lCIENUwhhi1Cgg7KXHSkwqU2yIQ0C5jwbGrDIOYTiwVC8ZDkaxBquFDWRd9zKlc+MkK8k85fOCmApJaoaNAapn3aVlOIIFWxRc8I77eAnss6sqrhqsSbBLK1NSWerQZNTBlKbdxToQTFivIsOTK1S10Sot4OAWDW2tAVjRnNFNYYAoVxY1NCdJBFEAXuQp+zwzE23cy2ClhJUtWiiHMtTGSaghW74xTnoGrBuuX0BSv7prsW0SCNmbBo6JSZa1XNOwMJJQ07CiFPc5cktB2camZxRlVKRbVtoyYlYbiyg2BFLVyMrNssyBYjo1LDDYNmtAUrBtYrSrjDGzUq8bAbNkWJtpqpcz6zrdMs/EQ1LnYfJc7ERQQiUIQ4gKJQQSAhCIREKMasiAAuBAqlKUppVQ+IKweGEmzAIAHpJI1LS15TjkXuSv3Gu2HnXGQLwHVOtbaTDSBqhHCoOU1Pac7CVDw8TCAMGEpWKAAAAAAAAAAAWAANAAIxYsmFDQIAEEEEEERK7VLSixWFFDWsUp9aIiIjWqWjWtDWwQQPTLEMMq3oYi5yyZM6TbbbdFGCVUqJNtt245S4itnOIjBklKmaLGLLhkbTOKNFpSmlt9fVnhTWYTGM1FWaxqwFH2dTBxjs3l94KQlYXbhbnLnTstqHXCXHMFFY411YKnBiIOYDQNEzKoW5KXuhzY1LeLqeEks7S2N4fmb7LB+ALxmYK4L4xaQdAOOyIYgxTHMZnZRMKVWVxiyltkxdBfGWBYcOWQZHsa/hCg0xnpBnZg3WMxGCW9aUwnLLsX0ZGCV8FNGeaCFmdFAh82VZ1TJccFL9wMBVEWlgfFiWViBMQw15BNDMxQQZaC66x72Gs1gIifg4pYpbjb05M9d6GgrAQPyG/UKIBAhFrC9rsOvl+X6/51GkjElBM68fNi0F2kC9jHbUayzcHSSNJXUki9G03akjNVqEpVrbWGoUmOS247nDBtWlFjCVRRebhUU012lSwbxTsN80Nk19JimsBHnIQOopgwo4gMJBqAsagz87dvKvPrrTOB9Eb6Jj3VmZ+Hx+Vfmvmtm1v0+v2nFf3Vr/z+p2e/f9n/9QOaFpdRBj81ZARwDYAEGgRhg1CgBgGEgkFYBmBH32jif/1pRAcoRdBGOUaF2FBQObQ6Azv4ehQpvQSiD70fsGSBsGmDYNMGwYvuRAbMHSQQKQ0A2khttA/cwI5poPKtCrGlq+p/+LuSKcKEhkwE+/g",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-23T06:46:31.524262Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-22_23_46_30-10025249199612656513'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0823050254'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-23T06:46:31.524262Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-22_23_46_30-10025249199612656513]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-22_23_46_30-10025249199612656513
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-22_23_46_30-10025249199612656513?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-22_23_46_30-10025249199612656513 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:34.016Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:34.753Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:34.800Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:34.937Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:34.963Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.073Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.126Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.226Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.260Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.299Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.333Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.379Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.414Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.441Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.473Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.510Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.556Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.591Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.626Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.662Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.695Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.733Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.768Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.811Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.851Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.889Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:35.925Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:36.096Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:36.162Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:36.188Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:36.214Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:36.226Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:36.251Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-f...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:36.310Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:36.310Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:36.384Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:36.424Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:46:36.496Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:47:03.744Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:47:09.879Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:48:34.189Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:48:34.227Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:51:49.517Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:51:49.593Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:51:49.632Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:51:49.704Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:51:49.749Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:51:49.755Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:51:49.774Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:51:49.803Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:51:49.809Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:51:49.831Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:51:49.883Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:51:49.909Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:54:02.664Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:54:02.744Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:54:02.797Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:54:02.867Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:42.984Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:43.058Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:43.108Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:43.184Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:46.447Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:46.514Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:46.573Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:46.611Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:46.636Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:46.672Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:46.719Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:46.757Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:46.838Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:49.244Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:49.316Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:49.393Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:49.461Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:49.528Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:49.597Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:51.936Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:52.002Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:52.074Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:52.197Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:56:52.227Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:57:43.784Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:57:43.832Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T06:57:43.873Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-22_23_46_30-10025249199612656513 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598165187608/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.07057738304138184 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: fd0e59dad17d4cbb9a1bd6c390b94f36 and timestamp: 1598165989.3078625:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 801
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598165187608/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598165187608/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.05904388427734375 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 802.230s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 13 mins 25.367 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 30s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/evvbwych25lpq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Jenkins build is back to normal :
beam_PerformanceTests_WordCountIT_Py37 #1852
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1852/display/redirect?page=changes>
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1851
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1851/display/redirect?page=changes>
Changes:
[zyichi] Fix PairWithRestrictionFn process no attribute signature error
[Boyuan Zhang] Handle split when truncate observes windows.
[noreply] [BEAM-10821] Add Python SDK typing improvements to CHANGES.md (#12693)
------------------------------------------
[...truncated 290.18 KB...]
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWbYSHeQACKN/////////////////////////////////////////////4AjT6gN24rhjZtCxsgI9g0k4g0UIZKZHpo0j1NMQPSbSepo0yMENANNBk9TQBpoA0GgAAA0ABoAaAAAA0AA0AGgAaB6hpiaNMjQaDTaQZIQ0ExGIJiaEaj1DT0mm1AANAaPUaNGmQABkAB6gaMgAAAAAABoAAAAAABoAAAAAAGSNigAAAATAJgARgBMjAACYAJgAEGBMRgEwmTAAAAAAAAATAmjIwIMAAAJoQep/qkGITACNMTTABMEaZMBGmmTAEwBGACYAIwARppiYATAAAjTAAJgAAmmAAIMIaNAwAJMABMAATAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAASJE0CEwgjI1T8hT9CJ6jQfooHqZqPUyGIxAADQDTQA0GmanqaAaAAA0NAAAAAAAAAAAAAAABlkv6+7mICpLmhTZbjokgE80AmirVq+LivDJXLrIX1i2uTJuxxwWbLzrjKH11FaSbqeiBtWrCueCEPmS7tmkEQgIUKBASNMSbSAZqaJ3yaS5ZMY13LnnqkpsY0JTEIJ2DDQBizvQWAoE8SEeArhoXwojtU72e3cuWLuG6s2s91MJaahhMRnHQ/4NNI8QaAROBarNM2xtTVOMyagaRbeXHRkWi38Z5zhMwHRnhyQWGIMXfSs8TeGK4mVorxkg7/Td5isK50JhMJwTkKVpKYTAnYT8fNTlARzvgI4nhOgbc5zkpV8WK03NnFkaVkMw87USnSEX38/dOvKYgligIxkoclEiUhsJmmSkhsbykENhxpCjckhktFzTIbTIxnXRnLE8Rnm0Co0iZasgaJqFCwmWFLeqhaEM5uMTOXVR1wJGZGD6eh23B/6w32HsnvaXmkxDy1j5Pt8fuJASy8t/1fh+qQW9kcm2XyL0HroePMRwljsd1t4RjBHslpibnaGlqQwwwwxYIIEEEEEIIIEESJ7xwQiEoCGAKI4AID36fUElJiij9Fwuc11ycjlC7lRVC6SJpq9hcspObJibqZCycTOqLQX9VKi6XioxPPlgFzmzmjmWl7FGtSXmnV4JY1dYoM21Vk4ZVow4G2M2WvOuQKuSKnSM0GYrFYpGEyRRQRBmFC3lNz3QKGTDjwyobBHEiX9/xwvjBUfukXERDAoLYoi4coJnGFSiApXaoFMyLsBz8HH8Rb4tU2a0zxxImcqGG9rvAYvxU5DO9B70Q3ZbzMSlG+/g5fyvh/TQGnlqfkUeymFy7koRnRG+vhfLLioGchIZ/VyWwT2tY3OP5WQle765eYmvvQVt2BmYMV9ixrjuzeEHknkmKJJaZJRQ1ZqUiJRahaDVknbihgSBSVigpCz6zn2tVDm5bcWbm8OU1Wjp++Ywi2K8Uu5ycLGufFjWF4g4xXJNZHXGzjU3xNDlmmzQScUN+iOLTUMjpnamvsdei2OoU77Z/ZQfkl2ASIL2GFRrRcYkHgYEk4YzhSIFCQsUawllddwSgS85DrZbnTUVNxF8oFJrkJVh0qkyosiqV/CjBkFapVl8SdxOe8nk1SnBK11ZVsLJjzKbQLLzd5WVYKgYX81b3pHYUxWpJWKsMQspkWAXWQVzE2b5Sz3UB54GoIEEul4NEhdofhWkIZE5MZXjVGX8GwFdel06hCojCQBKEwJIkwPB6/tuVMUANx4Z9QLTIOCBB1Kw9Pr2nrVvd1HGoyfM5aNS5y7smO3SwILo5SU6fOVk/kbJi/lVKqqIAn8A1Qh1wTAOFAKIz5F5E1QlIv5QrNM7MSGHuCgdEeCtNjYVSaljlrmKMzoDTXTWxAmOQFqAgUUOkzg4BpEGUNVBIZIYCWNu3Zxb91Hey8gx8harBY1xy4KuVTd9F1NSrUVYFiQKlLYdlYoVc3VJuus0kksK/7bdaTMZHCJ3cGsrnilx4RYGdI1ymnPmRUvFtqZmUdRK6ImRfoNA6TQRy6gmBtGCg0qE7VjcYBU5FVXF49sFdKPod0bcC3aouzPQFryCgbNOwhWLQLygJQbyYrYJgHAsLejSp8h+C8M3UU9OxloKZLe9aIEGHsvTMys7CEHAQIfAoBewvZwDlGXZUVRBSrb4TkNKTriQBMmVKcqsk9wNxgK00WRZMAU49vq6kieMhEgNp+boWoaANQDAtgFBNTreLAeVA5T5h0dXaIszvV2hNCKIDIwHgDkr+ROHaCkck1ezrYyy7gK3rRzA0Ns1c0soRkMtoq+fatNjAZclVHerWUIhQ15hFFHrFBB2wscKTCuqCsIhABaxoNPXRjmXjFgqFwynHvkGvaKF9F3rMqVz30pXhusvTdpgKEtYNKoNYz79O2mMIFYxRe8I8V3CT0M6wrshqwSbBLM1NSWergZNTBlE27ymjBMWPERacmWKjXFVV3DwCwa+3qCwaM5opYGAKleWtTQnQgiqAMTkKfs8MxNt3sugowkqNXCiHMuTGSaghXcAxznomtBveY0RSxdx3TFtUgjaGwaWkUM1ismnaGEkoadpVCnvM2WWi7eNTM4qzKiRdXuIy41abu2o2BFGr0ZmbhaiBYzoFLDDYNmvAUWDbxYi1VJVcrbTQAIgoD5REKF45jOZB1c8oUhXiKTI4ilKca3V1rXil2M41kDWtpMa+45LYRH5kkffffL8bzF0KcW3MoxEdllkUtLHlsZH7lRtqBym64x6Fd04pyCCwmLSGBQkiUGVa7uRciKh4eJhAGDCUrFAAAAAAAAAAALAAGgAEYsWTChoEACCCCCCN4bok9DJaaGhddXEWSbbbuuiTuuuLrr776ZpGDBllTBEXuWXLnSbbbbqqwSrlVJttu7JKXPq6c4iMGWUqTRaxZsMja5xVpNKU0tzutaeFNagmMZqqw1zWgKvs6uDjHbs4sQKQlaX7tbzNnTturHZCXHMFFg43StFTBjIOZDRNI1FWLeFHvRzY1LZXT8JJZ2nsbJ+ZvsUH4AuGZ4UgWjCqg6UcZjw8gwjJMZnKgmFVVZcgrJZemLgLRmgWHDmkGV7G68KVG2GekGdjDe5DGYJb9pTCcs2xiqy1mWwT4VLhElnuBFi0o2dHHtpmjZtisUYlcqywyZcryBMAxUpRNFM8oIMtDbcedzsD2vhET8GKWKsF/cnJnpOxUFYCBspva4ohEKEWLz2ux7Cp8v1/zrNUjAqAmdeNmwqC5VAuXzlyNdanD00jTV9CRiRtd8pI1K9UlKxbi01ShkktyO9wwbVxVawlWVYjdqqlLLitYNlTtOA0Nk19JimsBHnauHAVIYccYGEg/ALH4GfpbNuU5+2wNHr1RHqmJmvzU38flZZz5rp1dPfT6z6v7K2H4/NB+v3/aH/UTejfbwHwfiuICKAagAg+hGAD8FADAMJBIKwDMC2YCmCh/1uRAboQ1FDt0aNxFFRN7atBnHh0FGm6BKIPvR+wZIGwaYNg0wbBi+5EBtAdCCBSGgG0kNtoH7mBHNNB5VoVg0tb1P/xdyRThQkLYSHeQ",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-27T06:46:30.932652Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-26_23_46_29-1528657746125815905'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0827050318'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-27T06:46:30.932652Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-26_23_46_29-1528657746125815905]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-26_23_46_29-1528657746125815905
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-26_23_46_29-1528657746125815905?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-26_23_46_29-1528657746125815905 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:34.012Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:34.744Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:34.782Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:34.854Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:34.881Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.028Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.064Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.098Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.122Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.151Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.178Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.214Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.248Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.283Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.311Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.344Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.374Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.411Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.444Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.475Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.502Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.536Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.572Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.619Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.661Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.699Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:35.758Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:36.882Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:37.387Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:37.579Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:37.592Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:37.787Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:37.788Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-f...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:38.184Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:38.185Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:38.473Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:38.611Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:38.646Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:46:57.059Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:47:02.190Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 5 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:47:02.223Z: JOB_MESSAGE_DETAILED: Resized **** pool to 5, though goal was 10. This could be a quota issue.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:47:07.469Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:48:41.871Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:48:41.906Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:52:00.139Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:52:00.202Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:52:00.223Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:52:00.288Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:52:00.309Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:52:00.337Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:52:00.341Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:52:00.360Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:52:00.392Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:52:00.406Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:52:00.442Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:52:00.477Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:54:17.008Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:54:17.076Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:54:17.118Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:54:17.190Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:08.338Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:08.446Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:08.497Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:08.559Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:11.519Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:11.579Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:11.646Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:11.681Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:11.700Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:11.745Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:11.762Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:11.801Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:11.890Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:14.660Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:14.866Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:15.018Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:15.160Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:15.372Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:15.721Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:18.597Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:18.680Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:18.768Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:18.891Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:57:18.918Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:58:10.706Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:58:10.761Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T06:58:10.794Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-26_23_46_29-1528657746125815905 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598510787296/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.05666375160217285 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: bb552e89aa2d4251a0190f1bc5b7ffcc and timestamp: 1598511585.4203074:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 797
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598510787296/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598510787296/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.05598330497741699 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 798.614s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 13 mins 21.271 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 27s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/a46jfcngcp6lc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1850
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1850/display/redirect?page=changes>
Changes:
[leiyiz] implement query 12
[saavan] Fix author names
[noreply] [BEAM-10701] Add codecov config, fix paths hopefully (#12684)
[noreply] Merge pull request #12674 from [BEAM-8258] basic metric feature for
[noreply] Add skipped tests for Series.dt and Series.str methods (#12615)
[noreply] [BEAM-9547] Add a few more DataFrame operations (#12682)
------------------------------------------
[...truncated 289.76 KB...]
"python_side_input2-Write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWdb+P5QACKN/////////////////////////////////////////////4AjT6gN24rhjbNCxsgE9g0k4g0UKZkpkZo0jEwgybSepk0ZNGQyGjEDT1NAGg0AGgAAAaABoAZAAAAyaGgAAaABoPRGRkNNDIZDGkGSCARkYgmJk1Go9Q02pptQAAAPUBo9T1AAAANDQaZPUAAAAAAADQAAAAAANAAAAAABkjYoAAAAEwCYAEYATIwAAmACYABBgTEYBMJkwAAAAAAAAEwJoyMCDAAACaEHqf6pBiEwAjTE0wATBGmTARppkwBMARgAmACMAEaaYmAEwAAI0wACYAAJpgACDCGjQMACTAATAAEwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEiRNAhMIIxNU9qZPQmpo0ep4oGjajTammTTEaAA0A00BpkGjaamgGgAAA0AAADQAAADQAAAAAABipf1sqYgKcmZFJlqjPHAJpYBLFVq1sfFcMdYuMhfWLS49cco0YLFh1xrJ3101aKbyaeB6pUhXNBCHzHe2jQCIQEKFAgJGmJNpAM1NE5yaS5VMY13Xwr1iU2MaEpiEE7BhoAxZywWAoE8SEdArB4L4UUbNK+mtW7de9hvLFnOdTCWmoYTEZx0f+DTSPEmgETgWszTNU2ZrHFZNQNItvLjpCLRb6M854mYDpDxBILDEGLv5WeKvDFcTK0V4yQeApuMxWFc6EwmE4JyFK0lMJgTsJ+QmpygI5/wMcLoeibc5zkpV8OK03NnDkaVkMw8/USnSEX39BdOvKYgligIxkoclEiUhsJmmSkhsbykENhuyFG2JDJaLmmQ2mRjOUGcqTxGebIKjSJlqyBomoULCZYUt6yFoQzm4xM5ZVHYAkZkYOm0O4YP/WfBQ1pm2F+qMQ6sY2S70p30gJdaXE7E7R1AuFH5mYWx7wHjpeaOwYlzhb1dAjFiPZLTE3O0NLUhhhhhiwQQIIIIIQQQIIkT3jghEJQEMAURwAQHv0+oJKTFFH9BwOe11yUjky7sIqhcSJpq9hcspObJibqZCycLOqLwL+pFRcLpUYnXywDMc2UOXdLmOMKmNxmZEtVOUxkrpeHDKtCDLAwMyaDHKHAqhxlTJGZzMViqUjCa0UUEQbgoWsTdt3ChlA10YoegoxIl/f8cLpgpv2EXERDAoLYoi4cQStYVKICldqgUzIuYOFBx3QrqBaps1ZnjyRM5QMN7XfAxfipyHs9B70Qyq3WZahRffwdv8r4f0zx48tS8ifNpoHLtkCM6I3bourLWgGcdHabYyG1T3Vd32W5uXlvAymvhOfmisDkDMwYhHBwsbmvdEHUnUnBEktakooao1KhEosQsBqiTuhQYJApG0WGOtch0YpohjbN4Ifn8jqbGPduG4QKyJ8s02yUFfG2Ti10ZYsCqkWOq2hr72m9JoSppr0EmpDiIiTcUMxrXJxe8Jfy2+vU8UxX3exeeaXBIgtXYxF/LlEg6DDknBmcERAoGCiTVkrsjmCUCXnodbLc6aipuIvlApNcdKsOn0mVFkVSv4EYMgrVKsviTuJz3M8msU40rXWlWwsmPMptAsvN3lZVgqBhfzVvOI5hMVqSVirDELKZFgF14FcxNnFqWe6gOug0xAgk0fIzxlyd+FaQhkTZTI6ag27wa4VmkvHEIVEYSAJQlxJEmB42VDv1MgAN95KBQLDINKBBvKz9btW3s2Dg1fJoyfOlZbb3uP5LPl6QdV+Bwvg/D99RbxPidnvFuN3UgW8B1HBxlgPrAKIz5F5E1QlIv5MrNM7SSGHSlA9CeDtNjYVSaljlrmKiZkBpLpLYgTJIC8gIE87hMwOAaBBkDyoIzHDASxt273FvnUd+LyDHx1qsFjXGLjVcqm76Lq6lWoqwLEgVKWw7KxQq5uqTddZpJJYV/2260mYyOATu42srnil1sIsDOka5TTnzIqXi1aZmUdRK6ImRfoNA6TQRyygmBsmCg0qE7VjcYBU5BVXF49UK6UfR7c2gFqzPemacLXpqgbFKuhWLILqgJAbkpWwSgNCwt2ilZ5j6FoXt4dk4UtPVJebWCABh5L2fGWn4Eg0iAD0FAMAFq9+bIu+NBVcFLTE7PQfVn2ogCd3epNlcSeYC7Pz+wVxRLgU88vbXUR9jIW8CYoPlSMQwAYgGBcQKKcn2cUA6qBsoDDc3uURDub4wJgRRAdzAdAODfyBx9oKRwWr2ciMsu36t6+cuNDbNbNLKEZDLaKtnWpy5AGTWag75ayhEKGtKIoqOrTkHbCxpSUVlQVREIALOCDSygyTLjFgqFsyHJvkGtZKF9F3zNfK5rshXhuMvjepgKEtqGlUG1M+/TtpjCBWMUXvCPFdwE9DOsK7IasEmwSzNTUlnq4GTUwZRNu8powTFjxEWnJFio1w1Vdx+AWDX2lQWDRnNFLAwBUry1qaE6EEVQBichT9thmJtu9l0FGElRq4UQ5lyYyTUEK7fmOc9E2sG65fRFLF3TbsWzSCNkbBpaRQzWKyadoYSShp2lUKe5zZZaLt3aZnFWZUSLq+IjLjVpuLajYEUavRmZxC1ECxnRKWGGwbNeAosG0ixKyMMbJSsyMBs0C1NtNVru3r+uUln4yGpc7D5HnYiKnntVWteOXX695kDzz1Bjz7WyV0R+VI3333y/RdYudTVvZdGIjkkkalpX8xjKPuU3rMDaTjWOwrvGqbBBXTHpDAoRxKDKlZzVGxFQ8PEwgDBhKVigAAAAAAAAAAFgADQACMWLJhQ0CABBBBBBG8N0SehktNDQuuriLJNtt3XRJ3XXF11999M0jBgyypgiL3LLlzpNttt1VYJVyqk223dklLoFdOcRGDLKVJotYs2GRs84q0mlKaW22+1nhTWoJjGaysNc2sBV9nWwcU7d7ixApCVpfuFuc2dO26sdkJcYwUWDjbq0VMGMg5kNE0jUVYtyUe6HNjUt6t5hJLO09jen5m+ywfgC2ZXRQBeGFUB0Y5DJh1BhGOUytpiUVFVVxiqlh2UtBeGaBYcOaQZXsbfwxUaoz0wzswbrIYzBLfNKYTlm2MVWWsy2CfApcIks9wIsWlG90cerM0bNUrFGJXKssMmXK6gTAMVCQSxSuqCDFhtNdczcDuIERPwYpYqQX9uYmag5FOVgIHpDfVhPCIUIr3Pb7LNU/l+v+dXqEYFMEzLwZcKctqAW7rbka61OP00jTV9CRiRs+LUkalesSlYuItNYoZJLbDvcMG1cVWsJVlWI3CqpSy4rWDeqdpv2hsmumYpqwR6N8DSKwMaJQGEg+gK30GftmZmVbfdZmDwyiGXw+MJOfL5/S1T31a59r+33/FAqoVS79fuio/5/qSlpmdO9xgeh+7GgIkBiACD2EX4PoUAMAwkEgqAMwLjhqoKSluaIDNCDpoZmjTtQZs6e5LAZxobh7k24JRB96P2DJA2DTBsGmDYMX3IgNkDoQQKQ0A2khttA+lgRzTQeWaFYNLa+r/+LuSKcKEhrfx/KA",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-27T00:46:20.953237Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-26_17_46_19-14238271852110793962'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0827000956'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-27T00:46:20.953237Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-26_17_46_19-14238271852110793962]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-26_17_46_19-14238271852110793962
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-26_17_46_19-14238271852110793962?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-26_17_46_19-14238271852110793962 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:23.816Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-a.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.165Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.188Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.264Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.292Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.374Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.425Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.447Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.478Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.512Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.549Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.583Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.618Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.656Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.690Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.722Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.752Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.776Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.818Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.846Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.881Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.918Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.946Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:25.969Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.002Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.037Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.082Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.234Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.370Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.401Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.427Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.428Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.455Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-a...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.499Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.499Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.561Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.600Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:26.767Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:48.196Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:46:53.767Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:48:30.088Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:48:30.119Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:51:48.480Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:51:48.546Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:51:48.581Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:51:48.644Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:51:48.668Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:51:48.693Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:51:48.702Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:51:48.741Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:51:48.749Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:51:48.773Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:51:48.811Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:51:48.838Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:54:06.422Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:54:06.512Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:54:06.566Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:54:06.644Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:46.102Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:46.168Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:46.212Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:46.277Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:48.635Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:48.702Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:48.766Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:48.794Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:48.823Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:48.853Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:48.892Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:48.918Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:48.996Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:51.835Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:51.913Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:51.988Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:52.041Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:52.107Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:52.165Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:54.871Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:54.938Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:55.005Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:55.193Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:56:55.244Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:57:55.944Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:57:55.988Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-27T00:57:56.031Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-26_17_46_19-14238271852110793962 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598489177167/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.0621492862701416 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: 60efc2d6594849f09ec6854cf672295f and timestamp: 1598489980.6893268:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 803
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598489177167/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598489177167/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.048793792724609375 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 804.065s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 13 mins 26.546 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 26s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/qld2eamv33sek
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1849
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1849/display/redirect?page=changes>
Changes:
[sychen] Refactor to reuse computation+key
[Luke Cwik] [BEAM-10771] Fix whitespace error causing whitespace lint to fail
[noreply] [BEAM-4980] Upgrade to Netty 4.1.51.Final and Netty_tcnative
[noreply] [BEAM-10670] Make Twister2Runner opt-out for using an SDF powered Read
------------------------------------------
[...truncated 289.74 KB...]
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWWutV5IACKN/////////////////////////////////////////////4AjT6gN7cXBGzaFjZAJxjUTiDRQhkp6J4jSPU0xHqPSbSepk09I0ZDIDTQNPUAAAAAAAAaABoAAAAAGgaAABoAGgxGhoNNDQyDJ6QZIRkEZGEIyBTyj1DR6mm1AAAA0Bo9T1AAAAABo00AAAAAAADQAAAAAAAAAAAAAkwAEwABMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAADJ/qeoUp70UjNR6gAGg0AAaDQA0AAAAAADQAGgAAAAyABoAAADIAAGQAAAAAAAkwAEwABMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABIkTQITCTIyNU/I1T9CJo0Higeo8o9TJpk0xGgANANNAaZNGh6amgAAAANAAAA0AAAA0AAAADQABjphWd1MYFeXLCsy5VokhE80ImjsWLODHfGSyX2RQLFxcmsP1asN3dtfeZRAuurQTdT0QuppiXPDEIDJeXDUCIQEKFAgJGmJNpAM09A5uaS5RMY13Xwj1CU2MaEpiEE65k0AYs7yFgKBPEBGgWQ4L4QVbda/nuXt7avIrq7t5pp4S00zCYjOPHf4NJI8SbIInAtRmkbU2hqHFZNMNEtvLjoiLRb2M850mYDojxBILDEGLv5WeKvDFcTK0V4yQeApucxWFc6EwmE4JyFK0lMJgTsJ+RmpygI5/no4fQdC25znJSr4kVpubOJI0bIZh5+olOkIvv8LdOvKYgligIxkoclEiUhsJmkSkhsbykENhxZCjcEhktBzTIbTIxnJjOUJ4jPNmFRoky1ZA0DTKFhMsKW9ZC0IZzUYmcqqjsASMyMHSbLuGD/1ZvMXXPY3urvGPzm/8TtcbtpASbEt91Pf+qEW8j8jKsY9/D/zvFHXoSZ1u4WkI291gIjQaPS6nWbBa1rW7UpKUpSpSkpJ/r86g5G1QWhCNJavl9vyqRSqfCeh4PO6y5GRyRd2EVQumiaavYXLKTmyYm6mQsnDzqi4GDTiovlpUY2wFgGV5o5Y784twTGrLvLxCxVJTgLrLrqRsXbOqFWgB+4Muc3elSMJOkhoM5WLBSMJqxRQRBtihcxty3MKGUDzRjh0NWNEwsLjRgGGvBYRcZEUKgtiiL44wmeYVKICldqgUzIuXOHBwOgXTy1TZrzPIEiZyYYb2u+BJHqHofNsJZcMKisSxLs7qKQd77MvOU1MtMw67Lzm406peDS2RNGgaIkqVAzjo7PanV69PaU/cY7k4+S7kpL+2Y/E7UnsGZgxFM2dVfGz6g/E/EzRJLSpKKGrNShRqLkLQask7SUSCQLWVqpx1f1fNcqos2G2Fe5fC6Gpj6RvWcOtjfDLvHEiLk8SFxZXSqXhXrHOi1xtrinApoSJptkEnZDfIiQoyGQ0r3RoK9XYtjp1O9KuGtU8cuyCRBiywiNRLikg+i/EnEmcQRAoeIjTWEsqz4CUCXn4dbLc6aipuIvlApNcCVYdOpMqLIqlfwYwZBWqVZfEncTnup5NQpvytdaVbCyY8ym0Cy81eVlWCoGF/M282jl0xWpJWKsMQspkWAXXgVzE2cYpZ7qA1oNcQoJdDzmiQv0QRLSIMjemMrTSNq8C0FZhLp9CFRmIgCUJkSRJgd+U++5UxAA3Hgm1AtMguQIPRVnpdaw9Wpbtpi0ZDlSKNK5i/sGO1RAKnz8l0OlzVZOCNqzcFTKaqIAnABqhFrwmAcQAURnyLyJqhKRfyRWaR2kkMOjKB6I8FabGwqk1LHLWMdUzoDWXWWxAmRQFyAgUUPkzg4ZqEGUOVDIZIoSWNu3eYt66jvxeQY+BarBY1xq36uVTd9F1dSrUVYFiQKlLYdlYoVc3VJuus0UksK/7bdaTMZHBJ3b+srnil1sIsDOkaxTSnzAqXi2tMzKOoldETIv2TQOk0EcqoJgbNgoNGhO1Y3GAVOOVVxePahXSj8brm2AuW6Lwz0Ba9JUDd1rSFYtgtUBKDfTFbBMA8FhbaqUvkQgYhs9CkJ1stBRZP2LRDAxAF6PdJTUOQXCGECFALuGLaFYW+5KiaApKV98zE9GadiAJq1obCnJQENa4Wot1lmJkCmYCCXu4+hkLGBKzfwnnIbgOQDAsoE7MTTqzAPygYTZh5enuMrz5BtybkUYGpgPoHH38ccK0FI49q9nIDLLt8revnLDQ2zUzSyhGQy2is5rp82QBl1Wkd6tZQiJDZmEcdXUqCDtBY8UmFlUNgRiEC3iA1soMiy+YsFQvTKclAQbNsoYEXfs1srnwJSvJvsvzeJgKEtUNGoNUz79K2mMIFYxRe8I8V3BT2WdYV2Q1YJNglmampLPVwMmpgyibd5TQgmLHiItORLFRriKq7hYBYNbbVBYNGc0UsDAFSvLWpoToQRVAGJyFP2+GYm272XQUYSVGrhRDmXJjJNQQrt8Y5z0DVg3fLaApYu6a7FtEgjZmwaOiUM1ismnaGEkoadpVCnus2WWg7eLTM4qzKiRdXt4y41abm2o2BFGr0ZmbdaaBYzoVLDDYNmtAUWDbRYlZGGNmpWZGA2bItTbTVa7t7DrlJZ+MhqXOQ+Q5yIipztRWteZLtdc5kLnOqMdA89LaRIJkkggggMFVrF0KeW7KoxEfllkUtLXlsZV9yu63C9WfeY2Jd08p6GG0mCkUKhJGoMpsv6g9HIkIfwWMj75JJJJJJJJLwktklx48f0tlZAAAAfmIIfQDxp8+Yx2hEeIAADGIIYxjTGNttrkJaccelbiENgvXsgIAAAClOF2VEAABkAThHTEpQhDj0laXRaxZsMjaZxVotKU0txr6s8Ka0xMYzUVhrGrAVfa1MHFO3eYsQKQlaX7lbrNnTturHZCXGsFFg411aKmDGQcwGgaJpqsW6KPdjmxqW8XU8JJZ2lsbw/O32WD8IXpmaKgLhi0g6Echkg1BimSYzPVxMKVWFyCwl22YvAuGaBYcOaQZXsa/hio2oz04zswbvIYzBLetKYTlm2MVWVoruBTSH4QQszooEONhWeUy22Cl1sNBVVMJaHCtXrtQJhmOpKJo5mqCDHiuPNf9mFuMERPwYpYphwr2cmeo/HQVgIHSm/siiIRIRavvb7HMV/l+v+dTpIw64JnXiTYtBepAvcB65GstPhaSRpK+hIxI2nGKSNOvUJSsW3tNQoZJLcDvcMG1cVWsJVlWI3KqpSy4rWDeKdpvmhsmukYprAR5uDC4UoYUcUGEhCALIQGfplpaRePVWG7/KRkp7fdFTHw+Pyrsz8180v+n1+02r+6tl+Zyd/X7/s9P/51QQV0ECJyqoCNAcgAhBBGFEIFADAMJBIKwDMCy32iien7QiA6Qif9FukaB2Dp1QWdaDO/h5HqTeQlEH34/aMkDYNMGwaYNgxfdiA2YOhBApDQDaSG20D6OBHMtB5doVg0tX1n/xdyRThQkGutV5I",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-26T18:46:35.544501Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-26_11_46_34-12250844836443732077'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0826150316'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-26T18:46:35.544501Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-26_11_46_34-12250844836443732077]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-26_11_46_34-12250844836443732077
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-26_11_46_34-12250844836443732077?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-26_11_46_34-12250844836443732077 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:38.094Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:38.731Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:38.911Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:38.984Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.015Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.186Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.231Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.269Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.299Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.337Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.373Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.410Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.434Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.460Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.484Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.517Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.551Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.586Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.611Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.642Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.676Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.712Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.746Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.782Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.817Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.852Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:39.887Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:40.056Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:40.113Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:40.136Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:40.159Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:40.162Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:40.184Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-f...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:40.235Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:40.235Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:40.307Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:40.342Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:40.410Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:46:50.502Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:47:05.132Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 7 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:47:05.157Z: JOB_MESSAGE_DETAILED: Resized **** pool to 7, though goal was 10. This could be a quota issue.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:47:10.365Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:48:48.384Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:48:48.422Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:52:12.462Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:52:12.547Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:52:12.597Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:52:12.667Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:52:12.704Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:52:12.714Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:52:12.740Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:52:12.770Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:52:12.800Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:52:12.822Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:52:12.833Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:52:12.882Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:54:33.153Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:54:33.277Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:54:33.332Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:54:33.405Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:18.242Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:18.326Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:18.368Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:18.439Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:21.869Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:21.951Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:22.019Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:22.054Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:22.097Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:22.097Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:22.162Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:22.197Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:22.231Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:24.803Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:24.864Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:24.942Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:24.991Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:25.045Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:25.110Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:27.853Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:27.946Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:28.016Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:28.169Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:57:28.194Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:58:21.315Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:58:21.358Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T18:58:21.382Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-26_11_46_34-12250844836443732077 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598467591454/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.07944035530090332 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: 2086d4fa2a6840408127b0d2b8ca63de and timestamp: 1598468431.784637:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 840
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598467591454/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598467591454/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.062335968017578125 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 840.980s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 14 mins 4.668 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 15m 15s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/ovw2qrhiqxcrw
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1848
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1848/display/redirect?page=changes>
Changes:
[noreply] [BEAM-10675] Add Python GBK Load Tests for streaming on Dataflow
------------------------------------------
[...truncated 289.95 KB...]
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWdDf8yAACKN/////////////////////////////////////////////4AjT6gN2OrhjbNCxsgE9g0k4g0kKZkpkbI0m0TCaPSbSep6j1GmTCMmgzUD1PUAGTTQAAAAAaABoaAAAAAaBoaAAaABoPRGRkNPUNDEMnpBkpiATEYTSYmTVNqPUNMmm1AAAA0Bo9T1AAAANDRoaAAAAAAAANAAAAAAA0AAADQAAZI2KAAAABMAmABGAEyMAAJgAmAAQYExGATCZMAAAAAAAABMCaMjAgwAAAmhB6n+qQYhMAI0xNMAEwRpkwEaaZMATAEYAJgAjABGmmJgBMAACNMAAmAACaYAAgwho0DAAkwAEwABMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABIkQCEwhqMTVPaE9CanqNHqfooHqPKPU2pptTT0RoADQDTQNGQepsKaAAAAA0AAADQAAADQAAAAAABipg05QwgVpOAFVduKeN8TSviWGvXpxkN0x0ltcDyhbVHrbcUT9my42xc7yqzWiG+mnfdoogVM/AHjHf2zUCIQEKFAgJGmJNpAM0c879NJcNMY13XonpkpsY0JTEIJ1/JIAtRyoUA0CeKCOAUh0K4YRWquBNbuXLF/BfWbWcaN8sNEvmAzDxv+DQSPEGcETgWmzQNI15pnJ4tEMpZcWnSEWC3cZp0JMvHSHhyQVmAMHfyo8TcGC0mVIqwkg8BTtchUFU6SYTCcE5ClYSmEwJ1k/ITU5QEcTwMcbj9G25znJSq5EVJubORIy1wy/xKCU6YRddxbZ1YzAEsEBGElDkokSkNhM0CUkNjeMghsNwQo2ZIZLPc0yG0yMJzAzhk8BmmuCgykyxYgzzRKSsmVlNnWQsCGc9GBnMqg7AEjIi902d3C9/6Lv0ZZp1feqiyfGmE5Hd5XeQAmVRcLrz9JUC4EnlppVJv4f+j5ZK/iZNZY+5A+OEeyWmJudoaWpDDDDDFgggQQQQQgggQRInvHBCISgIYAojgAgPfp9QSUmKKP6Df9DqLl5HCLewioFyQmmrmFqxk5smJuhkLFxsyhVis4KhUqpQr1VimGLzi4q4laq2avRONypV8aeenHFh6+NBcJRvDSOKuPiocRxx5rzFnLNUjKncPhpwNnd4NhibruwaGTDHBih1+KFEwcHjxeL9Z6uioSIH2gpbRF04glYsoUQFNVigUzEubONBvuOtyWKbNnM8eSJnMBfua74Fq8ZOR9roffhGUU4vK1Innsdt3l/F+mdunl6vkz5lM+5hkgRfRm9eF5RY4AxpKS0et1+zR3Fc3mT5uUmO+1nPfPfmjg3kGMYGEUsWRD0ueznwnwlh5ISiCSRkxkqgzPaBQDJiRuJRII4thZLBJWjXy9tcIwyndCN5/G6etk33grIZQz8k46ciHkHSVj1cjC3lNsGmn2Rl14RgUTpYyK45ByO4R4lXg7Kyjy8ML/ey2urS8M1XlsL5pxWECCpXiD6+XJI57GDJGJMaHIgUNDszJyT2F6BJBDz0OplmZNRQ3EXSgUmt9KoOn0zKCuKJXb+L2IVilUXRJ2k57aeLTKd6VLrSqYVzHkU2gWPnrioovUAwu52zn0c2mKxJKtVBgFjMSvC68CtYmzcqWa6AOOA1g+gk0bHTxlud6BSQBcLJTI4aBtvhWA1mEvm0IahMBAEkSskCRA8TX8b1LGgDeeOhSCgxBxOIOxRfV7Nq7EHv37IHyvOlpbP3uH5LPl5Q6r8DjfB+H76lbpPktfultdxQgW6B0HBwlYPrAKIzZFxE1SSkXcIqNA7SSGHSlIehPB2GrqqhNSwy1DDEZkBqqqqWgTIIC6gIE87ZMwOEahBkDrT8ZjgfJYW7NbBu3Qc+XEGHfWKsWFcot6rVQ3dSurqVSii8sCBU02Q66hQqpuiTdVRlSSvr/tlthMwkb8nbvaiqeCXWwisMyRqFOhPnBU3C0qcjKXQStiJkXZzQOmaCOZUEwNcwUGWknYsLi8KnlVRaXD0gqppfjNobAC3anvzNOFK0xoGzVsIVC0C40BIDdlKliUBgUFORJV+YxCoLnYPqNlLRu8zOqBCAwgC9fwmG0MQcRCCBCQF8CpdBKY6QcCfgUxNT9BE9Rs5EAS1bUlNaQgIS6QVfXp7eSsFQQDBzqI+piFsAmqH40rQLwGgBgLcBRzzZvbwHxIKaEwdXZ5ZRr1W15LyJkFpgHsDg3cqcCwFI4LVzOWGV27xWdfOaGhtmnkljCMRjsFTnOpS40GTWKB36lFCIENMohhi1Wcg7WVsKSilp+uIQ+Bax4NXJjILurUC0LhkOReINNooXkVgL14qmvSFWSbXgG/S8UktiGWgNiZt2hZThCBVsUXO+PBbv087MrKq4arEmwSyNTUlmq0GTUwZSm3cU58ExYcBFhy5WqWuQqLeBeFe1NhQFY0ZjRTWF4KFcWNTQnSQRRAGByFP21+Ym27mWwUsJKlq0UQ5lqYyTUEK3eGGc882MG35rPFLB3TaMWvSCNcaplylJkrVc07AvklDTsKIU9tkxyz3ZuEzMKMipSLatlGPCrDa2UGqIpauRkZslooFhOjUr8Ng2akBSr2witKuL8a5SrxMBszixNtNVLu3ruuUyzcJDUugh8t0ERFDruqKUrNlWOvdW+667UW68xklhEelSN5554vROLVO0xTu/ItEbkkjaUljzVri92s7afZVbYtyBV8xpj79hMYkD7QjhaC6KW9TZCVDw8TCAMGEpWKAAAAAAAAAAAWAANAAIxYsmFDQIAEEEEEEbv2xJ52Kwzs622qIrk223bbEnbbaW23XXU5JF69jlTeiLnLHjzJNttt0UXpVSok223bilLiq2c4iL2OUqZosYsl+Rr8woytKU0tntNjO+mtETGM01WahsYCj7One5N2a2DACkJWF21W2yZk7Lah1wlyjBRWONorBU3sJBzgZ5lNFVC2xS9uObGpay3V8kszQ1dY/M32WD8AXDK4KgLow6AdFONyIcQYZjlMrKwlFDVdUYrpZclLAXRkgV+/kkGN6u08KUGkM9MM7MG3xGEvS3bSmE5ZNXBRjqMdYnv6bRElmuBFayxrZ+HSmZ9ekVCjArVUVmLHjcQJhGGpIJYZXGggxYLbHG8y+5iBET8VtKFD+DcmJmqNwzlQCB2Q4FIngECEWLvudlmK3zfZ/OrUEYVYEzKx8uHOWUAXLzLUai0eBoJGgrqSRgRr9ypI0atMlKtbKw0ykxSWzHc4YNq0osYSqKMBtVRTTXaVK9rKdhvGhsmumYppwRtBWw4irDFCXBggGICdiDG9k3Nyzp7bGvf2rJr7/hFT3x+Xzs9B9HNs5/X7fehTfhMr/VFR0n7/tLTf5vTsJEQIoog4MwGgAIMAfBBiEgDAGCAQCYAxgLdgncUtNcTwG50T/oxufTuQbt6e4KAY2EDqPai6hKIPvR+wZIGwaYNg0wbBi+5EBrgdJBApDQDaSG20D6WBHOtB5ZoVY0tj6v/4u5IpwoSGhv+ZAA==",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-26T12:46:21.678446Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-26_05_46_20-7280835498653275262'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0826100259'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-26T12:46:21.678446Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-26_05_46_20-7280835498653275262]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-26_05_46_20-7280835498653275262
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-26_05_46_20-7280835498653275262?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-26_05_46_20-7280835498653275262 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:24.917Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.041Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.080Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.154Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.183Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.282Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.328Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.364Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.399Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.427Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.461Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.495Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.519Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.541Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.562Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.601Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.634Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.659Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.691Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.737Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.768Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.803Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.838Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.870Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.905Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.937Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:26.961Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:27.158Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:27.228Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:27.271Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:27.272Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:27.294Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:27.306Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-f...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:27.364Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:27.364Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:27.433Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:27.459Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:27.516Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:38.280Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:54.317Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 9 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:46:54.343Z: JOB_MESSAGE_DETAILED: Resized **** pool to 9, though goal was 10. This could be a quota issue.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:47:22.821Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:48:28.919Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:48:28.956Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:51:48.714Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:51:48.858Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:51:48.884Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:51:48.965Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:51:48.992Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:51:49.013Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:51:49.027Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:51:49.047Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:51:49.084Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:51:49.096Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:51:49.127Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:51:49.154Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:54:16.519Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:54:16.581Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:54:16.632Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:54:16.698Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:56:57.019Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:56:57.063Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:56:57.107Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:56:57.151Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:56:59.585Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:56:59.664Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:56:59.735Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:56:59.770Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:56:59.793Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:56:59.813Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:56:59.866Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:56:59.900Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:56:59.972Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:02.675Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:02.733Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:02.803Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:02.868Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:02.934Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:03.008Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:05.592Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:05.664Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:05.738Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:05.867Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:05.913Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:53.564Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:53.607Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T12:57:53.643Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-26_05_46_20-7280835498653275262 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598445977822/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.06259560585021973 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: 10cf3a07eb344d69b81ca61b13194d8a and timestamp: 1598446776.5965533:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 798
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598445977822/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598445977822/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.06139039993286133 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 799.285s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 13 mins 21.964 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 21s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/dltmwfyl6pteq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1847
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1847/display/redirect?page=changes>
Changes:
[noreply] [BEAM-9680] Add Aggregation Count lesson to Go SDK katas (#12646)
------------------------------------------
[...truncated 289.89 KB...]
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWU4tn3EACKN/////////////////////////////////////////////4AjT6gN24rhjNtCxsgE9YUk4g0UIaSNHpo0m1NMQPSbSep6T1MTRkNANPUD1PUAGmjQAaAAABoAGgBkAAAABoaAAyABoMRkaDT1DQaDR6QZIQNEZGmQmJo1Go9Q09J6m1AANADIGj1PUAAAA0NBpk9QAAAAAAANAAAAAAA0aAAAaAABkjYoAAAAEwCYAEYATIwAAmACYABBgTEYBMJkwAAAAAAAAEwJoyMCDAAACaEHqf6pBiEwAjTE0wATBGmTARppkwBMARgAmACMAEaaYmAEwAAI0wACYAAJpgACDCGjQMACTAATAAEwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEiRNAhMJMjE1T2hT9CJ6jQfooHqPKPUyaZNMRoADQDTQBkGjaamgAAAANAAAA0AAAA0AAAAAAAYiX1bKmICnJmRSZaozRtFSVoliq1a2NivDHWLrIYFi0uPXnaNFtiw864yaBdNWkm6qTNfnnhXUbCIDHd2jQCIQEKFAgJGmJNpAM09A77NJcqmMa7n4R6pKbGNCUxCCdiySAMWcsFgKBPFhHgKwfC+HFGzSvalq3br3cN1Ys57p4S00zCYjOPG/4NJI8QbIInAtVmkahtDVOMyaYaJbeXHREWi30Z5zpMwHRHhyQWGIMXfis8TeGK4mVorxkg7/TdZisK50JhMJwTkKVpKYTAnYT8hNTlARz3gI4nQdC25znJSr4sVpubOLI0bIZh56olOkIvv5+6deUxBLFARjJQ5KJEpDYTNIlJDY3lIIbDjSFG4JDJaDmmQ2mRjOUGcqTxGebMKjRJlqyBoGmULCZYUt6uFoQzm4xM5ZVHXgkZkYOk2XbsH/q/fIqxTjK8Uhq/N47iSvH7SQEutLfdT2/mjFvJDJzK2Qfw/87wx99EudhutvCMWI9ktMTc7Q0tSGGGGGLBBAgggghBBAgiRPeOCEQlAQwBRHABAe/T6gkpMUUf0HC53WXJSOTLuviqF0sTTV7C5ZSc2TE3UyFk4mdUXwYNVKi6XioxPQFgGY50ocu8XcYYVYri8AszK5ypWLXVdBLByZVo4tDFxSyNSmsVUSiZjKViqUhCa4UUEQbkoWsPeN5ChlA48MQPtoxIl9fcgLhbTgqouIiFqgtiiLZwxK4wqUQFK7VApmRcwcSDg9At6WqbNSZ48kTOUDDe13oMX46cl7fRe/EMqt5mWoUYIMDcfM+L9NAfPMUvKmzaYPMuSBGdIblwXFlxQDOPj9Bq5HXp7Sq7jIcrIy3bmJv1zv2oKu9gzMGK6vX1p8YvqD8T8S9EktOkrVmpNXRmyLcOgNSSduKJBIFJWCsR9lkZRsoijdtsLJzOF0NXIU3er4d0beCbd+JERjvxYtdG1mMUyTfSa42MYnApoSZpsUEnVDfIji0tDJad7pbO93ktjqVfdmYNhWfFNrgkQWrsKjClxiQfRgSTiTOIIgUPENjVEqrb4CVhLz0OtludNRU3EXygUmuDKsOmUmVFkVSv4UYMgrVKsviTuJz3c8mqU4BWusKthZMeZTaBZebvKyrBUDC/mre9o5hMVqSVirDELKZFgF1wFcxNm9Us91AeeBphqCTS8CaMuzQQrSEMiclMjxnG4eHXCs0l06hCojCQDAZiwvgennPx6uDfQB1XqZ2ANQiJRE5yPi+TkvI0Xrbm/r7bve8RqHNWdgxK0oCsc/KdDpSikoMjYL4MqlU1BAFBgGoEUsCYBxABRGfIvImqEpF/JlZpHZyQw6MoHoTwVpr66qTUsctYxUTUQGkuktiBMFAX0BAmmdJqA35oEGQPqbGY4Wksbduxi3zqO+F5Bj4NqsFjXHLgK5VN30XVVKtRVgWJAqUth2VihVzdUm66zRSSwr/tt1pMxkcIndwKyueKXWQiwM6RrFNKfMipeLUpmZR1EroiZF+yaB0mgjllBMDZsFBo0J2rG4wCpx6quLx6gV0o/Gbk2oFqzNdmpMFryKgbFKuhWLILygJAbyUrYJQHAsLeopUeS0C0MXkU1OwloXqXnHQQwMQBef2SziHIUcQwgQrAvQWsYR2i7soKngpaZ9s9E9Fw6kATBhSXaqJQENcYSFZKrYS4FPQDNZSB8jIWkCZn/dQtwyAbgGBawKCdcObYA/Kw7T5h4eXtrZHyCZEyImoYGA+gchfx5w7QUjkGr2ciMsu36t66cuNDbNXNLKEZDLaKtn2rS44GTW5x4C1lCIUNaURRUdZmIO3FjhSUVlNqiINAs48GllBgsvGLBULZkOQgINayUMCLvWbCV1LkhXknWXpu0wFCW2DRqDbGffpW0xhArGKL3hHiu4SeyzrCuyGrBJsEszU1JZ6uBk1MGUTbvKaEExY8RFpyRYqNcVVXcPALBrbWoLBozmilgYAqV5a1NCdCCKoAxOQp+2wzE23ey6CjCSo1cKIcy5MZJqCFdvzHOegbaDecvoCli7luWLaJBGzNc0dEoZrFZNO0MJJQ07SqFPd5sstB28amZxVmVEi6vbxlxq03VtRriKNXozM2600CxnQqWGGwbNaAosG1ixKyMMbNSsyMBs2Ram2mq13T13WqSz8ZDUuch8jzkRFT7+sLWvGLr9g+xr779Bj8DjkldEglSOCCCAwUXmLmU4t/LoxEdkkjUtK/nMZRzlN+y1yk64x6Fd04pxra6Y1IWqEcSgyes7q7kRUPDxMIAwYSlYoAAAAAAAAAABYAA0AAjFiyYUNAgAQQQQQREoNUtJqUAmma15Sn1oiIiNapaNa0NbDDC7KsRRU1uxRF7lly50m2226qsEq5VSbbbuySlz6unOIjBllKk0WsWbDI2mcVaLSlNLcbnbTwprTExjNVWGsbaAq+zq4OMduxixApCVpfulu82dO26sdkJccwUWDjcq0VMGMg5kNA0TTVYt2Ue8HNjUthdRwklnaWvsH5m+xwfgC2ZXhQBfGFODpRx2QDyDCMcplcpiUTqqrjFVLD0pcBfGaBYcOaQZXr7nwpUagz0wzsgbzIYzBLfNKYTlm18VWWsy2CfCpcIks9wIsWjGxoY9SZoWahWKMSuVZYZMuV5Al+YqEglileUEGJDacedzbXsMIiZ1ilidt9bqE1KDsUxWAgfkN7WE0IhQivee72eap/N9n85Oci/pgmovHy4UxcnAt3HLkay0+HpJGkr6EjEjab1SRp16pKVi29pqlDJJbgd7hg2riq1hKsqxG6VVKWXFawbCnab9obJrpGKaoEedghRxUBhxKAwkGgCpoDP0TU1Ju/przJ/mGsx6/ZXZ33e/4WOe+Kxws+Xz+k+p+qld9/xQfn9fyh/v+c0TONECPxWkA2AbgAgzCMIGgVgGAYSCQUgGYFrwD0KH+25EByhE/6Kco0TqHLmitroDOODwPSm8BKIPvR+wZIGwaYNg0wbBi+5EBswdCCBSGgG0kNtoH0cCOaaDyzQrBpbb1f/xdyRThQkE4tn3EA==",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-26T06:46:33.644315Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-25_23_46_31-13003599906346747561'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0826050300'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-26T06:46:33.644315Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-25_23_46_31-13003599906346747561]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-25_23_46_31-13003599906346747561
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-25_23_46_31-13003599906346747561?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-25_23_46_31-13003599906346747561 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:36.889Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:37.713Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:37.766Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:37.881Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:37.911Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.023Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.057Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.079Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.114Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.149Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.184Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.217Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.251Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.285Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.319Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.351Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.388Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.421Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.445Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.478Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.524Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.559Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.591Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.633Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.666Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.707Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:38.728Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:39.015Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:39.086Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:39.120Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:39.121Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:39.157Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-f...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:39.157Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:39.207Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:39.207Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:39.276Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:39.310Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:46:39.341Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:47:07.926Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 9 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:47:07.957Z: JOB_MESSAGE_DETAILED: Resized **** pool to 9, though goal was 10. This could be a quota issue.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:47:11.623Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:47:34.794Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:48:46.343Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:48:46.364Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:19.265Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:19.307Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:19.330Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:19.379Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:19.398Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:19.422Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:19.438Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:19.446Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:19.472Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:19.506Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:19.526Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:19.637Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:22.333Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:22.379Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:22.417Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:54:22.460Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:00.604Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:00.682Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:00.733Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:00.802Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:03.239Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:03.305Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:03.371Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:03.407Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:03.452Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:03.457Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:03.520Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:03.552Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:03.621Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:06.199Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:06.258Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:06.320Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:06.374Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:06.446Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:06.518Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:10.247Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:10.328Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:10.433Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:10.544Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:57:10.581Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:58:08.483Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:58:08.529Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T06:58:08.561Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-25_23_46_31-13003599906346747561 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598424389157/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.06753802299499512 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: d598f2c371294559b75c5d0507b5d8bf and timestamp: 1598425192.6082213:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 803
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598424389157/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598424389157/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.06436467170715332 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 804.067s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 13 mins 26.935 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 35s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/b23qz45mtajgk
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1846
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1846/display/redirect?page=changes>
Changes:
[rehmanmuradali0] Adding key in Spark
[kmjung] Update BigQuery storage API source to v1
[kmjung] Make stream progress reporting work with v1 API
[kmjung] Clean up some leftovers
[kmjung] Address code review feedback
[noreply] Dataframe set_index is not index preserving (#12676)
------------------------------------------
[...truncated 290.27 KB...]
"python_side_input2-Write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWccO+ZwACKN/////////////////////////////////////////////4AjT6gNtJgjZtCxsgE9g0k4g0UIZKYniNI9TTEeo9JtJ6mTT0mRkNANNA09IAGgAAAAAGgAaAAAAABoGgA0BoAGgxGhoNNDQyDR6QZIIBGRhNJiaJPSPUNMmm1APUAGg0ADQAAAANGmmT1AAAANAAADQAAAAAAAAAANAACTAATAAEwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQep/qkGITACNMTTABMEaZMBGmmTAEwBGACYAIwARppiYATAAAjTAAJgAAmmAAIMIaNAwAJMABMAATAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAASJEARMIEyT1T8jVP0ImjQfooHqPKPUyaZNMRtQANANNDTRkGmaamgGmgAADQAAANAAAANAAAAAAAGTTF070YgK8mYFZlyrPHAJpYBLFYsU4EV8Y6S4yF9YuLj1tyrVgtWnXGsnfXXVoxvJp4HqKIVzQQh8x3tw1AiEBChQICRpiTaQDNPPPATSXJpjGu8c+9QlNjGhKYhBOw5JAGLOUCwFAnigjoFIeC+GFW3Wv5rl27ZvYby1bzrTwFhpmAwmYdH/g0kjxRnBE4FqM0jam0NQ4nHphoFlxadKRYLdxmnOky86U8SSCswhh8CVHi7gw2kypFWIkHgqdxlKgqnSTCYTgnIUrCUwmBOsn5CanKAjnvBxwuh6JtznOSlVw4qTc2cORoVwzBz1BKdMIuu6C2dWQwhLDARiJQ5KJEpDYTNIlJDY3kIIbDiiFG3JDJZ7mmQ2mRiOSGcmTwmaaIUGgTLFjDPNMpKyZWU2ddCwIZzMYWcoqDsgSMqL+nzu53/+rd5iq96211dot9cX/idnjdpICVYFvpP3fmiFvJDIy7CQfg/c7wx96EqdduNqCNwOVzPO9B0XTdV1zjjjjjrWsYxjGtaxi1/7UOh0DQgmGGDfcpvzdK0zI4foN/zusuQkciW9lFQLqImmrmFqyE5smJuhkLHwsygvAv5EqLhdKjE6+WAZbMnMHgXi63AMS2qxpXZXiWCrgsRWDcU6dXKtFGJdCG+ZEksSp0jNBnKxSUiCayUUEQbioXvB79zRB2coqO1FduG9vY2OkNurclmi5XoutglOEaq4JlomUKICmqxQKZjXLHCg33QrqZYps15nSEiZyQYLmuaCSMmeh8uwmHQYVFYliXV2UUgr32Jmep6ZaZf1mPmNvp1S4GlsiZswzRJUqBnHx+e1Or16e0qO4x3Jx8p25aZ9k195+FegZmDFZZMqs9tXxB9J9JkiSWlSUUNWalEsyMaFoNWSdqKJBIFrK5VI+w6vmuFUUa/bCwcvhdDUyFK3rKHWxngmXfiRFyd5GOY3WqxyvWONFrjaxycAmhJGm1QSdUN8iJGkIZDSvVIgb1di2OnU7su3aVXxTLEJEGDHCI1MuKSD4L8ScSZxBECh4iMNYSyrvYJQJeeh1MszJqKG4i6UCk1vpVB1KmZQVxRK7fxfjFYpVF0SdpOe5nj1CnelS68qmFcx5VNoFk5m4qKL6AYXcxZzaOWTFYklWqgwiyGNXi7ECtYmzi1LNdAHXQa4gQSYeNnjLk78K0hDImymR00Db/BshWaS8cQhURhIAlCYkkSYHelvtuVMQANx351QLTIOiBB5Kt9Lq2PqQu7Z4tGR5UkjTOYu65js0YCqc/JdDpc1WTcjaMm5U6nqiAJuAaoRS4JgHEAFEZsi4iapJSLuRKjSO2Ehh0xSHoTwthsbCoTUsUtYxVTMgNZdZbECY9AXkBAnncJmBwTUIMgeVBGY4YCWJuzZw7t0HNlxBi31irFiXGLeq1UN3UrralUoovWFAqabIddQoVU3RJuqo0EksC/7ZbYTMRG/J272oqnhl18IrDMkaxTpT5cVNwtrTlZS6CVsRMi7OaB0zQRyigmBosFBoUk7FicXip41UWlw9qFVNL6PXOIAuW570zTha9NUDarWUKxbBdUBIDfSlbBKA0LC3aqU3kQYYBq8ilJ10s9R5X1rRDAw/l5/bKTkOQdBDCACgF3DBrCL7hclRMwUpL+6biejOOpAE0aUVfUEn+GtkJU2yyzkxBTb/ArnYfMyFlAl5330DgNgHABgRoE/NTjmzgPqgXzph4eXqLsD3BNibEUWGhgPgHHXcacCwFI45q5nHjK7d4rOxnKjQ2zUyyyBGMyWCpzvUpcYDJrFA75ayhEKGmURRVdVnIO2ljSkopVBYEQgAt44Gtrgx7L5iwVC6ZDkHyDTbKF9F37NgK5sVIV5Jxl+b1LyklqhoUBqmbdpWU4ggVbFFzwDw279POzKyquGqxJsEsrU1JZqtBk1MGUpt3FOfBMWLCRYcgVqlrhqi3gXiv1uIoCsaMxoprC8KFcWNTQnSQRRAGFyFP22CYm27mWwUsJKlq0UQ5lqYyTUEK3eGKc881YN1yueKWHu+uxbRII0TYNDQKTLWq5p2BgJKGnYUQp7nLklnuzikzMKMqpSLattGTErDcWUGwIpauRlZtlpoFiOiUsENg2a0BSr+IitKuMEaKlXjYDZnFibaaqXefX9gplm4iGpc5D4/nIiKHntUWtecLs9e8yB556ox59rZLKI/Kkb7775fqusXOpq3ssjERySSNS0s+YxlX3a71uBtZxrHYV3jVNggspgJDAoRxKDKKXNTbEVDw8TCAMGEpWKAAAAAAAAAAAWAANAAIxYsmFDQIAEEEEEEbwWxJ52Owzs622qIrk223bbEnbbaW23XXU5ZF9+SVN8Rc5ZMmZJtttuii+VUqJNtt245S6BWznERfklKmaLGLLgkbTMKNBpSmlt9fVngTWmJjGairNY1YCj7OpfxLs2cOEFISsLtwtzlzJ2W1DrhLjGCisca6sFTfiIOXDPNA01ULclL3Q5salsrq+AkszS2Nk/M3wYPwBdMroqAvDCoB0c4zIB1BhGOUytriUUKsLjFhLTspaC8MsCwYMsgyPY1/DlBtRnphnaQ3WMxF8t20phOWXYw0ZKjJWJ7+m0RJZrgRWtCNnPxbWZn17UqFGFWqorMeTI6gTBMVSQSxSuqCDJw3Guue1A7r4RE/FiliiDF3ZiZqjkU5WAgekN/SJ4RChFm+9zss1X+b7P51agjBrgmZeOlwpy2gC7im2o1lp8DSSNJXUkjCjacWpI06tQlKtbaw1CkxyW3Hc4YNq0osYSqKMJuFRTTXaVK/ZU7DeNDZNdOxTWAjzcEHQUwYUcUGEhBgLIMGfomJiSd/TWmz9LRct7PbWZr3/D41+b+S6cXfP6fWdV/ZWx/E9P/n9fyg/v+c0MDdBACeqyAjAHAAIQIRhBBhQAwDCQSCsAzAjb7RxQf20ogOUIn/RTlGhdQ5c0NoWgzv4eB6U3gJRB96P2DJA2DTBsGmDYMX3IgNEHSQQKQ0A2khttA+mgRzDQeWaFWNLV9X/8XckU4UJDHDvmcA==",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-26T00:46:37.499554Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-25_17_46_36-5641805480378614175'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0826000648'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-26T00:46:37.499554Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-25_17_46_36-5641805480378614175]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-25_17_46_36-5641805480378614175
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-25_17_46_36-5641805480378614175?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-25_17_46_36-5641805480378614175 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:40.464Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-a.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:41.968Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.005Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.137Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.166Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.261Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.306Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.341Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.377Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.400Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.431Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.474Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.509Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.541Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.581Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.620Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.654Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.692Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.726Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.762Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.807Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.838Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.870Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.910Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.944Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:42.977Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:43.010Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:43.423Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:43.488Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:43.512Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:43.523Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:43.532Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:43.554Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-a...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:43.604Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:43.604Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:43.694Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:43.728Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:43.810Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:46:51.930Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:47:12.742Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:48:52.515Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:48:52.551Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:00.041Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:00.145Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:00.177Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:00.252Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:00.286Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:00.304Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:00.312Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:00.341Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:00.362Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:00.367Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:00.393Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:00.420Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:16.380Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:16.465Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:16.523Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:54:16.585Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:56:55.063Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:56:55.128Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:56:55.181Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:56:55.240Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:56:57.433Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:56:57.499Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:56:57.552Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:56:57.580Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:56:57.599Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:56:57.627Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:56:57.661Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:56:57.700Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:56:57.774Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:57:00.752Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:57:00.826Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:57:00.895Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:57:00.939Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:57:01.022Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:57:01.097Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:57:03.670Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:57:03.731Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:57:03.818Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:57:03.942Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:57:03.970Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:58:06.182Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:58:06.234Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-26T00:58:06.270Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-25_17_46_36-5641805480378614175 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598402793654/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.06614279747009277 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: c61791bb8898492da6f831f0cf8cace8 and timestamp: 1598403589.5030398:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 795
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598402793654/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598402793654/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.05433320999145508 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 796.367s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 13 mins 19.217 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 34s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/sygfgbgyeaalq
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1845
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1845/display/redirect>
Changes:
------------------------------------------
[...truncated 290.38 KB...]
"python_side_input2-Write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWW8sZzAACKN/////////////////////////////////////////////4AjT6gN2cudMdm0LGyAdvcGoniDRQhlJiemjSMTEeo9JtJ6mTTI0MjRoNNA09EAaaAaAAAAAABoAAANAA0BoDQAaABoHqGmJo00NDINNpBkhGkxMTBCbSaJPSPUNGmm1AAAA9Ro0aZAAAAAAaNAAAAAAAAAAAAAAAAAAAAABjU8SeVE9ATQ9TJiaaZNMhkYjJ6m0E09TCaYjIZA00GJoZMIaYjJhNGQaGhiGI9JobRGhoMjTQNMIyGgaaGmhoMQ0wEMhB6n+qQYhMAI0xNMAEwRpkwEaaZMATAEYAJgAjABGmmJgBMAACNMAAmAACaYAAgwho0DAAkwAEwABMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAABIkQBAIEyNU/I1T9CJ6jQeKB6maj1NqaYjEAANANNA0ZDI8ppoAAAA0NAAAAAAAAAAAAAAABlkxtrczCBLJvwsMuxURvieZ8TQ2bNrChwDHaLrIKyxdXHk3Yon7lxrrjKKy5VaCb6eh96mmBc78ArGO/umuEQgIUKBASNMSbSAZqaB3yaS5ZMY13PwT1SU2MaEpiEE61ioAxZ3cLAUCeHCNAtB4L4MRVWMGe7evW7+C+uVZpqYCw1DAYTEeM/waaR4czwicC1WaZtTaGqcZk1A0Sy4tOkIsFvozjniZedIeGJBWYQw99KjxFwYbSZUirGSDv1O5zFQVTpJhMJwTkKVhKYTAnWT8fNTlAR0Hf44vg+ibc5zkpVdDFSbmzoZGjXDMHQUEp0wi67iWzqymEJYYCMZKHJRIlIbCZpkpIbG8pBDYcaQo3BIZLQc0yG0yMZ14ZyxPCZxpBQaJMsWQNA1CkrJlZTZ1cLAhnNxhZy6oOuhIzIv+nn9uv/9ESETY/ezvDjFPbTAyfb5HcSAl1xcDq/D9Uot9I5NqukXwPnQ8eZvglzsV0twRjjdpLzPIm0vMTHHHHHHxjEIQhDGMQghwUBuVwxjOYBEYIyGh9kNG2JEXR8HntdcnI5Qt5UVAuliaauYWrKTmyYm6GQsnFxUKsVnDUKlVKFfVYphz/m1xF4GtVbCvji0aU+nKiNmzSVcxqXDilWfCOoIcwxSlNzCpkiM5mKxSUgCZIooIg2lQqym37gFDJhxoyoefihRMLC4wYJflrWUXCRA+oLYoi8coJnGFCiApqsUCmZF2E4kHH8VbwsU2a8zxxImcqGC5rvISR6p6Tz7P2N4wyKxLFODioozvnamKCqJlqXLsL/QdX1SpaDS2pMGAYIkqVAzzGY0Gs12yT29Wksfy8hLd31zEzN/ihrLwDMwYr7s7QzyyekHsnsnZEkuAIpQHAKnugXUxOMAcaR4NRmGnjZduy24ceL2XUaQCmlMzdL0Kfh71bn0LS3RfimHOThY9z41yX3at3pXrmkjsDZR6b8mhKGmyQScEOAiOM7oZHTvDu/yF5LZ6lTvtYJjW/JMLwkQXL8KjWC45IPQwBJw5nCkQKuQsWawlldeQSgS85DqZZimoobiLpQKTXISqDplMygriiV3Bi/IKxSqLok7Sc91PJqlPAKl1hVMK5jzKbQLLzdxUUX0Awu5qznEdiTFYklWqgwiymRXi62CtYmzeKWc6ANaDKH0Emh5zRGXaK0C0gDIXJjI00jaPAthXXJfOoQqEwEAShLySJMDwev7ySmKAG68M+oG4yDegQdSiOn17R1qzvGGNRk+ZKI1HnLeyY7dMArXRykr0+crKCI2LtBFU6oqIAoIA1QiVoTAOFAKIzpFxE1SSkXcoVGmdnJDD3BSHRnP2GzsqhNSxy1zDEZ0BsLsLYgTIIC8gIFFDpM4OGa5BkDyn4zHA+Sxt2b3DvnQc4XEGPkLFWLGuOXAVqobupXVVKpRResKBU02Q66hQqpuiTdVRopJYF/2y2wmYyOCTt4FRVPDLjwisMUjXKdOfMipuFtaczKXQStiJkXZ7QOmaCOXUEwNJgoNGknYsbi8VPIqi0uHtQqppfi9g2wF2qi/M9AWvSVA3LFtCsVAtUBIDgTFbBMA4FhbYkqXJgQuDJ1FPTsRaGmy/vbhCAw+l6ZqWnq4QbxCB+CgF6C5lBuUddVRMAUs1+E7Dys84EATFjS3KqpPsJcIOsM1kaS8FOvr+tcR9DIWoBrPzlE0DMBoAYEYBQzc82jQHtQOU+YdHV4irK8wDMmZFFBiYD0Bw7uROFYCkcNq5nJDK7d+rOuHMDQ2zVzSyhGQy2CtZrp82OBkyVI7xayhECG1MIYYtSoIOzFjhSYWlP2RCHwKseDY1cZBmAxYKheMhyNYg2qihrIvBZlSufGSFeK6zBN+l5SS1g0aA1jOu07KcYQKtii54B4beCnn4qyquGqxJsEszU1JZytBk1MGUpt3FOhBMWPCRYcmVqlroVRbwrxX6+2oCsaMTRTWF4UK4samhOkgiiAMLkKfs8ExNt3MtgpYSVLVoohzLUxkmoIVu/Mc56BrQbvmNAUsPcthi2iQRpGyaOiUmatVzTsDASUNOwohT3WbLLQdnGpmIozKlItq28ZcasNzZQbIilq5GZm3WogWM6JSwQ2DZrwFKv20VpVxgjSUq8jAbM8sTbTVS7p6zrNMs7GQ1LnYfJc7ERQQjiEIcYFCmIJOIQiARCjGpIQAKqgUSlKU0qgfEFWPDCTagEADskkalpb8pjIvcleqfcsOuMbAu+cU4+/bTCSB9QjhUGU2ndQchKh4eJhAGDCUrFAAAAAAAAAAALAAGgAEYsWTChoEACCCCCCN4LYk8/JYZ+fbbVEVybbbttiTtttLbbrrqc0i+/LKm+IucsuXFJtttuii+VUqJNtt25JS4itnOIi/LKVM0WMWbBI2mIo0WlKaW42NaeBNagmMZqqs1zWgKPs6t/GOze4cIKQlYXblbrNinZbUOuEuOYKKxxsKwVN+Mg5kNA0TUVQt0UvdjmxqW9XUcBJYtPZ3p+ZvscH4AvGZorgvDEpB0I47IhqDEMcxmclEwpVZXGLKXGzFwF4ZoFgwZpBlezseEKDajPSDOyBu8hjL5b5pTCcs2zhoytErwKaQ6+CFmdFAh42VZ5TJUwUvVBoKoi+Wh4WJZWoEwzDXkE0MzVBBloLrjXfYfbrIRE/Bilil/G3pyZ67sNBWAgekODaFEAgQi3ge12HXy/L9f86nSRhygmdePmxKC5SBexjlqNdanC00jTV1JIwo2m8UkalWqSlWtvYapSZJLcDucMG1aUWMJVFGE3KopprtKlfvVOw37Q2TX0mKawEeegA3iojDjjgwkIEBZAgz9LVry3P22Bm+eqK9UzNV+bnPj8rJO/NbPLfp9ftPq/urX/mgof1+/7Rf6jbUj/dw/D8wyAiwGgAIP4RgxAhQAwDCQSCsAzAjL/TRRf62ogNkIejiWyNI4Cjo21sbgZ4EOgpE3QJRB96P2DJA2DTBsGmDYMX3IgNIHSQQKQ0A2khttA/cwI5poPKtCrGlrep/+LuSKcKEg3ljOYA=",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-25T18:46:41.047593Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-25_11_46_39-7253857048984676632'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0825150324'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-25T18:46:41.047593Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-25_11_46_39-7253857048984676632]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-25_11_46_39-7253857048984676632
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-25_11_46_39-7253857048984676632?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-25_11_46_39-7253857048984676632 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:44.004Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-a.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:44.868Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:44.904Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:44.991Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.027Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.126Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.164Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.197Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.233Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.265Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.308Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.334Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.372Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.412Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.455Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.491Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.532Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.555Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.586Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.621Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.655Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.702Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.738Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.789Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.819Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.855Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:45.890Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:46.115Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:46.215Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:46.253Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:46.264Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:46.276Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:46.308Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-a...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:46.381Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:46.383Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:46.446Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:46.478Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:46:46.585Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:47:13.504Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:47:18.438Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:48:48.437Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:48:48.461Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:52:06.402Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:52:06.504Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:52:06.591Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:52:06.792Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:52:06.880Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:52:06.932Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:52:07.020Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:52:07.058Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:52:07.116Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:52:07.138Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:52:07.216Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:52:07.277Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:55:14.077Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:55:14.142Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:55:14.212Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:55:14.278Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:05.153Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:05.245Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:05.307Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:05.360Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:39.035Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:39.154Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:39.246Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:39.294Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:39.322Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:39.448Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:39.469Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:39.538Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:39.614Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:42.648Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:42.739Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:42.796Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:42.863Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:42.955Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:43.024Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:45.807Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:45.900Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:46.044Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:46.137Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:58:46.170Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:59:42.735Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:59:42.872Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T18:59:42.942Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-25_11_46_39-7253857048984676632 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598381197363/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.07443714141845703 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: d5e15eac891f40d0a5956e66ac2fc273 and timestamp: 1598382081.2802799:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 883
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598381197363/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598381197363/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.0652778148651123 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 884.497s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 14 mins 47.238 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 15m 55s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/vr5mid2qh3kv4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1844
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1844/display/redirect>
Changes:
------------------------------------------
[...truncated 291.25 KB...]
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWSc/MscACKN/////////////////////////////////////////////4AjT6gN26rd042bQsbIKT1Y1E4g0kKaekmJ6aNJhMRk9JtJ6mTTBM0jJpo9TQNPRAGQGjQaAAABoAGhoAADQNAyBoaDQDIAGgwmAjTIyMQyekGSENNEZMEI0CT0j1DRppkDQAGTTQ0aNNAAAAAANA0AAANAAADIADRoAAAAAAAAAAGJPZKEbQCYEYaAEwEwRgIwCYBGjAAEwAamRhDRoMAAAEyGg0BpoyYJiYATACMAAE0MTIGhLf6n6pFNtFD1PSaaMTQMQaNMmAIyGhpk0aZGmIAxGTBMmAhpoZBoaaBkYJhDEMTBGTTQaaMCNMjQwmgYmQZNA00AJMABMAATAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAASJE0CE0aTE2k1T8plP0JHqNB4oHqPKPU0aYT0RkADQDTQBk0aNimgAAAANAAAA0AAAA0AAAADQABlUw9reDGBNLlhYZeV6ZHCidwnjs2bWDHfmS0X2RQrF4uTW369d13dwPvMphXMrGm6opc2qqJdDohCZL28NYIhAQoUCAkaYk2kAzLoHf5pLhJjGu6+FeqSmxjQlMQgnX8igDFnfAsBQJ4wJABaDQvLivcsYFF5fX1u9iuru5nmXCWmUwmIzDx3+DTSPEmyCJwLVZpmobM1Tjc3KGiW3lx0RFot5Gec6TMB0R4gkFhiDF4ArPFXhiuJlaK8ZIPA02+QrCudCYTCcE5ClaSmEwJ2E/IzU5QEc/z0cPidC25znJSr4sVpubOLI0bIZh5+olOkIvv6C6decYgligIxkoclEiUhsJmmSkhsbziCGw6cQo2pIZLQc0yG0yMZygzhE8RnmkFRoky1ZoaBlKFhMsKW9aC0IZzcYmcqqjsQSMiMHSbLuGD/0Hi1tzoFeIZYVwXRsnMdLyIATScpLvUlbYi5UfoZxPHuYc+z646ME0aEir8D4kWGnJ6ho+94accccccpCEAAAAEIQAGG3yIqgaCJ4hBTEAJCE+iFWWFxT9DwOd11yMjki7kxVC6aJpq9hcs4nNkxN1MhZvDzKi0GHHFRfMBUY4ISwDLZg8IeCaYOFtYc12PqxJWp1kqgRvQZZTXJqOrGoUUuHZHETyPCs9ZykaCd4+EnA2d3g1OY3XdgoZMPQDKhrq8aJg4PIjALpobKLjIicoLYoi9OUE7zBUogKV2qBTM1cucODf8RbotU2bWZ5AkTOUDDe13wAN8CEjjUOUWCQIxSAmRLSsrR3i4zdxGKhZcwtuFjkEyYCXiEAUYpIkISGFCEGNHR2u3G63yPDtkjnurn5nyzc/Q0f5rHxtBjGBhBNbXANypvOcCcCazyQkGQwo+kMWsLyQcEfLddOmgqPPajo+XyGYzeSexyWOoGP73NZznOUv+vt410N6p9nk1kUzykUowz9FpN2u2W9MqwiLqidKmRVHIMh0keJRsO0Eg2tjtGYkuBtkvnnHlS/eyfUBAgnUZM97Lokc3jHkjAmMsIgT+shjMJMNvbgSUIbyHWy3MmoqbiL5QKTW/lWHT6TKiyKpX8CMGaK1SrL4k7ic9xPN1Sm+K11tVsLJjyKbQLO5u8rKsFQML+at76jl0xWpJWKsMQs4zVgF18FcxNm6Us91BBADMHIJdK0GmQv0wxLSIMjenMsBqG25y2FZlLp9CFRmIgElJQSBIgeibquOlzAAkfTUpQxGIMZxBpKDl/BeO++clxzZ8p1pU9y7CbxGExZALd2tF25eWSE8kZS1vJWm1MBAE8gGYAtTBEAywAjoz5F5E1QlIv5IrNM7SSGHRlA9EeDtNjYVSaljlrqOuaEBsLsLYgTFIC1AQKaXyaAcI1iDKGqdIZInCWNu3d4t46jvxeQY9/arBY1xy3yuVTd9F1hSrUVYFiQKlLYdlYoVc3VJuus0UksK/7bdaTMZHAJ3b6srnil1wIsDMka5TTnzIqXi1KZGUdRK6ImRfsmgdJoI5VQTA0mCg0aE7VjcYBU49VXF49QK6UfjdsbQF5cpvTRSFr1BQN3YtoVi4DAoCUG/nK2CcB4LC4K6YPUeAnCppFmRhC17VNUDEH0GDoXxopmnfyDGH0OoSgRYTqnpmicKwE4gpmcpKaB7lOyEASlTY2a1oOj7gHp7VsMQSgFTOjsmZRUGIXoCcqaWuXBWAuAMBDgVlHTr4gBwShmqTBoaW2Furc7qyVkUKFJgG8Bbfx5wbQUjsDV7OQGWXb1W9hOWGhtmrklnBGaZ1otLPcfPiAZdZqGbWsoREhtTiOOvq1JB2sseKTi0p1kRhwFzEg2MmMUy/YsFQvjKcXCQbVwoYUXgM14row0pXkX2YBvUjKEtYNGoNYz79O2mMIFYxRe8I8V3AT2WZYV2Q1YJNglkampLPVwMmpgyibd5TQgmLHiItORLFRriqq7g4BYNfaVBYNGY0UsDAFSvLWpoToQRVAGJyFP2+GYm272XQUYSVGrhRDmXJjJNQQrt6Y5z0DWg3PLaApYu6bZi2aQRpGwaOiUMliawpCBWCypCkIeUWYU8y6khz8oJA9OXyQ6DDKmlMI29tRsCKNXoyM4xZUCxnQqWGGwbNeAosG0ixKyMMaSlZmsBs2Ram2mq13b1/XaSz8ZDUuch8hzkRFVjcgta86Xb7BrHNa2sxsLz0ttEhnSSGGGEw14GLpU8tvAoxEfllkUtLflsZX9yZtxz1h95kES7p5TznW0wUicoSRqDKrT+qvR0GGIyBx2DTTQAAAAAAAEAAQgBEgQIzUIPgiIiIkrOUtKbEIppc6BSmrREREc5S0c5wc6KKJ+dYjjmW/GoxCQSJMICIiIiI8eRDAg8CIiIjsKBuEo400wwxEkAPmpWAtYsmGRs8wq0WlKaW122tPCmsomMZqqw1zWgKvtauDjXbu8WIFIStL9utxkzJ23VjshLjmCiwcbZWipgxkHMhoGiZVWLcFHuRzY1LdrqmEkszT2N2fnb5iD8IdMM8ArAtGMqB0k4jFiBBjDJOZ3phOKlWVyCyl3BOXgWieBYcOSQZz2Nt4YqNQZ6cZ2YNzmmMwS3jSmE5ZNjFVnViZoKae+4ELNCKBDTZVolUtxgqbcEAKq5cYA0WJpoECYRjrSieOeBQQZWK8egf9l0GUCIn4MUsVOw99QTRWfjpKwEDZTgWhTEIkIt3/t5rMzfL9f86vURhTAmheJnxlJeqAvsM85GusvB00jTV9CRiRs90pIy16pKVi4y01ShmyW1He4YNq4qtYSrKsRt1VSllxWsG7U7TetDZNdIxTrBG1buGMOQyolgYIB4AYXgGN8p2dlWeeuKtzm4WboaKCo6X6fW6U32TU6ao+/4qUlUkUfqrrK39/2ur/8vsHbDh1FXAHCGAXAAg7A96DwEoBgDBAIBIAYwEPjmoV1ffTwF50D/lq8+wZAvX2F8YgY0aGgTyLQCUQffj9oyQNg0wbBpg2DF92IDSB0IIFIaAbSQ22gfRwI5poPLtCsGlres/+LuSKcKEgTn5ljg",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-25T12:46:33.824245Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-25_05_46_32-11226913448634491822'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0825100258'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-25T12:46:33.824245Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-25_05_46_32-11226913448634491822]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-25_05_46_32-11226913448634491822
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-25_05_46_32-11226913448634491822?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-25_05_46_32-11226913448634491822 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:36.433Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.060Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.097Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.261Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.300Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.406Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.455Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.490Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.521Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.555Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.589Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.624Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.660Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.706Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.741Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.775Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.822Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.859Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.898Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.930Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.964Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:37.992Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.072Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.113Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.155Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.190Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.226Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.386Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.452Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.501Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.543Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-f...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.635Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.660Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.702Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.731Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.773Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.808Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:46:38.877Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:47:07.697Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 9 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:47:07.736Z: JOB_MESSAGE_DETAILED: Resized **** pool to 9, though goal was 10. This could be a quota issue.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:47:13.042Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:47:13.101Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:48:47.396Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:48:47.436Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:52:02.420Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:52:02.492Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:52:02.525Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:52:02.595Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:52:02.630Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:52:02.653Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:52:02.667Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:52:02.686Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:52:02.723Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:52:02.740Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:52:02.756Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:52:02.817Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:54:17.955Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:54:18.173Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:54:18.232Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:54:18.312Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:06.009Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:06.088Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:06.143Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:06.220Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:08.934Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:09.001Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:09.172Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:09.233Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:09.237Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:09.286Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:09.304Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:09.353Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:09.422Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:12.271Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:12.343Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:12.414Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:12.469Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:12.527Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:12.592Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:15.365Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:15.434Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:15.507Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:15.682Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:57:15.719Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:58:04.818Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:58:04.853Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T12:58:04.888Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-25_05_46_32-11226913448634491822 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598359590048/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.061821937561035156 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: e9430d50f7874c1ba34fd3e5f14d3732 and timestamp: 1598360390.601246:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 800
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598359590048/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598359590048/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.059340476989746094 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 801.156s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 13 mins 23.823 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 33s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/mcc5b6jnee7sc
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1843
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1843/display/redirect?page=changes>
Changes:
[noreply] [BEAM-10777] Add two blog posts detailing changes to the type hints
------------------------------------------
[...truncated 291.01 KB...]
"python_side_input2-Write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWZpMUXMACKN/////////////////////////////////////////////4AjT6gNslgjZtCxsgE9YVE4g0UIZKYniNI9TTEeo9JtJ6nqNPSZGQ0A00DT1AAAAAAAAGgAaAAAAABoGhoABoAGgxGRoNNDQyDR6QZIINEZMEIyaJNqNDRppkA0GgA0Bo00AAAADQGgAAAAAAAaNAAAAAAAAAAAAACTAATAAEwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAQep/qkGITACNMTTABMEaZMBGmmTAEwBGACYAIwARppiYATAAAjTAAJgAAmmAAIMIaNAwAJMABMAATAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAASJE0CE0yTIyNU/I1T9CJ6jQeKB6jyj1NqaZNMRoADQDTQNGQZPJqaAaAAADQAAANAAAANAAAAAADQZVMO1upjAml38WGXa9MjhRO4Tx2bNrEx4JktF5kUKxdXJrD1eu6/v33msphXMrQzfUUugqqiXQ6IQmTAumsEQgIUKBASNMSbSAZpZ54CaS5RMY13fwz0yU2MaEpiEE65kkAYs5YLAUCeJCPgWhAF8KK9yxhUXb163gRX1/czbSwlppGExGYdF/k0UjxRsAicC02aJszZGmcZm6QaBbeXHRkWi3kZxzpMwHRniSQWGIMXgSs8XeGK4mVorxkg8FTb5SsK50JhMJwTkKVpKYTAnYT8jNTlARz3g44nQdC25znJSr4sVpubOLI0LIZh56olOkIvv4d068hiCWKAjGShyUSJSGwmaJKSGxvIQQ2HGkKNsSGSz3NMhtMjGcmM5QniM42IVGgTLVmhnmkULCZYUt62FoQzmoxM5VVHYgkZUYOl2HcsH/q1eoquzDa7PEW/ucBxe1x+2kBJsC3/U935oRb2OyUqwjoAQHO8UbexJnXLjagjcDvfN870HRdN1WYeeeeefa1jGMY1rWMWv/ax0CoZ+EyAYN4y28t0fI6dkPQ8DndVcjI5Iu7GKoXTxNNXsLlkJzZMTdTIWbxMypWCt4ShUVahYK7VMOf86uGufsVessETjqKlw5cTQXElJu/VcOqFWhC6HwF8CLyp5lUJIaTQViyUjCasUUEQbYoXcpuW5hQyYa+MqIHV40TDw+OGILpobSLjIicoLYojBOUE7WFSiApXaoFMzVy5xIN/0C3Raps20zyBImcmGG9rvwSR6h6LzbGWdhhkViWKd3hRRvfOzLzrZMtOv67Hzm41CpcDS2ZM2YZokqVAzjY3P6rWbBPa1Dc4/lZCS7kpL+yZ+89CvgMzBisMmVVfWr8g/k/kyRJLTJKKGrNShRiLkLQask7SUSCQLW1upxtg1ki5VRRr9uK/zOH0NVHUjfModbGeGXeeLEXN5j7kxutUuKvWudHrza3JOCTQkDTaoJO6G/REfRkMjpnyjN73dy2WoU70q4aVTxy7EJEGDHCI1IuMSD8L+ScSZxBECh4iMNYSyrPoJQJefh1stzJqKm4i+UCk1v5Vh1CkyosiqV/AjBmitUqy+JO4nPcTzdMpvitdcVbCyY8qm0Cyc1eVlWCoGF/M282jl0xWpJWKsMQshmrALsAK5ibN0pZzqA++DMHIJchndMhephiWkQZG2cyvmobV4FsKzKXzyEKjMRAEoTEkiTA78p9t0piQBufBOKBaZB2QIPZVrpdaxdWF3jPGIx/LkEaVzV3YMdqiAVPn5PodKRVk4I2jJwVMpqogCcAGqEUuCYBxABRGdIvImqEpF/JFZonayQw6QoHojwtpr66qTUsctUx1zQgNhdhbECY1AYEBAppeJoBxRrEGUQKdIZInEsbdu7xbx1HNl5Bj39qsFjXHLfK5VN30XWVKtRVgWJAqUth2VihVzdUm66zQSSwr/tt1pMxkcAndvqyueKXXQiwMyRqlNGfMCpeLZ0yso6iV0RMi/YNA6TQRyqgmBsWCg0KE7VjcYBU49VXF49mFdKPotY2gF25TgGikLXpKgb+xbQrFwF9QEoODOVsE4DQsLfrpS+TBhgGr2KQnXCz9Fk5haIYGIEvR7ZKbhyDsIYQQUAvAYNYRfcLmqJmCkpX3TUT0Zt3IAmjShr6clAw1shKk2WWcmIKagW654HzMhZAJWc98+5DYByAYFmAnpmbdWcB/UC+cMPT2+Rdffae2JsRRYaGA/AcK/jzg2gpHCavZyAyy7eq3sJyw0Ns08ssgRmmS0VrN9PnxYMuq1DvVrKERIbU4jjr6lSQdoLGlJxaU6yIw4C5jAbGTGNZgsWCoXjKcdCQbVwoYUXhM1sroxEpXknmYRwEwFCWoGhUGoZ1+jbTGECsYoveEeK7gJ7DMsK7IasEmwSytTUlnK4GTUwZRNu8pnwTFjxEWnIlio1xVVdwcAsGrtKgsGjMaKWBgCpXlrU0J0IIqgDE5Cn7fDMTbd7LoKMJKjVwohzLkxkmoIV29Mc555qQbnls8UsXdtZi2SQRsTXNDQKGWxWTTtDCSUNO0qhT3GXJLPdvGpmYVZVRIur2sZMatNvbUa4ijV6MrNqtJAsZ0KlhhsGzVgKLBtIsSsjDGxUrM1gNmwLU201Wu8ev69SWdjIalzkPkOciIqgg1Fa15ou3mIGOgggrMgha2W2iQzpJDDDCYa77F0qauDfUYiPSyyKWlvy2Mr+5NBcc2w81j8S75qmudbTEpE5QkjUGVWntQbGVDw8TCAMGEpWKAAAAAAAAAAAWAANAAIxYsmFDQIAEEEEEERKzlLSmxCKaXOfUqBaIiIjnKWjnODnRRRPTrEccy3o4i9yyZMyTbbbdVWCVcqpNtt3ZspcNXTnERgySlSaLWLLhkbLMKtBpSmlttbUnhTWkJjGaasNU1ICr7Wng4x27vFiBSErS/brcZcydt1Y7IS45gosHGsrRUwYyDmAzzQNJVi3BR7kc2NS3a6rhJLM0dfdn52+zwfhC8Z3xWBgGPqB0Q4vHB9BjzJOZ2zCcVKsrkFlL9+ctBgGWBYcOWQZHr63hyo2Yz04ztAbnNMZglvGlMJyy6+KrI+JoAUyLzgQs0IoEQGyrPqpbjBVBcD4Kq5cXxALE0z6BMUY60onjnfUEGViutfe9lz+UCIn4MUsVOw71BNFZ6OkrAQQSnCtCmIRIRbwfb7DMzfL9f86nURipgTQvGT4+ktqAvYhtyNVaXB0UjRV9CRiRst0pI0q9MlKxbW00yhmyW2He4YNq4qtYSrKsRt1VSllxWsG7U7TetDZNdKxTWAjzlPDsKUMMJEGEhBgLIMGfplpaQefVWW0BKRcp7PbWJn3/D416a+S6bXfP6fWcV/ZWx/E7Pfn9fyf/v+dUDe6iCE7VUBGAOQAQbhGEEGFADAMJBIKwDMCzX6iif/tpRAdIRP+inSNA7h06oLQtBngA9D1JvQSiD78ftGSBsGmDYNMGwYvuxAbEHQggUhoBtJDbaB9JAjmWg8u0KwaWp6z/4u5IpwoSE0mKLm",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-25T06:46:28.183549Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-24_23_46_26-17649069668830347684'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0825050320'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-25T06:46:28.183549Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-24_23_46_26-17649069668830347684]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-24_23_46_26-17649069668830347684
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-24_23_46_26-17649069668830347684?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-24_23_46_26-17649069668830347684 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:31.527Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.156Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.238Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.312Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.344Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.440Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.483Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.517Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.544Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.590Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.625Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.650Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.678Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.713Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.750Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.784Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.820Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.856Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.893Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.928Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.961Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:32.997Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.035Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.073Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.100Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.143Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.178Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.332Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.389Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.413Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.434Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.447Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.471Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-f...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.529Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.529Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.587Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.623Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:33.659Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:46:43.742Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:47:06.599Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:48:39.962Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:48:39.995Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:51:50.515Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:51:50.600Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:51:50.629Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:51:50.720Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:51:50.753Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:51:50.776Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:51:50.781Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:51:50.816Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:51:50.835Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:51:50.852Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:51:50.888Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:51:50.916Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:54:08.288Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:54:08.385Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:54:08.441Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:54:08.515Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:47.586Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:47.667Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:47.728Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:47.797Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:49.909Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:49.994Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:50.069Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:50.105Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:50.120Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:50.161Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:50.187Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:50.227Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:50.292Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:52.844Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:52.931Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:53.012Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:53.063Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:53.131Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:53.209Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:55.562Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:55.639Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:55.852Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:55.918Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:56:55.953Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:57:49.641Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:57:49.684Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T06:57:49.719Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-24_23_46_26-17649069668830347684 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598337984285/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.06971955299377441 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: 4bb177c5241b4655ad12398c7363004b and timestamp: 1598338759.799575:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 775
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598337984285/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598337984285/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.05524754524230957 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 775.997s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 12 mins 58.659 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 1s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/mqxd3thj5tquk
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1842
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1842/display/redirect?page=changes>
Changes:
[Robin Qiu] Propagate BigQuery streaming insert throttled time to Dataflow worker in
[noreply] Merge pull request #12415 from [BEAM-10603] Add the RecordingManager and
------------------------------------------
[...truncated 292.31 KB...]
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWeKoJmUACKN/////////////////////////////////////////////4AjT6gN24uCNm0LGyCk9g1E4g0UIZKYniNI9TTEeo9JtJ6nqNMjJkNANNBk9QAaAAAAAANAAaAGgAAANA0A0AGgAaB6hoaDTQ0Gg02kGSEaBMTBTIyaE0j1DR6mm1AAAAyaNGmQAAAAAaZPUAAAAAAAAAAAAAAAAAAAAAGSNigAAAATAJgARgBMjAACYAJgAEGBMRgEwmTAAAAAAAAATAmjIwIMAAAJoQep/qkGITACNMTTABMEaZMBGmmTAEwBGACYAIwARppiYATAAAjTAAJgAAmmAAIMIaNAwAJMABMAATAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAASJE0CEwgjI1T8plP0JHqNB4oHqZqPUyaYjEAANANNA0ZGmmynqaAaAAA0NAAAAAAAAAAAAANAAGXTG2N1MQE0uaFdluOmSAUTwCeK6urGDFfGSwXWQvrFtcmUdjjgvLxrrjKX1zK0Q3dFMD1VUK6IIQ+ZL22awRCAhQoEBI0xJtIBmponf5pLlUxjXdugeqSmxjQlMQgnXMigDFneQsBQJ4cI0CwHgvgxHar39Fu5cs3sN3eWs31MJaahhMRnHRf4NNI8SaAROBarNM2hszVOLyagaRbeXHSEWi3sZ5zpMwHSHiCQWGIMXgCs8VeGK4mVorxkg8DTcZisK50JhMJwTkKVpKYTAnYT8hNTlARz/go4fhehbc5zkpV8SK03NnEkaVkMw8/USnSEX38K6deUxBLFARjJQ5KJEpDYTNMlJDY3lIIbDpxCjbEhktFzTIbTIxnYRnKk8RnmyCo0iZasgaJqFCwmWFLetBaEM5qMTOWVR2AJGZGD6mh3HB/6tXiLrvrbXR4jH9xfpPr8TsJAS7At50fd+qEW6kcg5YSMAIDmd/K3kS51y32kIx4j2S0xNztDS1IYYYYYsEECCCCCEEECCJE944IRCUBDAFEcAEB79PqCSkxRR+j4HO665KRyZdygqhdNE01ewuWUnNkxN1MhZOHnVF4F/USoulpUYmvlgFzmTmjmHi3GHFrLczK8MsVi2DEsXiWjkirQg1y2IS12tKpmKqdIzQZysVFIwmTKKCINrULWV3HcgoZQONGWD0EcSJg4PGC/MEz9hFxEQwKC2KIuHKidxhUogKV2qBTMi7IcKDjeGt2WqbNeZ48kTOUDDe13wJI9E9B49d6HYYNFYlh3d4UUb3nrTH5pqZaVf1GPkOF0ypcDS2BM2YZokqVAzkJDOaiS1qeyqG2xkpjZbs+eY9kz952pvgMzBisMmVVfWr8g/k/kyRJLRpKKGrNShRqNkC0GrJO0FFAkCkq3DyFg1PLsiqLNftBX+TwJXT6CkbtlELY7vTDzJxNweeFb2Nyql2VyTiR1ZtbgnBJocc02qCTuhvERwqMhjtG+UZvd7qWu0qnbcwjSqeCYYhIgwY4JGpFwyQfhfSTijOJIgURExxrCWVZ9BKBLzsOtludNRU3EXygUmuOlWHT6TKiyKpX8CMGQVqlWXxJ3E57meTVKb8rXW1WwsmPMptAsvNXlZVgqBhfzNvNo7KmK1JKxVhiFlMiwC68CuYmzdqWe6gNaDMIEEujZ5TIXaX4VpCGROTmVpqG1eBZCuuS7dQhURhIAlCYkkSYHc8/22ymHAG17s4oFpkHZAg9lWuf07F0qnuGeKRk+Rx0aVyl3VMdeiAQ/NyMrz+UrKEI2jKEKmU1UQBQgBqhFrgmAcSAURnyLyJqhKRfyZWaZ2okMPclA6M8HabGwqk1LHLXMUZoQGuuutiBMegLyAgU0uk0A4RrEGUPKgkMkMBLG3bvMW9dRzZeQY+OtVgsa4xb9XKpu+i6wpVqKsCxIFSlsOysUKubqk3XWaSSWFf9tutJmMjgE7t/WVzxS40IsDOka5TTnzAqXi2lMzKOoldETIv0GgdJoI5ZQTA2TBQaVCdqxuMAqceqri8e0CulH0W3NqBbtU3popC16YoG8r2UKxaBaoCUG+nK2CcBwLC2xpS+LBhgGr2KQnXCztFl/WtEMDECXl9stNxBB2EMIIKAXYMGsKvj7iqJmClnPumoqVm3cgCaNKGvpyUDDWuFqTZZZiYgpqBbrngfMyEaA5nPfPOA2AcAGBZQJ2Zm3VmAf1AvnDD09vkZX32ntibEUYGhgPwHIX8ecG0FI5Bq9nIjLLt8revnLjQ2zVzSyhGQy2isZxqU+OBlydQ7xayhEKGxOIoo9VpIO0FjhScWFQXQiEAFrDBr6yMey+YsFQuGU5B8g2LRQvou/ZliujAlK8i6y/N6mAoS1g0qg1jPv07aYwgVjFF7wjxXcBPQzrCuyGrBJsEszU1JZ6uBk1MGUTbvKaMExY8RFpyRYqNcRVXcHALBr7WoLBozmilgYAqV5a1NCdCCKoAxOQp+0wzE23ey6CjCSo1cKIcy5MZJqCFdvjHOeia0G65fRFLF3XbsWzSCNkbBpaRQzWF5hR8CsFlSFHw4oswp5l1I/nRQSBycukiBuAqaUvm4tqNgRRq9GZnFLUQLGdCpYYbBs14CiwbWLErIwxslKzIwGzQLU201Wu7+t65SWfjIalzkPkeciIqee1Ra14xdnrHmQPPPVmPPuOS2UR+dJH333y/G1i6VOLezCMRHZZZFLSz5TGR+5M9agcruuMbCu7cU5BBZTBSGBQkiUGVWHdTciKh4eJhAGDCUrFAAAAAAAAAAALAAGgAEYsWTChoEACCCCCCN4bok9DJaaGhddXEWSbbbuuiTuuuLrr776ZpGDBllTBEXuWXLnSbbbbqqwSrlVJttu7JKXCV05xEYMspUmi1izYZGzzirSaUppbbb608Ka1BMYzVVhrmtAVfa1cHFu3eYsQKQlaX7hbnNnTturHZCXGMFFg426tFTBjIOYDRNI1FWLclHuhzY1LeLqmEks7T2N4fnb7NB+ELhnaKwLwxKgdFOOyAagxDJOZ3JhOKlXS5BdJeNnLgLwzQLDhzSDK9jb+GKjaDPSjOzhushjMEt60phOWbYxVZWiZ4FNLdgBCzQigQ8bpWf1S2mCp60GgqjMBaHhXmmagTCMVaUTxTtUEGXhtuNd9iButhET8GKWKoMbcoJorOxUlYCB6U39gUwiFCLN97XYdfN8v1/zkqiMKYE0Lw58SkuVAXMBy5GutTg6aRpq+hIxI2e7UkaleqSlYuKtNUoZJLbDvcMG1cVWsJVlWI3CqpSy4rWDeKdpvmhsmvpsU1gI81Tw7ClDCDhgwkIMBZBgz8rlzKPPprLaA80Z5vZ7axM+/4fGvTXyXTa75/T6ziv7K2P4/M7+v3/Z7/T7qgb3MQQ/FVQEcA4ABBuEYUQYUAMAwkEgrAMwLLfKKJ7/WhEB0hFT8W6RoHcT8+6s60Gd+D0KBN6CUQffj9oyQNg0wbBpg2DF92IDZA6EECkNANpIbbQP3UCOZaDyzQrBpa3qv/i7kinChIcVQTMo",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-25T00:46:34.523788Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-24_17_46_33-8733151788777511898'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0825000824'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-25T00:46:34.523788Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-24_17_46_33-8733151788777511898]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-24_17_46_33-8733151788777511898
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-24_17_46_33-8733151788777511898?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-24_17_46_33-8733151788777511898 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:37.404Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:38.864Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:38.902Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:38.974Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.015Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.109Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.151Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.173Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.199Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.233Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.260Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.295Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.330Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.364Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.401Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.423Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.457Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.481Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.517Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.541Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.572Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.608Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.641Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.670Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.722Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.751Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:39.782Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:40Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:40.065Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:40.101Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:40.112Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:40.134Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:40.144Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-f...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:40.202Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:40.202Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:40.256Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:40.289Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:40.326Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:46:47.616Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:47:14.934Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 6 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:47:14.964Z: JOB_MESSAGE_DETAILED: Resized **** pool to 6, though goal was 10. This could be a quota issue.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:47:20.410Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 7 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:47:20.437Z: JOB_MESSAGE_DETAILED: Resized **** pool to 7, though goal was 10. This could be a quota issue.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:47:25.992Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 9 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:47:26.022Z: JOB_MESSAGE_DETAILED: Resized **** pool to 9, though goal was 10. This could be a quota issue.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:48:15.028Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:48:51.335Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:48:51.371Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:52:09.696Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:52:09.757Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:52:09.778Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:52:09.847Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:52:09.870Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:52:09.896Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:52:09.897Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:52:09.912Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:52:09.945Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:52:09.954Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:52:09.986Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:52:10.011Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:54:53.216Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:54:53.293Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:54:53.351Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:54:53.412Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:42.533Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:42.594Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:42.642Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:42.698Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:44.918Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:44.978Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:45.082Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:45.106Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:45.126Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:45.154Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:45.192Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:45.228Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:45.310Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:47.679Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:47.751Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:47.809Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:47.880Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:47.929Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:47.986Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:50.703Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:50.768Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:50.852Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:50.953Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:57:50.986Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:58:42.066Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:58:42.106Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-25T00:58:42.133Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-24_17_46_33-8733151788777511898 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598316390786/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.07180428504943848 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: 9a25ee0023924912b21d30d4fabe8c77 and timestamp: 1598317224.2084987:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 833
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598316390786/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598316390786/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.058439016342163086 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 833.914s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 13 mins 56.385 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 15m 6s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/wjpi3hlzuqpqe
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1841
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1841/display/redirect?page=changes>
Changes:
[Udi Meiri] Silence red PYTHONPATH warning from tox
[tobiasz.kedzierski] [BEAM-10682] Add workflow to run Java tests on Linux/Windows/Mac
[Kamil Wasilewski] [BEAM-10524] JSON decoder for ReadFromBigQuery supports repeatable
------------------------------------------
[...truncated 289.67 KB...]
"python_side_input2-Write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWXPJSEcACKN/////////////////////////////////////////////4AjT6gNtFgjZtCxsgpPWFROINFCGSmJ4jSPU0xHqPSbSepk0yNGQ0A00DT0gBo0AaAAAAGgAaAAAAABoGhoDQGgAaDEaGg00NDINHpBkgg0Q0wQjIEaj1DTJpkA0GgaDQGjTQAAAANBo9TQAAADQAAGjQAAAAAAAAAAAAAkwAEwABMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEHqf6pBiEwAjTE0wATBGmTARppkwBMARgAmACMAEaaYmAEwAAI0wACYAAJpgACDCGjQMACTAATAAEwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEiRNAhMJMjI1T8ptU/Qkeo0Higeo8o9TJpk0xGgANANNA0ZBpmmpoBoAAANAAAA0AAAA0AAAAAAAZZMGzvBjAlkwxXZcq0VnCeZwmjsWLOJjvjWsl5kUKxcXW1t6rVdd3b7zWUQrlVo5up6HQU0xLndEITWvLhqBEICFCgQEjTEm0gGaegd/mkuTTGNd2556hKbGNCUxCCdfyaAMWcuFgKBPFBHwLIgC+GFW3Xv57l7e2ryK6u7ed6eEtNMwmIzjov8GkkeJNkETgWozSNqbQ1Di8mmGiW3lx0ZFot7Gec4TMB0Z4gkFhiDF4ArPFXhiuJlaK8ZIPA03GYrCudCYTCcE5ClaSmEwJ2E/ITU5QEc74KOH0HQtuc5yUq+JFabmziSNGyGYedqJTpCL7+funXlMQSxQEYyUOSiRKQ2EzSJSQ2N5SCGw6cQo25IZLQc0yG0yMZyQzkyeIzzZhUaJMtWQNA0yhYTLClvWgtCGczGJnKKo7CEjMjB0uy7jg/9Xr3EWH1tru6RL23wHF7PH7SQEswLfynu/NGLeyGSmGEg+B853hj74JY7Bc7aEYw79mud6Doum6rNvPPPPPtaxjGMa1rGLX/tQ6JSNDCZAMG95ffG6XkclkPQcHnNZchI5Eu7EKoXTRNNXsLllJzZMTdTIWTh51RgBhyRUXi+VGN+EsAr5o4ZzEBfxBjVhtXiyzD4JlkpdZlTFqtnVyrRheDMlzIa6ppFTpWNBnKxYKVAmslFBEG3qFzK7ruwUMoGvjLCB1WNEwcHjxgF0sNpFxkROUFsURfHKiZrCpRAUrtUCmZFyxw4OB0C3Zaps15njyRM5IMN7XfQkjKHovLsZlyGGRWJYpzdFFICO7E1P1FMtOv6zHzHB6hUuBpbMmbMM0SVKgZx8fn9VrNgntanucfyshK9uXmvZN/egqzwDMwYrjJlWXlq9IPZPZMkSS0ySihqzUo0Ui3C0GrJO2FDgkC1tfq8fZNZJxaqINftxY+Zw+hqpCmb5lCrYrwTTrxYaNdZGNY3asRivWt9Hrza3RN+TQkjTaoJOaG/REjSkMjpnilP8deC2WoU7sxBNKx4ppiEiDBjhEaqXGJB6F/JOHM4YiBQsNFGsJZWnkEoEvPQ62W501FTcRfKBSa4Eqw6fSZUWRVK/gxgyCtUqy+JO4nPczyahTfla62q2Fkx5lNoFl5m8rKsFQML+Yt5pHLJitSSsVYYhZTIsAuvgrmJs3alnuoD74MocgkyGLorF6iGJaRBkbZjI+aRtvgWgrOJdPIQqMxEAShMSSJMDvS/23SmJAG5788oFpkHJAg7lXul1bN1KtvGeMRkeXJI07mruuY7NIAq/PyfQ6UmrKCI2jKCKoVFUQBQQBqhELgmAcMAURnyLyJqhKRfyJWaR2okMOkKB6E8JabGwqk1LHLWMdUzoDXXXWxAmNQGBAQKKHiZwcI1CDIIFOrGtE4ljbt3mLeuo5ovIMfAtVgsa4xb9XKpu+i6wpVqKsCxIFSlsOysUKubqk3XWaKSWFf9tutJmMjgk7t/WVzxS64EWBnSNYppT5cVLxbWmZlHUSuiJkX7JoHSaCOUUEwNmwUGjQnasbjAKnGqq4vHtQrpR9Frm2AuW6Lwz0Ba9NUDd17SFYtgvqAkBvpitgmAaFhb9VKfyYEMA1dxTE7AWfpMt61ohAYfS8/tlZ2FIOQhA/BQC8hg1g18ZGqiZgpWY905D9GdcyAJo0o6+pJPsJb4OqtllqJiCnH1/XOg+ZkLQBMT3voW4bANwDAiwKCbnXFqAe1AvnjDs7vETY3mAbE2IokNDAegOOv404VoKRxzV7OPGWXb5W9gOVGhtmpmllCMhltFZzzUpsUDJrFI71ayhESGzMI46uq0EHayxpSYWVOsCMOAt4wGvlBjWXzFgqF6ZDjoSDZtlDCi79mvFc+BIV5N5l+bxMBQlqho1BqmffpW0xhArGKL3hHiu4KeyzrCuyGrBJsEszU1JZ6uBk1MGUTbvKaEExY8RFpyBYqNcRVXcLALBrbaoLBozmilgYAqV5a1NCdCCKoAxOQp+2wzE23ey6CjCSo1cKIcy5MZJqCFdvjHOegasG65XQFLF3XXYtokEbM2DR0ShmsMDCkIFQLKkKQhqizCmlXSkOfFBWDZi8SHP4CpZDCbi2o2BFGr0ZmcUtNAsZ0KlhhsGzWgKLBtosSsjDGzUrMjAbNkWptpqtd39d12ks/GQ1Lm4fH83ERVBBqi1rxC7WbgY6CCCoyCFrZLSJDMlaGGGEw1X2LoU1cGYRiI9JJWUtLXlsZV9yWC25td5rH4l3TVNc60mJSJyhWjUGU2XtTbGVDw8TCAMGEpWKAAAAAAAAAAAWAANAAIxYsmFDQIAEEEEEERKjlLSivCKKHOfUqBaIiIjnKWjnODnRRRPTLEccq3o4i9yy5c6TbbbdVWCVcqpNtt3ZJS59XTnERgyylSaLWLNhkbTOKtFpSmlt9fVnhTWmJjGaisNY1YCr7Opg4t27zFiBSErS/cLc5s6dt1Y7IS4xgosHGurRUwYyDlw0DRNNVi3JR7oc2NS3i6phJLO0tjeH5m+zQfgC9Mz4qAwDH0g6QcVjg+gx5rTGZsomFKrC6wsJdvzFoMAzQLDhzSDK9jX8MVG1GemGdnDdZDGYJb1pTCcs2xiqyviWAFMi84ELM6KBEBsK0KmS2wUwWw+CqqXF8QCvLK+gTCMdSQTRzPqCDLRXGvvey5/KhET8GKWKXYN7OTPUejoKwEEEhv7IoiESEWr72+wzkvy/X/OrUkYUoJnXjJsfQW0gXuA25GstPhaSRpK+hIxI2m7UkadeoSlYuKtNQoZJLbjvcMG1cVWsJVlWI3CqpSy4rWDeKdpvmhsmulYprAR5yADkKcMMJMGEhAgLIEGfomZmSdfTXWz5LxMv7PbXJv3/D42Kc+S6dXfP6fWeV/ZWx/E/Qfn9fyh/v+cUT/dg/CfrKAigG4AIP4RgxAhQAwDCQSCsAzAi79SRQ/22IgOEIf/RDhGicw4cUVrWgzwAdh6U3YJRB96P2DJA2DTBsGmDYMX3IgNmDoQQKQ0A2khttA+kgRzDQeWaFYNLV9X/8XckU4UJBzyUhHA==",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-24T18:46:29.209972Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-24_11_46_27-14398290832772796054'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0824154410'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-24T18:46:29.209972Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-24_11_46_27-14398290832772796054]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-24_11_46_27-14398290832772796054
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-24_11_46_27-14398290832772796054?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-24_11_46_27-14398290832772796054 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:31.607Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-b.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:32.845Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:32.889Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.079Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.107Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.213Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.264Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.290Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.325Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.358Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.393Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.420Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.457Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.491Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.525Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.561Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.596Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.633Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.665Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.692Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.725Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.760Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.791Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.825Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.863Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.903Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:33.945Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:34.158Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:34.222Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:34.250Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:34.261Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:34.287Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:34.297Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-b...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:34.354Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:34.354Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:34.412Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:34.448Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:34.521Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:44.468Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:46:58.421Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:48:34.443Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:48:34.478Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:53:12.792Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:53:12.864Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:53:12.906Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:53:12.971Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:53:13.005Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:53:13.031Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:53:13.043Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:53:13.053Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:53:13.083Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:53:13.094Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:53:13.105Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:53:13.160Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:54:11.267Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:54:11.334Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:54:11.387Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:54:11.436Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:49.991Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:50.046Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:50.090Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:50.159Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:52.675Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:52.724Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:52.779Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:52.817Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:52.835Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:52.870Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:52.895Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:52.931Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:52.993Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:55.596Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:55.668Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:55.730Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:55.777Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:55.840Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:55.906Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:58.517Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:58.581Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:58.695Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:58.830Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:56:58.859Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:57:50.327Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:57:50.419Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T18:57:50.451Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-24_11_46_27-14398290832772796054 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598294785142/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.06347227096557617 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: c11658f2839248a2a722f22304ed7cd0 and timestamp: 1598295583.104606:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 797
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598294785142/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598294785142/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.05551004409790039 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 798.554s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Daemon **** Thread 3,5,main]) completed. Took 13 mins 21.7 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 23s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/tmabp4jjocx2e
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1840
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1840/display/redirect>
Changes:
------------------------------------------
[...truncated 290.79 KB...]
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWcH7YY8ACKN/////////////////////////////////////////////4AjT6gNtFYY2bQsbIBPYNJOINFCGSmR6aNI9TTED0m0nqaNMTRkNANNA09EADQAGgAAA0ABoAaAAAA00A0NADIAGjTahoxDTQ0Gg02kGSCARkYgmJo1Tyj0jR6mm1Gg9QAaD1GjRpkAAaAAZANNAAAANBpoAGgAAAAAAGgAAAAAAkwAEwABMAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEHqf6pBiEwAjTE0wATBGmTARppkwBMARgAmACMAEaaYmAEwAAI0wACYAAJpgACDCGjQMACTAATAAEwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEiRNAhMIIyNU/IU/Qieo0Higepmo9TJpiMQAA0A00DRkGmynqaAAAANDQAAAAAAAAAAAAAAAZRL+tvRiApyZgUmWqM8cAmlgEsVWrWvorwx1i4yF9YtLjyLlGjBYsOuNZO+umrSzdTTwPVKkK5oIQ+Y7u0aARCAhQoEBI0xJtIBmponfJpLl0xjXc+heqSmxjQlMQgnYsRAGLOXCwFAnjAjoFYPBfECjZpXs1q3br3cN1Ys6BqYS01DCYjOPGf4NNI8OaAROBarNM2hszVOMyagaRbeXHSEWi30Z5z5MwHSHhiQWGIMXfSs8ReGK4mVorxkg79Tc5isK50JhMJwTkKVpKYTAnYT8fNTlAR0Hf44ng+ibc5zkpV8WK03NnFkaVkMw9BUSnSEX38O6deUxBLFARjJQ5KJEpDYTNMlJDY3lIIbDjSFG3JDJaLmmQ2mRjOvDOXJ4jPNkFRpEy1ZA0TUKFhMsKW9XC0IZzkYmcwqjroSMyMH09Dt2D/1iv8NZfa0vdJh3lrGSEpye4kBLLy4XWnP1SC4Edlmy+Oeg9dHx5vACWOyXe4BG7Hf81z+bznTdVnXHHHHHWtYxjGNa1jFr/2gcdUGkBMMMGWy++t0/Ucfh9Hwef11ykjlS7lhVC6WJpq9hcspObJibqZCycTOqLwL+PKi4XSoxOvlgGYzR4Q8E8Xb4xKZwa8EsfW9UKhXS6OIVY4O2BCbLzjczDimZlJkjM5mKxVKRhMgUUEQbgoWslu27hQyIa6MmHoKMSJf3/Ii4YKb9hFxEQwKC2KIvDkhK1hUogKV2qBTMi7CcODj+It4WqbOlTPHEiZywYb2u8hi/FTk89m/eiG8rdZlqFF9/ByvlfD+mhPHmaXkT+ymj802QIzOG5cFxZaqBnGxui10fs09xWN5kuZk5XuzEz7pv70FadgZmDFgYsa67s3hB5J5JiiSWpSUUNWalIiEWoWg1ZJ28oUEgUfY63G2jX9CKVQxuW6Fn5/Gk9bpqfwWMGtiPFMuchCRbnyLuwvVci1ce1jtibO8JviaHNNNmgk4ocJEcimoZTUu1NfcDfC2mrU77Z/ZVzyTLAJEF7DEIwJSJIPAwhJwpnCEQKDhIg1hLK87glAl5yHWy3OmoqbiL5QKTXISrDplJlRZFUr+DGDIK1SrL4k7ic91PJqlOAVrrCrYWTHmU2gWXnLysqwVAwv5u3nUdiTFaklYqwxCymRYBdbBXMTZvFLPdQHXQaYgQSYeiTxlyd+FaQhkTZTI6agyvg1wrr0unEIVEYSAJQmBJEmB4Jj7bxTGgDd+GfUC0yDggQdSsXU7Fq69a31RyCMhzuajUpJd2jEpSwK30svJ9SSVk/kbJi/lVKqqIAn8A1QhlwTAOEAKIz5F5E1QlIv5UrNM7OSGHuCgdGeBtNjYVSaljlrmKiZkBpLpLYgTGIC8gIE87hMwOAaBBkDyoIzHDASxt273FvnUc6XkGPkLVYLGuOXAVyqbvouqqVairAsSBUpbDsrFCrm6pN11mkklhX/bbrSZjI4JO7gVlc8UuPCLAzpGuU0580Kl4tpTMyjqJXREyL9BoHSaCOYUEwNkwUGlQnasbjAKnIqq4vHtArpR+N3BtQLVme7M04WvUlA2KVdCsWQXVASA3kpWwSgNCwt2ilT5T8F4Zuop6dkLSUyW9q0QQMPZS81Kz0GQcBBB8CgF9C9nAOV2i1RVEFKtpydhZOecSAJkypTlVknuCuUBAtFkUTAFOvb6upI+ZkLYA2n/fQtQ0AagGBEgUE3PN4oB5UDlPmHR1doezu9XaE0IocMjAeAOSv5E4VoKRyTV7OTGWXb9W9cOZGhtmrmllCMhltFWxWry4sGTIVB3q1lCIUNaURRUdanIO3FjSkorKgqiIQAWcEGlr4xjLxiwVC2ZDjXyDWslC+i71mTK5rkhXiOMvTdpgKEtYNKoNYz79O2mMIFYxRe8I8V3BT0M6wrshqwSbBLM1NSWergZNTBlE27ymjBMWPERacoWKjXFVV3CwCwa+1qCwaM5opYGAKleWtTQnQgiqAMTkKfs8MxNt3sugowkqNXCiHMuTGSaghXb8xznomtBu+Z0RSxdy3DFs0gjZGwaWkUM1ismnaGEkoadpVCnus2WWi7eNTM4qzKiRdXtoy41abm2o2BFGr0ZmbZaiBYzolLDDYNmvAUWDaxYlZGGNkpWZGA2aBam2mq13T1nWaSz8ZDUueh8nz0RFTz2srWvPl1868yB556gx59rZK6I/Kkb7775fousXOpq3uBRiI5JJGpaV/LYyj7lN6zA2k41jsK7pqmwQV0vkhgUI4lBlSs5rDYioeHiYQBgwlKxQAAAAAAAAAACwABoABGLFkwoaBAAgggggjeG6JPQyWmhoXXVxFkm227rok7rri66+++maRgwZZUwRF7lly50m2226qsEq5VSbbbuySlw1dOcRGDLKVJotYs2GRs84q0mlKaW33GtPCmtQTGM1VYa5rQFX2dXBxjt3uLECkJWl+5W6zZ07bqx2QlxzBRYONwrRUwYyDmg0TSNRVi3RR7sc2NS3q6jhJLO09jen5m+xwfgC2ZXRQBeGFUB0w4vGh1BhGOUytpiUVFVVxiqlh2UtBeGaBYcOaQZXsbjwhUbQZ6QZ2QN3kMZglvmlMJyzbGKrLWZbBPg0uESWe4EWLSje6OPaTNGzaFYoxK5Vlhky5XUCYBioSCWKV1QQZSG011zPQO7CERPwYpYqQX9uYmag5FOVgIHpDe1hPCIUIr3ntdj2FP5fr/nW6hGBTBMy8GXCnLagFu425GutThaaRpq+hIxI2e8UkaleqSlYttaapQySW3He4YNq4qtYSrKsRuVVSllxWsG9U7TftDZNfSYprAR6CrhwFSGKEiDCQfgFj8DOXbNuY5+uwtHr0w/p901YJv3/D42ad+S6eXfP6fWfV/ZWw/H5oP1+/7Q/6ib0b7eg+D8V1ARADUAEH0IwAfgoAYBhIJBWAZgRODpgof9b0QG6ELRQzdGjcRRUTe3LQZxgdBRpugSiD70fsGSBsGmDYNMGwYvuRAbIHQggUhoBtJDbaB+5gRzbQeVaFYNLW9T/8XckU4UJDB+2GPA=",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-24T12:46:22.666171Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-24_05_46_21-4734129608145900368'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0824100317'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-24T12:46:22.666171Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-24_05_46_21-4734129608145900368]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-24_05_46_21-4734129608145900368
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-24_05_46_21-4734129608145900368?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-24_05_46_21-4734129608145900368 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:25.226Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.065Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.105Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.175Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.196Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.310Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.347Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.371Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.400Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.422Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.450Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.472Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.499Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.532Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.568Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.604Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.639Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.671Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.705Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.729Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.753Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.788Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.824Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.860Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.897Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.928Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:26.962Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:27.291Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:27.358Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:27.382Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:27.393Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:27.409Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:27.419Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-f...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:27.481Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:27.481Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:27.548Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:27.581Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:27.617Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:46:40.441Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:47:00.095Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 9 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:47:00.123Z: JOB_MESSAGE_DETAILED: Resized **** pool to 9, though goal was 10. This could be a quota issue.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:47:37.084Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:48:28.222Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:48:28.259Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:05.776Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:05.873Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:05.907Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:05.975Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:06.018Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:06.042Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:06.050Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:06.056Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:06.112Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:06.119Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:06.145Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:06.274Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:09.750Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:09.817Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:09.869Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:54:09.937Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:48.484Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:48.546Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:48.595Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:48.664Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:50.821Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:50.895Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:50.958Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:50.985Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:51.012Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:51.034Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:51.088Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:51.122Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:51.192Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:53.908Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:53.985Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:54.053Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:54.107Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:54.176Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:54.245Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:57.206Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:57.285Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:57.419Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:58.126Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:56:58.160Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:57:49.184Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:57:49.225Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T12:57:49.254Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-24_05_46_21-4734129608145900368 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598273178688/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.05425429344177246 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: 7e73a20505744345ae3839dfe5621e41 and timestamp: 1598273973.2899864:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 794
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598273178688/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598273178688/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.061908721923828125 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 795.121s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 13 mins 17.725 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 18s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/iot6n3v5dr5ae
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1839
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1839/display/redirect>
Changes:
------------------------------------------
[...truncated 290.66 KB...]
"python_side_input2-Write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWfarnlIACKN/////////////////////////////////////////////4AjT6gN26rdTHZtCxsgpPWFROINJCmaKYnpo0mEzUNPSbSepo9TJkbUNBo09QNPUADTTQAAAAAAAaAAAADQMQaNDTQBoAGjTahkwjT1D0mmQ08kGSEaNEZMEI0aFPKPUNMmmQAAANA0aZAAAAAA00AAAAAAAAAAAAAAAAAAAAABSVP2Sj1BDATJoHqGJgEaZMEyaDCMJkyYAEGEwDQmRoxGmhk9CMRkw0NEGmTEaBiDAmE0YmQMmmmBGmhoaGAQep/qkGITACNMTTABMEaZMBGmmTAEwBGACYAIwARppiYATAAAjTAAJgAAmmAAIMIaNAwAJMABMAATAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAASJEAjUwhqMjVPym1T9BR6jQeKB6maj1Mmm0jEAANA000AZGmmyamgAAADQ0AAAAAAAAAAAAA0AAZZMS1uZjAmlzAsMvq9MjhRO4Tx2bNrDjwDJaL7IoVi+XJk369d17etfeZTCuZWgG8opdBVVEuh0QhMl/fGsEQgIUKBASNMSbSAZp6J3+aS5VMY13boHqEpsY0JTEIJ1rJIAxZ3cLAUCeHCNAtCAL4MV7ljBovrt23fxXl7czTTwlhpmEvM48b/g0kjxBoBE4FqM0jaGzNQ3WPTDMWXFp0hFgt7Gec6TMB0h4ckFZeF/gCo8TcF9pMqRViJB4Gnb5SoKp0kwmE4JyFKwlMJgTrJ+QmpygI5/wUcToeibc5zkpVcWKk3NnFkZq4Zh5+glOmEXXcO2dWQvCV8BGIlDkokSkNhM0iUkNjeQghsOnEKNsSGS0XNMhtMjEdhGcqTvM82QUGYmWLGGiaZSVkysps60FgQzm4vZyyoOwBIyowfU0O44P/WjCRVqnV2Ac2KtlipHwc/woATKkpDtUFNUi5EfnGamPfw/9b0aLCiZNExl3B99DxZfqzZs51+0UUUUUdDGIQhCGMYhEk5x+FXRQKwCFsAgGYSFGu7ZgW+j4HO6y5ORwi3lBUC6aJpq5hashObJiboZCx8TOoMAMOmlRfLSoxthLAMvzJzBy8BbhmNTzX2Y0vb8uooLdtkaS4SjfHQ7U4i3VE3nDz3etBZ6kZk7h9iTgbOZg2uU2/cAoZMPNGVEDq8aJiYnGDCLpobKLjIicoLYoi6coJ3mFCiApqsUCmY12Q4cHHcRbssU2a8zx5ImcoGG5rvYBE1xBr2zfxPgQBJ0Iyf6MstR4dlf/xOOwyiBYonuI2MqPSgIEbCYMAMEhG4AxtDodTtdvu0eBXuJmOlmZjxTc58p/80sK9gxjAwszy82N8WvpyslZPJ5IdERZZBRBZ6zE70B4AgoJ21BXQB0QZfb7SMY+pVs42AhAOAYaml8ssa1dEjrKeL8846yLCNdZKMVR1ijUm3ZR+5MtvyMCidKGRWnIOR0geJJ6OzWwe3qCwuBLebJN5GcGssXpnFQQIKVWOPsBSZHPoxJIxBjMCIFDsIsyUktkfASYIedh1MszpqKG4i6UCk1x8qg6fTMoK4oldwIwYxWKVRdEnaTnuJ49Qp35UutqphXMeVTaBZObuKijBQDC7mrO+o7KmKxJKtVBeLIY1gF14FaxNm7Us90Aa0GYOQS6FjaZC/TDEtIgyN6cytNQ2jwLYV1yXj6EKjMRAEmJUSBIgeWbo+ImyYA4fmapgnMQcTiDuVol+5b+3C8asZY+R6coe/SqjvGHgqgFh6+dlpeVSFBkZY8wZVqtuBAFBgGcBFKAiAZgAURnyLiJqklIu4RUaR2okMPclIdGeDsNjYVCalilrGOuaEBsLsLYgTHoDAgIFNL5NAOKaxBlECnSGSJxLE3ZvL966DvxcQYuPsVYsS41b9Wqhu6ldYUqlFGBXoFTTZDrqFCqm6JN1VGZJLCv+2W2EzERwCdu/qKp3y44IrDOkaxTpT5gVNwtpTlZS6CVsRMi7QaB0zQRyygmBsmCgzUk7FicYBU8gqLS4e0Cqml+M1zagX1ym/NFIWvSFA3ti2hWLgLVASg4E5WwTgPBYW2ulZ6C8KQtdxVkYktW8TM6nEMDCAKanphpDkHEQwgQmAwQUrYR0vsa4FWAUwzoKKIlmjkQBLFlTdK4hAQ15hLAuS3QlQKigIJQ5j7GIXEBm1oadkFwDIAwFzApZ9o3ugCtMHRqYOzu9sbY+V1cS4iYhYYB9A5G7kDg2ApHItXM5IZXbvlZ185caG2amWWQIxmSwVrNdOnxgMuqVDvFrKERIbU4jjr6jSQdmLHik4tKdZEYcBcxwNjVxj2YDFgqF0ynIQkG1cKGFF4LMqV0YUpXkn2YJv0wFJLVDNQGqZ92lZTiCBVsUXPCO+3gJ6GdWVVw1WJNgllampLPVoMmpgylNu4p0YJixXkWHJlapa4qot4OAWDW2tAVjRnNFNYYAoVxY1NCdJBFEAXuQp+0wzE23cy2ClhJUtWiiHMtTGSaghW74xTnomrBueX0RSv7rrsWzSCNkbBmzFJlrMDCkIFYLKkKQh5RZizzLqSHNygkD05fJDm4SppTCbeyg2BFLVyMrOMWmgWI6JSww2DZrQFKwbWK0q4wxslKvGwGzQLE201UuZ9b1ymWfiIalzkPkuciIogg1Ba15ku31kDHQQQVmQQvPS20SGdJIYYYTDXaxdKnlwb4jER+WWRS0t+Uxlf3JoLjnrD7zGxLvHlPOdbTDSJyhJGoMqtP6e9GVDw8TCAMGEpWKAAAAAAAAAAAWAANAAIxYsmFDQIAEEEEEERKzlLSmxCKaXOapUC0RERHOUtHOcHOiiifnWI45lvxxFzlkyZ0m2226KMEqpUSbbbtxylw1bOcRGDJKVM0WMWXDI2ecUZmlKaW219WeFNaYmMZqKs1jVgKPtamDdOzeX3gpCVhdt1uMudOy2odcJcawUVjjXVgqcGIg5gNEzGmqhbgpe5HNjUt4uqYSSztLY3h+dvs0H4QumdorAwDI1A6CcZkA1BkTJOZ3phOKlWVyCyl62cvAwDLAsOHLIMj2NfwpQbQZ6UZ2cNzjMRglvWlMJyy7F9GRomgBTR33AhZoRQIgNlWd1S3GCqC4GgqrlxaIBYmmagTFMdaUTxztUEGWivnmv+w5ushET8GKWKnYl2gmis/HSVgIIJTg2hTEIkIt4Htdh183y/X/OpVEYswJoXjp8jSXqgLuE9ajWWnwdJI0ldSSL0bPdqSNOrUJSrXGWGoUmOS2w7nDBtWlFjCVRRebdUU012lSwbxTsN80Nk19NimlBG09dDiH4ZASYMEAvASrwY00zZ9J1+FnXP/uY+75T1mn6H5/S10X1UNFH2+/4apKNIqpP1S037/tP/mzeogo4QIpLGcIsBkACEED4QLwmAMAYIBAJADGAueIeBT/67HgNzohtFNz6hyDZs3uqcGNig7CoRdglEH34/aMkDYNMGwaYNgxfdiA2QOkggUhoBtJDbaB+6gRzTQeWaFWNLV9V/8XckU4UJD2q55SA==",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-24T06:46:40.526914Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-23_23_46_39-11870257176693943788'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0824050319'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-24T06:46:40.526914Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-23_23_46_39-11870257176693943788]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-23_23_46_39-11870257176693943788
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-23_23_46_39-11870257176693943788?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-23_23_46_39-11870257176693943788 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:42.877Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:43.577Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:43.613Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:43.777Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:43.804Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:43.883Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:43.931Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:43.962Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:43.997Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.032Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.058Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.081Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.106Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.136Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.172Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.198Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.224Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.262Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.290Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.323Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.356Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.393Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.421Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.464Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.500Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.535Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.564Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.785Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.841Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.875Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.887Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.911Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.911Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-f...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.971Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:44.971Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:45.040Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:45.066Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:46:45.147Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:47:10.911Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:47:14.286Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:48:43.364Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:48:43.389Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:51:58.567Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:51:58.622Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:51:58.647Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:51:58.716Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:51:58.739Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:51:58.773Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:51:58.780Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:51:58.792Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:51:58.826Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:51:58.832Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:51:58.858Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:51:58.892Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:54:13.023Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:54:13.082Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:54:13.125Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:54:13.185Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:49.636Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:49.696Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:49.745Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:49.817Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:51.862Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:51.922Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:51.996Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:52.027Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:52.046Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:52.081Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:52.105Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:52.144Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:52.218Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:54.607Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:54.665Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:54.747Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:54.794Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:54.854Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:54.911Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:57.251Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:57.322Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:57.401Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:57.485Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:56:57.516Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:57:45.550Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:57:45.587Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T06:57:45.623Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-23_23_46_39-11870257176693943788 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598251596715/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.06810641288757324 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: ede32645e83143aabef0133125609f4e and timestamp: 1598252388.9067647:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 791
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598251596715/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598251596715/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.05318903923034668 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 792.769s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 13 mins 15.852 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 29s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/jy6vsgyeb6w7o
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1838
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1838/display/redirect>
Changes:
------------------------------------------
[...truncated 290.18 KB...]
"python_side_input2-Write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWTFF7DkACKN/////////////////////////////////////////////4AjT6gN24uCNmyLGyAT1hSTiDRQhkpieI0j1NMR6j0m0nojTI0ZDQDTQNPUAGgBoAAAANAAaAGgAAANADQ0AMgAaB6hoaDTQ0GQ02kGSCDRMRiT0jRojSPUNMmm1A0AADRo0aZAAAANAGjTQAAAAAAANAAAAAAAAAAAAABkjYoAAAAEwCYAEYATIwAAmACYABBgTEYBMJkwAAAAAAAAEwJoyMCDAAACaEHqf6pBiEwAjTE0wATBGmTARppkwBMARgAmACMAEaaYmAEwAAI0wACYAAJpgACDCGjQMACTAATAAEwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEiRNAhMJNqMTVPyFP0IIB6agaaeo9TJpiMQAA0A00AZBpsp6mgAAADQ0AAAAAAAAAAAAAAAGWTFVZcxgWJsyK7L2SmVwoncJ47q6qwY78y1F5kUKxerlyb0kjry8feaymFdhWQN3RS6CzZiXQ6IQmW+vTWCIQEKFAgJGmJNpAM1NI75NJcumMa7l0D1iU2MaEpiEE67kUAYs7yFgKBPEBHwKhAF8IJLdfAovbly1fRXd5bznUwlpqGExGgeM/waaR4c2gROBazNM1TbGscVk1Azlt5cdGRaLjo0TniZgOjPDEgsMQYu+lZ4i8MVxMrRXjJB36m7zFYVzoTCYTgnIUrSUwmBOwn4+anKAjn+/xw/B9C25znJSr4kVpubOJIz2QzDz9RKdIRffwrp15TEEsUBGMlDkokSkNhM0yUkNjeUghsOLIUbkkMlpOaZDaZGM66M5cniNE2oVGcmWrIGkahQsJlhS3qoWhDOcjEzmFUdcCRmRg+ntO24P/VyOiq/7G13dot9b4CSleN2UgJdgW+6fvnqIW8kMk4YSD8H7neDL3sS5165WwIxoj2S0xNztDS1IYYYYYsEECCCCCEEECCJE944IRCUBDAFEcAEB79PqCSkxRR+i4HPbC5SRypdywqhdJE01ewuWUnNkxN1MhZOHoVGAGHUSovF8qMb8JYBX5o5k5iAv4kxq35q8YWVcBE11RUGpdnIlWhC2KxczNBM0qNc6SGgzlYslIwmSKKCINsULeU3LcwoZMNfGVEDpI0TBweNGAXWIakXGRE5QWxRFw5QTtYVKICldqgUzIuwHCg5DhrjC1TZrTPHEiZywYb2u8BJHpnovJsPS6DCorEsS6uyikDe+xMfqBTLTr+sx8p1LUKlwNLZEzZhmiSpUDOPj89qpHXp7So7jHcnHy3a9Ex7Zn8TtUegZmDFaZMqu9tXxB9J9JkiSWlSUUNWalEsyNnC0GrJO1FEgkCka7VY+xavm2dVFGv2wsPM4UpqdFSt6yh1sZ35h3koi5u/EuTG61aNVyLeQ1ptbmnAJoco02qCTqhvkRxKQhkNK9UiCjrwWw06nccQjSreGYYhIgwY4NGplxSQfBfyTiTOIIgUPERhrCWVh7BKBLzkOtluhNRU3EXygUmuryrDjKTKiyKpX8CMGQVqlWXxJ3E57yeTWKceVrqyrYWTHmU2gWXnLysqwVAwv5u3vSOwpitSSsVYYhZTIsAutArmJs41S0XUB98GwHIJtGxlMpephiWkQZG2czPmyNr8C0Fdel28hCozEQBKExJIkwO76PvuFMSANv3pxQLTIOiBB5KudHq2TqVTdM8YjJcvlI0yTXdcxK0YCq8/JynRk1ZQhG0ZQhU6nqiAKEANUIpcEwDiACiNGReRNUJSL+VKzTOzEhh7goHRHgbTZ2VUmpY5bBjkNCA1111sQJjkBgQECml4mgHCNYgzCBTpTLE4LmRIdAk0FGjLiIKE2lQmAGY58dHLi1EieOQK3ypsZkIKlLYdlYoVc3VJuuszpJYV/2260mYyOATu4+srnilyARYGhI2CmnPmhUvFq0zMo6iV0RMi/aNA6TQRzCgmBtWCgz0J2rG4wCpyKquLx6oV0o/F7o24F7bpvjRSFr0xQN5XtIVi2C+oCYG/nK2CcBoWFvyJTePBhgGryKUnXiz9Hl/YtEMDD+Xm90tNw5B0EMIAKAXkMGsKvjboqJmClnHvmomUm3UgCaNKKvgUn+Gt0LU2yy0kxBTT/BLnYfQyFmAcTnwn24bANwDAtAE7Mzbm0gPqgXzhh4eXqLsL3UGxNiKLDQwHwDkr+RODaCkck1ezkxll2/VvWzmRobZrZpZQjIZbRVZ1qU+LBmyVkd4tZQiJDVOI45NVpIO0ljSk4qU66EYcBbxoNfWhjmX7FgqFwzHHwkGq2UMKLwGZUroxExXkXmYBvkwFCWuGeoNc0b9O2mMIFYxRe8I8V3AT2mhYV2Q1YJNglmampLRVwMmpgyibd5TSgmLHiItOULFRriKq7g4BYNjb1BYNGg0UsDAFSvLWpoToQRVAGJyFP2eGYm272XQUYSVGrhRDmXJjJNQQrt+Y5z0jXg3vM6QpYu47pi2yQRtTZM+coZrFZNO0MJJQ07SqFPeZsstJ28WmaBVmVEi6vcRlxq03dtRsiKNXozM3C1ECxnQqWGGwbNiAosG3ixKyMMbVSsyMBs2ham2mq13P1fWKS0cZDUudh8nzsRFUEGqLWvErtdZAx0EEFZkELWzWkSGdJYYYYTDI+xdKmrgzCMRHpppVLS15TGSe5YgtubXeax+Jd21TXOtJgpE5QljUGWantTbGVDw8TCAMGEpWKAAAAAAAAAAAWAANAAIxYsmFDQIAEEEEEERKzlLSmvCKaXOfUqBaIiIjnKWjnODnRRRPTrEcdhb0cRe5ZcuhJtttuqrBKuVUm227skpcJXTnERgyylSaLWLNhkbbQKs7SlNLc7rXnhTWoJjGaysNg14Cr7Otg4p277FiBSErS/dreZtCdt1Y7IS6YwUWDjdK0VMGMg5oNIzmoqxbwo96ObGpb5dPwkloaezvj8zfYoPwBcM74rAwDDsg6KcXjw+gwzLOZ22BOLKrpcoukvH5y0GAZoFhw5pBlezuvCFRqjPSDOxhvchjMEuOaUwnLNs4qstZlsE+BS4RJaLgRYs8b7Sx6szSs1SsUYlcqywyZcr6BMIx1phPHO+oIMtFetfe9hz+uBET8GKWLLsVcoJorPR0lYCCCY4FQpiESEWr/2s3m7Hy/X/OrWSMKwCaF42fDpLbIFzENuRsLU4OmkaavoSMSNtxqkjUr1iUrFuLTWKGSS3I73DBtXFVrCVZViN2qqUsuK1g3ynab9obJr6TFNYCPOVAOgpgwwkwYSEGAsgwZ+Zw45Lv6q22fvPF+f2+6tTPw+PysE18102u+n1+04r+6tj+f1Oz37/s//qBzQwV2EAPzV0BGANwAQggjCiDCgBgGEgkFYBmBaL9RxP/61ogOUImginKNC6igoHNqWgzwAeBQpvASiD70fsGSBsGmDYNMGwYvuRAbUHQggUhoBtJDbaB+5gRzbQeVaFYNLX9T/8XckU4UJAxRew5A=",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-24T00:46:31.108998Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-23_17_46_29-9827459840762801037'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0824000554'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-24T00:46:31.108998Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-23_17_46_29-9827459840762801037]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-23_17_46_29-9827459840762801037
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-23_17_46_29-9827459840762801037?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-23_17_46_29-9827459840762801037 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:34.434Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.297Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.333Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.412Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.438Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.541Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.587Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.622Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.649Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.685Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.716Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.748Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.795Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.830Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.861Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.888Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.922Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.957Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:35.995Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.030Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.054Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.085Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.120Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.192Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.225Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.263Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.296Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.449Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.525Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.549Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.560Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.583Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.597Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-f...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.653Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.653Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.712Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.747Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:46:36.795Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:47:02.956Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:47:05.230Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:48:42.319Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:48:42.359Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:52:02.528Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:52:02.608Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:52:02.643Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:52:02.701Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:52:02.736Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:52:02.761Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:52:02.792Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:52:02.831Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:52:02.868Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:52:02.888Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:52:02.896Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:52:02.956Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:54:21.179Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:54:21.252Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:54:21.310Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:54:21.386Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:01.306Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:01.372Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:01.439Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:01.514Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:03.766Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:04.025Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:04.091Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:04.112Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:04.141Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:04.167Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:04.192Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:04.244Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:04.318Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:06.648Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:06.712Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:06.774Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:06.840Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:06.909Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:06.971Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:09.517Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:09.587Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:09.750Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:09.805Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:09.839Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:58.130Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:58.174Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-24T00:57:58.205Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-23_17_46_29-9827459840762801037 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598229987312/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.06815385818481445 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: 6f5be9c5d81c45e7b839b1350766d071 and timestamp: 1598230780.3068867:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 792
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598229987312/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598229987312/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.05867767333984375 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 793.511s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 13 mins 16.197 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 24s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/2abm6gmy2ut7g
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1837
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1837/display/redirect>
Changes:
------------------------------------------
[...truncated 290.83 KB...]
"python_side_input2-Write/Write/WriteImpl/FinalizeWrite": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWWYnPLEACKN/////////////////////////////////////////////4AjT6gN24rhjZtCxsgpPYNROINFCGSEemjSPU0xDT0m0nqaNMjRkNANNA09Q0A0AAaAAADQAGgBoAAADEGjQ0AMgAaB6hppoNNDQZDTaQZIRoEZMEIyaNRqPUYmmmQAAAPUaNGmQABoAABo9TQAAAAAAAAAAAAAAaAAAAAABkjYoAAAAEwCYAEYATIwAAmACYABBgTEYBMJkwAAAAAAAAEwJoyMCDAAACaEHqf6pBiEwAjTE0wATBGmTARppkwBMARgAmACMAEaaYmAEwAAI0wACYAAJpgACDCGjQMACTAATAAEwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEiRNAhMIIyNU/KZT9CR6jQfooHqZqPUyaYjEAANANNAGQabKepoAAAA0NAAAAAAAAAAAAAAABl0wbG6mICWTEFZlurRHAJ5oBNFXr2MXFfmOwXWQvrFtceUdq1YL29a64yh9cqtEN5PRA9TTCueCEPmO+tmoEQgIUKBASNMSbSAZqaJ3yaS5dMY13LoHqkpsY0JTEIJ13DQBizvIWAoE8QEaBYDwXwgq2q2BPbuXLN9DeXtrOdTCWmoYTEZx4z/BppHhzQCJwLVZpm2NqapxuTUDSLby46Mi0W+jPOeJmA6M8MSCwxBi76VniLwxXEytFeMkHfqbrMVhXOhMJhOCchStJTCYE7Cfj5qcoCOf7/HE8H0LbnOclKvixWm5s4sjSshmHn6iU6Qi+/h3TrymIJYoCMZKHJRIlIbCZpkpIbG8pBDYdKIUbgkMlouaZDaZGM66M5cniM82gVGkTLVkDRNQoWEywpb1YLQhnORiZzCqOuBIzIwfT0O24P/WG/w9k9rS90iIemsZIS3I7SQEwvLhdT30DgXAjss2Xxz4Hzn+HN4ATB2OLuARuwj2S0xNztDS1IYYYYYsEECCCCCEEECCJE944IRCUBDAFEcAEB79PqCSkxRR+i4PPa65SRypdywqhdJE01ewuWUnNkxN1MhZOJnVF4F/USoulpUYmvlgFzmjiHMvFuKNNRbiscV8BmmWCtQvEvThlWghy2MQsfbUYx3NKnSoaDKViuUqBMmUUEQbWoW8ruO5BQygcaMsHoKsSJg4PGi6YJX7KLiIhgUFsURfHKiZxhUogKV2qBTMi7AcODj+It6WqbNtM8cSJnLBhva7wGL8VOPzfP+9EN0W1m8VKr7+OzHlfD+mMePK1vIo9lMjyzkgRnQm7dF1ZcVAzjY3Ra6P2ae4q28yXLycv2/TMzU3+J+su4MzBivsWNceGbyg9E9ExRJLUpKKGrNRwtaLULQask4ooYEgUfYq1G2fX86JVQ5ue6Fm5vGldbpqdwWMItifBMukhCxbpIxbC9VuLVx7WO2Js7um/JoSZps0EnJDhIiRpiGU1LvTH+/Xwtpq1O62gWVb8UywCRBewxCNYKSJB5GEJOGM4UiBQkLEmsJZXXgEoEvOQ62W501FTcRfKBSa5CVYdLpMqLIqlfwYwZBWqVZfEncTnu55NUpwCtdXVbCyY8ym0Cy85eVlWCoGF/N295R2FMVqSVirDELKZFgF1oFcxNm9Us91Aa0GUQIJNGzyiMu0PwrSEMicmMjTSMx4NkK69Lx1CFRGEgCUJgSRJgd70/feKY0AbvvzygWmQcUCDsVh6XWtPVrO+qGQRkOZJovcou7BiWpQFa6GXlelKKygSNkxgSqVTVEAUCAaoQ64JgHCgFEZ8i8iaoSkX8qVmmdmJDD3BQOiPA2mxsKpNSxy1zFVM6A1l1lsQJkEBeQECih0mcHCNQgyB5UEZjhgJY27dnFvnUd7LyDHyFqsFjXHLgK5VN30XVFKtRVgWJAqUth2VihVzdUm66zSSSwr/tt1pMxkcEndwKyueKXHhFgZ0jXKac+aFS8W2pmZR1EroiZF+g0DpNBHMKCYG0YKDSoTtWNxgFTkVVcXj2wV0o/F7k24Fu1RfGegLXpigb2tZQrFoFqgJAb+YrYJgHAsLbVSo8mAC8M3YU5Oxlo6XMe1aIMGH0vP7pedhCDiIMPwUAvoXs4Jzu14VFUAUu2985DSs65EATJlSXOqJPsHcoKsNFluJgCnH1/XUgfQyFrAbT3woWoaANQDAtoE/Nzre3APSgc54w6uzvEWZ4qrQmhFEBkYDyByV/InCtBSOSavZyYyy7fq3rZzI0Ns1c0soRkMtorGdalNjQZMnSO9WsoRChsTCKKrqtBB2kscKTCwqCuIhABax4NbWhkGX7FgqFwyHIvkGxaKF9F4DMsVz3ZCvDdZgG+TAUJawaVQaxn36dtMYQKxii94R4ruCnoZ1hXZDVgk2CWZqaks9XAyamDKJt3lNGCYseIi05QsVGuKqruFgFg19vUFg0ZzRSwMAVK8tamhOhBFUAYnIU/Z4ZibbvZdBRhJUauFEOZcmMk1BCu35jnPRNaDeczoili7juWLapBG0Ng0tIoZrC8wo+BUCypCj4cUWYU0q6UfxhQRhyYukiBt1UshfN1bUbAijV6MzOMWogWM6FSww2DZrwFFg28WJWRhjaKVmRgNmgWptpqtdz9Z1iks/GQ1LnYfJ87ERU89qi1rxS7PWvMgeeeqMefccksoj8yRvvvvl+q1i6FOLezKMRHZJI1LSz5bGVfcletQOVnXGNhXeOKcggspi0hgUI4lBlNh3U3IioeHiYQBgwlKxQAAAAAAAAAACwABoABGLFkwoaBAAgggggjeG6JPQyWmhoXXVxFkm227rok7rri66+++maRgwZZUwRF7lly50m2226qsEq5VSbbbuySlw1dOcRGDLKVJotYs2GRtc4q0mlKaW43OtPCmtQTGM1VYa5rQFX2dXBxrt2cWIFIStL90t3mzp23VjshLjmCiwcblWipgxkHNBomkairFuyj3g5salsrqGEks7T2Nk/M32KD8AXDM0VAXhkqQdFONyIagyRjmMzkomFKq64xXS9bMXAXhmgWHDmkGV7G58IVG2GekGdjDeZDGYJb5pTCcs2xiqytErwKaW7ACFmdFAh411Z/TJaYKXrQaCqqYC0PCtLK1AmEYqkgmimaoIMvDbca7m4G64ERPwYpYpgwbk5M9R2KgrAQPSHAsCiEQoRZv/a7HsJfl+v+dWpIwpQTOvHzZKguUgXLrlyNdanC00jTV9CRiRtd6pI1K9UlKxcZaapQySW4He4YNq4qtYSrKsRulVSllxWsGyp2m/aGya+kxTWAj0FVDiHsYoSQMJCAAWQAM/O2bct09dgaPnoiPRNe6vzfw+Pyss58106u+n1+08r+6th+f1P0H7/tD/qJvRv96D8PzXEBEgNQAQfwjBCACgBgGEgkFYBmBbcHSxQ/6KRAboQ1FDt0aNyFFRN7etBnGB1FGm6hKIPvR+wZIGwaYNg0wbBi+5EBtAdCCBSGgG0kNtoH7mBHNtB5VoVg0tb1P/xdyRThQkGYnPLE",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-23T18:46:32.889978Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-23_11_46_31-8898380291330454735'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0823150256'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-23T18:46:32.889978Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-23_11_46_31-8898380291330454735]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-23_11_46_31-8898380291330454735
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-23_11_46_31-8898380291330454735?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-23_11_46_31-8898380291330454735 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:36.063Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:36.927Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:36.970Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.032Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.061Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.165Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.209Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.244Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.279Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.311Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.342Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.372Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.406Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.431Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.466Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.502Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.536Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.592Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.627Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.661Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.685Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.721Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.751Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.787Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.819Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.853Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:37.887Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:38.042Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:38.110Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:38.143Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:38.155Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:38.179Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:38.190Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-f...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:38.248Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:38.248Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:38.317Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:38.356Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:46:38.391Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:47:05.048Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:47:08.138Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:48:39.545Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:48:39.594Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:51:57.602Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:51:57.682Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:51:57.706Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:51:57.779Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:51:57.804Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:51:57.824Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:51:57.839Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:51:57.864Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:51:57.892Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:51:57.905Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:51:57.943Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:51:57.977Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:54:21.800Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:54:21.888Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:54:21.949Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:54:22.022Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:56:59.788Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:56:59.860Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:56:59.926Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:56:59.996Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:02.339Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:02.408Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:02.480Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:02.516Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:02.529Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:02.567Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:02.621Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:02.646Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:02.715Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:04.873Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:04.939Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:05.024Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:05.077Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:05.152Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:05.220Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:07.542Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:07.630Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:07.712Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:07.837Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:57:07.876Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:58:03.403Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:58:03.461Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T18:58:03.497Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-23_11_46_31-8898380291330454735 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598208389286/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.06461787223815918 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: ad7d0bc79ac4469391859c4ab63fe48a and timestamp: 1598209206.1238163:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 816
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598208389286/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598208389286/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.06682324409484863 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 817.407s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 13 mins 40.606 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 49s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/tqoeqeodmqml6
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org
Build failed in Jenkins: beam_PerformanceTests_WordCountIT_Py37
#1836
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/1836/display/redirect>
Changes:
------------------------------------------
[...truncated 291.13 KB...]
"output_name": "out",
"step_name": "SideInput-s20"
}
},
"output_info": [
{
"encoding": {
"@type": "kind:windowed_value",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "FastPrimitivesCoder$QlpoOTFBWSZTWYQR6NMAAEDXwH8QgCEJAEBAv279AmAAIABqEqnqGgaABpoyNAGVHpGgMgDQBk0oR6IeoECBiqqU5NY23ndshzT2UPUOGrg42YPi9VyA8lbwwPJgtghxs5Qq1aWwExCDeMa0RHC2QigTCdizz1nx+LuSKcKEhCCPRpg=",
"component_encodings": [],
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
}
],
"is_pair_like": true,
"pipeline_proto_coder_id": "ref_Coder_FastPrimitivesCoder_5"
},
{
"@type": "kind:global_window"
}
],
"is_wrapper": true
},
"output_name": "None",
"user_name": "Write/Write/WriteImpl/FinalizeWrite.out"
}
],
"parallel_input": {
"@type": "OutputReference",
"output_name": "out",
"step_name": "s7"
},
"serialized_fn": "QlpoOTFBWSZTWchwHzQACKN/////////////////////////////////////////////4AjT6gN2cudMdm0LGyAdvXConiDRQhlJiemjSNNMR6j0m0nqeo0yNNNDQDTQNPRAGmgaAaAAAAABoAAANAA0AA00AyABoHqGJpo00NBoNNpBkiBPTUxME0mJok9I9Q0aabUAAAD1GjRpkAAAAABo0AAAAAAAAAAAAAAAAAAAAAGNTxJ5SNBDI9EaYg0NNDTTINpNMmj1HpBiGmTCZMhpgRpgRiBpoxDCZNHqaGJtIM0QNBppoBpoyAaaGBGCAaNDRkyaEHqf6pBiEwAjTE0wATBGmTARppkwBMARgAmACMAEaaYmAEwAAI0wACYAAJpgACDCGjQMACTAATAAEwAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAAEiRNAhMJMjI1T2jVP0InqNB4oHqZqPU2ppiMQAA0A00DRkHqbKepoAAAA0NAAAAAAAAAAAAAAABlkxtW5loEsm/Cw5diojYJ5mCZtmzVhNwDHUXnIayxdXHk3oomW7b7zrlFZcqtBN9PQyCmmFc7IRWMd/dNcIhAQoUCAkaYk2kAzT0Dvk0lzCYxrufQvUJTYxoSmIQTrWKgDizu4WAoE8OEfAqEAXwYiuWMGe7evWr+G+t3M008JaaZhMRnHjP8GkkeHNgETgWozSNmbI1DjsmmGiW3lx0hFot7GedATMB0h4YkFhiDF30rPEXhiuJlaK8ZIO/U3GYrCudCYTCcE5ClaSmEwJ2E/HzU5QEcPv8cXwfRNuc5yUq+NFabmzjSNGyGYeHUSnSEX38S6deUxBLFARjJQ5KJEpDYTNIlJDY3lIIbDjyFG3JDJaDmmQ2mRjOvDOYJ4jPNiFRoky1ZA0DTKFhMsKW9XC0IZzsYmcyqjroSMyMH09h27B/6w3yHskyzvLnEPjWPkpXkdtICXXFwOr7/1SC30hk2y6QfQ+9HxZm+iXOx3W3hGMNiU0zxptPylBxxxxx8YxCEIQxjEIIb9CblVkxmMAjMEZCw0ONpakiPo+B0GquVkcsXcuKoXSxNNXsLllJzZMTdTIWTi51SsFbwlCoq1CwV2qYZvNriLh2KvwWssMquLLpynEtFQaTacUqz4PC2GnDdzCq0jmYSZIjOZisUlIgmSKKCINpULmU2/cAoZMOvjKiBkTUTCwuMGCWS1rSLaRCxQW4oi8coJnXCpRAUrtUCmZF2E4kHI8VbstU2aszxxImcuGG9rvISR6p6TzbP1uIwyKxLFOTmoo0vvamPzU0y1Lp2F/nOC1SpaDS2pMGAYIkqVAzzGY0Gs12yT29Wkcfy8hLdz1TE1N/egrLyDMwYr7w8Vx6ZPaD4T4TwiSW/EUoDgFUtEFuMQGAONJpjWkw08amzeW3CXhUzgNIBXTmZ6l5vZmJJdoEWYcInwzDrJQsa68a6L7vCRyvXNZDYGyjU39NDmGmyQSckOAiONTEMjp3mmNL9ei2epU7zaBYwnjmF4SILl+FRrBcckHsYEk4YzhSIFW4WJNYSyuvQJQJech1stzpqKm4i+UCk1yUqw6ZSZUWRVK/gRgyCtUqy+JO4nPczyahTfla6wq2Fkx5lNoFl528rKsFQML+ct55HYkxWpJWKsMQspkWAXWwVzE2btSz3UB98GUMQSaHnNEZeorQrSEONdmMj5pG0eBaCuuS+eQhTTCQBKEvJIkwO/6vtIqYoAbrwT6gcDIOKBB3KwynXtPWrO8YY1GS5vMRqHPW9kxK0oCtdLKScpz1ZQJGxeIEqlU1RAFAgGqEOtCYBwoBRGfIvImqEpF/LFZpHZyQw9wUDozwNpr66qTUsctUtiM6A2F2FuIEyCAwICBRQ8TODhmuQZBApkZjhYSxt27zFvXUc8XkGPkrVYLGuQW/VyqbvouqqVairAsSBUpbDsrFCrm6pN11miklhX/bbrSZjI4BO7f1lc8UuRCLAzpGqU0p82Kl4tnTMyjqJXREyL9g0DpNBHMqCYGxYKDRoTtWNxgFTk1VcXj2YV0o/G6xtALtyi/M9AWvSVA27FpCsXAX1ASA4ExW4JgHQsLfiSo8mAC4MncU5OxloaXLzLgIMGH4vR7paerZBxEGH8KARwXMoJ0jLsqJgClm3vnYaTnnIgCYsaS6VRJ+g7jBVhmsiyXgp1+aLXMfMyFrAbT85QtQzAagGBFAUE3PN4sB8UDpPmHZ3eYizPVVZkzIogMTAewOFfyZwbQUjhNXs5QZZdvlb1w5oaG2amaWUIyGW0VWa6fNjgZMlSO8WsoRChqmDWxalQQdmLjpSYVKZZDQwC5jwbGrjIOYDiwVC8ZDkaxBquFDWReC5lSufGSFeK85gm/TAUJbUNGoNqZ9+lbTGECsYoveEeK7gJ7DOsK7IasEmwSzNTUlnq4GTUwZRNu8poQTFjxEWnKlio1xlVdwcAsGrtKgsGjOaKWBgCpXlrU0J0IIqgDE5Cn7PDMTbd7LoKMJKjVwohzLkxkmoIV2+Mc56BtYN1zWgKWLuWsxbJII2JrmjolDNYrJp2hhJKGnaVQp7nNlloO3j0zOKsyokXV7aMuNWm4tqNcRRq9GZm2WmgWM6JSww2DZqwFFg2kWJWRhjYqVmRgNmwLU201Wu6es6zSWfjIalz8PlOfiIqIRKEIcQFEoIJAQhEIiFGNWRAAXAgVSlKU0qofEFYPDCTZgEAD0kkalpa8pxyL3JYLjHbDzrj8K751TrGWkwkhYoRtUHKantQdaVDw8TCAMGEpWKAAAAAAAAAAAWAANAAIxYsmFDQIAEEEEEERK7FLSixWFFDGPqVAtERERjFLRjGBjIYYXplhrZVvNiL3LLlzpNttt1VYJVyqk223dklLiK6c4iMGWUqTRaxZsMjZZxVotKU0tvrbWeFNaYmMZqKw1TawFX2dTBxzt3mLECkJWl+4W5zZ07bqx2QlyDBRYONZWipgxkHNhoGiaarFuSj3Q5salvF1HCSWdpa+8PzN9jg/AF4zPiuDAMSkHQjjsiH0GIY5jM7KJhSqyuMWUtvzF0GAZoFhw5pBlevreEKjZjPSDOyBushjMEt60phOWbXxVZXxLACmkPMBCzOigRAbKs8pkuOCmC4HwVRFhfEAsSyvoEwy2vIJmzPqCDLQ3XX3vYY/rIRE/BxSxSzG3pyZ67zaCsBBBIcGoUQiFCLWB7XYdfL8v1/zqdJGHKCZ14+bEoLtIF7GO3I1Vp8HSSNJX0JGJGy3akjTr1CUrFtrTUKGSS2473DBtXFVrCVZViNwqqUsuK1g3inab5obJr6TFNYCPPVUOIqAw444MJCAAWQAM/Q2bct19lgZvvpiPTNe6vzc58PjZZ35LZ5b8/p9Z9X9la/8fmg/X7/tD/qJvRtLwH8fiuICJAagAg0CMEIAKAGAYSCQVgGYEVgKWKH/W5EBuhDUUO3Ro3IUVE3trgDOPDsKNN2CUQfej9gyQNg0wbBpg2DF9yIDYg6EECkNANpIbbQP3MCOcaDyrQrBpbX1P/xdyRThQkMhwHzQA==",
"user_name": "Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite"
}
}
],
"type": "JOB_TYPE_BATCH"
}
apache_beam.runners.dataflow.internal.apiclient: INFO: Create job: <Job
createTime: '2020-08-23T12:46:30.457160Z'
currentStateTime: '1970-01-01T00:00:00Z'
id: '2020-08-23_05_46_29-7094010668391870518'
location: 'us-central1'
name: 'performance-tests-wordcount-python37-batch-1gb0823100313'
projectId: 'apache-beam-testing'
stageStates: []
startTime: '2020-08-23T12:46:30.457160Z'
steps: []
tempFiles: []
type: TypeValueValuesEnum(JOB_TYPE_BATCH, 1)>
apache_beam.runners.dataflow.internal.apiclient: INFO: Created job with id: [2020-08-23_05_46_29-7094010668391870518]
apache_beam.runners.dataflow.internal.apiclient: INFO: Submitted job: 2020-08-23_05_46_29-7094010668391870518
apache_beam.runners.dataflow.internal.apiclient: INFO: To access the Dataflow monitoring console, please navigate to https://console.cloud.google.com/dataflow/jobs/us-central1/2020-08-23_05_46_29-7094010668391870518?project=apache-beam-testing
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-23_05_46_29-7094010668391870518 is in state JOB_STATE_RUNNING
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:33.285Z: JOB_MESSAGE_BASIC: Worker configuration: n1-standard-1 in us-central1-f.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.367Z: JOB_MESSAGE_DETAILED: Expanding CoGroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.406Z: JOB_MESSAGE_DEBUG: Combiner lifting skipped for step Write/Write/WriteImpl/GroupByKey: GroupByKey not followed by a combiner.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.508Z: JOB_MESSAGE_DETAILED: Expanding GroupByKey operations into optimizable parts.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.543Z: JOB_MESSAGE_DETAILED: Lifting ValueCombiningMappingFns into MergeBucketsMappingFns
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.646Z: JOB_MESSAGE_DEBUG: Annotating graph with Autotuner information.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.705Z: JOB_MESSAGE_DETAILED: Fusing adjacent ParDo, Read, Write, and Flatten operations
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.750Z: JOB_MESSAGE_DETAILED: Fusing consumer Split into Read/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.784Z: JOB_MESSAGE_DETAILED: Fusing consumer PairWIthOne into Split
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.812Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial into PairWIthOne
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.840Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Reify into GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.873Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/GroupByKey/Write into GroupAndSum/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.907Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine into GroupAndSum/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.928Z: JOB_MESSAGE_DETAILED: Fusing consumer GroupAndSum/Combine/Extract into GroupAndSum/Combine
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.957Z: JOB_MESSAGE_DETAILED: Fusing consumer Format into GroupAndSum/Combine/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:34.991Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WindowInto(WindowIntoFn) into Format
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.025Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/WriteBundles/WriteBundles into Write/Write/WriteImpl/WindowInto(WindowIntoFn)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.063Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Pair into Write/Write/WriteImpl/WriteBundles/WriteBundles
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.096Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Reify into Write/Write/WriteImpl/Pair
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.129Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/Write into Write/Write/WriteImpl/GroupByKey/Reify
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.164Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/GroupByKey/GroupByWindow into Write/Write/WriteImpl/GroupByKey/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.187Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/Extract into Write/Write/WriteImpl/GroupByKey/GroupByWindow
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.222Z: JOB_MESSAGE_DETAILED: Fusing consumer Write/Write/WriteImpl/InitializeWrite into Write/Write/WriteImpl/DoOnce/Read
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.260Z: JOB_MESSAGE_DEBUG: Workflow config is missing a default resource spec.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.293Z: JOB_MESSAGE_DEBUG: Adding StepResource setup and teardown to workflow graph.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.325Z: JOB_MESSAGE_DEBUG: Adding workflow start and stop steps.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.359Z: JOB_MESSAGE_DEBUG: Assigning stage ids.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.641Z: JOB_MESSAGE_DEBUG: Executing wait step start35
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.711Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.745Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.757Z: JOB_MESSAGE_DEBUG: Starting **** pool setup.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.791Z: JOB_MESSAGE_BASIC: Starting 10 ****s in us-central1-f...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.815Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.868Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.885Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Create
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.950Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:35.975Z: JOB_MESSAGE_DEBUG: Value "GroupAndSum/GroupByKey/Session" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:36.045Z: JOB_MESSAGE_BASIC: Executing operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:46:51.662Z: JOB_MESSAGE_WARNING: Your project already contains 100 Dataflow-created metric descriptors and Stackdriver will not create new Dataflow custom metrics for this job. Each unique user-defined metric name (independent of the DoFn in which it is defined) produces a new metric descriptor. To delete old / unused metric descriptors see https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.list and https://developers.google.com/apis-explorer/#p/monitoring/v3/monitoring.projects.metricDescriptors.delete
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:47:05.593Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 9 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:47:05.634Z: JOB_MESSAGE_DETAILED: Resized **** pool to 9, though goal was 10. This could be a quota issue.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:47:10.895Z: JOB_MESSAGE_DETAILED: Autoscaling: Raised the number of ****s to 10 based on the rate of progress in the currently running stage(s).
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:48:44.565Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:48:44.596Z: JOB_MESSAGE_DETAILED: Workers have started successfully.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:52:03.107Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/DoOnce/Read+Write/Write/WriteImpl/InitializeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:52:03.194Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/DoOnce/Read.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:52:03.226Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/InitializeWrite.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:52:03.308Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:52:03.341Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:52:03.375Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:52:03.405Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:52:03.433Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:52:03.439Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:52:03.478Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/WriteBundles/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:52:03.510Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:52:03.541Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(InitializeWrite.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:54:20.777Z: JOB_MESSAGE_BASIC: Finished operation Read/Read+Split+PairWIthOne+GroupAndSum/GroupByKey+GroupAndSum/Combine/Partial+GroupAndSum/GroupByKey/Reify+GroupAndSum/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:54:20.920Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:54:21.002Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:54:21.170Z: JOB_MESSAGE_BASIC: Executing operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:00.726Z: JOB_MESSAGE_BASIC: Finished operation GroupAndSum/GroupByKey/Read+GroupAndSum/Combine+GroupAndSum/Combine/Extract+Format+Write/Write/WriteImpl/WindowInto(WindowIntoFn)+Write/Write/WriteImpl/WriteBundles/WriteBundles+Write/Write/WriteImpl/Pair+Write/Write/WriteImpl/GroupByKey/Reify+Write/Write/WriteImpl/GroupByKey/Write
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:00.810Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:00.873Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Close
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:00.953Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:03.580Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/GroupByKey/Read+Write/Write/WriteImpl/GroupByKey/GroupByWindow+Write/Write/WriteImpl/Extract
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:03.652Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/Extract.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:03.720Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:03.752Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:03.767Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:03.834Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:03.844Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:03.905Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize/_UnpickledSideInput(Extract.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:03.989Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:06.543Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/PreFinalize/PreFinalize
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:06.616Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/PreFinalize.out" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:06.700Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:06.755Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0)
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:06.824Z: JOB_MESSAGE_DEBUG: Value "Write/Write/WriteImpl/FinalizeWrite/_UnpickledSideInput(PreFinalize.out.0).output" materialized.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:06.884Z: JOB_MESSAGE_BASIC: Executing operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:09.505Z: JOB_MESSAGE_BASIC: Finished operation Write/Write/WriteImpl/FinalizeWrite/FinalizeWrite
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:09.597Z: JOB_MESSAGE_DEBUG: Executing success step success33
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:09.739Z: JOB_MESSAGE_DETAILED: Cleaning up.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:09.872Z: JOB_MESSAGE_DEBUG: Starting **** pool teardown.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:57:09.907Z: JOB_MESSAGE_BASIC: Stopping **** pool...
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:58:02.164Z: JOB_MESSAGE_DETAILED: Autoscaling: Resized **** pool from 10 to 0.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:58:02.210Z: JOB_MESSAGE_BASIC: Worker pool stopped.
apache_beam.runners.dataflow.dataflow_runner: INFO: 2020-08-23T12:58:02.244Z: JOB_MESSAGE_DEBUG: Tearing down pending resources...
apache_beam.runners.dataflow.dataflow_runner: INFO: Job 2020-08-23_05_46_29-7094010668391870518 is in state JOB_STATE_DONE
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results*-of-*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598186786830/results[^/\\\\]*\\-of\\-[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.06090593338012695 seconds.
apache_beam.testing.pipeline_verifiers: INFO: Find 30 files in gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results*-of-*:
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00000-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00001-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00002-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00003-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00004-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00005-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00006-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00007-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00008-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00009-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00010-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00011-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00012-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00013-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00014-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00015-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00016-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00017-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00018-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00019-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00020-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00021-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00022-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00023-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00024-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00025-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00026-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00027-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00028-of-00030
gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results-00029-of-00030
apache_beam.testing.pipeline_verifiers: INFO: Read from given path gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results*-of-*, 26186927 lines, checksum: ea0ca2e5ee4ea5f218790f28d0b9fe7d09d8d710.
google.auth._default: DEBUG: Checking None for explicit credentials as part of auth process...
google.auth._default: DEBUG: Checking Cloud SDK credentials as part of auth process...
google.auth._default: DEBUG: Cloud SDK credentials not found on disk; not using them
google.auth._default: DEBUG: Checking for App Engine runtime as part of auth process...
google.auth._default: DEBUG: No App Engine library was found so cannot authentication via App Engine Identity Credentials.
google.auth.transport._http_client: DEBUG: Making request: GET http://169.254.169.254
google.auth.transport._http_client: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/project/project-id
urllib3.util.retry: DEBUG: Converted retries value: 3 -> Retry(total=3, connect=None, read=None, redirect=None, status=None)
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/default/?recursive=true
urllib3.connectionpool: DEBUG: Starting new HTTP connection (1): metadata.google.internal:80
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/default/?recursive=true HTTP/1.1" 200 144
google.auth.transport.requests: DEBUG: Making request: GET http://metadata.google.internal/computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token
urllib3.connectionpool: DEBUG: http://metadata.google.internal:80 "GET /computeMetadata/v1/instance/service-accounts/844138762903-compute@developer.gserviceaccount.com/token HTTP/1.1" 200 221
urllib3.connectionpool: DEBUG: Starting new HTTPS connection (1): bigquery.googleapis.com:443
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance HTTP/1.1" 200 None
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "GET /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Load test results for test: e3715b3f2b5848dbabc406c0377963d6 and timestamp: 1598187578.1807082:
apache_beam.testing.load_tests.load_test_metrics_utils: INFO: Metric: runtime Value: 791
urllib3.connectionpool: DEBUG: https://bigquery.googleapis.com:443 "POST /bigquery/v2/projects/apache-beam-testing/datasets/beam_performance/tables/wordcount_py37_pkb_results/insertAll HTTP/1.1" 200 None
apache_beam.testing.load_tests.load_test_metrics_utils: ERROR: no such field.
apache_beam.io.filesystem: DEBUG: Listing files in 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results'
apache_beam.io.filesystem: DEBUG: translate_pattern: 'gs://temp-storage-for-end-to-end-tests/py-it-cloud/output/1598186786830/results*' -> 'gs://temp\\-storage\\-for\\-end\\-to\\-end\\-tests/py\\-it\\-cloud/output/1598186786830/results[^/\\\\]*'
apache_beam.io.gcp.gcsio: INFO: Starting the size estimation of the input
apache_beam.io.gcp.gcsio: INFO: Finished listing 30 files in 0.06565999984741211 seconds.
--------------------- >> end captured logging << ---------------------
----------------------------------------------------------------------
XML: nosetests-runPerformanceTest-df-py37.xml
----------------------------------------------------------------------
XML: <https://ci-beam.apache.org/job/beam_PerformanceTests_WordCountIT_Py37/ws/src/sdks/python/nosetests.xml>
----------------------------------------------------------------------
Ran 1 test in 791.833s
FAILED (errors=1)
> Task :sdks:python:test-suites:dataflow:py37:runPerformanceTest FAILED
:sdks:python:test-suites:dataflow:py37:runPerformanceTest (Thread[Execution **** for ':',5,main]) completed. Took 13 mins 14.602 secs.
FAILURE: Build failed with an exception.
* What went wrong:
Execution failed for task ':sdks:python:test-suites:dataflow:py37:runPerformanceTest'.
> Process 'command 'sh'' finished with non-zero exit value 1
* Try:
Run with --stacktrace option to get the stack trace. Run with --debug option to get more log output. Run with --scan to get full insights.
* Get more help at https://help.gradle.org
Deprecated Gradle features were used in this build, making it incompatible with Gradle 6.0.
Use '--warning-mode all' to show the individual deprecation warnings.
See https://docs.gradle.org/5.2.1/userguide/command_line_interface.html#sec:command_line_warnings
BUILD FAILED in 14m 21s
5 actionable tasks: 5 executed
Publishing build scan...
https://gradle.com/s/q2sb3clzd7ji4
Build step 'Invoke Gradle script' changed build result to FAILURE
Build step 'Invoke Gradle script' marked build as failure
---------------------------------------------------------------------
To unsubscribe, e-mail: builds-unsubscribe@beam.apache.org
For additional commands, e-mail: builds-help@beam.apache.org