You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@beam.apache.org by Apache Jenkins Server <je...@builds.apache.org> on 2018/03/24 11:54:35 UTC
Build failed in Jenkins: beam_PerformanceTests_TFRecordIOIT_HDFS #3
See <https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/3/display/redirect>
------------------------------------------
[...truncated 78.84 KB...]
at com.google.cloud.dataflow.worker.DataflowBatchWorkerHarness$WorkerThread.call(DataflowBatchWorkerHarness.java:114)
at com.google.cloud.dataflow.worker.DataflowBatchWorkerHarness$WorkerThread.call(DataflowBatchWorkerHarness.java:101)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.IllegalStateException: Not a valid TFRecord. Fewer than 12 bytes.
at org.apache.beam.sdk.repackaged.com.google.common.base.Preconditions.checkState(Preconditions.java:444)
at org.apache.beam.sdk.io.TFRecordIO$TFRecordCodec.read(TFRecordIO.java:631)
at org.apache.beam.sdk.io.TFRecordIO$TFRecordSource$TFRecordReader.readNextRecord(TFRecordIO.java:526)
at org.apache.beam.sdk.io.CompressedSource$CompressedReader.readNextRecord(CompressedSource.java:426)
at org.apache.beam.sdk.io.FileBasedSource$FileBasedReader.advanceImpl(FileBasedSource.java:473)
at org.apache.beam.sdk.io.OffsetBasedSource$OffsetBasedReader.advance(OffsetBasedSource.java:267)
at com.google.cloud.dataflow.worker.WorkerCustomSources$BoundedReaderIterator.advance(WorkerCustomSources.java:602)
... 14 more
java.io.IOException: Failed to advance reader of source: hdfs://146.148.62.9:9000/TEXTIO_IT__1521892046445-00001-of-00003.tfrecord range [0, 19333314)
at com.google.cloud.dataflow.worker.WorkerCustomSources$BoundedReaderIterator.advance(WorkerCustomSources.java:605)
at com.google.cloud.dataflow.worker.util.common.worker.ReadOperation$SynchronizedReaderIterator.advance(ReadOperation.java:379)
at com.google.cloud.dataflow.worker.util.common.worker.ReadOperation.runReadLoop(ReadOperation.java:185)
at com.google.cloud.dataflow.worker.util.common.worker.ReadOperation.start(ReadOperation.java:150)
at com.google.cloud.dataflow.worker.util.common.worker.MapTaskExecutor.execute(MapTaskExecutor.java:74)
at com.google.cloud.dataflow.worker.BatchDataflowWorker.executeWork(BatchDataflowWorker.java:381)
at com.google.cloud.dataflow.worker.BatchDataflowWorker.doWork(BatchDataflowWorker.java:353)
at com.google.cloud.dataflow.worker.BatchDataflowWorker.getAndPerformWork(BatchDataflowWorker.java:284)
at com.google.cloud.dataflow.worker.DataflowBatchWorkerHarness$WorkerThread.doWork(DataflowBatchWorkerHarness.java:134)
at com.google.cloud.dataflow.worker.DataflowBatchWorkerHarness$WorkerThread.call(DataflowBatchWorkerHarness.java:114)
at com.google.cloud.dataflow.worker.DataflowBatchWorkerHarness$WorkerThread.call(DataflowBatchWorkerHarness.java:101)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.IllegalStateException: Invalid data
at org.apache.beam.sdk.repackaged.com.google.common.base.Preconditions.checkState(Preconditions.java:444)
at org.apache.beam.sdk.io.TFRecordIO$TFRecordCodec.read(TFRecordIO.java:642)
at org.apache.beam.sdk.io.TFRecordIO$TFRecordSource$TFRecordReader.readNextRecord(TFRecordIO.java:526)
at org.apache.beam.sdk.io.CompressedSource$CompressedReader.readNextRecord(CompressedSource.java:426)
at org.apache.beam.sdk.io.FileBasedSource$FileBasedReader.advanceImpl(FileBasedSource.java:473)
at org.apache.beam.sdk.io.OffsetBasedSource$OffsetBasedReader.advance(OffsetBasedSource.java:267)
at com.google.cloud.dataflow.worker.WorkerCustomSources$BoundedReaderIterator.advance(WorkerCustomSources.java:602)
... 14 more
java.io.IOException: Failed to advance reader of source: hdfs://146.148.62.9:9000/TEXTIO_IT__1521892046445-00001-of-00003.tfrecord range [0, 19333314)
at com.google.cloud.dataflow.worker.WorkerCustomSources$BoundedReaderIterator.advance(WorkerCustomSources.java:605)
at com.google.cloud.dataflow.worker.util.common.worker.ReadOperation$SynchronizedReaderIterator.advance(ReadOperation.java:379)
at com.google.cloud.dataflow.worker.util.common.worker.ReadOperation.runReadLoop(ReadOperation.java:185)
at com.google.cloud.dataflow.worker.util.common.worker.ReadOperation.start(ReadOperation.java:150)
at com.google.cloud.dataflow.worker.util.common.worker.MapTaskExecutor.execute(MapTaskExecutor.java:74)
at com.google.cloud.dataflow.worker.BatchDataflowWorker.executeWork(BatchDataflowWorker.java:381)
at com.google.cloud.dataflow.worker.BatchDataflowWorker.doWork(BatchDataflowWorker.java:353)
at com.google.cloud.dataflow.worker.BatchDataflowWorker.getAndPerformWork(BatchDataflowWorker.java:284)
at com.google.cloud.dataflow.worker.DataflowBatchWorkerHarness$WorkerThread.doWork(DataflowBatchWorkerHarness.java:134)
at com.google.cloud.dataflow.worker.DataflowBatchWorkerHarness$WorkerThread.call(DataflowBatchWorkerHarness.java:114)
at com.google.cloud.dataflow.worker.DataflowBatchWorkerHarness$WorkerThread.call(DataflowBatchWorkerHarness.java:101)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.IllegalStateException: Invalid data
at org.apache.beam.sdk.repackaged.com.google.common.base.Preconditions.checkState(Preconditions.java:444)
at org.apache.beam.sdk.io.TFRecordIO$TFRecordCodec.read(TFRecordIO.java:642)
at org.apache.beam.sdk.io.TFRecordIO$TFRecordSource$TFRecordReader.readNextRecord(TFRecordIO.java:526)
at org.apache.beam.sdk.io.CompressedSource$CompressedReader.readNextRecord(CompressedSource.java:426)
at org.apache.beam.sdk.io.FileBasedSource$FileBasedReader.advanceImpl(FileBasedSource.java:473)
at org.apache.beam.sdk.io.OffsetBasedSource$OffsetBasedReader.advance(OffsetBasedSource.java:267)
at com.google.cloud.dataflow.worker.WorkerCustomSources$BoundedReaderIterator.advance(WorkerCustomSources.java:602)
... 14 more
java.io.IOException: Failed to advance reader of source: hdfs://146.148.62.9:9000/TEXTIO_IT__1521892046445-00000-of-00003.tfrecord range [0, 19222204)
at com.google.cloud.dataflow.worker.WorkerCustomSources$BoundedReaderIterator.advance(WorkerCustomSources.java:605)
at com.google.cloud.dataflow.worker.util.common.worker.ReadOperation$SynchronizedReaderIterator.advance(ReadOperation.java:379)
at com.google.cloud.dataflow.worker.util.common.worker.ReadOperation.runReadLoop(ReadOperation.java:185)
at com.google.cloud.dataflow.worker.util.common.worker.ReadOperation.start(ReadOperation.java:150)
at com.google.cloud.dataflow.worker.util.common.worker.MapTaskExecutor.execute(MapTaskExecutor.java:74)
at com.google.cloud.dataflow.worker.BatchDataflowWorker.executeWork(BatchDataflowWorker.java:381)
at com.google.cloud.dataflow.worker.BatchDataflowWorker.doWork(BatchDataflowWorker.java:353)
at com.google.cloud.dataflow.worker.BatchDataflowWorker.getAndPerformWork(BatchDataflowWorker.java:284)
at com.google.cloud.dataflow.worker.DataflowBatchWorkerHarness$WorkerThread.doWork(DataflowBatchWorkerHarness.java:134)
at com.google.cloud.dataflow.worker.DataflowBatchWorkerHarness$WorkerThread.call(DataflowBatchWorkerHarness.java:114)
at com.google.cloud.dataflow.worker.DataflowBatchWorkerHarness$WorkerThread.call(DataflowBatchWorkerHarness.java:101)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1142)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:617)
at java.lang.Thread.run(Thread.java:745)
Caused by: java.lang.IllegalStateException: Not a valid TFRecord. Fewer than 12 bytes.
at org.apache.beam.sdk.repackaged.com.google.common.base.Preconditions.checkState(Preconditions.java:444)
at org.apache.beam.sdk.io.TFRecordIO$TFRecordCodec.read(TFRecordIO.java:631)
at org.apache.beam.sdk.io.TFRecordIO$TFRecordSource$TFRecordReader.readNextRecord(TFRecordIO.java:526)
at org.apache.beam.sdk.io.CompressedSource$CompressedReader.readNextRecord(CompressedSource.java:426)
at org.apache.beam.sdk.io.FileBasedSource$FileBasedReader.advanceImpl(FileBasedSource.java:473)
at org.apache.beam.sdk.io.OffsetBasedSource$OffsetBasedReader.advance(OffsetBasedSource.java:267)
at com.google.cloud.dataflow.worker.WorkerCustomSources$BoundedReaderIterator.advance(WorkerCustomSources.java:602)
... 14 more
Workflow failed. Causes: S03:TFRecordIO.Read/Read+Transform bytes to strings/Map+Calculate hashcode/WithKeys/AddKeys/Map+Calculate hashcode/Combine.perKey(Hashing)/GroupByKey+Calculate hashcode/Combine.perKey(Hashing)/Combine.GroupedValues/Partial+Calculate hashcode/Combine.perKey(Hashing)/GroupByKey/Reify+Calculate hashcode/Combine.perKey(Hashing)/GroupByKey/Write failed., A work item was attempted 4 times without success. Each time the worker eventually lost contact with the service. The work item was attempted on:
tfrecordioit0writethenrea-03240451-eb64-harness-3ctb,
tfrecordioit0writethenrea-03240451-eb64-harness-h06g,
tfrecordioit0writethenrea-03240451-eb64-harness-48v0,
tfrecordioit0writethenrea-03240451-eb64-harness-3ctb
at org.apache.beam.runners.dataflow.TestDataflowRunner.run(TestDataflowRunner.java:134)
at org.apache.beam.runners.dataflow.TestDataflowRunner.run(TestDataflowRunner.java:90)
at org.apache.beam.runners.dataflow.TestDataflowRunner.run(TestDataflowRunner.java:55)
at org.apache.beam.sdk.Pipeline.run(Pipeline.java:311)
at org.apache.beam.sdk.testing.TestPipeline.run(TestPipeline.java:346)
at org.apache.beam.sdk.testing.TestPipeline.run(TestPipeline.java:328)
at org.apache.beam.sdk.io.tfrecord.TFRecordIOIT.writeThenReadAll(TFRecordIOIT.java:127)
at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke(Method.java:498)
at org.junit.runners.model.FrameworkMethod$1.runReflectiveCall(FrameworkMethod.java:50)
at org.junit.internal.runners.model.ReflectiveCallable.run(ReflectiveCallable.java:12)
at org.junit.runners.model.FrameworkMethod.invokeExplosively(FrameworkMethod.java:47)
at org.junit.internal.runners.statements.InvokeMethod.evaluate(InvokeMethod.java:17)
at org.apache.beam.sdk.testing.TestPipeline$1.evaluate(TestPipeline.java:317)
at org.apache.beam.sdk.testing.TestPipeline$1.evaluate(TestPipeline.java:317)
at org.junit.rules.RunRules.evaluate(RunRules.java:20)
at org.junit.runners.ParentRunner.runLeaf(ParentRunner.java:325)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:78)
at org.junit.runners.BlockJUnit4ClassRunner.runChild(BlockJUnit4ClassRunner.java:57)
at org.junit.runners.ParentRunner$3.run(ParentRunner.java:290)
at org.apache.maven.surefire.junitcore.pc.Scheduler$1.run(Scheduler.java:410)
at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
at java.util.concurrent.FutureTask.run(FutureTask.java:266)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
[INFO]
[INFO] Results:
[INFO]
[ERROR] Errors:
[ERROR] TFRecordIOIT.writeThenReadAll:127 Runtime java.io.IOException: Failed to adv...
[INFO]
[ERROR] Tests run: 1, Failures: 0, Errors: 1, Skipped: 0
[INFO]
[INFO]
[INFO] --- maven-dependency-plugin:3.0.2:analyze-only (default) @ beam-sdks-java-io-file-based-io-tests ---
[INFO] No dependency problems found
[INFO]
[INFO] --- maven-failsafe-plugin:2.21.0:verify (default) @ beam-sdks-java-io-file-based-io-tests ---
[INFO] Failsafe report directory: <https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/runs/9fb5402c/beam/sdks/java/io/file-based-io-tests/target/failsafe-reports>
[INFO] ------------------------------------------------------------------------
[INFO] BUILD FAILURE
[INFO] ------------------------------------------------------------------------
[INFO] Total time: 07:27 min
[INFO] Finished at: 2018-03-24T11:54:34Z
[INFO] Final Memory: 91M/882M
[INFO] ------------------------------------------------------------------------
[ERROR] Failed to execute goal org.apache.maven.plugins:maven-failsafe-plugin:2.21.0:verify (default) on project beam-sdks-java-io-file-based-io-tests: There are test failures.
[ERROR]
[ERROR] Please refer to <https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/runs/9fb5402c/beam/sdks/java/io/file-based-io-tests/target/failsafe-reports> for the individual test results.
[ERROR] Please refer to dump files (if any exist) [date]-jvmRun[N].dump, [date].dumpstream and [date]-jvmRun[N].dumpstream.
[ERROR] -> [Help 1]
org.apache.maven.lifecycle.LifecycleExecutionException: Failed to execute goal org.apache.maven.plugins:maven-failsafe-plugin:2.21.0:verify (default) on project beam-sdks-java-io-file-based-io-tests: There are test failures.
Please refer to <https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/runs/9fb5402c/beam/sdks/java/io/file-based-io-tests/target/failsafe-reports> for the individual test results.
Please refer to dump files (if any exist) [date]-jvmRun[N].dump, [date].dumpstream and [date]-jvmRun[N].dumpstream.
at org.apache.maven.lifecycle.internal.MojoExecutor.execute (MojoExecutor.java:213)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute (MojoExecutor.java:154)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute (MojoExecutor.java:146)
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject (LifecycleModuleBuilder.java:117)
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject (LifecycleModuleBuilder.java:81)
at org.apache.maven.lifecycle.internal.builder.singlethreaded.SingleThreadedBuilder.build (SingleThreadedBuilder.java:51)
at org.apache.maven.lifecycle.internal.LifecycleStarter.execute (LifecycleStarter.java:128)
at org.apache.maven.DefaultMaven.doExecute (DefaultMaven.java:309)
at org.apache.maven.DefaultMaven.doExecute (DefaultMaven.java:194)
at org.apache.maven.DefaultMaven.execute (DefaultMaven.java:107)
at org.apache.maven.cli.MavenCli.execute (MavenCli.java:955)
at org.apache.maven.cli.MavenCli.doMain (MavenCli.java:290)
at org.apache.maven.cli.MavenCli.main (MavenCli.java:194)
at sun.reflect.NativeMethodAccessorImpl.invoke0 (Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke (NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke (DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke (Method.java:498)
at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced (Launcher.java:289)
at org.codehaus.plexus.classworlds.launcher.Launcher.launch (Launcher.java:229)
at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode (Launcher.java:415)
at org.codehaus.plexus.classworlds.launcher.Launcher.main (Launcher.java:356)
Caused by: org.apache.maven.plugin.MojoFailureException: There are test failures.
Please refer to <https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/runs/9fb5402c/beam/sdks/java/io/file-based-io-tests/target/failsafe-reports> for the individual test results.
Please refer to dump files (if any exist) [date]-jvmRun[N].dump, [date].dumpstream and [date]-jvmRun[N].dumpstream.
at org.apache.maven.plugin.surefire.SurefireHelper.throwException (SurefireHelper.java:240)
at org.apache.maven.plugin.surefire.SurefireHelper.reportExecution (SurefireHelper.java:112)
at org.apache.maven.plugin.failsafe.VerifyMojo.execute (VerifyMojo.java:192)
at org.apache.maven.plugin.DefaultBuildPluginManager.executeMojo (DefaultBuildPluginManager.java:134)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute (MojoExecutor.java:208)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute (MojoExecutor.java:154)
at org.apache.maven.lifecycle.internal.MojoExecutor.execute (MojoExecutor.java:146)
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject (LifecycleModuleBuilder.java:117)
at org.apache.maven.lifecycle.internal.LifecycleModuleBuilder.buildProject (LifecycleModuleBuilder.java:81)
at org.apache.maven.lifecycle.internal.builder.singlethreaded.SingleThreadedBuilder.build (SingleThreadedBuilder.java:51)
at org.apache.maven.lifecycle.internal.LifecycleStarter.execute (LifecycleStarter.java:128)
at org.apache.maven.DefaultMaven.doExecute (DefaultMaven.java:309)
at org.apache.maven.DefaultMaven.doExecute (DefaultMaven.java:194)
at org.apache.maven.DefaultMaven.execute (DefaultMaven.java:107)
at org.apache.maven.cli.MavenCli.execute (MavenCli.java:955)
at org.apache.maven.cli.MavenCli.doMain (MavenCli.java:290)
at org.apache.maven.cli.MavenCli.main (MavenCli.java:194)
at sun.reflect.NativeMethodAccessorImpl.invoke0 (Native Method)
at sun.reflect.NativeMethodAccessorImpl.invoke (NativeMethodAccessorImpl.java:62)
at sun.reflect.DelegatingMethodAccessorImpl.invoke (DelegatingMethodAccessorImpl.java:43)
at java.lang.reflect.Method.invoke (Method.java:498)
at org.codehaus.plexus.classworlds.launcher.Launcher.launchEnhanced (Launcher.java:289)
at org.codehaus.plexus.classworlds.launcher.Launcher.launch (Launcher.java:229)
at org.codehaus.plexus.classworlds.launcher.Launcher.mainWithExitCode (Launcher.java:415)
at org.codehaus.plexus.classworlds.launcher.Launcher.main (Launcher.java:356)
[ERROR]
[ERROR] Re-run Maven using the -X switch to enable full debug logging.
[ERROR]
[ERROR] For more information about the errors and possible solutions, please read the following articles:
[ERROR] [Help 1] http://cwiki.apache.org/confluence/display/MAVEN/MojoFailureException
STDERR:
2018-03-24 11:54:34,402 9fb5402c MainThread beam_integration_benchmark(1/1) ERROR Error during benchmark beam_integration_benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 624, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 527, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 159, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 90, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2018-03-24 11:54:34,403 9fb5402c MainThread beam_integration_benchmark(1/1) INFO Cleaning up benchmark beam_integration_benchmark
2018-03-24 11:54:34,404 9fb5402c MainThread beam_integration_benchmark(1/1) INFO Running: kubectl --kubeconfig=<https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/config-filebasedioithdfs-1521890089911> delete -f <https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/src/.test-infra/kubernetes/hadoop/SmallITCluster/hdfs-single-datanode-cluster.yml>
2018-03-24 11:54:34,858 9fb5402c MainThread beam_integration_benchmark(1/1) INFO Running: kubectl --kubeconfig=<https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/config-filebasedioithdfs-1521890089911> delete -f <https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/src/.test-infra/kubernetes/hadoop/SmallITCluster/hdfs-single-datanode-cluster-for-local-dev.yml>
2018-03-24 11:54:35,013 9fb5402c MainThread beam_integration_benchmark(1/1) ERROR Exception running benchmark
Traceback (most recent call last):
File "<https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 758, in RunBenchmarkTask
RunBenchmark(spec, collector)
File "<https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 624, in RunBenchmark
DoRunPhase(spec, collector, detailed_timer)
File "<https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/PerfKitBenchmarker/perfkitbenchmarker/pkb.py",> line 527, in DoRunPhase
samples = spec.BenchmarkRun(spec)
File "<https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/PerfKitBenchmarker/perfkitbenchmarker/linux_benchmarks/beam_integration_benchmark.py",> line 159, in Run
job_type=job_type)
File "<https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/PerfKitBenchmarker/perfkitbenchmarker/providers/gcp/gcp_dpb_dataflow.py",> line 90, in SubmitJob
assert retcode == 0, "Integration Test Failed."
AssertionError: Integration Test Failed.
2018-03-24 11:54:35,014 9fb5402c MainThread beam_integration_benchmark(1/1) ERROR Benchmark 1/1 beam_integration_benchmark (UID: beam_integration_benchmark0) failed. Execution will continue.
2018-03-24 11:54:35,014 9fb5402c MainThread beam_integration_benchmark(1/1) INFO Benchmark run statuses:
---------------------------------------------------------------------------------
Name UID Status Failed Substatus
---------------------------------------------------------------------------------
beam_integration_benchmark beam_integration_benchmark0 FAILED
---------------------------------------------------------------------------------
Success rate: 0.00% (0/1)
2018-03-24 11:54:35,014 9fb5402c MainThread beam_integration_benchmark(1/1) INFO Complete logs can be found at: <https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/runs/9fb5402c/pkb.log>
2018-03-24 11:54:35,014 9fb5402c MainThread beam_integration_benchmark(1/1) INFO Completion statuses can be found at: <https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/runs/9fb5402c/completion_statuses.json>
Build step 'Execute shell' marked build as failure
Build failed in Jenkins: beam_PerformanceTests_TFRecordIOIT_HDFS #4
Posted by Apache Jenkins Server <je...@builds.apache.org>.
See <https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/4/display/redirect?page=changes>
Changes:
[alex] Correct BigQuery.write JavaDoc example
[rober] Correct EventTime encoding. time.(Time).Unix() returns seconds not
[tgroh] Enable Proto Round Trips in the DirectRunner Tests
[lcwik] [BEAM-3249, BEAM-3457] Make Apache Rat a dependency of the Java and
[sidhom] Add Java bounded read overrides
[herohde] [BEAM-3817] Switch BQ write to not use side input
[lcwik] [BEAM-3457] Exclude generated and data files part of the Go SDK.
[sidhom] Add ExecutableStagePayload to make aid runner stage reconstruction
[sidhom] Fix typo
[coheigea] Replacing size() == 0 with isEmpty()
[markliu] [BEAM-3841] Fix TestDataflowRunner.run to run_pipeline
[coheigea] Put the String literal first when comparing it to an Object
[ankurgoenka] Send error response for process bundle progress instead of throwing and
[tgroh] Use InstructionRequestHandler in RemoteEnvironment
[ekirpichov] BigQueryIO.read().fromQuery() improvements
[szewinho] [BEAM-3060] Fixing mvn dependency issue when runnning filebasedIOIT
[lukasz.gajowy] [BEAM-3798] Remove error check on dataflow when getting batch job state
[tgroh] Add ExecutableStage -> ProcessBundleDescriptor
[tgroh] Clean up GrpcFnServer Constructor Signature
[tgroh] Add a RemoteExecutionTest
[tgroh] Clean up a couple of SdkHarnessClient Signatures
[tgroh] Separate RemoteInputDestinations from SdkHarnessClient
[XuMingmin] [SQL] Add support for DOT expression (#4863)
[herohde] Add TODO to revert Go IO to use side input
[jb] [BEAM-3500] "Attach" JDBC connection to the bundle (improve the pooling)
[jb] [BEAM-3500] Test if the user provides both withDataSourceConfiguration()
[jb] [BEAM-3500] Wrap the datasource as a poolable datasource and expose
[jb] [BEAM-3500] Add commons-pool2 dependency
[jb] [BEAM-3500] Only expose max number of connections in the pool to the
[jb] [BEAM-3500] Cleanup pool configuration parameters
[jb] [BEAM-3500] Remove dataSourceFactory
[jb] [BEAM-3500] Remove unecessary check on dataSourceConfiguration
[szewinho] Jax-api added to dependencyManagement.
[szewinho] Jaxb-api added to beam root pom.xml
[szewinho] Removed empty line
[Pablo] Fixing counter string name.
[tgroh] Add Network Utility Methods
[lcwik] Fix missing license header in test.
[aaltay] Enable input subscription in Python streaming wordcount (#4864)
[tgroh] fixup! Add Network Utility Methods
[tgroh] Remove Future#set(null) in fn-execution
[herohde] [BEAM-3869] Fix Go Dataflow break due to zero timestamps
[axelmagn] Add a generic interface for the state service.
[herohde] Remove WindowedValue on PCollections for Go SDK
[cclauss] [BEAM-3872] Change Exception.message --> Exception.args[0]
[alan.myrvold] [BEAM-3249] Publish java artifacts using gradle
[herohde] CR: Fix comments to remove old windowing
[lcwik] Fix minor checkstyle/findbugs errors
[mariagh] Add support for streaming side inputs in the DirectRunner
[andreas.ehrencrona] [BEAM-2264] Credentials were not being reused between GCS calls
[ccy] Add support for PaneInfo in WindowedValues
[axelmagn] Write unit tests for GrpcStateService.
[XuMingmin] Bump calcite and avatica versions (#4887)
[axelmagn] Fix StateRequestHandler interface to be idiomatic
[herohde] Add Go support for universal runners, incl Flink
[herohde] CR: Fixed comments for job service helper functions
[iemejia] Add missing ASF license to ExecutableStageTranslation file
[szewinho] Updated gradle build to use jaxb-api. Jaxb-api version set to 2.2.12
[mariand] Switched AvroIO default codec to snappyCodec().
[yifanzou] [BEAM-3840] Get python mobile-gaming automating on core runners
[coheigea] Remove "i == numSplits" condition, which can never be reached as i is
[sidhom] [BEAM-3565] Clean up ExecutableStage
[wcn] Fix incorrect read of atomic counter.
[tgroh] Add a suborder parameter to Networks#topologicalOrder
[herohde] [BEAM-3893] Add fallback to unauthenticated access for GCS IO
[robertwb] [BEAM-3865] Fix watermark hold handling bug.
[robertwb] [BEAM-2927] Python support for dataflow portable side inputs over Fn API
[Pablo] Fixing check for sideinput_io_metrics experiment flag.
[herohde] CR: fix typo
[herohde] [BEAM-3897] Add Go wordcount example with multi output DoFns
[ccy] Replace side inputs when applying PTransformOverrides
[tgroh] Add QueryablePipeline#forTransforms
[herohde] Allow manual override of Go worker binary
[aaltay] [BEAM-3861] Improve test infra in Python SDK for streaming end-to-end
[iemejia] Remove testing package-info from main package for GCP IO
[iemejia] Update maven failsafe/surefire plugin to version 2.21.0
[iemejia] [BEAM-3873] Update commons-compress to version 1.16.1 (fix
[iemejia] Remove maven warnings
[tgroh] Add Side Inputs to ExecutableStage
[herohde] [BEAM-3866] Remove windowed value requirement for External
[lcwik] [BEAM-3903] Build the Go SDK examples using gradle
[markliu] Clean up terminal state check in TestDataflowRunner
[robertwb] Add mapper microbenchmark.
[szewinho] Scope removed from root pom.xml and jaxb-api dependency set to
[Pablo] Updating dataflow API protocol buffers.
[tgroh] Make RemoteEnvironment public
[boyuanz] Add cython annotation to make DistributionAccumulator faster
[tgroh] Move getId to the top-level pipeline node
[tgroh] Add FusedPipeline#toPipeline
[robertwb] Remove from docs generation.
[wcn] BEAM-3894: Add Protocol Buffer support
[altay] Convert big query writes to beam.io.WriteToBigQuery
[aaltay] Improve test util for PubSub exist check (#4922)
[XuMingmin] [BEAM-3851] Option to preserve element timestamp while publishing to
[ankurgoenka] Send worker_id in all channels to runner Adding
[wcn] Updating generated files.
[ehudm] Reduce precommit test timeouts for Java and Go.
[altay] More graceful fallback when grpc is not present.
[markliu] [BEAM-3861] Complete streaming wordcount test in Python SDK
[aaltay] [BEAM-3738] Enable py3 lint and cleanup tox.ini. (#4877)
------------------------------------------
Started by timer
[EnvInject] - Loading node environment variables.
Building remotely on beam2 (beam) in workspace <https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/>
> git rev-parse --is-inside-work-tree # timeout=10
Fetching changes from the remote Git repository
> git config remote.origin.url https://github.com/apache/beam.git # timeout=10
Fetching upstream changes from https://github.com/apache/beam.git
> git --version # timeout=10
> git fetch --tags --progress https://github.com/apache/beam.git +refs/heads/*:refs/remotes/origin/* +refs/pull/${ghprbPullId}/*:refs/remotes/origin/pr/${ghprbPullId}/*
> git rev-parse origin/master^{commit} # timeout=10
Checking out Revision 8b72e5af513e5182c403f3b67ff6d16395c281db (origin/master)
> git config core.sparsecheckout # timeout=10
> git checkout -f 8b72e5af513e5182c403f3b67ff6d16395c281db
Commit message: "Merge pull request #4940: Update generated files"
> git rev-list --no-walk 5b6ca47fec0b5b720ad5afb9274dc0d418545b43 # timeout=10
Cleaning workspace
> git rev-parse --verify HEAD # timeout=10
Resetting working tree
> git reset --hard # timeout=10
> git clean -fdx # timeout=10
[EnvInject] - Executing scripts and injecting environment variables after the SCM step.
[EnvInject] - Injecting as environment variables the properties content
SPARK_LOCAL_IP=127.0.0.1
[EnvInject] - Variables injected successfully.
[beam_PerformanceTests_TFRecordIOIT_HDFS] $ /bin/bash -xe /tmp/jenkins8246568391384583897.sh
+ gcloud container clusters get-credentials io-datastores --zone=us-central1-a --verbosity=debug
DEBUG: Running gcloud.container.clusters.get-credentials with Namespace(__calliope_internal_deepest_parser=ArgumentParser(prog='gcloud.container.clusters.get-credentials', usage=None, description='See https://cloud.google.com/container-engine/docs/kubectl for\nkubectl documentation.', version=None, formatter_class=<class 'argparse.HelpFormatter'>, conflict_handler='error', add_help=False), account=None, api_version=None, authority_selector=None, authorization_token_file=None, cmd_func=<bound method Command.Run of <googlecloudsdk.calliope.backend.Command object at 0x7f6fef241510>>, command_path=['gcloud', 'container', 'clusters', 'get-credentials'], configuration=None, credential_file_override=None, document=None, format=None, h=None, help=None, http_timeout=None, log_http=None, name='io-datastores', project=None, quiet=None, trace_email=None, trace_log=None, trace_token=None, user_output_enabled=None, verbosity='debug', version=None, zone='us-central1-a').
WARNING: Accessing a Container Engine cluster requires the kubernetes commandline
client [kubectl]. To install, run
$ gcloud components install kubectl
Fetching cluster endpoint and auth data.
DEBUG: Saved kubeconfig to /home/jenkins/.kube/config
kubeconfig entry generated for io-datastores.
INFO: Display format "default".
[beam_PerformanceTests_TFRecordIOIT_HDFS] $ /bin/bash -xe /tmp/jenkins3974261244662825482.sh
+ cp /home/jenkins/.kube/config <https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/config-filebasedioithdfs-1521890089911>
[beam_PerformanceTests_TFRecordIOIT_HDFS] $ /bin/bash -xe /tmp/jenkins5848965985188276448.sh
+ kubectl --kubeconfig=<https://builds.apache.org/job/beam_PerformanceTests_TFRecordIOIT_HDFS/ws/config-filebasedioithdfs-1521890089911> create namespace filebasedioithdfs-1521890089911
Error from server (AlreadyExists): namespaces "filebasedioithdfs-1521890089911" already exists
Build step 'Execute shell' marked build as failure