You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "Elon Azoulay (Jira)" <ji...@apache.org> on 2019/09/17 23:31:00 UTC
[jira] [Updated] (HUDI-261) Failed to create deltacommit.inflight
file, duplicate timestamp issue
[ https://issues.apache.org/jira/browse/HUDI-261?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Elon Azoulay updated HUDI-261:
------------------------------
Description:
{{Hudi jobs started failing with }}
{{Found commits after time :20190916210221, please rollback greater commits first}}
This occured after a "Failed to create deltacommit inflight file" exception:
{{Exception in thread "main" org.apache.hudi.exception.HoodieUpsertException: Failed to upsert for commit time 20190916210221 at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:177) at org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:353) at org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:228) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:123) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.main(HoodieDeltaStreamer.java:290) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by: org.apache.hudi.exception.HoodieIOException: Failed to create file gs://****************/.hoodie/20190916210223.deltacommit.inflight at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:391) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInMetaPath(HoodieActiveTimeline.java:371) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.saveToInflight(HoodieActiveTimeline.java:359) at org.apache.hudi.HoodieWriteClient.saveWorkloadProfileMetadataToInflight(HoodieWriteClient.java:417) at org.apache.hudi.HoodieWriteClient.upsertRecordsInternal(HoodieWriteClient.java:440) at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:172) ... 14 more Caused by: java.io.IOException: com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException: 412 Precondition Failed { "code" : 412, "errors" : [
{ "domain" : "global", "location" : "If-Match", "locationType" : "header", "message" : "Precondition Failed", "reason" : "conditionNotMet" }
], "message" : "Precondition Failed" } at com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.waitForCompletionAndThrowIfUploadFailed(AbstractGoogleAsyncWriteChannel.java:367) at com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.close(AbstractGoogleAsyncWriteChannel.java:238) at java.nio.channels.Channels$1.close(Channels.java:178) at java.io.FilterOutputStream.close(FilterOutputStream.java:159) at com.google.cloud.hadoop.fs.gcs.GoogleHadoopOutputStream.close(GoogleHadoopOutputStream.java:127) at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at org.apache.hudi.common.io.storage.SizeAwareFSDataOutputStream.close(SizeAwareFSDataOutputStream.java:66) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:388) ... 19 more Caused by: com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException: 412 Precondition Failed}}
was:
{{Hudi jobs started failing with }}
{{Found commits after time :20190916210221, please rollback greater commits first}}
This occured after a "Failed to create deltacommit inflight file" exception:
{{Exception in thread "main" org.apache.hudi.exception.HoodieUpsertException: Failed to upsert for commit time 20190916210221 at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:177) at org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:353) at org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:228) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:123) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.main(HoodieDeltaStreamer.java:290) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by: org.apache.hudi.exception.HoodieIOException: Failed to create file gs://css-data-warehouse-hudi-ingest/hudi/data/hudi_ingest_raw/order_central_updates_latest/.hoodie/20190916210223.deltacommit.inflight at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:391) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInMetaPath(HoodieActiveTimeline.java:371) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.saveToInflight(HoodieActiveTimeline.java:359) at org.apache.hudi.HoodieWriteClient.saveWorkloadProfileMetadataToInflight(HoodieWriteClient.java:417) at org.apache.hudi.HoodieWriteClient.upsertRecordsInternal(HoodieWriteClient.java:440) at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:172) ... 14 more Caused by: java.io.IOException: com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException: 412 Precondition Failed \{ "code" : 412, "errors" : [ { "domain" : "global", "location" : "If-Match", "locationType" : "header", "message" : "Precondition Failed", "reason" : "conditionNotMet" } ], "message" : "Precondition Failed" } at com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.waitForCompletionAndThrowIfUploadFailed(AbstractGoogleAsyncWriteChannel.java:367) at com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.close(AbstractGoogleAsyncWriteChannel.java:238) at java.nio.channels.Channels$1.close(Channels.java:178) at java.io.FilterOutputStream.close(FilterOutputStream.java:159) at com.google.cloud.hadoop.fs.gcs.GoogleHadoopOutputStream.close(GoogleHadoopOutputStream.java:127) at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at org.apache.hudi.common.io.storage.SizeAwareFSDataOutputStream.close(SizeAwareFSDataOutputStream.java:66) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:388) ... 19 more Caused by: com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException: 412 Precondition Failed}}
> Failed to create deltacommit.inflight file, duplicate timestamp issue
> ---------------------------------------------------------------------
>
> Key: HUDI-261
> URL: https://issues.apache.org/jira/browse/HUDI-261
> Project: Apache Hudi (incubating)
> Issue Type: Bug
> Components: deltastreamer
> Reporter: Elon Azoulay
> Priority: Major
>
> {{Hudi jobs started failing with }}
> {{Found commits after time :20190916210221, please rollback greater commits first}}
>
> This occured after a "Failed to create deltacommit inflight file" exception:
> {{Exception in thread "main" org.apache.hudi.exception.HoodieUpsertException: Failed to upsert for commit time 20190916210221 at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:177) at org.apache.hudi.utilities.deltastreamer.DeltaSync.writeToSink(DeltaSync.java:353) at org.apache.hudi.utilities.deltastreamer.DeltaSync.syncOnce(DeltaSync.java:228) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.sync(HoodieDeltaStreamer.java:123) at org.apache.hudi.utilities.deltastreamer.HoodieDeltaStreamer.main(HoodieDeltaStreamer.java:290) at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method) at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62) at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43) at java.lang.reflect.Method.invoke(Method.java:498) at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52) at org.apache.spark.deploy.SparkSubmit$.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:894) at org.apache.spark.deploy.SparkSubmit$.doRunMain$1(SparkSubmit.scala:198) at org.apache.spark.deploy.SparkSubmit$.submit(SparkSubmit.scala:228) at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:137) at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala) Caused by: org.apache.hudi.exception.HoodieIOException: Failed to create file gs://****************/.hoodie/20190916210223.deltacommit.inflight at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:391) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInMetaPath(HoodieActiveTimeline.java:371) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.saveToInflight(HoodieActiveTimeline.java:359) at org.apache.hudi.HoodieWriteClient.saveWorkloadProfileMetadataToInflight(HoodieWriteClient.java:417) at org.apache.hudi.HoodieWriteClient.upsertRecordsInternal(HoodieWriteClient.java:440) at org.apache.hudi.HoodieWriteClient.upsert(HoodieWriteClient.java:172) ... 14 more Caused by: java.io.IOException: com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException: 412 Precondition Failed { "code" : 412, "errors" : [
> { "domain" : "global", "location" : "If-Match", "locationType" : "header", "message" : "Precondition Failed", "reason" : "conditionNotMet" }
> ], "message" : "Precondition Failed" } at com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.waitForCompletionAndThrowIfUploadFailed(AbstractGoogleAsyncWriteChannel.java:367) at com.google.cloud.hadoop.repackaged.gcs.com.google.cloud.hadoop.util.AbstractGoogleAsyncWriteChannel.close(AbstractGoogleAsyncWriteChannel.java:238) at java.nio.channels.Channels$1.close(Channels.java:178) at java.io.FilterOutputStream.close(FilterOutputStream.java:159) at com.google.cloud.hadoop.fs.gcs.GoogleHadoopOutputStream.close(GoogleHadoopOutputStream.java:127) at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at org.apache.hadoop.fs.FSDataOutputStream$PositionCache.close(FSDataOutputStream.java:72) at org.apache.hadoop.fs.FSDataOutputStream.close(FSDataOutputStream.java:106) at org.apache.hudi.common.io.storage.SizeAwareFSDataOutputStream.close(SizeAwareFSDataOutputStream.java:66) at org.apache.hudi.common.table.timeline.HoodieActiveTimeline.createFileInPath(HoodieActiveTimeline.java:388) ... 19 more Caused by: com.google.cloud.hadoop.repackaged.gcs.com.google.api.client.googleapis.json.GoogleJsonResponseException: 412 Precondition Failed}}
--
This message was sent by Atlassian Jira
(v8.3.4#803005)