You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by da...@apache.org on 2024/10/23 04:18:26 UTC
(hudi) branch revert-12136-apply-write_ignore_failed-when-write-data created (now 31202125d8b)
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a change to branch revert-12136-apply-write_ignore_failed-when-write-data
in repository https://gitbox.apache.org/repos/asf/hudi.git
at 31202125d8b Revert "[HUDI-8400] apply 'write.ignore.failed' when write data failed (#12136)"
This branch includes the following new commits:
new 31202125d8b Revert "[HUDI-8400] apply 'write.ignore.failed' when write data failed (#12136)"
The 1 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
(hudi) 01/01: Revert "[HUDI-8400] apply 'write.ignore.failed' when write data failed (#12136)"
Posted by da...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
danny0405 pushed a commit to branch revert-12136-apply-write_ignore_failed-when-write-data
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 31202125d8b9343eb074f6f1fc88fbe0db1662ba
Author: Danny Chan <yu...@gmail.com>
AuthorDate: Wed Oct 23 12:18:18 2024 +0800
Revert "[HUDI-8400] apply 'write.ignore.failed' when write data failed (#12136)"
This reverts commit a9194edd526e733863fef8a1fc1e945898498735.
---
.../java/org/apache/hudi/config/HoodieWriteConfig.java | 18 ------------------
.../java/org/apache/hudi/io/HoodieAppendHandle.java | 1 -
.../java/org/apache/hudi/io/HoodieCreateHandle.java | 1 -
.../java/org/apache/hudi/io/HoodieMergeHandle.java | 2 --
.../java/org/apache/hudi/io/HoodieWriteHandle.java | 6 ------
.../hudi/io/storage/row/HoodieRowDataCreateHandle.java | 5 -----
.../org/apache/hudi/configuration/FlinkOptions.java | 1 -
.../java/org/apache/hudi/util/FlinkWriteClients.java | 3 +--
8 files changed, 1 insertion(+), 36 deletions(-)
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
index dd5ee628f54..09df47b4452 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/config/HoodieWriteConfig.java
@@ -811,12 +811,6 @@ public class HoodieWriteConfig extends HoodieConfig {
*/
public static final String WRITES_FILEID_ENCODING = "_hoodie.writes.fileid.encoding";
- public static final ConfigProperty<Boolean> IGNORE_FAILED = ConfigProperty
- .key("hoodie.write.ignore.failed")
- .defaultValue(false)
- .sinceVersion("")
- .withDocumentation("Flag to indicate whether to ignore any non exception error (e.g. writestatus error).");
-
private ConsistencyGuardConfig consistencyGuardConfig;
private FileSystemRetryConfig fileSystemRetryConfig;
@@ -2794,13 +2788,6 @@ public class HoodieWriteConfig extends HoodieConfig {
return metadataConfig.getSecondaryIndexParallelism();
}
- /**
- * Whether to ignore the write failed.
- */
- public boolean getIgnoreWriteFailed() {
- return getBooleanOrDefault(IGNORE_FAILED);
- }
-
public static class Builder {
protected final HoodieWriteConfig writeConfig = new HoodieWriteConfig();
@@ -3343,11 +3330,6 @@ public class HoodieWriteConfig extends HoodieConfig {
return this;
}
- public Builder withWriteIgnoreFailed(boolean ignoreFailedWriteData) {
- writeConfig.setValue(IGNORE_FAILED, String.valueOf(ignoreFailedWriteData));
- return this;
- }
-
protected void setDefaults() {
writeConfig.setDefaultValue(MARKERS_TYPE, getDefaultMarkersType(engineType));
// Check for mandatory properties
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
index b70415e06f7..cb0c7dd283f 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieAppendHandle.java
@@ -499,7 +499,6 @@ public class HoodieAppendHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
// for a single record
writeStatus.markFailure(record, t, recordMetadata);
LOG.error("Error writing record " + record, t);
- ignoreWriteFailed(t);
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
index 93f4716fae7..12406927ae6 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieCreateHandle.java
@@ -170,7 +170,6 @@ public class HoodieCreateHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O
// for a single record
writeStatus.markFailure(record, t, recordMetadata);
LOG.error("Error writing record " + record, t);
- ignoreWriteFailed(t);
}
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
index f1c1473dc64..1bf6f6b0138 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieMergeHandle.java
@@ -304,7 +304,6 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
HoodieUpsertException failureEx = new HoodieUpsertException("mismatched partition path, record partition: "
+ newRecord.getPartitionPath() + " but trying to insert into partition: " + partitionPath);
writeStatus.markFailure(newRecord, failureEx, recordMetadata);
- ignoreWriteFailed(failureEx);
return false;
}
try {
@@ -334,7 +333,6 @@ public class HoodieMergeHandle<T, I, K, O> extends HoodieWriteHandle<T, I, K, O>
} catch (Exception e) {
LOG.error("Error writing record " + newRecord, e);
writeStatus.markFailure(newRecord, e, recordMetadata);
- ignoreWriteFailed(e);
}
return false;
}
diff --git a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
index efb266dd0c4..a251a5f7614 100644
--- a/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
+++ b/hudi-client/hudi-client-common/src/main/java/org/apache/hudi/io/HoodieWriteHandle.java
@@ -174,12 +174,6 @@ public abstract class HoodieWriteHandle<T, I, K, O> extends HoodieIOHandle<T, I,
// NO_OP
}
- protected void ignoreWriteFailed(Throwable throwable) {
- if (config.getIgnoreWriteFailed()) {
- throw new HoodieException(throwable.getMessage(), throwable);
- }
- }
-
/**
* Perform the actual writing of the given record into the backing file.
*/
diff --git a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
index dce64fce11b..b08e814d15c 100644
--- a/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
+++ b/hudi-client/hudi-flink-client/src/main/java/org/apache/hudi/io/storage/row/HoodieRowDataCreateHandle.java
@@ -31,7 +31,6 @@ import org.apache.hudi.common.model.IOType;
import org.apache.hudi.common.util.HoodieTimer;
import org.apache.hudi.common.util.Option;
import org.apache.hudi.config.HoodieWriteConfig;
-import org.apache.hudi.exception.HoodieException;
import org.apache.hudi.exception.HoodieIOException;
import org.apache.hudi.exception.HoodieInsertException;
import org.apache.hudi.storage.HoodieStorage;
@@ -143,11 +142,7 @@ public class HoodieRowDataCreateHandle implements Serializable {
? HoodieRecordDelegate.create(recordKey, partitionPath, null, newRecordLocation) : null;
writeStatus.markSuccess(recordDelegate, Option.empty());
} catch (Throwable t) {
- LOG.error("Failed to write : key is " + recordKey + ", data is " + rowData, t);
writeStatus.markFailure(recordKey, partitionPath, t);
- if (!writeConfig.getIgnoreWriteFailed()) {
- throw new HoodieException(t.getMessage(), t);
- }
}
} catch (Throwable ge) {
writeStatus.setGlobalError(ge);
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
index 8d4a3f78b7e..cc7f5f6783a 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/configuration/FlinkOptions.java
@@ -439,7 +439,6 @@ public class FlinkOptions extends HoodieConfig {
.key("write.ignore.failed")
.booleanType()
.defaultValue(false)
- .withFallbackKeys("hoodie.write.ignore.failed")
.withDescription("Flag to indicate whether to ignore any non exception error (e.g. writestatus error). within a checkpoint batch. \n"
+ "By default false. Turning this on, could hide the write status errors while the flink checkpoint moves ahead. \n"
+ "So, would recommend users to use this with caution.");
diff --git a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java
index f25367392fe..e9d0310d475 100644
--- a/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java
+++ b/hudi-flink-datasource/hudi-flink/src/main/java/org/apache/hudi/util/FlinkWriteClients.java
@@ -225,8 +225,7 @@ public class FlinkWriteClients {
.withAutoCommit(false)
.withAllowOperationMetadataField(conf.getBoolean(FlinkOptions.CHANGELOG_ENABLED))
.withProps(flinkConf2TypedProperties(conf))
- .withSchema(getSourceSchema(conf).toString())
- .withWriteIgnoreFailed(conf.get(FlinkOptions.IGNORE_FAILED));
+ .withSchema(getSourceSchema(conf).toString());
Option<HoodieLockConfig> lockConfig = getLockConfig(conf);
if (lockConfig.isPresent()) {