You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by co...@apache.org on 2022/08/09 14:48:12 UTC
[hudi] 12/48: [HUDI-4516] fix Task not serializable error when run HoodieCleaner after one failure (#6265)
This is an automated email from the ASF dual-hosted git repository.
codope pushed a commit to branch release-0.12.0
in repository https://gitbox.apache.org/repos/asf/hudi.git
commit 92843dc4095e497d684d51318708611163d6fd8b
Author: 冯健 <fe...@gmail.com>
AuthorDate: Wed Aug 3 07:09:33 2022 +0800
[HUDI-4516] fix Task not serializable error when run HoodieCleaner after one failure (#6265)
Co-authored-by: jian.feng <ji...@shopee.com>
---
.../main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java | 4 +++-
.../java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java | 4 +++-
.../org/apache/hudi/common/util/collection/ExternalSpillableMap.java | 2 +-
.../java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java | 4 ++--
4 files changed, 9 insertions(+), 5 deletions(-)
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java b/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java
index 0e6d78f1f2..8032a04df4 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/DefaultSizeEstimator.java
@@ -18,12 +18,14 @@
package org.apache.hudi.common.util;
+import java.io.Serializable;
+
/**
* Default implementation of size-estimator that uses Twitter's ObjectSizeCalculator.
*
* @param <T>
*/
-public class DefaultSizeEstimator<T> implements SizeEstimator<T> {
+public class DefaultSizeEstimator<T> implements SizeEstimator<T>, Serializable {
@Override
public long sizeEstimate(T t) {
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java
index 78f7b1b9bb..88a0f70cd8 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/HoodieRecordSizeEstimator.java
@@ -26,12 +26,14 @@ import org.apache.avro.Schema;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
+import java.io.Serializable;
+
/**
* Size Estimator for Hoodie record payload.
*
* @param <T>
*/
-public class HoodieRecordSizeEstimator<T extends HoodieRecordPayload> implements SizeEstimator<HoodieRecord<T>> {
+public class HoodieRecordSizeEstimator<T extends HoodieRecordPayload> implements SizeEstimator<HoodieRecord<T>>, Serializable {
private static final Logger LOG = LogManager.getLogger(HoodieRecordSizeEstimator.class);
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java
index a6e8d5cfb3..8d2707d604 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/ExternalSpillableMap.java
@@ -52,7 +52,7 @@ import java.util.stream.Stream;
* map may occupy more memory than is available, resulting in OOM. However, if the spill threshold is too low, we spill
* frequently and incur unnecessary disk writes.
*/
-public class ExternalSpillableMap<T extends Serializable, R extends Serializable> implements Map<T, R> {
+public class ExternalSpillableMap<T extends Serializable, R extends Serializable> implements Map<T, R>, Serializable {
// Find the actual estimated payload size after inserting N records
private static final int NUMBER_OF_RECORDS_TO_ESTIMATE_PAYLOAD_SIZE = 100;
diff --git a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java
index c4fbd3a8cb..67b4ffabe2 100644
--- a/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java
+++ b/hudi-common/src/main/java/org/apache/hudi/common/util/collection/RocksDBBasedMap.java
@@ -29,12 +29,12 @@ import java.util.Set;
/**
* A map's implementation based on RocksDB.
*/
-public final class RocksDBBasedMap<K extends Serializable, R extends Serializable> implements Map<K, R> {
+public final class RocksDBBasedMap<K extends Serializable, R extends Serializable> implements Map<K, R>, Serializable {
private static final String COL_FAMILY_NAME = "map_handle";
private final String rocksDbStoragePath;
- private RocksDBDAO rocksDBDAO;
+ private transient RocksDBDAO rocksDBDAO;
private final String columnFamilyName;
public RocksDBBasedMap(String rocksDbStoragePath) {