You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "Ethan Guo (Jira)" <ji...@apache.org> on 2022/03/15 21:35:00 UTC
[jira] [Created] (HUDI-3638) Task not serializable due to ZookeeperBasedLockProvider in clean action
Ethan Guo created HUDI-3638:
-------------------------------
Summary: Task not serializable due to ZookeeperBasedLockProvider in clean action
Key: HUDI-3638
URL: https://issues.apache.org/jira/browse/HUDI-3638
Project: Apache Hudi
Issue Type: Task
Reporter: Ethan Guo
{code:java}
22/03/14 10:58:05 WARN CleanActionExecutor: Failed to perform previous clean operation, instant: [==>20220314105144254__clean__REQUESTED]
org.apache.spark.SparkException: Task not serializable
at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:416)
at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:406)
at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:162)
at org.apache.spark.SparkContext.clean(SparkContext.scala:2477)
at org.apache.spark.rdd.RDD.$anonfun$mapPartitions$1(RDD.scala:860)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
at org.apache.spark.rdd.RDD.withScope(RDD.scala:414)
at org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:859)
at org.apache.spark.api.java.JavaRDDLike.mapPartitionsToPair(JavaRDDLike.scala:188)
at org.apache.spark.api.java.JavaRDDLike.mapPartitionsToPair$(JavaRDDLike.scala:183)
at org.apache.spark.api.java.AbstractJavaRDDLike.mapPartitionsToPair(JavaRDDLike.scala:45)
at org.apache.hudi.client.common.HoodieSparkEngineContext.mapPartitionsToPairAndReduceByKey(HoodieSparkEngineContext.java:116)
at org.apache.hudi.table.action.clean.CleanActionExecutor.clean(CleanActionExecutor.java:141)
at org.apache.hudi.table.action.clean.CleanActionExecutor.runClean(CleanActionExecutor.java:197)
at org.apache.hudi.table.action.clean.CleanActionExecutor.runPendingClean(CleanActionExecutor.java:176)
at org.apache.hudi.table.action.clean.CleanActionExecutor.lambda$execute$6(CleanActionExecutor.java:238)
at java.util.ArrayList.forEach(ArrayList.java:1259)
at org.apache.hudi.table.action.clean.CleanActionExecutor.execute(CleanActionExecutor.java:232)
at org.apache.hudi.table.HoodieSparkCopyOnWriteTable.clean(HoodieSparkCopyOnWriteTable.java:339)
at org.apache.hudi.client.BaseHoodieWriteClient.clean(BaseHoodieWriteClient.java:780)
at org.apache.hudi.client.BaseHoodieWriteClient.clean(BaseHoodieWriteClient.java:737)
at org.apache.hudi.async.AsyncCleanerService.lambda$startService$0(AsyncCleanerService.java:55)
at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)
at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
at java.lang.Thread.run(Thread.java:748)
Caused by: java.io.NotSerializableException: org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider
Serialization stack:
- object not serializable (class: org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider, value: org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider@2e4da831)
- field (class: org.apache.hudi.client.transaction.lock.LockManager, name: lockProvider, type: interface org.apache.hudi.common.lock.LockProvider)
- object (class org.apache.hudi.client.transaction.lock.LockManager, org.apache.hudi.client.transaction.lock.LockManager@60d74cfd)
- field (class: org.apache.hudi.client.transaction.TransactionManager, name: lockManager, type: class org.apache.hudi.client.transaction.lock.LockManager)
- object (class org.apache.hudi.client.transaction.TransactionManager, org.apache.hudi.client.transaction.TransactionManager@54e830d4)
- field (class: org.apache.hudi.table.action.clean.CleanActionExecutor, name: txnManager, type: class org.apache.hudi.client.transaction.TransactionManager)
- object (class org.apache.hudi.table.action.clean.CleanActionExecutor, org.apache.hudi.table.action.clean.CleanActionExecutor@71c368a0)
- element of array (index: 0)
- array (class [Ljava.lang.Object;, size 1)
- field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;)
- object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.hudi.table.action.clean.CleanActionExecutor, functionalInterfaceMethod=org/apache/hudi/common/function/SerializablePairFlatMapFunction.call:(Ljava/lang/Object;)Ljava/util/stream/Stream;, implementation=invokeSpecial org/apache/hudi/table/action/clean/CleanActionExecutor.lambda$clean$59ddfded$1:(Ljava/util/Iterator;)Ljava/util/stream/Stream;, instantiatedMethodType=(Ljava/util/Iterator;)Ljava/util/stream/Stream;, numCaptured=1])
- writeReplace data (class: java.lang.invoke.SerializedLambda)
- object (class org.apache.hudi.table.action.clean.CleanActionExecutor$$Lambda$3706/863841076, org.apache.hudi.table.action.clean.CleanActionExecutor$$Lambda$3706/863841076@5e1f7ca3)
- element of array (index: 0)
- array (class [Ljava.lang.Object;, size 1)
- field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;)
- object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.hudi.client.common.HoodieSparkEngineContext, functionalInterfaceMethod=org/apache/spark/api/java/function/PairFlatMapFunction.call:(Ljava/lang/Object;)Ljava/util/Iterator;, implementation=invokeStatic org/apache/hudi/client/common/HoodieSparkEngineContext.lambda$mapPartitionsToPairAndReduceByKey$4e9dbab9$1:(Lorg/apache/hudi/common/function/SerializablePairFlatMapFunction;Ljava/util/Iterator;)Ljava/util/Iterator;, instantiatedMethodType=(Ljava/util/Iterator;)Ljava/util/Iterator;, numCaptured=1])
- writeReplace data (class: java.lang.invoke.SerializedLambda)
- object (class org.apache.hudi.client.common.HoodieSparkEngineContext$$Lambda$3709/1146198397, org.apache.hudi.client.common.HoodieSparkEngineContext$$Lambda$3709/1146198397@7aff92c3)
- element of array (index: 0)
- array (class [Ljava.lang.Object;, size 1)
- field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;)
- object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=interface org.apache.spark.api.java.JavaRDDLike, functionalInterfaceMethod=scala/Function1.apply:(Ljava/lang/Object;)Ljava/lang/Object;, implementation=invokeStatic org/apache/spark/api/java/JavaRDDLike.$anonfun$mapPartitionsToPair$1:(Lorg/apache/spark/api/java/function/PairFlatMapFunction;Lscala/collection/Iterator;)Lscala/collection/Iterator;, instantiatedMethodType=(Lscala/collection/Iterator;)Lscala/collection/Iterator;, numCaptured=1])
- writeReplace data (class: java.lang.invoke.SerializedLambda)
- object (class org.apache.spark.api.java.JavaRDDLike$$Lambda$3710/1144734313, org.apache.spark.api.java.JavaRDDLike$$Lambda$3710/1144734313@4a33e3f8)
at org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:41)
at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:47)
at org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:101)
at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:413)
... 26 more {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)