You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "ASF GitHub Bot (Jira)" <ji...@apache.org> on 2022/03/23 16:05:00 UTC
[jira] [Updated] (HUDI-3638) Task not serializable due to ZookeeperBasedLockProvider in clean action
[ https://issues.apache.org/jira/browse/HUDI-3638?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
ASF GitHub Bot updated HUDI-3638:
---------------------------------
Labels: pull-request-available (was: )
> Task not serializable due to ZookeeperBasedLockProvider in clean action
> -----------------------------------------------------------------------
>
> Key: HUDI-3638
> URL: https://issues.apache.org/jira/browse/HUDI-3638
> Project: Apache Hudi
> Issue Type: Task
> Reporter: Ethan Guo
> Priority: Major
> Labels: pull-request-available
>
>
> {code:java}
> 22/03/14 10:58:05 WARN CleanActionExecutor: Failed to perform previous clean operation, instant: [==>20220314105144254__clean__REQUESTED]
> org.apache.spark.SparkException: Task not serializable
> at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:416)
> at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:406)
> at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:162)
> at org.apache.spark.SparkContext.clean(SparkContext.scala:2477)
> at org.apache.spark.rdd.RDD.$anonfun$mapPartitions$1(RDD.scala:860)
> at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
> at org.apache.spark.rdd.RDD.withScope(RDD.scala:414)
> at org.apache.spark.rdd.RDD.mapPartitions(RDD.scala:859)
> at org.apache.spark.api.java.JavaRDDLike.mapPartitionsToPair(JavaRDDLike.scala:188)
> at org.apache.spark.api.java.JavaRDDLike.mapPartitionsToPair$(JavaRDDLike.scala:183)
> at org.apache.spark.api.java.AbstractJavaRDDLike.mapPartitionsToPair(JavaRDDLike.scala:45)
> at org.apache.hudi.client.common.HoodieSparkEngineContext.mapPartitionsToPairAndReduceByKey(HoodieSparkEngineContext.java:116)
> at org.apache.hudi.table.action.clean.CleanActionExecutor.clean(CleanActionExecutor.java:141)
> at org.apache.hudi.table.action.clean.CleanActionExecutor.runClean(CleanActionExecutor.java:197)
> at org.apache.hudi.table.action.clean.CleanActionExecutor.runPendingClean(CleanActionExecutor.java:176)
> at org.apache.hudi.table.action.clean.CleanActionExecutor.lambda$execute$6(CleanActionExecutor.java:238)
> at java.util.ArrayList.forEach(ArrayList.java:1259)
> at org.apache.hudi.table.action.clean.CleanActionExecutor.execute(CleanActionExecutor.java:232)
> at org.apache.hudi.table.HoodieSparkCopyOnWriteTable.clean(HoodieSparkCopyOnWriteTable.java:339)
> at org.apache.hudi.client.BaseHoodieWriteClient.clean(BaseHoodieWriteClient.java:780)
> at org.apache.hudi.client.BaseHoodieWriteClient.clean(BaseHoodieWriteClient.java:737)
> at org.apache.hudi.async.AsyncCleanerService.lambda$startService$0(AsyncCleanerService.java:55)
> at java.util.concurrent.CompletableFuture$AsyncSupply.run(CompletableFuture.java:1604)
> at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
> at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
> at java.lang.Thread.run(Thread.java:748)
> Caused by: java.io.NotSerializableException: org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider
> Serialization stack:
> - object not serializable (class: org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider, value: org.apache.hudi.client.transaction.lock.ZookeeperBasedLockProvider@2e4da831)
> - field (class: org.apache.hudi.client.transaction.lock.LockManager, name: lockProvider, type: interface org.apache.hudi.common.lock.LockProvider)
> - object (class org.apache.hudi.client.transaction.lock.LockManager, org.apache.hudi.client.transaction.lock.LockManager@60d74cfd)
> - field (class: org.apache.hudi.client.transaction.TransactionManager, name: lockManager, type: class org.apache.hudi.client.transaction.lock.LockManager)
> - object (class org.apache.hudi.client.transaction.TransactionManager, org.apache.hudi.client.transaction.TransactionManager@54e830d4)
> - field (class: org.apache.hudi.table.action.clean.CleanActionExecutor, name: txnManager, type: class org.apache.hudi.client.transaction.TransactionManager)
> - object (class org.apache.hudi.table.action.clean.CleanActionExecutor, org.apache.hudi.table.action.clean.CleanActionExecutor@71c368a0)
> - element of array (index: 0)
> - array (class [Ljava.lang.Object;, size 1)
> - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;)
> - object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.hudi.table.action.clean.CleanActionExecutor, functionalInterfaceMethod=org/apache/hudi/common/function/SerializablePairFlatMapFunction.call:(Ljava/lang/Object;)Ljava/util/stream/Stream;, implementation=invokeSpecial org/apache/hudi/table/action/clean/CleanActionExecutor.lambda$clean$59ddfded$1:(Ljava/util/Iterator;)Ljava/util/stream/Stream;, instantiatedMethodType=(Ljava/util/Iterator;)Ljava/util/stream/Stream;, numCaptured=1])
> - writeReplace data (class: java.lang.invoke.SerializedLambda)
> - object (class org.apache.hudi.table.action.clean.CleanActionExecutor$$Lambda$3706/863841076, org.apache.hudi.table.action.clean.CleanActionExecutor$$Lambda$3706/863841076@5e1f7ca3)
> - element of array (index: 0)
> - array (class [Ljava.lang.Object;, size 1)
> - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;)
> - object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.hudi.client.common.HoodieSparkEngineContext, functionalInterfaceMethod=org/apache/spark/api/java/function/PairFlatMapFunction.call:(Ljava/lang/Object;)Ljava/util/Iterator;, implementation=invokeStatic org/apache/hudi/client/common/HoodieSparkEngineContext.lambda$mapPartitionsToPairAndReduceByKey$4e9dbab9$1:(Lorg/apache/hudi/common/function/SerializablePairFlatMapFunction;Ljava/util/Iterator;)Ljava/util/Iterator;, instantiatedMethodType=(Ljava/util/Iterator;)Ljava/util/Iterator;, numCaptured=1])
> - writeReplace data (class: java.lang.invoke.SerializedLambda)
> - object (class org.apache.hudi.client.common.HoodieSparkEngineContext$$Lambda$3709/1146198397, org.apache.hudi.client.common.HoodieSparkEngineContext$$Lambda$3709/1146198397@7aff92c3)
> - element of array (index: 0)
> - array (class [Ljava.lang.Object;, size 1)
> - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;)
> - object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=interface org.apache.spark.api.java.JavaRDDLike, functionalInterfaceMethod=scala/Function1.apply:(Ljava/lang/Object;)Ljava/lang/Object;, implementation=invokeStatic org/apache/spark/api/java/JavaRDDLike.$anonfun$mapPartitionsToPair$1:(Lorg/apache/spark/api/java/function/PairFlatMapFunction;Lscala/collection/Iterator;)Lscala/collection/Iterator;, instantiatedMethodType=(Lscala/collection/Iterator;)Lscala/collection/Iterator;, numCaptured=1])
> - writeReplace data (class: java.lang.invoke.SerializedLambda)
> - object (class org.apache.spark.api.java.JavaRDDLike$$Lambda$3710/1144734313, org.apache.spark.api.java.JavaRDDLike$$Lambda$3710/1144734313@4a33e3f8)
> at org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:41)
> at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:47)
> at org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:101)
> at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:413)
> ... 26 more {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)