You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hudi.apache.org by "sivabalan narayanan (Jira)" <ji...@apache.org> on 2022/03/09 03:15:00 UTC
[jira] [Created] (HUDI-3587) SparkUpgradeDowngradeHelper not serialization exception
sivabalan narayanan created HUDI-3587:
-----------------------------------------
Summary: SparkUpgradeDowngradeHelper not serialization exception
Key: HUDI-3587
URL: https://issues.apache.org/jira/browse/HUDI-3587
Project: Apache Hudi
Issue Type: Bug
Reporter: sivabalan narayanan
{code:java}
[AWS CodeBuild Plugin] - object (class org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter$$Lambda$138/1591629766, org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter$$Lambda$138/1591629766@18e13131)
[AWS CodeBuild Plugin] - field (class: org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1, name: fun$1, type: interface org.apache.spark.api.java.function.Function)
[AWS CodeBuild Plugin] - object (class org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1, <function1>)
[AWS CodeBuild Plugin] at org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:40)
[AWS CodeBuild Plugin] at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:46)
[AWS CodeBuild Plugin] at org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:100)
[AWS CodeBuild Plugin] at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:400)
[AWS CodeBuild Plugin] ... 20 more
[AWS CodeBuild Plugin] Exception in thread "main" org.apache.hudi.exception.HoodieException: Failed to run Test Suite
[AWS CodeBuild Plugin] at org.apache.hudi.integ.testsuite.HoodieTestSuiteJob.runTestSuite(HoodieTestSuiteJob.java:208)
[AWS CodeBuild Plugin] at org.apache.hudi.integ.testsuite.HoodieTestSuiteJob.main(HoodieTestSuiteJob.java:170)
[AWS CodeBuild Plugin] at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
[AWS CodeBuild Plugin] at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:62)
[AWS CodeBuild Plugin] at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
[AWS CodeBuild Plugin] at java.lang.reflect.Method.invoke(Method.java:498)
[AWS CodeBuild Plugin] at org.apache.spark.deploy.JavaMainApplication.start(SparkApplication.scala:52)
[AWS CodeBuild Plugin] at org.apache.spark.deploy.SparkSubmit.org$apache$spark$deploy$SparkSubmit$$runMain(SparkSubmit.scala:845)
[AWS CodeBuild Plugin] at org.apache.spark.deploy.SparkSubmit.doRunMain$1(SparkSubmit.scala:161)
[AWS CodeBuild Plugin] at org.apache.spark.deploy.SparkSubmit.submit(SparkSubmit.scala:184)
[AWS CodeBuild Plugin] at org.apache.spark.deploy.SparkSubmit.doSubmit(SparkSubmit.scala:86)
[AWS CodeBuild Plugin] at org.apache.spark.deploy.SparkSubmit$$anon$2.doSubmit(SparkSubmit.scala:920)
[AWS CodeBuild Plugin] at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:929)
[AWS CodeBuild Plugin] at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)
[AWS CodeBuild Plugin] Caused by: java.util.concurrent.ExecutionException: org.apache.hudi.exception.HoodieException: org.apache.spark.SparkException: Task not serializable
[AWS CodeBuild Plugin] at java.util.concurrent.FutureTask.report(FutureTask.java:122)
[AWS CodeBuild Plugin] at java.util.concurrent.FutureTask.get(FutureTask.java:206)
[AWS CodeBuild Plugin] at org.apache.hudi.integ.testsuite.dag.scheduler.DagScheduler.execute(DagScheduler.java:113)
[AWS CodeBuild Plugin] at org.apache.hudi.integ.testsuite.dag.scheduler.DagScheduler.schedule(DagScheduler.java:68)
[AWS CodeBuild Plugin] at org.apache.hudi.integ.testsuite.HoodieTestSuiteJob.runTestSuite(HoodieTestSuiteJob.java:203)
[AWS CodeBuild Plugin] ... 13 more
[AWS CodeBuild Plugin] Caused by: org.apache.hudi.exception.HoodieException: org.apache.spark.SparkException: Task not serializable
[AWS CodeBuild Plugin] at org.apache.hudi.integ.testsuite.dag.scheduler.DagScheduler.executeNode(DagScheduler.java:146)
[AWS CodeBuild Plugin] at org.apache.hudi.integ.testsuite.dag.scheduler.DagScheduler.lambda$execute$0(DagScheduler.java:105)
[AWS CodeBuild Plugin] at java.util.concurrent.Executors$RunnableAdapter.call(Executors.java:511)
[AWS CodeBuild Plugin] at java.util.concurrent.FutureTask.run(FutureTask.java:266)
[AWS CodeBuild Plugin] at java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1149)
[AWS CodeBuild Plugin] at java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:624)
[AWS CodeBuild Plugin] at java.lang.Thread.run(Thread.java:748)
[AWS CodeBuild Plugin] Caused by: org.apache.spark.SparkException: Task not serializable
[AWS CodeBuild Plugin] at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:403)
[AWS CodeBuild Plugin] at org.apache.spark.util.ClosureCleaner$.org$apache$spark$util$ClosureCleaner$$clean(ClosureCleaner.scala:393)
[AWS CodeBuild Plugin] at org.apache.spark.util.ClosureCleaner$.clean(ClosureCleaner.scala:162)
[AWS CodeBuild Plugin] at org.apache.spark.SparkContext.clean(SparkContext.scala:2326)
[AWS CodeBuild Plugin] at org.apache.spark.rdd.RDD$$anonfun$map$1.apply(RDD.scala:371)
[AWS CodeBuild Plugin] at org.apache.spark.rdd.RDD$$anonfun$map$1.apply(RDD.scala:370)
[AWS CodeBuild Plugin] at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
[AWS CodeBuild Plugin] at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:112)
[AWS CodeBuild Plugin] at org.apache.spark.rdd.RDD.withScope(RDD.scala:363)
[AWS CodeBuild Plugin] at org.apache.spark.rdd.RDD.map(RDD.scala:370)
[AWS CodeBuild Plugin] at org.apache.spark.api.java.JavaRDDLike$class.map(JavaRDDLike.scala:93)
[AWS CodeBuild Plugin] at org.apache.spark.api.java.AbstractJavaRDDLike.map(JavaRDDLike.scala:45)
[AWS CodeBuild Plugin] at org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter.getNextBatch(HoodieTestSuiteWriter.java:138)
[AWS CodeBuild Plugin] at org.apache.hudi.integ.testsuite.dag.nodes.SparkInsertNode.execute(SparkInsertNode.scala:52)
[AWS CodeBuild Plugin] at org.apache.hudi.integ.testsuite.dag.scheduler.DagScheduler.executeNode(DagScheduler.java:139)
[AWS CodeBuild Plugin] ... 6 more
[AWS CodeBuild Plugin] Caused by: java.io.NotSerializableException: org.apache.hudi.table.upgrade.SparkUpgradeDowngradeHelper
[AWS CodeBuild Plugin] Serialization stack:
[AWS CodeBuild Plugin] - object not serializable (class: org.apache.hudi.table.upgrade.SparkUpgradeDowngradeHelper, value: org.apache.hudi.table.upgrade.SparkUpgradeDowngradeHelper@77e85a58)
[AWS CodeBuild Plugin] - field (class: org.apache.hudi.client.BaseHoodieWriteClient, name: upgradeDowngradeHelper, type: interface org.apache.hudi.table.upgrade.SupportsUpgradeDowngrade)
[AWS CodeBuild Plugin] - object (class org.apache.hudi.client.SparkRDDWriteClient, org.apache.hudi.client.SparkRDDWriteClient@50c0b7d2)
[AWS CodeBuild Plugin] - field (class: org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter, name: writeClient, type: class org.apache.hudi.client.SparkRDDWriteClient)
[AWS CodeBuild Plugin] - object (class org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter, org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter@7b38a6c2)
[AWS CodeBuild Plugin] - element of array (index: 0)
[AWS CodeBuild Plugin] - array (class [Ljava.lang.Object;, size 1)
[AWS CodeBuild Plugin] - field (class: java.lang.invoke.SerializedLambda, name: capturedArgs, type: class [Ljava.lang.Object;)
[AWS CodeBuild Plugin] - object (class java.lang.invoke.SerializedLambda, SerializedLambda[capturingClass=class org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter, functionalInterfaceMethod=org/apache/spark/api/java/function/Function.call:(Ljava/lang/Object;)Ljava/lang/Object;, implementation=invokeSpecial org/apache/hudi/integ/testsuite/HoodieTestSuiteWriter.lambda$getNextBatch$c5590ce6$1:(Lorg/apache/hudi/common/model/HoodieRecord;)Lorg/apache/avro/generic/GenericRecord;, instantiatedMethodType=(Lorg/apache/hudi/common/model/HoodieRecord;)Lorg/apache/avro/generic/GenericRecord;, numCaptured=1])
[AWS CodeBuild Plugin] - writeReplace data (class: java.lang.invoke.SerializedLambda)
[AWS CodeBuild Plugin] - object (class org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter$$Lambda$138/1591629766, org.apache.hudi.integ.testsuite.HoodieTestSuiteWriter$$Lambda$138/1591629766@18e13131)
[AWS CodeBuild Plugin] - field (class: org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1, name: fun$1, type: interface org.apache.spark.api.java.function.Function)
[AWS CodeBuild Plugin] - object (class org.apache.spark.api.java.JavaPairRDD$$anonfun$toScalaFunction$1, <function1>)
[AWS CodeBuild Plugin] at org.apache.spark.serializer.SerializationDebugger$.improveException(SerializationDebugger.scala:40)
[AWS CodeBuild Plugin] at org.apache.spark.serializer.JavaSerializationStream.writeObject(JavaSerializer.scala:46)
[AWS CodeBuild Plugin] at org.apache.spark.serializer.JavaSerializerInstance.serialize(JavaSerializer.scala:100)
[AWS CodeBuild Plugin] at org.apache.spark.util.ClosureCleaner$.ensureSerializable(ClosureCleaner.scala:400)
[AWS CodeBuild Plugin] ... 20 more
[AWS CodeBuild Plugin]
[AWS CodeBuild Plugin] 2022/03/09 03:00:06 Command did not exit successfully sh run-intig-test-v2.sh HudiIntegTestsManualTest-23-2022-03-09 MERGE_ON_READ cow-spark-simple.yaml test.properties hudi-integ-test-bundle-0.11.0-SNAPSHOT.jar exit status 1
[AWS CodeBuild Plugin] 2022/03/09 03:00:06 Phase complete: BUILD State: FAILED
[AWS CodeBuild Plugin] 2022/03/09 03:00:06 Phase context status code: COMMAND_EXECUTION_ERROR Message: Error while executing command: sh run-intig-test-v2.sh HudiIntegTestsManualTest-23-2022-03-09 MERGE_ON_READ cow-spark-simple.yaml test.properties hudi-integ-test-bundle-0.11.0-SNAPSHOT.jar. Reason: exit status 1
[AWS CodeBuild Plugin] 2022/03/09 03:00:06 Entering phase POST_BUILD
[AWS CodeBuild Plugin] 2022/03/09 03:00:06 Phase complete: POST_BUILD State: SUCCEEDED
[AWS CodeBuild Plugin] 2022/03/09 03:00:06 Phase context status code: Message: {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)