You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "Yuming Wang (JIRA)" <ji...@apache.org> on 2019/08/11 23:07:00 UTC
[jira] [Updated] (SPARK-28688) Incompatible java.util.ArrayList
[ https://issues.apache.org/jira/browse/SPARK-28688?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
Yuming Wang updated SPARK-28688:
--------------------------------
Summary: Incompatible java.util.ArrayList (was: incompatible java.util.ArrayList)
> Incompatible java.util.ArrayList
> --------------------------------
>
> Key: SPARK-28688
> URL: https://issues.apache.org/jira/browse/SPARK-28688
> Project: Spark
> Issue Type: Sub-task
> Components: SQL, Tests
> Affects Versions: 3.0.0
> Reporter: Yuming Wang
> Priority: Major
> Attachments: screenshot-1.png, screenshot-2.png
>
>
> {{2 TESTS FAILED}} from {{org.apache.spark.sql.hive.client.VersionsSuite}}
> {noformat}
> - 3.0: sql read hive materialized view *** FAILED *** (1 second, 292 milliseconds)
> - 3.1: sql read hive materialized view *** FAILED *** (1 second, 312 milliseconds)
> {noformat}
> Logs:
> {noformat}
> 06:54:07.608 WARN org.apache.hadoop.hive.ql.exec.Utilities: Cannot create class org.apache.hive.storage.jdbc.JdbcInputFormat for hive.vectorized.row.serde.inputformat.excludes checks
> 06:54:07.670 WARN org.apache.hadoop.hive.ql.Driver: Hive-on-MR is deprecated in Hive 2 and may not be available in the future versions. Consider using a different execution engine (i.e. spark, tez) or using Hive 1.X releases.
> java.lang.RuntimeException: java.lang.NoSuchFieldException: parentOffset
> at org.apache.hadoop.hive.ql.exec.SerializationUtilities$ArrayListSubListSerializer.<init>(SerializationUtilities.java:389)
> at org.apache.hadoop.hive.ql.exec.SerializationUtilities$1.create(SerializationUtilities.java:235)
> at org.apache.hive.com.esotericsoftware.kryo.pool.KryoPoolQueueImpl.borrow(KryoPoolQueueImpl.java:51)
> at org.apache.hadoop.hive.ql.exec.SerializationUtilities.borrowKryo(SerializationUtilities.java:279)
> at org.apache.hadoop.hive.ql.exec.Utilities.setBaseWork(Utilities.java:571)
> at org.apache.hadoop.hive.ql.exec.Utilities.setMapWork(Utilities.java:563)
> at org.apache.hadoop.hive.ql.exec.Utilities.setMapRedWork(Utilities.java:555)
> at org.apache.hadoop.hive.ql.exec.mr.ExecDriver.execute(ExecDriver.java:362)
> at org.apache.hadoop.hive.ql.exec.mr.MapRedTask.execute(MapRedTask.java:149)
> at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:205)
> at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:97)
> at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2479)
> at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:2150)
> at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1826)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1567)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1556)
> at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$runHive$1(HiveClientImpl.scala:786)
> at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$withHiveState$1(HiveClientImpl.scala:310)
> at org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:244)
> at org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:243)
> at org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:293)
> at org.apache.spark.sql.hive.client.HiveClientImpl.runHive(HiveClientImpl.scala:766)
> at org.apache.spark.sql.hive.client.HiveClientImpl.runSqlHive(HiveClientImpl.scala:753)
> at org.apache.spark.sql.hive.client.VersionsSuite.$anonfun$new$72(VersionsSuite.scala:626)
> at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
> at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
> at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
> at org.scalatest.Transformer.apply(Transformer.scala:22)
> at org.scalatest.Transformer.apply(Transformer.scala:20)
> at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
> at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:149)
> at org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184)
> at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196)
> at org.scalatest.SuperEngine.runTestImpl(Engine.scala:289)
> at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196)
> at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178)
> at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:56)
> at org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221)
> at org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214)
> at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:56)
> at org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229)
> at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:396)
> at scala.collection.immutable.List.foreach(List.scala:392)
> at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384)
> at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:379)
> at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:461)
> at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229)
> at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228)
> at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
> at org.scalatest.Suite.run(Suite.scala:1147)
> at org.scalatest.Suite.run$(Suite.scala:1129)
> at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
> at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233)
> at org.scalatest.SuperEngine.runImpl(Engine.scala:521)
> at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233)
> at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232)
> at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:56)
> at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
> at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
> at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
> at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:56)
> at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:314)
> at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:507)
> at sbt.ForkMain$Run$2.call(ForkMain.java:296)
> at sbt.ForkMain$Run$2.call(ForkMain.java:286)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:834)
> Caused by: java.lang.NoSuchFieldException: parentOffset
> at java.base/java.lang.Class.getDeclaredField(Class.java:2412)
> at org.apache.hadoop.hive.ql.exec.SerializationUtilities$ArrayListSubListSerializer.<init>(SerializationUtilities.java:383)
> ... 68 more
> 06:54:07.780 ERROR org.apache.hadoop.hive.ql.exec.Task: Job Submission failed with exception 'java.lang.RuntimeException(java.lang.NoSuchFieldException: parentOffset)'
> java.lang.RuntimeException: java.lang.NoSuchFieldException: parentOffset
> at org.apache.hadoop.hive.ql.exec.SerializationUtilities$ArrayListSubListSerializer.<init>(SerializationUtilities.java:389)
> at org.apache.hadoop.hive.ql.exec.SerializationUtilities$1.create(SerializationUtilities.java:235)
> at org.apache.hive.com.esotericsoftware.kryo.pool.KryoPoolQueueImpl.borrow(KryoPoolQueueImpl.java:51)
> at org.apache.hadoop.hive.ql.exec.SerializationUtilities.borrowKryo(SerializationUtilities.java:279)
> at org.apache.hadoop.hive.ql.exec.Utilities.setBaseWork(Utilities.java:571)
> at org.apache.hadoop.hive.ql.exec.Utilities.setMapWork(Utilities.java:563)
> at org.apache.hadoop.hive.ql.exec.Utilities.setMapRedWork(Utilities.java:555)
> at org.apache.hadoop.hive.ql.exec.mr.ExecDriver.execute(ExecDriver.java:362)
> at org.apache.hadoop.hive.ql.exec.mr.MapRedTask.execute(MapRedTask.java:149)
> at org.apache.hadoop.hive.ql.exec.Task.executeTask(Task.java:205)
> at org.apache.hadoop.hive.ql.exec.TaskRunner.runSequential(TaskRunner.java:97)
> at org.apache.hadoop.hive.ql.Driver.launchTask(Driver.java:2479)
> at org.apache.hadoop.hive.ql.Driver.execute(Driver.java:2150)
> at org.apache.hadoop.hive.ql.Driver.runInternal(Driver.java:1826)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1567)
> at org.apache.hadoop.hive.ql.Driver.run(Driver.java:1556)
> at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$runHive$1(HiveClientImpl.scala:786)
> at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$withHiveState$1(HiveClientImpl.scala:310)
> at org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:244)
> at org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:243)
> at org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:293)
> at org.apache.spark.sql.hive.client.HiveClientImpl.runHive(HiveClientImpl.scala:766)
> at org.apache.spark.sql.hive.client.HiveClientImpl.runSqlHive(HiveClientImpl.scala:753)
> at org.apache.spark.sql.hive.client.VersionsSuite.$anonfun$new$72(VersionsSuite.scala:626)
> at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
> at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
> at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
> at org.scalatest.Transformer.apply(Transformer.scala:22)
> at org.scalatest.Transformer.apply(Transformer.scala:20)
> at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
> at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:149)
> at org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184)
> at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196)
> at org.scalatest.SuperEngine.runTestImpl(Engine.scala:289)
> at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196)
> at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178)
> at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:56)
> at org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221)
> at org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214)
> at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:56)
> at org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229)
> at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:396)
> at scala.collection.immutable.List.foreach(List.scala:392)
> at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384)
> at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:379)
> at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:461)
> at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229)
> at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228)
> at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
> at org.scalatest.Suite.run(Suite.scala:1147)
> at org.scalatest.Suite.run$(Suite.scala:1129)
> at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
> at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233)
> at org.scalatest.SuperEngine.runImpl(Engine.scala:521)
> at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233)
> at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232)
> at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:56)
> at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
> at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
> at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
> at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:56)
> at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:314)
> at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:507)
> at sbt.ForkMain$Run$2.call(ForkMain.java:296)
> at sbt.ForkMain$Run$2.call(ForkMain.java:286)
> at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> at java.base/java.lang.Thread.run(Thread.java:834)
> Caused by: java.lang.NoSuchFieldException: parentOffset
> at java.base/java.lang.Class.getDeclaredField(Class.java:2412)
> at org.apache.hadoop.hive.ql.exec.SerializationUtilities$ArrayListSubListSerializer.<init>(SerializationUtilities.java:383)
> ... 68 more
> 06:54:07.780 ERROR org.apache.hadoop.hive.ql.Driver: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask. java.lang.NoSuchFieldException: parentOffset
> 06:54:07.792 ERROR org.apache.spark.sql.hive.client.HiveClientImpl:
> ======================
> HIVE FAILURE OUTPUT
> ======================
> SET spark.sql.test.key=1
> OK
> Query ID = root_20190811065407_baceadf3-30bb-4884-a815-72cbb02d429d
> Total jobs = 1
> Launching Job 1 out of 1
> Number of reduce tasks is set to 0 since there's no reduce operator
> Job Submission failed with exception 'java.lang.RuntimeException(java.lang.NoSuchFieldException: parentOffset)'
> FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask. java.lang.NoSuchFieldException: parentOffset
> ======================
> END HIVE FAILURE OUTPUT
> ======================
> [info] - 3.0: sql read hive materialized view *** FAILED *** (1 second, 292 milliseconds)
> [info] org.apache.spark.sql.execution.QueryExecutionException: FAILED: Execution Error, return code 1 from org.apache.hadoop.hive.ql.exec.mr.MapRedTask. java.lang.NoSuchFieldException: parentOffset
> [info] at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$runHive$1(HiveClientImpl.scala:790)
> [info] at org.apache.spark.sql.hive.client.HiveClientImpl.$anonfun$withHiveState$1(HiveClientImpl.scala:310)
> [info] at org.apache.spark.sql.hive.client.HiveClientImpl.liftedTree1$1(HiveClientImpl.scala:244)
> [info] at org.apache.spark.sql.hive.client.HiveClientImpl.retryLocked(HiveClientImpl.scala:243)
> [info] at org.apache.spark.sql.hive.client.HiveClientImpl.withHiveState(HiveClientImpl.scala:293)
> [info] at org.apache.spark.sql.hive.client.HiveClientImpl.runHive(HiveClientImpl.scala:766)
> [info] at org.apache.spark.sql.hive.client.HiveClientImpl.runSqlHive(HiveClientImpl.scala:753)
> [info] at org.apache.spark.sql.hive.client.VersionsSuite.$anonfun$new$72(VersionsSuite.scala:626)
> [info] at org.scalatest.OutcomeOf.outcomeOf(OutcomeOf.scala:85)
> [info] at org.scalatest.OutcomeOf.outcomeOf$(OutcomeOf.scala:83)
> [info] at org.scalatest.OutcomeOf$.outcomeOf(OutcomeOf.scala:104)
> [info] at org.scalatest.Transformer.apply(Transformer.scala:22)
> [info] at org.scalatest.Transformer.apply(Transformer.scala:20)
> [info] at org.scalatest.FunSuiteLike$$anon$1.apply(FunSuiteLike.scala:186)
> [info] at org.apache.spark.SparkFunSuite.withFixture(SparkFunSuite.scala:149)
> [info] at org.scalatest.FunSuiteLike.invokeWithFixture$1(FunSuiteLike.scala:184)
> [info] at org.scalatest.FunSuiteLike.$anonfun$runTest$1(FunSuiteLike.scala:196)
> [info] at org.scalatest.SuperEngine.runTestImpl(Engine.scala:289)
> [info] at org.scalatest.FunSuiteLike.runTest(FunSuiteLike.scala:196)
> [info] at org.scalatest.FunSuiteLike.runTest$(FunSuiteLike.scala:178)
> [info] at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterEach$$super$runTest(SparkFunSuite.scala:56)
> [info] at org.scalatest.BeforeAndAfterEach.runTest(BeforeAndAfterEach.scala:221)
> [info] at org.scalatest.BeforeAndAfterEach.runTest$(BeforeAndAfterEach.scala:214)
> [info] at org.apache.spark.SparkFunSuite.runTest(SparkFunSuite.scala:56)
> [info] at org.scalatest.FunSuiteLike.$anonfun$runTests$1(FunSuiteLike.scala:229)
> [info] at org.scalatest.SuperEngine.$anonfun$runTestsInBranch$1(Engine.scala:396)
> [info] at scala.collection.immutable.List.foreach(List.scala:392)
> [info] at org.scalatest.SuperEngine.traverseSubNodes$1(Engine.scala:384)
> [info] at org.scalatest.SuperEngine.runTestsInBranch(Engine.scala:379)
> [info] at org.scalatest.SuperEngine.runTestsImpl(Engine.scala:461)
> [info] at org.scalatest.FunSuiteLike.runTests(FunSuiteLike.scala:229)
> [info] at org.scalatest.FunSuiteLike.runTests$(FunSuiteLike.scala:228)
> [info] at org.scalatest.FunSuite.runTests(FunSuite.scala:1560)
> [info] at org.scalatest.Suite.run(Suite.scala:1147)
> [info] at org.scalatest.Suite.run$(Suite.scala:1129)
> [info] at org.scalatest.FunSuite.org$scalatest$FunSuiteLike$$super$run(FunSuite.scala:1560)
> [info] at org.scalatest.FunSuiteLike.$anonfun$run$1(FunSuiteLike.scala:233)
> [info] at org.scalatest.SuperEngine.runImpl(Engine.scala:521)
> [info] at org.scalatest.FunSuiteLike.run(FunSuiteLike.scala:233)
> [info] at org.scalatest.FunSuiteLike.run$(FunSuiteLike.scala:232)
> [info] at org.apache.spark.SparkFunSuite.org$scalatest$BeforeAndAfterAll$$super$run(SparkFunSuite.scala:56)
> [info] at org.scalatest.BeforeAndAfterAll.liftedTree1$1(BeforeAndAfterAll.scala:213)
> [info] at org.scalatest.BeforeAndAfterAll.run(BeforeAndAfterAll.scala:210)
> [info] at org.scalatest.BeforeAndAfterAll.run$(BeforeAndAfterAll.scala:208)
> [info] at org.apache.spark.SparkFunSuite.run(SparkFunSuite.scala:56)
> [info] at org.scalatest.tools.Framework.org$scalatest$tools$Framework$$runSuite(Framework.scala:314)
> [info] at org.scalatest.tools.Framework$ScalaTestTask.execute(Framework.scala:507)
> [info] at sbt.ForkMain$Run$2.call(ForkMain.java:296)
> [info] at sbt.ForkMain$Run$2.call(ForkMain.java:286)
> [info] at java.base/java.util.concurrent.FutureTask.run(FutureTask.java:264)
> [info] at java.base/java.util.concurrent.ThreadPoolExecutor.runWorker(ThreadPoolExecutor.java:1128)
> [info] at java.base/java.util.concurrent.ThreadPoolExecutor$Worker.run(ThreadPoolExecutor.java:628)
> [info] at java.base/java.lang.Thread.run(Thread.java:834)
> {noformat}
> https://github.com/apache/hive/blob/ae4df627952610dbec029b099f0964908b3a4f25/ql/src/java/org/apache/hadoop/hive/ql/exec/SerializationUtilities.java#L381-L383
> !screenshot-2.png!
> !screenshot-1.png!
--
This message was sent by Atlassian JIRA
(v7.6.14#76016)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org