You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "leesf (Jira)" <ji...@apache.org> on 2022/03/16 01:07:00 UTC
[jira] [Commented] (SPARK-29091) spark-shell don't support added jar's class as Serde class
[ https://issues.apache.org/jira/browse/SPARK-29091?page=com.atlassian.jira.plugin.system.issuetabpanels:comment-tabpanel&focusedCommentId=17507293#comment-17507293 ]
leesf commented on SPARK-29091:
-------------------------------
any updates here? we also encountered this problem.
> spark-shell don't support added jar's class as Serde class
> -----------------------------------------------------------
>
> Key: SPARK-29091
> URL: https://issues.apache.org/jira/browse/SPARK-29091
> Project: Spark
> Issue Type: Bug
> Components: SQL
> Affects Versions: 2.3.4, 2.4.4, 3.0.0
> Reporter: angerszhu
> Priority: Major
>
> {code:java}
> Welcome to
> ____ __
> / __/__ ___ _____/ /__
> _\ \/ _ \/ _ `/ __/ '_/
> /___/ .__/\_,_/_/ /_/\_\ version 3.0.0-SNAPSHOT
> /_/Using Scala version 2.12.8 (Java HotSpot(TM) 64-Bit Server VM, Java 1.8.0_191)
> Type in expressions to have them evaluated.
> Type :help for more information.scala> spark.sql("add jar /Users/angerszhu/.m2/repository/org/apache/hive/hcatalog/hive-hcatalog-core/2.3.6/hive-hcatalog-core-2.3.6.jar")
> 19/09/16 07:38:01 main WARN ObjectStore: Version information not found in metastore. hive.metastore.schema.verification is not enabled so recording the schema version 1.2.0
> 19/09/16 07:38:01 main WARN ObjectStore: Failed to get database default, returning NoSuchObjectException
> res0: org.apache.spark.sql.DataFrame = [result: int]scala> spark.sql("CREATE TABLE addJar27(key string) ROW FORMAT SERDE 'org.apache.hive.hcatalog.data.JsonSerDe'")
> 19/09/16 07:38:05 main WARN HiveMetaStore: Location: file:/Users/angerszhu/Documents/project/AngersZhu/spark/spark-warehouse/addjar27 specified for non-external table:addjar27
> res1: org.apache.spark.sql.DataFrame = []scala> spark.sql("select * from addJar27").show
> 19/09/16 07:38:08 main WARN ObjectStore: Failed to get database global_temp, returning NoSuchObjectException
> java.lang.RuntimeException: java.lang.ClassNotFoundException: org.apache.hive.hcatalog.data.JsonSerDe
> at org.apache.hadoop.hive.ql.plan.TableDesc.getDeserializerClass(TableDesc.java:74)
> at org.apache.spark.sql.hive.execution.HiveTableScanExec.addColumnMetadataToConf(HiveTableScanExec.scala:123)
> at org.apache.spark.sql.hive.execution.HiveTableScanExec.hadoopConf$lzycompute(HiveTableScanExec.scala:101)
> at org.apache.spark.sql.hive.execution.HiveTableScanExec.hadoopConf(HiveTableScanExec.scala:98)
> at org.apache.spark.sql.hive.execution.HiveTableScanExec.hadoopReader$lzycompute(HiveTableScanExec.scala:110)
> at org.apache.spark.sql.hive.execution.HiveTableScanExec.hadoopReader(HiveTableScanExec.scala:105)
> at org.apache.spark.sql.hive.execution.HiveTableScanExec.$anonfun$doExecute$1(HiveTableScanExec.scala:188)
> at org.apache.spark.util.Utils$.withDummyCallSite(Utils.scala:2488)
> at org.apache.spark.sql.hive.execution.HiveTableScanExec.doExecute(HiveTableScanExec.scala:188)
> at org.apache.spark.sql.execution.SparkPlan.$anonfun$execute$1(SparkPlan.scala:189)
> at org.apache.spark.sql.execution.SparkPlan.$anonfun$executeQuery$1(SparkPlan.scala:227)
> at org.apache.spark.rdd.RDDOperationScope$.withScope(RDDOperationScope.scala:151)
> at org.apache.spark.sql.execution.SparkPlan.executeQuery(SparkPlan.scala:224)
> at org.apache.spark.sql.execution.SparkPlan.execute(SparkPlan.scala:185)
> at org.apache.spark.sql.execution.SparkPlan.getByteArrayRdd(SparkPlan.scala:329)
> at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:421)
> at org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:47)
> at org.apache.spark.sql.Dataset.collectFromPlan(Dataset.scala:3382)
> at org.apache.spark.sql.Dataset.$anonfun$head$1(Dataset.scala:2509)
> at org.apache.spark.sql.Dataset.$anonfun$withAction$1(Dataset.scala:3372)
> at org.apache.spark.sql.execution.SQLExecution$.$anonfun$withNewExecutionId$4(SQLExecution.scala:100)
> at org.apache.spark.sql.execution.SQLExecution$.withSQLConfPropagated(SQLExecution.scala:160)
> at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:87)
> at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3368)
> at org.apache.spark.sql.Dataset.head(Dataset.scala:2509)
> at org.apache.spark.sql.Dataset.take(Dataset.scala:2716)
> at org.apache.spark.sql.Dataset.getRows(Dataset.scala:290)
> at org.apache.spark.sql.Dataset.showString(Dataset.scala:327)
> at org.apache.spark.sql.Dataset.show(Dataset.scala:792)
> at org.apache.spark.sql.Dataset.show(Dataset.scala:751)
> at org.apache.spark.sql.Dataset.show(Dataset.scala:760)
> ... 47 elided
> Caused by: java.lang.ClassNotFoundException: org.apache.hive.hcatalog.data.JsonSerDe
> at scala.reflect.internal.util.AbstractFileClassLoader.findClass(AbstractFileClassLoader.scala:72)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:424)
> at java.lang.ClassLoader.loadClass(ClassLoader.java:357)
> at java.lang.Class.forName0(Native Method)
> at java.lang.Class.forName(Class.java:348)
> at org.apache.hadoop.hive.ql.plan.TableDesc.getDeserializerClass(TableDesc.java:71)
> ... 77 morescala>
> {code}
--
This message was sent by Atlassian Jira
(v8.20.1#820001)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org