You are viewing a plain text version of this content. The canonical link for it is here.
Posted to issues@spark.apache.org by "huanghuai (JIRA)" <ji...@apache.org> on 2018/10/15 10:25:00 UTC
[jira] [Updated] (SPARK-25723) spark sql External DataSource
[ https://issues.apache.org/jira/browse/SPARK-25723?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]
huanghuai updated SPARK-25723:
------------------------------
Description: (was: {color:#333333}*spark.read()*{color}
{color:#333333}*.format("com.myself.datasource")*{color}
{color:#333333}*.option("ur","xxxx")*{color}
{color:#333333}*.load()*{color}
{color:#FF0000}*.show()*{color}
{color:#FF0000}*Driver stacktrace:*{color}
{color:#FF0000} *at*{color} org.apache.spark.scheduler.DAGScheduler.org$apache$spark$scheduler$DAGScheduler$$failJobAndIndependentStages(DAGScheduler.scala:1599) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1587) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.scheduler.DAGScheduler$$anonfun$abortStage$1.apply(DAGScheduler.scala:1586) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) ~[scala-library-2.11.8.jar:?]
at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:48) ~[scala-library-2.11.8.jar:?]
at org.apache.spark.scheduler.DAGScheduler.abortStage(DAGScheduler.scala:1586) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:831) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.scheduler.DAGScheduler$$anonfun$handleTaskSetFailed$1.apply(DAGScheduler.scala:831) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at scala.Option.foreach(Option.scala:257) ~[scala-library-2.11.8.jar:?]
at org.apache.spark.scheduler.DAGScheduler.handleTaskSetFailed(DAGScheduler.scala:831) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.doOnReceive(DAGScheduler.scala:1820) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1769) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.scheduler.DAGSchedulerEventProcessLoop.onReceive(DAGScheduler.scala:1758) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.util.EventLoop$$anon$1.run(EventLoop.scala:48) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.scheduler.DAGScheduler.runJob(DAGScheduler.scala:642) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2027) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2048) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.SparkContext.runJob(SparkContext.scala:2067) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.execution.SparkPlan.executeTake(SparkPlan.scala:363) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.execution.CollectLimitExec.executeCollect(limit.scala:38) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.Dataset.org$apache$spark$sql$Dataset$$collectFromPlan(Dataset.scala:3272) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2484) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.Dataset$$anonfun$head$1.apply(Dataset.scala:2484) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.Dataset$$anonfun$52.apply(Dataset.scala:3253) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.execution.SQLExecution$.withNewExecutionId(SQLExecution.scala:77) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.Dataset.withAction(Dataset.scala:3252) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.Dataset.head(Dataset.scala:2484) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.Dataset.take(Dataset.scala:2698) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.Dataset.showString(Dataset.scala:254) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.Dataset.show(Dataset.scala:723) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.Dataset.show(Dataset.scala:682) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.Dataset.show(Dataset.scala:691) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
{color:#FF0000}*Caused by: scala.MatchError: 23.25 (of class java.lang.Double)*{color}
{color:#FF0000} *at*{color} org.apache.spark.sql.catalyst.CatalystTypeConverters$StringConverter$.toCatalystImpl(CatalystTypeConverters.scala:276) ~[spark-catalyst_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.catalyst.CatalystTypeConverters$StringConverter$.toCatalystImpl(CatalystTypeConverters.scala:275) ~[spark-catalyst_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.catalyst.CatalystTypeConverters$CatalystTypeConverter.toCatalyst(CatalystTypeConverters.scala:103) ~[spark-catalyst_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.catalyst.CatalystTypeConverters$$anonfun$createToCatalystConverter$2.apply(CatalystTypeConverters.scala:379) ~[spark-catalyst_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.execution.RDDConversions$$anonfun$rowToRowRdd$1$$anonfun$apply$3.apply(ExistingRDD.scala:60) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.execution.RDDConversions$$anonfun$rowToRowRdd$1$$anonfun$apply$3.apply(ExistingRDD.scala:57) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at scala.collection.Iterator$$anon$11.next(Iterator.scala:409) ~[scala-library-2.11.8.jar:?]
at org.apache.spark.sql.catalyst.expressions.GeneratedClass$GeneratedIteratorForCodegenStage1.processNext(Unknown Source) ~[?:?]
at org.apache.spark.sql.execution.BufferedRowIterator.hasNext(BufferedRowIterator.java:43) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.execution.WholeStageCodegenExec$$anonfun$10$$anon$1.hasNext(WholeStageCodegenExec.scala:614) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:253) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.sql.execution.SparkPlan$$anonfun$2.apply(SparkPlan.scala:247) ~[spark-sql_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:830) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.rdd.RDD$$anonfun$mapPartitionsInternal$1$$anonfun$apply$25.apply(RDD.scala:830) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.rdd.MapPartitionsRDD.compute(MapPartitionsRDD.scala:38) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.rdd.RDD.computeOrReadCheckpoint(RDD.scala:324) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.rdd.RDD.iterator(RDD.scala:288) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.scheduler.ResultTask.runTask(ResultTask.scala:87) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.scheduler.Task.run(Task.scala:109) ~[spark-core_2.11-2.3.0.jar:2.3.0]
at org.apache.spark.executor.Executor$TaskRunner.run(Executor.scala:345) ~[spark-core_2.11-2.3.0.jar:2.3.0]
... 3 more
---------------------------------------------------------------------------------------------------------------
if i use dataset.show() this error will occured.
if i use dataset.collectAsList() ,it's ok.
I debug at follwing method,
{color:#FF0000}*org.apache.spark.sql.execution.RDDConversions.*{color}
{color:#FF0000}*rowToRowRdd(data: RDD[Row], outputTypes: Seq[DataType])*{color}
if found , everytime , when i show {*outputTypes: Seq[DataType]*}*, it is different like this:*
*at first time:*
DoubleType
IntegerType
StringType
*maby second or third time:*
DoubleType
StringType
IntegerType
The order of datatype(schema) is wrong， so when it execute
{mutableRow(i) = converters(i)(r(i))}
will get a wrong converter , and scala's matcherror will occured..)
Summary: spark sql External DataSource (was: dataset.show() , scala.MatchError: 23.25 (of class java.lang.Double))
> spark sql External DataSource
> -----------------------------
>
> Key: SPARK-25723
> URL: https://issues.apache.org/jira/browse/SPARK-25723
> Project: Spark
> Issue Type: Question
> Components: SQL
> Affects Versions: 2.3.2
> Environment: local mode
> Reporter: huanghuai
> Priority: Major
>
--
This message was sent by Atlassian JIRA
(v7.6.3#76005)
---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org