[jira] [Resolved] (SPARK-16666) Kryo encoder for custom complex classes

["Sean Zhong (JIRA)" <ji...@apache.org>]

     [ https://issues.apache.org/jira/browse/SPARK-16666?page=com.atlassian.jira.plugin.system.issuetabpanels:all-tabpanel ]

Sean Zhong resolved SPARK-16666.
--------------------------------
    Resolution: Not A Problem

> Kryo encoder for custom complex classes
> ---------------------------------------
>
>                 Key: SPARK-16666
>                 URL: https://issues.apache.org/jira/browse/SPARK-16666
>             Project: Spark
>          Issue Type: Question
>          Components: SQL
>    Affects Versions: 1.6.2
>            Reporter: Sam
>
> I'm trying to create a dataset with some geo data using spark and esri. If `Foo` only have `Point` field, it'll work but if I add some other fields beyond a `Point`, I get ArrayIndexOutOfBoundsException.
> {code:scala}
>     import com.esri.core.geometry.Point
>     import org.apache.spark.sql.{Encoder, Encoders, SQLContext}
>     import org.apache.spark.{SparkConf, SparkContext}
>     
>     object Main {
>     
>       case class Foo(position: Point, name: String)
>     
>       object MyEncoders {
>         implicit def PointEncoder: Encoder[Point] = Encoders.kryo[Point]
>     
>         implicit def FooEncoder: Encoder[Foo] = Encoders.kryo[Foo]
>       }
>     
>       def main(args: Array[String]): Unit = {
>         val sc = new SparkContext(new SparkConf().setAppName("app").setMaster("local"))
>         val sqlContext = new SQLContext(sc)
>         import MyEncoders.{FooEncoder, PointEncoder}
>         import sqlContext.implicits._
>         Seq(new Foo(new Point(0, 0), "bar")).toDS.show
>       }
>     }
> {code}
> {noformat}
> Exception in thread "main" java.lang.ArrayIndexOutOfBoundsException: 1
>     at org.apache.spark.sql.execution.Queryable$$anonfun$formatString$1$$anonfun$apply$2.apply(Queryable.scala:71)
>     at org.apache.spark.sql.execution.Queryable$$anonfun$formatString$1$$anonfun$apply$2.apply(Queryable.scala:70) 
>     at scala.collection.TraversableLike$WithFilter$$anonfun$foreach$1.apply(TraversableLike.scala:772) 
>     at scala.collection.mutable.ResizableArray$class.foreach(ResizableArray.scala:59) 
>     at scala.collection.mutable.ArrayBuffer.foreach(ArrayBuffer.scala:47) 
>     at scala.collection.TraversableLike$WithFilter.foreach(TraversableLike.scala:771) 
>     at org.apache.spark.sql.execution.Queryable$$anonfun$formatString$1.apply(Queryable.scala:70) 
>     at org.apache.spark.sql.execution.Queryable$$anonfun$formatString$1.apply(Queryable.scala:69) 
>     at scala.collection.mutable.ArraySeq.foreach(ArraySeq.scala:73) 
>     at org.apache.spark.sql.execution.Queryable$class.formatString(Queryable.scala:69) 
>     at org.apache.spark.sql.Dataset.formatString(Dataset.scala:65) 
>     at org.apache.spark.sql.Dataset.showString(Dataset.scala:263) 
>     at org.apache.spark.sql.Dataset.show(Dataset.scala:230) 
>     at org.apache.spark.sql.Dataset.show(Dataset.scala:193) 
>     at org.apache.spark.sql.Dataset.show(Dataset.scala:201) 
>     at Main$.main(Main.scala:24) 
>     at Main.main(Main.scala)
> {noformat}



--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org