[jira] [Created] (SPARK-4709) Spark SQL support for Parquet with timestamp type field

["Felix Cheung (JIRA)" <ji...@apache.org>]

Felix Cheung created SPARK-4709:
-----------------------------------

             Summary: Spark SQL support for Parquet with timestamp type field
                 Key: SPARK-4709
                 URL: https://issues.apache.org/jira/browse/SPARK-4709
             Project: Spark
          Issue Type: Bug
          Components: SQL
    Affects Versions: 1.1.0
            Reporter: Felix Cheung
            Priority: Critical


Have a data set on Parquet format (created by Hive) with a field of the timestamp type. Reading this causes an exception:

val sqlContext = new org.apache.spark.sql.SQLContext(sc)
val p = sqlContext.parquetFile("hdfs:///data/parquetdata")

java.lang.RuntimeException: Potential loss of precision: cannot convert INT96
	at scala.sys.package$.error(package.scala:27)
	at org.apache.spark.sql.parquet.ParquetTypesConverter$.toPrimitiveDataType(ParquetTypes.scala:61)
	at org.apache.spark.sql.parquet.ParquetTypesConverter$.toDataType(ParquetTypes.scala:113)
	at org.apache.spark.sql.parquet.ParquetTypesConverter$$anonfun$convertToAttributes$1.apply(ParquetTypes.scala:314)
	at org.apache.spark.sql.parquet.ParquetTypesConverter$$anonfun$convertToAttributes$1.apply(ParquetTypes.scala:311)
	at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
	at scala.collection.TraversableLike$$anonfun$map$1.apply(TraversableLike.scala:244)
	at scala.collection.Iterator$class.foreach(Iterator.scala:727)
	at scala.collection.AbstractIterator.foreach(Iterator.scala:1157)
	at scala.collection.IterableLike$class.foreach(IterableLike.scala:72)
	at scala.collection.AbstractIterable.foreach(Iterable.scala:54)
	at scala.collection.TraversableLike$class.map(TraversableLike.scala:244)
	at scala.collection.AbstractTraversable.map(Traversable.scala:105)
	at org.apache.spark.sql.parquet.ParquetTypesConverter$.convertToAttributes(ParquetTypes.scala:310)
	at org.apache.spark.sql.parquet.ParquetTypesConverter$.readSchemaFromFile(ParquetTypes.scala:441)
	at org.apache.spark.sql.parquet.ParquetRelation.<init>(ParquetRelation.scala:66)
	at org.apache.spark.sql.SQLContext.parquetFile(SQLContext.scala:141)
	at $iwC$$iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:17)
	at $iwC$$iwC$$iwC$$iwC$$iwC.<init>(<console>:22)
	at $iwC$$iwC$$iwC$$iwC.<init>(<console>:24)
	at $iwC$$iwC$$iwC.<init>(<console>:26)
	at $iwC$$iwC.<init>(<console>:28)
	at $iwC.<init>(<console>:30)
	at <init>(<console>:32)
	at .<init>(<console>:36)
	at .<clinit>(<console>)
	at .<init>(<console>:7)
	at .<clinit>(<console>)
	at $print(<console>)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.spark.repl.SparkIMain$ReadEvalPrint.call(SparkIMain.scala:846)
	at org.apache.spark.repl.SparkIMain$Request.loadAndRun(SparkIMain.scala:1119)
	at org.apache.spark.repl.SparkIMain.loadAndRunReq$1(SparkIMain.scala:672)
	at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:703)
	at org.apache.spark.repl.SparkIMain.interpret(SparkIMain.scala:667)
	at org.apache.spark.repl.SparkILoop.reallyInterpret$1(SparkILoop.scala:819)
	at org.apache.spark.repl.SparkILoop.interpretStartingWith(SparkILoop.scala:864)
	at org.apache.spark.repl.SparkILoop.command(SparkILoop.scala:776)
	at org.apache.spark.repl.SparkILoop.processLine$1(SparkILoop.scala:619)
	at org.apache.spark.repl.SparkILoop.innerLoop$1(SparkILoop.scala:627)
	at org.apache.spark.repl.SparkILoop.loop(SparkILoop.scala:632)
	at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply$mcZ$sp(SparkILoop.scala:959)
	at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:907)
	at org.apache.spark.repl.SparkILoop$$anonfun$process$1.apply(SparkILoop.scala:907)
	at scala.tools.nsc.util.ScalaClassLoader$.savingContextLoader(ScalaClassLoader.scala:135)
	at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:907)
	at org.apache.spark.repl.SparkILoop.process(SparkILoop.scala:1002)
	at org.apache.spark.repl.Main$.main(Main.scala:31)
	at org.apache.spark.repl.Main.main(Main.scala)
	at sun.reflect.NativeMethodAccessorImpl.invoke0(Native Method)
	at sun.reflect.NativeMethodAccessorImpl.invoke(NativeMethodAccessorImpl.java:57)
	at sun.reflect.DelegatingMethodAccessorImpl.invoke(DelegatingMethodAccessorImpl.java:43)
	at java.lang.reflect.Method.invoke(Method.java:606)
	at org.apache.spark.deploy.SparkSubmit$.launch(SparkSubmit.scala:331)
	at org.apache.spark.deploy.SparkSubmit$.main(SparkSubmit.scala:75)
	at org.apache.spark.deploy.SparkSubmit.main(SparkSubmit.scala)




--
This message was sent by Atlassian JIRA
(v6.3.4#6332)

---------------------------------------------------------------------
To unsubscribe, e-mail: issues-unsubscribe@spark.apache.org
For additional commands, e-mail: issues-help@spark.apache.org