[jira] [Created] (PHOENIX-3751) spark 2.1 with Phoenix 4.10 load data as dataframe fail

["Nan Xu (JIRA)" <ji...@apache.org>]

Nan Xu created PHOENIX-3751:
-------------------------------

             Summary: spark 2.1 with Phoenix 4.10 load data as dataframe fail
                 Key: PHOENIX-3751
                 URL: https://issues.apache.org/jira/browse/PHOENIX-3751
             Project: Phoenix
          Issue Type: Bug
    Affects Versions: 4.10.0
         Environment: HBase 1.14
spark 2.10
phoenix: 4.10
            Reporter: Nan Xu


create phoenix table: 
create table phoenix.quote (  
sym Varchar not null, 
src varchar,
kdbPublishTime time not null,
location varchar,
bid DOUBLE,
ask DOUBLE,
bsize unsigned_int,
asize unsigned_int, 
srcTime time,
layer varchar,
expiryTime time,
quoteId varchar,
recvTime time, 
distTime  time,
"TIME" time
CONSTRAINT quote_pk PRIMARY KEY (sym, src, kdbPublishTime)) COMPRESSION='SNAPPY', DATA_BLOCK_ENCODING='FAST_DIFF', VERSIONS=1000

insert data:
SYM	SRC	KDBPUBLISHTIME	LOCATION	BID	ASK	BSIZE	ASIZE	SRCTIME	           LAYER	EXPIRYTIME	QUOTEID	      RECVTIME	   DISTTIME	TIME
6AH7	cme1	03:42:59	N	       0.7471	0.7506	20	25	03:42:59	   (null)	    (null)	(null)	       03:42:59	    (null)	03:42:59
6AH7	cme1	03:42:59	N	       0.7474	0.7506	25	25	03:42:59	   (null)	    (null)	(null)	       03:42:59	    (null)	03:42:59

val spark = SparkSession
    .builder()
    .appName("load_avro")
    .master("local[1]")
    .config("spark.sql.warehouse.dir", "file:/tmp/spark-warehouse")
    .getOrCreate()
 val df = spark.sqlContext.phoenixTableAsDataFrame("PHOENIX.QUOTE", Seq("SYM","SRC", "EXPIRYTIME"), zkUrl = Some("a1.cluster:2181"))
  df.show(100)

problem is in PhoenixRDD:140

 val rowSeq = columns.map { case (name, sqlType) =>
        val res = pr.resultMap(name)
          // Special handling for data types
          if (dateAsTimestamp && (sqlType == 91 || sqlType == 19)) { // 91 is the defined type for Date and 19 for UNSIGNED_DATE
            new java.sql.Timestamp(res.asInstanceOf[java.sql.Date].getTime)
          } else if (sqlType == 92 || sqlType == 18) { // 92 is the defined type for Time and 18 for UNSIGNED_TIME
            new java.sql.Timestamp(res.asInstanceOf[java.sql.Time].getTime)
          } else {
            res
          }
      }
res.asInstanceOf[java.sql.Time].getTime could be null and get NPE.



--
This message was sent by Atlassian JIRA
(v6.3.15#6346)