You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by yh...@apache.org on 2015/05/07 19:05:14 UTC
spark git commit: [SPARK-7330] [SQL] avoid NPE at jdbc rdd
Repository: spark
Updated Branches:
refs/heads/master 4f87e9562 -> ed9be06a4
[SPARK-7330] [SQL] avoid NPE at jdbc rdd
Thank nadavoosh point this out in #5590
Author: Daoyuan Wang <da...@intel.com>
Closes #5877 from adrian-wang/jdbcrdd and squashes the following commits:
cc11900 [Daoyuan Wang] avoid NPE in jdbcrdd
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ed9be06a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ed9be06a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ed9be06a
Branch: refs/heads/master
Commit: ed9be06a4797bbb678355b361054c8872ac20b75
Parents: 4f87e95
Author: Daoyuan Wang <da...@intel.com>
Authored: Thu May 7 10:05:01 2015 -0700
Committer: Yin Huai <yh...@databricks.com>
Committed: Thu May 7 10:05:01 2015 -0700
----------------------------------------------------------------------
.../org/apache/spark/sql/jdbc/JDBCRDD.scala | 8 ++++++-
.../org/apache/spark/sql/jdbc/JDBCSuite.scala | 25 ++++++++++++++++++++
2 files changed, 32 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ed9be06a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JDBCRDD.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JDBCRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JDBCRDD.scala
index 325a326..1a5083d 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JDBCRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/jdbc/JDBCRDD.scala
@@ -362,7 +362,13 @@ private[sql] class JDBCRDD(
conversions(i) match {
case BooleanConversion => mutableRow.setBoolean(i, rs.getBoolean(pos))
case DateConversion =>
- mutableRow.update(i, DateUtils.fromJavaDate(rs.getDate(pos)))
+ // DateUtils.fromJavaDate does not handle null value, so we need to check it.
+ val dateVal = rs.getDate(pos)
+ if (dateVal != null) {
+ mutableRow.update(i, DateUtils.fromJavaDate(dateVal))
+ } else {
+ mutableRow.update(i, null)
+ }
case DecimalConversion =>
val decimalVal = rs.getBigDecimal(pos)
if (decimalVal == null) {
http://git-wip-us.apache.org/repos/asf/spark/blob/ed9be06a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
index 3ec17d3..021affa 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/jdbc/JDBCSuite.scala
@@ -104,6 +104,8 @@ class JDBCSuite extends FunSuite with BeforeAndAfter {
).executeUpdate()
conn.prepareStatement("insert into test.timetypes values ('12:34:56', "
+ "'1996-01-01', '2002-02-20 11:22:33.543543543')").executeUpdate()
+ conn.prepareStatement("insert into test.timetypes values ('12:34:56', "
+ + "null, '2002-02-20 11:22:33.543543543')").executeUpdate()
conn.commit()
sql(
s"""
@@ -127,6 +129,23 @@ class JDBCSuite extends FunSuite with BeforeAndAfter {
|OPTIONS (url '$url', dbtable 'TEST.FLTTYPES', user 'testUser', password 'testPass')
""".stripMargin.replaceAll("\n", " "))
+ conn.prepareStatement(
+ s"""
+ |create table test.nulltypes (a INT, b BOOLEAN, c TINYINT, d BINARY(20), e VARCHAR(20),
+ |f VARCHAR_IGNORECASE(20), g CHAR(20), h BLOB, i CLOB, j TIME, k DATE, l TIMESTAMP,
+ |m DOUBLE, n REAL, o DECIMAL(40, 20))
+ """.stripMargin.replaceAll("\n", " ")).executeUpdate()
+ conn.prepareStatement("insert into test.nulltypes values ("
+ + "null, null, null, null, null, null, null, null, null, "
+ + "null, null, null, null, null, null)").executeUpdate()
+ conn.commit()
+ sql(
+ s"""
+ |CREATE TEMPORARY TABLE nulltypes
+ |USING org.apache.spark.sql.jdbc
+ |OPTIONS (url '$url', dbtable 'TEST.NULLTYPES', user 'testUser', password 'testPass')
+ """.stripMargin.replaceAll("\n", " "))
+
// Untested: IDENTITY, OTHER, UUID, ARRAY, and GEOMETRY types.
}
@@ -254,6 +273,7 @@ class JDBCSuite extends FunSuite with BeforeAndAfter {
val rows = TestSQLContext.jdbc(urlWithUserAndPass, "TEST.TIMETYPES").collect()
val cachedRows = TestSQLContext.jdbc(urlWithUserAndPass, "TEST.TIMETYPES").cache().collect()
assert(rows(0).getAs[java.sql.Date](1) === java.sql.Date.valueOf("1996-01-01"))
+ assert(rows(1).getAs[java.sql.Date](1) === null)
assert(cachedRows(0).getAs[java.sql.Date](1) === java.sql.Date.valueOf("1996-01-01"))
}
@@ -266,6 +286,11 @@ class JDBCSuite extends FunSuite with BeforeAndAfter {
assert(cachedRows(0).getAs[java.sql.Date](1) === java.sql.Date.valueOf("1996-01-01"))
}
+ test("test types for null value") {
+ val rows = TestSQLContext.jdbc(urlWithUserAndPass, "TEST.NULLTYPES").collect()
+ assert((0 to 14).forall(i => rows(0).isNullAt(i)))
+ }
+
test("H2 floating-point types") {
val rows = sql("SELECT * FROM flttypes").collect()
assert(rows(0).getDouble(0) === 1.00000000000000022) // Yes, I meant ==.
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org