You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@kudu.apache.org by jd...@apache.org on 2016/03/02 04:46:47 UTC
incubator-kudu git commit: KUDU-1360. [spark]
KuduRelation.getKuduValue doesn't check nulls
Repository: incubator-kudu
Updated Branches:
refs/heads/master 8eec83f95 -> dd6dbda35
KUDU-1360. [spark] KuduRelation.getKuduValue doesn't check nulls
Fixes the bug and adds a small test.
We should have a "all types all options" kind of test.
Change-Id: I8bfd3144806b0b84bea00e8eec77a6ba7698379b
Reviewed-on: http://gerrit.cloudera.org:8080/2381
Tested-by: Kudu Jenkins
Reviewed-by: Todd Lipcon <to...@apache.org>
Project: http://git-wip-us.apache.org/repos/asf/incubator-kudu/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-kudu/commit/dd6dbda3
Tree: http://git-wip-us.apache.org/repos/asf/incubator-kudu/tree/dd6dbda3
Diff: http://git-wip-us.apache.org/repos/asf/incubator-kudu/diff/dd6dbda3
Branch: refs/heads/master
Commit: dd6dbda35246722dd28e565252fd4f23c2c8dcad
Parents: 8eec83f
Author: Jean-Daniel Cryans <jd...@apache.org>
Authored: Tue Mar 1 16:26:58 2016 -0800
Committer: Jean-Daniel Cryans <jd...@gerrit.cloudera.org>
Committed: Wed Mar 2 03:42:17 2016 +0000
----------------------------------------------------------------------
.../src/main/scala/org/kududb/spark/DefaultSource.scala | 12 +++++++++---
.../test/scala/org/kududb/spark/DefaultSourceTest.scala | 6 +++++-
.../src/test/scala/org/kududb/spark/TestContext.scala | 11 +++++++++--
3 files changed, 23 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/dd6dbda3/java/kudu-spark/src/main/scala/org/kududb/spark/DefaultSource.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/main/scala/org/kududb/spark/DefaultSource.scala b/java/kudu-spark/src/main/scala/org/kududb/spark/DefaultSource.scala
index 50cb9c6..447b059 100644
--- a/java/kudu-spark/src/main/scala/org/kududb/spark/DefaultSource.scala
+++ b/java/kudu-spark/src/main/scala/org/kududb/spark/DefaultSource.scala
@@ -116,7 +116,7 @@ class KuduRelation(val tableName: String,
/**
* Build the RDD to scan rows.
*
- * @param requiredColumns clumns that are being requested by the requesting query
+ * @param requiredColumns columns that are being requested by the requesting query
* @param filters filters that are being applied by the requesting query
* @return RDD will all the results from Kudu
*/
@@ -128,8 +128,14 @@ class KuduRelation(val tableName: String,
}
private def getKuduValue(row: RowResult, columnName: String): Any = {
- val columnType = kuduSchemaColumnMap.getOrElse(columnName,
- throw new IllegalArgumentException(s"Couldn't find column '$columnName'")).getType
+ val columnSchema = kuduSchemaColumnMap.getOrElse(columnName,
+ throw new IllegalArgumentException(s"Couldn't find column '$columnName'"))
+
+ if (columnSchema.isNullable && row.isNull(columnName)) {
+ return null
+ }
+
+ val columnType = columnSchema.getType
columnType match {
case Type.BINARY => row.getBinary(columnName)
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/dd6dbda3/java/kudu-spark/src/test/scala/org/kududb/spark/DefaultSourceTest.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/kududb/spark/DefaultSourceTest.scala b/java/kudu-spark/src/test/scala/org/kududb/spark/DefaultSourceTest.scala
index 17cea07..483b2e2 100644
--- a/java/kudu-spark/src/test/scala/org/kududb/spark/DefaultSourceTest.scala
+++ b/java/kudu-spark/src/test/scala/org/kududb/spark/DefaultSourceTest.scala
@@ -35,6 +35,10 @@ class DefaultSourceTest extends FunSuite with TestContext {
Map("kudu.table" -> tableName, "kudu.master" -> miniCluster.getMasterAddresses))
.registerTempTable(tableName)
- assert(sqlContext.sql("SELECT * FROM " + tableName).collectAsList().size() == rowCount)
+ val results = sqlContext.sql("SELECT * FROM " + tableName).collectAsList()
+ assert(results.size() == rowCount)
+
+ assert(results.get(0).isNullAt(2))
+ assert(!results.get(1).isNullAt(2))
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-kudu/blob/dd6dbda3/java/kudu-spark/src/test/scala/org/kududb/spark/TestContext.scala
----------------------------------------------------------------------
diff --git a/java/kudu-spark/src/test/scala/org/kududb/spark/TestContext.scala b/java/kudu-spark/src/test/scala/org/kududb/spark/TestContext.scala
index fa24f52..9876282 100644
--- a/java/kudu-spark/src/test/scala/org/kududb/spark/TestContext.scala
+++ b/java/kudu-spark/src/test/scala/org/kududb/spark/TestContext.scala
@@ -39,7 +39,7 @@ trait TestContext extends BeforeAndAfterAll { self: Suite =>
val columns = ImmutableList.of(
new ColumnSchemaBuilder("key", Type.INT32).key(true).build(),
new ColumnSchemaBuilder("c1_i", Type.INT32).build(),
- new ColumnSchemaBuilder("c2_s", Type.STRING).build())
+ new ColumnSchemaBuilder("c2_s", Type.STRING).nullable(true).build())
new Schema(columns)
}
@@ -75,7 +75,14 @@ trait TestContext extends BeforeAndAfterAll { self: Suite =>
val row = insert.getRow
row.addInt(0, i)
row.addInt(1, i)
- row.addString(2, i.toString)
+
+ // Sprinkling some nulls so that queries see them.
+ if (i % 2 == 0) {
+ row.addString(2, i.toString)
+ } else {
+ row.setNull(2)
+ }
+
kuduSession.apply(insert)
}
}