You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/12/02 23:15:47 UTC
spark git commit: [SPARK-4676][SQL] JavaSchemaRDD.schema may throw
NullType MatchError if sql has null
Repository: spark
Updated Branches:
refs/heads/master 69b6fed20 -> 106642760
[SPARK-4676][SQL] JavaSchemaRDD.schema may throw NullType MatchError if sql has null
val jsc = new org.apache.spark.api.java.JavaSparkContext(sc)
val jhc = new org.apache.spark.sql.hive.api.java.JavaHiveContext(jsc)
val nrdd = jhc.hql("select null from spark_test.for_test")
println(nrdd.schema)
Then the error is thrown as follows:
scala.MatchError: NullType (of class org.apache.spark.sql.catalyst.types.NullType$)
at org.apache.spark.sql.types.util.DataTypeConversions$.asJavaDataType(DataTypeConversions.scala:43)
Author: YanTangZhai <ha...@tencent.com>
Author: yantangzhai <ty...@163.com>
Author: Michael Armbrust <mi...@databricks.com>
Closes #3538 from YanTangZhai/MatchNullType and squashes the following commits:
e052dff [yantangzhai] [SPARK-4676] [SQL] JavaSchemaRDD.schema may throw NullType MatchError if sql has null
4b4bb34 [yantangzhai] [SPARK-4676] [SQL] JavaSchemaRDD.schema may throw NullType MatchError if sql has null
896c7b7 [yantangzhai] fix NullType MatchError in JavaSchemaRDD when sql has null
6e643f8 [YanTangZhai] Merge pull request #11 from apache/master
e249846 [YanTangZhai] Merge pull request #10 from apache/master
d26d982 [YanTangZhai] Merge pull request #9 from apache/master
76d4027 [YanTangZhai] Merge pull request #8 from apache/master
03b62b0 [YanTangZhai] Merge pull request #7 from apache/master
8a00106 [YanTangZhai] Merge pull request #6 from apache/master
cbcba66 [YanTangZhai] Merge pull request #3 from apache/master
cdef539 [YanTangZhai] Merge pull request #1 from apache/master
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/10664276
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/10664276
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/10664276
Branch: refs/heads/master
Commit: 10664276007beca3843638e558f504cad44b1fb3
Parents: 69b6fed
Author: YanTangZhai <ha...@tencent.com>
Authored: Tue Dec 2 14:12:48 2014 -0800
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Tue Dec 2 14:15:12 2014 -0800
----------------------------------------------------------------------
.../org/apache/spark/sql/api/java/DataType.java | 5 ++++
.../org/apache/spark/sql/api/java/NullType.java | 27 ++++++++++++++++++++
.../scala/org/apache/spark/sql/package.scala | 10 ++++++++
.../sql/types/util/DataTypeConversions.scala | 1 +
.../spark/sql/api/java/JavaSQLSuite.scala | 16 ++++++++++++
5 files changed, 59 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/10664276/sql/core/src/main/java/org/apache/spark/sql/api/java/DataType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/DataType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/DataType.java
index c383540..c69bbd5 100644
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/DataType.java
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/DataType.java
@@ -83,6 +83,11 @@ public abstract class DataType {
public static final ShortType ShortType = new ShortType();
/**
+ * Gets the NullType object.
+ */
+ public static final NullType NullType = new NullType();
+
+ /**
* Creates an ArrayType by specifying the data type of elements ({@code elementType}).
* The field of {@code containsNull} is set to {@code true}.
*/
http://git-wip-us.apache.org/repos/asf/spark/blob/10664276/sql/core/src/main/java/org/apache/spark/sql/api/java/NullType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/NullType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/NullType.java
new file mode 100644
index 0000000..6d5ecdf
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/NullType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing null and NULL values.
+ *
+ * {@code NullType} is represented by the singleton object {@link DataType#NullType}.
+ */
+public class NullType extends DataType {
+ protected NullType() {}
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/10664276/sql/core/src/main/scala/org/apache/spark/sql/package.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/package.scala b/sql/core/src/main/scala/org/apache/spark/sql/package.scala
index 51dad54..1fd8e62 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/package.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/package.scala
@@ -266,6 +266,16 @@ package object sql {
/**
* :: DeveloperApi ::
*
+ * The data type representing `NULL` values.
+ *
+ * @group dataType
+ */
+ @DeveloperApi
+ val NullType = catalyst.types.NullType
+
+ /**
+ * :: DeveloperApi ::
+ *
* The data type for collections of multiple values.
* Internally these are represented as columns that contain a ``scala.collection.Seq``.
*
http://git-wip-us.apache.org/repos/asf/spark/blob/10664276/sql/core/src/main/scala/org/apache/spark/sql/types/util/DataTypeConversions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/types/util/DataTypeConversions.scala b/sql/core/src/main/scala/org/apache/spark/sql/types/util/DataTypeConversions.scala
index d425815..4160a80 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/types/util/DataTypeConversions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/types/util/DataTypeConversions.scala
@@ -62,6 +62,7 @@ protected[sql] object DataTypeConversions {
case IntegerType => JDataType.IntegerType
case LongType => JDataType.LongType
case ShortType => JDataType.ShortType
+ case NullType => JDataType.NullType
case arrayType: ArrayType => JDataType.createArrayType(
asJavaDataType(arrayType.elementType), arrayType.containsNull)
http://git-wip-us.apache.org/repos/asf/spark/blob/10664276/sql/core/src/test/scala/org/apache/spark/sql/api/java/JavaSQLSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/api/java/JavaSQLSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/api/java/JavaSQLSuite.scala
index c9012c9..8afc3a9 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/api/java/JavaSQLSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/api/java/JavaSQLSuite.scala
@@ -68,6 +68,22 @@ class JavaSQLSuite extends FunSuite {
javaSqlCtx.sql("SELECT * FROM people").collect()
}
+ test("schema with null from JavaBeans") {
+ val person = new PersonBean
+ person.setName("Michael")
+ person.setAge(29)
+
+ val rdd = javaCtx.parallelize(person :: Nil)
+ val schemaRDD = javaSqlCtx.applySchema(rdd, classOf[PersonBean])
+
+ schemaRDD.registerTempTable("people")
+ val nullRDD = javaSqlCtx.sql("SELECT null FROM people")
+ val structFields = nullRDD.schema.getFields()
+ assert(structFields.size == 1)
+ assert(structFields(0).getDataType().isInstanceOf[NullType])
+ assert(nullRDD.collect.head.row === Seq(null))
+ }
+
test("all types in JavaBeans") {
val bean = new AllTypesBean
bean.setStringField("")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org