You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/08/01 20:15:00 UTC
[1/2] [SPARK-2179][SQL] A minor refactoring Java data type APIs (2179
follow-up).
Repository: spark
Updated Branches:
refs/heads/master 8d338f64c -> c41fdf04f
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/scala/org/apache/spark/sql/types/util/DataTypeConversions.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/types/util/DataTypeConversions.scala b/sql/core/src/main/scala/org/apache/spark/sql/types/util/DataTypeConversions.scala
index d1aa3c8..77353f4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/types/util/DataTypeConversions.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/types/util/DataTypeConversions.scala
@@ -18,7 +18,7 @@
package org.apache.spark.sql.types.util
import org.apache.spark.sql._
-import org.apache.spark.sql.api.java.types.{DataType => JDataType, StructField => JStructField}
+import org.apache.spark.sql.api.java.{DataType => JDataType, StructField => JStructField}
import scala.collection.JavaConverters._
@@ -74,37 +74,37 @@ protected[sql] object DataTypeConversions {
* Returns the equivalent DataType in Scala for the given DataType in Java.
*/
def asScalaDataType(javaDataType: JDataType): DataType = javaDataType match {
- case stringType: org.apache.spark.sql.api.java.types.StringType =>
+ case stringType: org.apache.spark.sql.api.java.StringType =>
StringType
- case binaryType: org.apache.spark.sql.api.java.types.BinaryType =>
+ case binaryType: org.apache.spark.sql.api.java.BinaryType =>
BinaryType
- case booleanType: org.apache.spark.sql.api.java.types.BooleanType =>
+ case booleanType: org.apache.spark.sql.api.java.BooleanType =>
BooleanType
- case timestampType: org.apache.spark.sql.api.java.types.TimestampType =>
+ case timestampType: org.apache.spark.sql.api.java.TimestampType =>
TimestampType
- case decimalType: org.apache.spark.sql.api.java.types.DecimalType =>
+ case decimalType: org.apache.spark.sql.api.java.DecimalType =>
DecimalType
- case doubleType: org.apache.spark.sql.api.java.types.DoubleType =>
+ case doubleType: org.apache.spark.sql.api.java.DoubleType =>
DoubleType
- case floatType: org.apache.spark.sql.api.java.types.FloatType =>
+ case floatType: org.apache.spark.sql.api.java.FloatType =>
FloatType
- case byteType: org.apache.spark.sql.api.java.types.ByteType =>
+ case byteType: org.apache.spark.sql.api.java.ByteType =>
ByteType
- case integerType: org.apache.spark.sql.api.java.types.IntegerType =>
+ case integerType: org.apache.spark.sql.api.java.IntegerType =>
IntegerType
- case longType: org.apache.spark.sql.api.java.types.LongType =>
+ case longType: org.apache.spark.sql.api.java.LongType =>
LongType
- case shortType: org.apache.spark.sql.api.java.types.ShortType =>
+ case shortType: org.apache.spark.sql.api.java.ShortType =>
ShortType
- case arrayType: org.apache.spark.sql.api.java.types.ArrayType =>
+ case arrayType: org.apache.spark.sql.api.java.ArrayType =>
ArrayType(asScalaDataType(arrayType.getElementType), arrayType.isContainsNull)
- case mapType: org.apache.spark.sql.api.java.types.MapType =>
+ case mapType: org.apache.spark.sql.api.java.MapType =>
MapType(
asScalaDataType(mapType.getKeyType),
asScalaDataType(mapType.getValueType),
mapType.isValueContainsNull)
- case structType: org.apache.spark.sql.api.java.types.StructType =>
+ case structType: org.apache.spark.sql.api.java.StructType =>
StructType(structType.getFields.map(asScalaStructField))
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
----------------------------------------------------------------------
diff --git a/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java b/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
index 8ee4591..3c92906 100644
--- a/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
+++ b/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaApplySchemaSuite.java
@@ -28,9 +28,6 @@ import org.junit.Assert;
import org.junit.Before;
import org.junit.Test;
-import org.apache.spark.sql.api.java.types.DataType;
-import org.apache.spark.sql.api.java.types.StructField;
-import org.apache.spark.sql.api.java.types.StructType;
import org.apache.spark.api.java.JavaRDD;
import org.apache.spark.api.java.JavaSparkContext;
import org.apache.spark.api.java.function.Function;
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaSideDataTypeConversionSuite.java
----------------------------------------------------------------------
diff --git a/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaSideDataTypeConversionSuite.java b/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaSideDataTypeConversionSuite.java
index 96a5039..d099a48 100644
--- a/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaSideDataTypeConversionSuite.java
+++ b/sql/core/src/test/java/org/apache/spark/sql/api/java/JavaSideDataTypeConversionSuite.java
@@ -24,8 +24,6 @@ import org.junit.Assert;
import org.junit.Test;
import org.apache.spark.sql.types.util.DataTypeConversions;
-import org.apache.spark.sql.api.java.types.DataType;
-import org.apache.spark.sql.api.java.types.StructField;
public class JavaSideDataTypeConversionSuite {
public void checkDataType(DataType javaDataType) {
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/test/scala/org/apache/spark/sql/api/java/ScalaSideDataTypeConversionSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/api/java/ScalaSideDataTypeConversionSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/api/java/ScalaSideDataTypeConversionSuite.scala
index 46de6fe..ff1debf 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/api/java/ScalaSideDataTypeConversionSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/api/java/ScalaSideDataTypeConversionSuite.scala
@@ -20,12 +20,13 @@ package org.apache.spark.sql.api.java
import org.apache.spark.sql.types.util.DataTypeConversions
import org.scalatest.FunSuite
-import org.apache.spark.sql._
+import org.apache.spark.sql.{DataType => SDataType, StructField => SStructField}
+import org.apache.spark.sql.{StructType => SStructType}
import DataTypeConversions._
class ScalaSideDataTypeConversionSuite extends FunSuite {
- def checkDataType(scalaDataType: DataType) {
+ def checkDataType(scalaDataType: SDataType) {
val javaDataType = asJavaDataType(scalaDataType)
val actual = asScalaDataType(javaDataType)
assert(scalaDataType === actual, s"Converted data type ${actual} " +
@@ -34,48 +35,52 @@ class ScalaSideDataTypeConversionSuite extends FunSuite {
test("convert data types") {
// Simple DataTypes.
- checkDataType(StringType)
- checkDataType(BinaryType)
- checkDataType(BooleanType)
- checkDataType(TimestampType)
- checkDataType(DecimalType)
- checkDataType(DoubleType)
- checkDataType(FloatType)
- checkDataType(ByteType)
- checkDataType(IntegerType)
- checkDataType(LongType)
- checkDataType(ShortType)
+ checkDataType(org.apache.spark.sql.StringType)
+ checkDataType(org.apache.spark.sql.BinaryType)
+ checkDataType(org.apache.spark.sql.BooleanType)
+ checkDataType(org.apache.spark.sql.TimestampType)
+ checkDataType(org.apache.spark.sql.DecimalType)
+ checkDataType(org.apache.spark.sql.DoubleType)
+ checkDataType(org.apache.spark.sql.FloatType)
+ checkDataType(org.apache.spark.sql.ByteType)
+ checkDataType(org.apache.spark.sql.IntegerType)
+ checkDataType(org.apache.spark.sql.LongType)
+ checkDataType(org.apache.spark.sql.ShortType)
// Simple ArrayType.
- val simpleScalaArrayType = ArrayType(StringType, true)
+ val simpleScalaArrayType =
+ org.apache.spark.sql.ArrayType(org.apache.spark.sql.StringType, true)
checkDataType(simpleScalaArrayType)
// Simple MapType.
- val simpleScalaMapType = MapType(StringType, LongType)
+ val simpleScalaMapType =
+ org.apache.spark.sql.MapType(org.apache.spark.sql.StringType, org.apache.spark.sql.LongType)
checkDataType(simpleScalaMapType)
// Simple StructType.
- val simpleScalaStructType = StructType(
- StructField("a", DecimalType, false) ::
- StructField("b", BooleanType, true) ::
- StructField("c", LongType, true) ::
- StructField("d", BinaryType, false) :: Nil)
+ val simpleScalaStructType = SStructType(
+ SStructField("a", org.apache.spark.sql.DecimalType, false) ::
+ SStructField("b", org.apache.spark.sql.BooleanType, true) ::
+ SStructField("c", org.apache.spark.sql.LongType, true) ::
+ SStructField("d", org.apache.spark.sql.BinaryType, false) :: Nil)
checkDataType(simpleScalaStructType)
// Complex StructType.
- val complexScalaStructType = StructType(
- StructField("simpleArray", simpleScalaArrayType, true) ::
- StructField("simpleMap", simpleScalaMapType, true) ::
- StructField("simpleStruct", simpleScalaStructType, true) ::
- StructField("boolean", BooleanType, false) :: Nil)
+ val complexScalaStructType = SStructType(
+ SStructField("simpleArray", simpleScalaArrayType, true) ::
+ SStructField("simpleMap", simpleScalaMapType, true) ::
+ SStructField("simpleStruct", simpleScalaStructType, true) ::
+ SStructField("boolean", org.apache.spark.sql.BooleanType, false) :: Nil)
checkDataType(complexScalaStructType)
// Complex ArrayType.
- val complexScalaArrayType = ArrayType(complexScalaStructType, true)
+ val complexScalaArrayType =
+ org.apache.spark.sql.ArrayType(complexScalaStructType, true)
checkDataType(complexScalaArrayType)
// Complex MapType.
- val complexScalaMapType = MapType(complexScalaStructType, complexScalaArrayType, false)
+ val complexScalaMapType =
+ org.apache.spark.sql.MapType(complexScalaStructType, complexScalaArrayType, false)
checkDataType(complexScalaMapType)
}
}
[2/2] git commit: [SPARK-2179][SQL] A minor refactoring Java data
type APIs (2179 follow-up).
Posted by ma...@apache.org.
[SPARK-2179][SQL] A minor refactoring Java data type APIs (2179 follow-up).
It is a follow-up PR of SPARK-2179 (https://issues.apache.org/jira/browse/SPARK-2179). It makes package names of data type APIs more consistent across languages (Scala: `org.apache.spark.sql`, Java: `org.apache.spark.sql.api.java`, Python: `pyspark.sql`).
Author: Yin Huai <hu...@cse.ohio-state.edu>
Closes #1712 from yhuai/javaDataType and squashes the following commits:
62eb705 [Yin Huai] Move package-info.
add4bcb [Yin Huai] Make the package names of data type classes consistent across languages by moving all Java data type classes to package sql.api.java.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/c41fdf04
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/c41fdf04
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/c41fdf04
Branch: refs/heads/master
Commit: c41fdf04f4beebe36379396b0c4fff3ab7ad3cf4
Parents: 8d338f6
Author: Yin Huai <hu...@cse.ohio-state.edu>
Authored: Fri Aug 1 11:14:53 2014 -0700
Committer: Michael Armbrust <mi...@databricks.com>
Committed: Fri Aug 1 11:14:53 2014 -0700
----------------------------------------------------------------------
.../apache/spark/sql/api/java/ArrayType.java | 68 +++++++
.../apache/spark/sql/api/java/BinaryType.java | 27 +++
.../apache/spark/sql/api/java/BooleanType.java | 27 +++
.../org/apache/spark/sql/api/java/ByteType.java | 27 +++
.../org/apache/spark/sql/api/java/DataType.java | 190 +++++++++++++++++++
.../apache/spark/sql/api/java/DecimalType.java | 27 +++
.../apache/spark/sql/api/java/DoubleType.java | 27 +++
.../apache/spark/sql/api/java/FloatType.java | 27 +++
.../apache/spark/sql/api/java/IntegerType.java | 27 +++
.../org/apache/spark/sql/api/java/LongType.java | 27 +++
.../org/apache/spark/sql/api/java/MapType.java | 78 ++++++++
.../apache/spark/sql/api/java/ShortType.java | 27 +++
.../apache/spark/sql/api/java/StringType.java | 27 +++
.../apache/spark/sql/api/java/StructField.java | 76 ++++++++
.../apache/spark/sql/api/java/StructType.java | 58 ++++++
.../spark/sql/api/java/TimestampType.java | 27 +++
.../apache/spark/sql/api/java/package-info.java | 21 ++
.../spark/sql/api/java/types/ArrayType.java | 68 -------
.../spark/sql/api/java/types/BinaryType.java | 27 ---
.../spark/sql/api/java/types/BooleanType.java | 27 ---
.../spark/sql/api/java/types/ByteType.java | 27 ---
.../spark/sql/api/java/types/DataType.java | 190 -------------------
.../spark/sql/api/java/types/DecimalType.java | 27 ---
.../spark/sql/api/java/types/DoubleType.java | 27 ---
.../spark/sql/api/java/types/FloatType.java | 27 ---
.../spark/sql/api/java/types/IntegerType.java | 27 ---
.../spark/sql/api/java/types/LongType.java | 27 ---
.../spark/sql/api/java/types/MapType.java | 78 --------
.../spark/sql/api/java/types/ShortType.java | 27 ---
.../spark/sql/api/java/types/StringType.java | 27 ---
.../spark/sql/api/java/types/StructField.java | 76 --------
.../spark/sql/api/java/types/StructType.java | 59 ------
.../spark/sql/api/java/types/TimestampType.java | 27 ---
.../spark/sql/api/java/types/package-info.java | 22 ---
.../spark/sql/api/java/JavaSQLContext.scala | 60 +++---
.../spark/sql/api/java/JavaSchemaRDD.scala | 1 -
.../org/apache/spark/sql/package-info.java | 21 --
.../sql/types/util/DataTypeConversions.scala | 30 +--
.../sql/api/java/JavaApplySchemaSuite.java | 3 -
.../java/JavaSideDataTypeConversionSuite.java | 2 -
.../java/ScalaSideDataTypeConversionSuite.scala | 59 +++---
41 files changed, 872 insertions(+), 882 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/ArrayType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/ArrayType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/ArrayType.java
new file mode 100644
index 0000000..b73a371
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/ArrayType.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing Lists.
+ * An ArrayType object comprises two fields, {@code DataType elementType} and
+ * {@code boolean containsNull}. The field of {@code elementType} is used to specify the type of
+ * array elements. The field of {@code containsNull} is used to specify if the array has
+ * {@code null} values.
+ *
+ * To create an {@link ArrayType},
+ * {@link DataType#createArrayType(DataType)} or
+ * {@link DataType#createArrayType(DataType, boolean)}
+ * should be used.
+ */
+public class ArrayType extends DataType {
+ private DataType elementType;
+ private boolean containsNull;
+
+ protected ArrayType(DataType elementType, boolean containsNull) {
+ this.elementType = elementType;
+ this.containsNull = containsNull;
+ }
+
+ public DataType getElementType() {
+ return elementType;
+ }
+
+ public boolean isContainsNull() {
+ return containsNull;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ ArrayType arrayType = (ArrayType) o;
+
+ if (containsNull != arrayType.containsNull) return false;
+ if (!elementType.equals(arrayType.elementType)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = elementType.hashCode();
+ result = 31 * result + (containsNull ? 1 : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/BinaryType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/BinaryType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/BinaryType.java
new file mode 100644
index 0000000..7daad60
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/BinaryType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing byte[] values.
+ *
+ * {@code BinaryType} is represented by the singleton object {@link DataType#BinaryType}.
+ */
+public class BinaryType extends DataType {
+ protected BinaryType() {}
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/BooleanType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/BooleanType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/BooleanType.java
new file mode 100644
index 0000000..5a1f527
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/BooleanType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing boolean and Boolean values.
+ *
+ * {@code BooleanType} is represented by the singleton object {@link DataType#BooleanType}.
+ */
+public class BooleanType extends DataType {
+ protected BooleanType() {}
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/ByteType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/ByteType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/ByteType.java
new file mode 100644
index 0000000..e5cdf06
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/ByteType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing byte and Byte values.
+ *
+ * {@code ByteType} is represented by the singleton object {@link DataType#ByteType}.
+ */
+public class ByteType extends DataType {
+ protected ByteType() {}
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/DataType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/DataType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/DataType.java
new file mode 100644
index 0000000..3eccdde
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/DataType.java
@@ -0,0 +1,190 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+import java.util.HashSet;
+import java.util.List;
+import java.util.Set;
+
+/**
+ * The base type of all Spark SQL data types.
+ *
+ * To get/create specific data type, users should use singleton objects and factory methods
+ * provided by this class.
+ */
+public abstract class DataType {
+
+ /**
+ * Gets the StringType object.
+ */
+ public static final StringType StringType = new StringType();
+
+ /**
+ * Gets the BinaryType object.
+ */
+ public static final BinaryType BinaryType = new BinaryType();
+
+ /**
+ * Gets the BooleanType object.
+ */
+ public static final BooleanType BooleanType = new BooleanType();
+
+ /**
+ * Gets the TimestampType object.
+ */
+ public static final TimestampType TimestampType = new TimestampType();
+
+ /**
+ * Gets the DecimalType object.
+ */
+ public static final DecimalType DecimalType = new DecimalType();
+
+ /**
+ * Gets the DoubleType object.
+ */
+ public static final DoubleType DoubleType = new DoubleType();
+
+ /**
+ * Gets the FloatType object.
+ */
+ public static final FloatType FloatType = new FloatType();
+
+ /**
+ * Gets the ByteType object.
+ */
+ public static final ByteType ByteType = new ByteType();
+
+ /**
+ * Gets the IntegerType object.
+ */
+ public static final IntegerType IntegerType = new IntegerType();
+
+ /**
+ * Gets the LongType object.
+ */
+ public static final LongType LongType = new LongType();
+
+ /**
+ * Gets the ShortType object.
+ */
+ public static final ShortType ShortType = new ShortType();
+
+ /**
+ * Creates an ArrayType by specifying the data type of elements ({@code elementType}).
+ * The field of {@code containsNull} is set to {@code false}.
+ */
+ public static ArrayType createArrayType(DataType elementType) {
+ if (elementType == null) {
+ throw new IllegalArgumentException("elementType should not be null.");
+ }
+
+ return new ArrayType(elementType, false);
+ }
+
+ /**
+ * Creates an ArrayType by specifying the data type of elements ({@code elementType}) and
+ * whether the array contains null values ({@code containsNull}).
+ */
+ public static ArrayType createArrayType(DataType elementType, boolean containsNull) {
+ if (elementType == null) {
+ throw new IllegalArgumentException("elementType should not be null.");
+ }
+
+ return new ArrayType(elementType, containsNull);
+ }
+
+ /**
+ * Creates a MapType by specifying the data type of keys ({@code keyType}) and values
+ * ({@code keyType}). The field of {@code valueContainsNull} is set to {@code true}.
+ */
+ public static MapType createMapType(DataType keyType, DataType valueType) {
+ if (keyType == null) {
+ throw new IllegalArgumentException("keyType should not be null.");
+ }
+ if (valueType == null) {
+ throw new IllegalArgumentException("valueType should not be null.");
+ }
+
+ return new MapType(keyType, valueType, true);
+ }
+
+ /**
+ * Creates a MapType by specifying the data type of keys ({@code keyType}), the data type of
+ * values ({@code keyType}), and whether values contain any null value
+ * ({@code valueContainsNull}).
+ */
+ public static MapType createMapType(
+ DataType keyType,
+ DataType valueType,
+ boolean valueContainsNull) {
+ if (keyType == null) {
+ throw new IllegalArgumentException("keyType should not be null.");
+ }
+ if (valueType == null) {
+ throw new IllegalArgumentException("valueType should not be null.");
+ }
+
+ return new MapType(keyType, valueType, valueContainsNull);
+ }
+
+ /**
+ * Creates a StructField by specifying the name ({@code name}), data type ({@code dataType}) and
+ * whether values of this field can be null values ({@code nullable}).
+ */
+ public static StructField createStructField(String name, DataType dataType, boolean nullable) {
+ if (name == null) {
+ throw new IllegalArgumentException("name should not be null.");
+ }
+ if (dataType == null) {
+ throw new IllegalArgumentException("dataType should not be null.");
+ }
+
+ return new StructField(name, dataType, nullable);
+ }
+
+ /**
+ * Creates a StructType with the given list of StructFields ({@code fields}).
+ */
+ public static StructType createStructType(List<StructField> fields) {
+ return createStructType(fields.toArray(new StructField[0]));
+ }
+
+ /**
+ * Creates a StructType with the given StructField array ({@code fields}).
+ */
+ public static StructType createStructType(StructField[] fields) {
+ if (fields == null) {
+ throw new IllegalArgumentException("fields should not be null.");
+ }
+ Set<String> distinctNames = new HashSet<String>();
+ for (StructField field: fields) {
+ if (field == null) {
+ throw new IllegalArgumentException(
+ "fields should not contain any null.");
+ }
+
+ distinctNames.add(field.getName());
+ }
+ if (distinctNames.size() != fields.length) {
+ throw new IllegalArgumentException("fields should have distinct names.");
+ }
+
+ return new StructType(fields);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/DecimalType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/DecimalType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/DecimalType.java
new file mode 100644
index 0000000..bc54c07
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/DecimalType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing java.math.BigDecimal values.
+ *
+ * {@code DecimalType} is represented by the singleton object {@link DataType#DecimalType}.
+ */
+public class DecimalType extends DataType {
+ protected DecimalType() {}
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/DoubleType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/DoubleType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/DoubleType.java
new file mode 100644
index 0000000..f0060d0
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/DoubleType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing double and Double values.
+ *
+ * {@code DoubleType} is represented by the singleton object {@link DataType#DoubleType}.
+ */
+public class DoubleType extends DataType {
+ protected DoubleType() {}
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/FloatType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/FloatType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/FloatType.java
new file mode 100644
index 0000000..4a6a37f
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/FloatType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing float and Float values.
+ *
+ * {@code FloatType} is represented by the singleton object {@link DataType#FloatType}.
+ */
+public class FloatType extends DataType {
+ protected FloatType() {}
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/IntegerType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/IntegerType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/IntegerType.java
new file mode 100644
index 0000000..bfd7049
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/IntegerType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing int and Integer values.
+ *
+ * {@code IntegerType} is represented by the singleton object {@link DataType#IntegerType}.
+ */
+public class IntegerType extends DataType {
+ protected IntegerType() {}
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/LongType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/LongType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/LongType.java
new file mode 100644
index 0000000..af13a46
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/LongType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing long and Long values.
+ *
+ * {@code LongType} is represented by the singleton object {@link DataType#LongType}.
+ */
+public class LongType extends DataType {
+ protected LongType() {}
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/MapType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/MapType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/MapType.java
new file mode 100644
index 0000000..063e6b3
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/MapType.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing Maps. A MapType object comprises two fields,
+ * {@code DataType keyType}, {@code DataType valueType}, and {@code boolean valueContainsNull}.
+ * The field of {@code keyType} is used to specify the type of keys in the map.
+ * The field of {@code valueType} is used to specify the type of values in the map.
+ * The field of {@code valueContainsNull} is used to specify if map values have
+ * {@code null} values.
+ * For values of a MapType column, keys are not allowed to have {@code null} values.
+ *
+ * To create a {@link MapType},
+ * {@link DataType#createMapType(DataType, DataType)} or
+ * {@link DataType#createMapType(DataType, DataType, boolean)}
+ * should be used.
+ */
+public class MapType extends DataType {
+ private DataType keyType;
+ private DataType valueType;
+ private boolean valueContainsNull;
+
+ protected MapType(DataType keyType, DataType valueType, boolean valueContainsNull) {
+ this.keyType = keyType;
+ this.valueType = valueType;
+ this.valueContainsNull = valueContainsNull;
+ }
+
+ public DataType getKeyType() {
+ return keyType;
+ }
+
+ public DataType getValueType() {
+ return valueType;
+ }
+
+ public boolean isValueContainsNull() {
+ return valueContainsNull;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ MapType mapType = (MapType) o;
+
+ if (valueContainsNull != mapType.valueContainsNull) return false;
+ if (!keyType.equals(mapType.keyType)) return false;
+ if (!valueType.equals(mapType.valueType)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = keyType.hashCode();
+ result = 31 * result + valueType.hashCode();
+ result = 31 * result + (valueContainsNull ? 1 : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/ShortType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/ShortType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/ShortType.java
new file mode 100644
index 0000000..7d7604b
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/ShortType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing short and Short values.
+ *
+ * {@code ShortType} is represented by the singleton object {@link DataType#ShortType}.
+ */
+public class ShortType extends DataType {
+ protected ShortType() {}
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/StringType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/StringType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/StringType.java
new file mode 100644
index 0000000..f4ba0c0
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/StringType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing String values.
+ *
+ * {@code StringType} is represented by the singleton object {@link DataType#StringType}.
+ */
+public class StringType extends DataType {
+ protected StringType() {}
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/StructField.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/StructField.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/StructField.java
new file mode 100644
index 0000000..b48e2a2
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/StructField.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * A StructField object represents a field in a StructType object.
+ * A StructField object comprises three fields, {@code String name}, {@code DataType dataType},
+ * and {@code boolean nullable}. The field of {@code name} is the name of a StructField.
+ * The field of {@code dataType} specifies the data type of a StructField.
+ * The field of {@code nullable} specifies if values of a StructField can contain {@code null}
+ * values.
+ *
+ * To create a {@link StructField},
+ * {@link DataType#createStructField(String, DataType, boolean)}
+ * should be used.
+ */
+public class StructField {
+ private String name;
+ private DataType dataType;
+ private boolean nullable;
+
+ protected StructField(String name, DataType dataType, boolean nullable) {
+ this.name = name;
+ this.dataType = dataType;
+ this.nullable = nullable;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public DataType getDataType() {
+ return dataType;
+ }
+
+ public boolean isNullable() {
+ return nullable;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ StructField that = (StructField) o;
+
+ if (nullable != that.nullable) return false;
+ if (!dataType.equals(that.dataType)) return false;
+ if (!name.equals(that.name)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ int result = name.hashCode();
+ result = 31 * result + dataType.hashCode();
+ result = 31 * result + (nullable ? 1 : 0);
+ return result;
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/StructType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/StructType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/StructType.java
new file mode 100644
index 0000000..a4b501e
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/StructType.java
@@ -0,0 +1,58 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+import java.util.Arrays;
+
+/**
+ * The data type representing Rows.
+ * A StructType object comprises an array of StructFields.
+ *
+ * To create an {@link StructType},
+ * {@link DataType#createStructType(java.util.List)} or
+ * {@link DataType#createStructType(StructField[])}
+ * should be used.
+ */
+public class StructType extends DataType {
+ private StructField[] fields;
+
+ protected StructType(StructField[] fields) {
+ this.fields = fields;
+ }
+
+ public StructField[] getFields() {
+ return fields;
+ }
+
+ @Override
+ public boolean equals(Object o) {
+ if (this == o) return true;
+ if (o == null || getClass() != o.getClass()) return false;
+
+ StructType that = (StructType) o;
+
+ if (!Arrays.equals(fields, that.fields)) return false;
+
+ return true;
+ }
+
+ @Override
+ public int hashCode() {
+ return Arrays.hashCode(fields);
+ }
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/TimestampType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/TimestampType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/TimestampType.java
new file mode 100644
index 0000000..06d44c7
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/TimestampType.java
@@ -0,0 +1,27 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.spark.sql.api.java;
+
+/**
+ * The data type representing java.sql.Timestamp values.
+ *
+ * {@code TimestampType} is represented by the singleton object {@link DataType#TimestampType}.
+ */
+public class TimestampType extends DataType {
+ protected TimestampType() {}
+}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/package-info.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/package-info.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/package-info.java
new file mode 100644
index 0000000..67007a9
--- /dev/null
+++ b/sql/core/src/main/java/org/apache/spark/sql/api/java/package-info.java
@@ -0,0 +1,21 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+/**
+ * Allows the execution of relational queries, including those expressed in SQL using Spark.
+ */
+package org.apache.spark.sql.api.java;
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/ArrayType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/ArrayType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/ArrayType.java
deleted file mode 100644
index 17334ca..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/ArrayType.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * The data type representing Lists.
- * An ArrayType object comprises two fields, {@code DataType elementType} and
- * {@code boolean containsNull}. The field of {@code elementType} is used to specify the type of
- * array elements. The field of {@code containsNull} is used to specify if the array has
- * {@code null} values.
- *
- * To create an {@link ArrayType},
- * {@link org.apache.spark.sql.api.java.types.DataType#createArrayType(DataType)} or
- * {@link org.apache.spark.sql.api.java.types.DataType#createArrayType(DataType, boolean)}
- * should be used.
- */
-public class ArrayType extends DataType {
- private DataType elementType;
- private boolean containsNull;
-
- protected ArrayType(DataType elementType, boolean containsNull) {
- this.elementType = elementType;
- this.containsNull = containsNull;
- }
-
- public DataType getElementType() {
- return elementType;
- }
-
- public boolean isContainsNull() {
- return containsNull;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- ArrayType arrayType = (ArrayType) o;
-
- if (containsNull != arrayType.containsNull) return false;
- if (!elementType.equals(arrayType.elementType)) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = elementType.hashCode();
- result = 31 * result + (containsNull ? 1 : 0);
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/BinaryType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/BinaryType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/BinaryType.java
deleted file mode 100644
index 6170317..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/BinaryType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * The data type representing byte[] values.
- *
- * {@code BinaryType} is represented by the singleton object {@link DataType#BinaryType}.
- */
-public class BinaryType extends DataType {
- protected BinaryType() {}
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/BooleanType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/BooleanType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/BooleanType.java
deleted file mode 100644
index 8fa24d8..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/BooleanType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * The data type representing boolean and Boolean values.
- *
- * {@code BooleanType} is represented by the singleton object {@link DataType#BooleanType}.
- */
-public class BooleanType extends DataType {
- protected BooleanType() {}
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/ByteType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/ByteType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/ByteType.java
deleted file mode 100644
index 2de3297..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/ByteType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * The data type representing byte and Byte values.
- *
- * {@code ByteType} is represented by the singleton object {@link DataType#ByteType}.
- */
-public class ByteType extends DataType {
- protected ByteType() {}
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/DataType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/DataType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/DataType.java
deleted file mode 100644
index f84e5a4..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/DataType.java
+++ /dev/null
@@ -1,190 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-import java.util.HashSet;
-import java.util.List;
-import java.util.Set;
-
-/**
- * The base type of all Spark SQL data types.
- *
- * To get/create specific data type, users should use singleton objects and factory methods
- * provided by this class.
- */
-public abstract class DataType {
-
- /**
- * Gets the StringType object.
- */
- public static final StringType StringType = new StringType();
-
- /**
- * Gets the BinaryType object.
- */
- public static final BinaryType BinaryType = new BinaryType();
-
- /**
- * Gets the BooleanType object.
- */
- public static final BooleanType BooleanType = new BooleanType();
-
- /**
- * Gets the TimestampType object.
- */
- public static final TimestampType TimestampType = new TimestampType();
-
- /**
- * Gets the DecimalType object.
- */
- public static final DecimalType DecimalType = new DecimalType();
-
- /**
- * Gets the DoubleType object.
- */
- public static final DoubleType DoubleType = new DoubleType();
-
- /**
- * Gets the FloatType object.
- */
- public static final FloatType FloatType = new FloatType();
-
- /**
- * Gets the ByteType object.
- */
- public static final ByteType ByteType = new ByteType();
-
- /**
- * Gets the IntegerType object.
- */
- public static final IntegerType IntegerType = new IntegerType();
-
- /**
- * Gets the LongType object.
- */
- public static final LongType LongType = new LongType();
-
- /**
- * Gets the ShortType object.
- */
- public static final ShortType ShortType = new ShortType();
-
- /**
- * Creates an ArrayType by specifying the data type of elements ({@code elementType}).
- * The field of {@code containsNull} is set to {@code false}.
- */
- public static ArrayType createArrayType(DataType elementType) {
- if (elementType == null) {
- throw new IllegalArgumentException("elementType should not be null.");
- }
-
- return new ArrayType(elementType, false);
- }
-
- /**
- * Creates an ArrayType by specifying the data type of elements ({@code elementType}) and
- * whether the array contains null values ({@code containsNull}).
- */
- public static ArrayType createArrayType(DataType elementType, boolean containsNull) {
- if (elementType == null) {
- throw new IllegalArgumentException("elementType should not be null.");
- }
-
- return new ArrayType(elementType, containsNull);
- }
-
- /**
- * Creates a MapType by specifying the data type of keys ({@code keyType}) and values
- * ({@code keyType}). The field of {@code valueContainsNull} is set to {@code true}.
- */
- public static MapType createMapType(DataType keyType, DataType valueType) {
- if (keyType == null) {
- throw new IllegalArgumentException("keyType should not be null.");
- }
- if (valueType == null) {
- throw new IllegalArgumentException("valueType should not be null.");
- }
-
- return new MapType(keyType, valueType, true);
- }
-
- /**
- * Creates a MapType by specifying the data type of keys ({@code keyType}), the data type of
- * values ({@code keyType}), and whether values contain any null value
- * ({@code valueContainsNull}).
- */
- public static MapType createMapType(
- DataType keyType,
- DataType valueType,
- boolean valueContainsNull) {
- if (keyType == null) {
- throw new IllegalArgumentException("keyType should not be null.");
- }
- if (valueType == null) {
- throw new IllegalArgumentException("valueType should not be null.");
- }
-
- return new MapType(keyType, valueType, valueContainsNull);
- }
-
- /**
- * Creates a StructField by specifying the name ({@code name}), data type ({@code dataType}) and
- * whether values of this field can be null values ({@code nullable}).
- */
- public static StructField createStructField(String name, DataType dataType, boolean nullable) {
- if (name == null) {
- throw new IllegalArgumentException("name should not be null.");
- }
- if (dataType == null) {
- throw new IllegalArgumentException("dataType should not be null.");
- }
-
- return new StructField(name, dataType, nullable);
- }
-
- /**
- * Creates a StructType with the given list of StructFields ({@code fields}).
- */
- public static StructType createStructType(List<StructField> fields) {
- return createStructType(fields.toArray(new StructField[0]));
- }
-
- /**
- * Creates a StructType with the given StructField array ({@code fields}).
- */
- public static StructType createStructType(StructField[] fields) {
- if (fields == null) {
- throw new IllegalArgumentException("fields should not be null.");
- }
- Set<String> distinctNames = new HashSet<String>();
- for (StructField field: fields) {
- if (field == null) {
- throw new IllegalArgumentException(
- "fields should not contain any null.");
- }
-
- distinctNames.add(field.getName());
- }
- if (distinctNames.size() != fields.length) {
- throw new IllegalArgumentException("fields should have distinct names.");
- }
-
- return new StructType(fields);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/DecimalType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/DecimalType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/DecimalType.java
deleted file mode 100644
index 9250491..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/DecimalType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * The data type representing java.math.BigDecimal values.
- *
- * {@code DecimalType} is represented by the singleton object {@link DataType#DecimalType}.
- */
-public class DecimalType extends DataType {
- protected DecimalType() {}
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/DoubleType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/DoubleType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/DoubleType.java
deleted file mode 100644
index 3e86917..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/DoubleType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * The data type representing double and Double values.
- *
- * {@code DoubleType} is represented by the singleton object {@link DataType#DoubleType}.
- */
-public class DoubleType extends DataType {
- protected DoubleType() {}
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/FloatType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/FloatType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/FloatType.java
deleted file mode 100644
index fa860d4..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/FloatType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * The data type representing float and Float values.
- *
- * {@code FloatType} is represented by the singleton object {@link DataType#FloatType}.
- */
-public class FloatType extends DataType {
- protected FloatType() {}
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/IntegerType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/IntegerType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/IntegerType.java
deleted file mode 100644
index bd973ec..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/IntegerType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * The data type representing int and Integer values.
- *
- * {@code IntegerType} is represented by the singleton object {@link DataType#IntegerType}.
- */
-public class IntegerType extends DataType {
- protected IntegerType() {}
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/LongType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/LongType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/LongType.java
deleted file mode 100644
index e002333..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/LongType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * The data type representing long and Long values.
- *
- * {@code LongType} is represented by the singleton object {@link DataType#LongType}.
- */
-public class LongType extends DataType {
- protected LongType() {}
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/MapType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/MapType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/MapType.java
deleted file mode 100644
index 94936e2..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/MapType.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * The data type representing Maps. A MapType object comprises two fields,
- * {@code DataType keyType}, {@code DataType valueType}, and {@code boolean valueContainsNull}.
- * The field of {@code keyType} is used to specify the type of keys in the map.
- * The field of {@code valueType} is used to specify the type of values in the map.
- * The field of {@code valueContainsNull} is used to specify if map values have
- * {@code null} values.
- * For values of a MapType column, keys are not allowed to have {@code null} values.
- *
- * To create a {@link MapType},
- * {@link org.apache.spark.sql.api.java.types.DataType#createMapType(DataType, DataType)} or
- * {@link org.apache.spark.sql.api.java.types.DataType#createMapType(DataType, DataType, boolean)}
- * should be used.
- */
-public class MapType extends DataType {
- private DataType keyType;
- private DataType valueType;
- private boolean valueContainsNull;
-
- protected MapType(DataType keyType, DataType valueType, boolean valueContainsNull) {
- this.keyType = keyType;
- this.valueType = valueType;
- this.valueContainsNull = valueContainsNull;
- }
-
- public DataType getKeyType() {
- return keyType;
- }
-
- public DataType getValueType() {
- return valueType;
- }
-
- public boolean isValueContainsNull() {
- return valueContainsNull;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- MapType mapType = (MapType) o;
-
- if (valueContainsNull != mapType.valueContainsNull) return false;
- if (!keyType.equals(mapType.keyType)) return false;
- if (!valueType.equals(mapType.valueType)) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = keyType.hashCode();
- result = 31 * result + valueType.hashCode();
- result = 31 * result + (valueContainsNull ? 1 : 0);
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/ShortType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/ShortType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/ShortType.java
deleted file mode 100644
index 98f9507..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/ShortType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * The data type representing short and Short values.
- *
- * {@code ShortType} is represented by the singleton object {@link DataType#ShortType}.
- */
-public class ShortType extends DataType {
- protected ShortType() {}
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/StringType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/StringType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/StringType.java
deleted file mode 100644
index b8e7dbe..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/StringType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * The data type representing String values.
- *
- * {@code StringType} is represented by the singleton object {@link DataType#StringType}.
- */
-public class StringType extends DataType {
- protected StringType() {}
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/StructField.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/StructField.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/StructField.java
deleted file mode 100644
index 54e9c11..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/StructField.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * A StructField object represents a field in a StructType object.
- * A StructField object comprises three fields, {@code String name}, {@code DataType dataType},
- * and {@code boolean nullable}. The field of {@code name} is the name of a StructField.
- * The field of {@code dataType} specifies the data type of a StructField.
- * The field of {@code nullable} specifies if values of a StructField can contain {@code null}
- * values.
- *
- * To create a {@link StructField},
- * {@link org.apache.spark.sql.api.java.types.DataType#createStructField(String, DataType, boolean)}
- * should be used.
- */
-public class StructField {
- private String name;
- private DataType dataType;
- private boolean nullable;
-
- protected StructField(String name, DataType dataType, boolean nullable) {
- this.name = name;
- this.dataType = dataType;
- this.nullable = nullable;
- }
-
- public String getName() {
- return name;
- }
-
- public DataType getDataType() {
- return dataType;
- }
-
- public boolean isNullable() {
- return nullable;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- StructField that = (StructField) o;
-
- if (nullable != that.nullable) return false;
- if (!dataType.equals(that.dataType)) return false;
- if (!name.equals(that.name)) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- int result = name.hashCode();
- result = 31 * result + dataType.hashCode();
- result = 31 * result + (nullable ? 1 : 0);
- return result;
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/StructType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/StructType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/StructType.java
deleted file mode 100644
index 33a42f4..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/StructType.java
+++ /dev/null
@@ -1,59 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-import java.util.Arrays;
-import java.util.List;
-
-/**
- * The data type representing Rows.
- * A StructType object comprises an array of StructFields.
- *
- * To create an {@link StructType},
- * {@link org.apache.spark.sql.api.java.types.DataType#createStructType(java.util.List)} or
- * {@link org.apache.spark.sql.api.java.types.DataType#createStructType(StructField[])}
- * should be used.
- */
-public class StructType extends DataType {
- private StructField[] fields;
-
- protected StructType(StructField[] fields) {
- this.fields = fields;
- }
-
- public StructField[] getFields() {
- return fields;
- }
-
- @Override
- public boolean equals(Object o) {
- if (this == o) return true;
- if (o == null || getClass() != o.getClass()) return false;
-
- StructType that = (StructType) o;
-
- if (!Arrays.equals(fields, that.fields)) return false;
-
- return true;
- }
-
- @Override
- public int hashCode() {
- return Arrays.hashCode(fields);
- }
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/TimestampType.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/TimestampType.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/TimestampType.java
deleted file mode 100644
index 6529577..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/TimestampType.java
+++ /dev/null
@@ -1,27 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.spark.sql.api.java.types;
-
-/**
- * The data type representing java.sql.Timestamp values.
- *
- * {@code TimestampType} is represented by the singleton object {@link DataType#TimestampType}.
- */
-public class TimestampType extends DataType {
- protected TimestampType() {}
-}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/java/org/apache/spark/sql/api/java/types/package-info.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/package-info.java b/sql/core/src/main/java/org/apache/spark/sql/api/java/types/package-info.java
deleted file mode 100644
index f169ac6..0000000
--- a/sql/core/src/main/java/org/apache/spark/sql/api/java/types/package-info.java
+++ /dev/null
@@ -1,22 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-
-/**
- * Allows users to get and create Spark SQL data types.
- */
-package org.apache.spark.sql.api.java.types;
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/scala/org/apache/spark/sql/api/java/JavaSQLContext.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/java/JavaSQLContext.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/java/JavaSQLContext.scala
index c1c18a0..809dd03 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/java/JavaSQLContext.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/java/JavaSQLContext.scala
@@ -23,9 +23,8 @@ import org.apache.hadoop.conf.Configuration
import org.apache.spark.annotation.{DeveloperApi, Experimental}
import org.apache.spark.api.java.{JavaRDD, JavaSparkContext}
-import org.apache.spark.sql.api.java.types.{StructType => JStructType}
import org.apache.spark.sql.json.JsonRDD
-import org.apache.spark.sql._
+import org.apache.spark.sql.{SQLContext, StructType => SStructType}
import org.apache.spark.sql.catalyst.expressions.{AttributeReference, GenericRow, Row => ScalaRow}
import org.apache.spark.sql.parquet.ParquetRelation
import org.apache.spark.sql.execution.{ExistingRdd, SparkLogicalPlan}
@@ -104,9 +103,9 @@ class JavaSQLContext(val sqlContext: SQLContext) {
* provided schema. Otherwise, there will be runtime exception.
*/
@DeveloperApi
- def applySchema(rowRDD: JavaRDD[Row], schema: JStructType): JavaSchemaRDD = {
+ def applySchema(rowRDD: JavaRDD[Row], schema: StructType): JavaSchemaRDD = {
val scalaRowRDD = rowRDD.rdd.map(r => r.row)
- val scalaSchema = asScalaDataType(schema).asInstanceOf[StructType]
+ val scalaSchema = asScalaDataType(schema).asInstanceOf[SStructType]
val logicalPlan =
SparkLogicalPlan(ExistingRdd(scalaSchema.toAttributes, scalaRowRDD))(sqlContext)
new JavaSchemaRDD(sqlContext, logicalPlan)
@@ -133,7 +132,7 @@ class JavaSQLContext(val sqlContext: SQLContext) {
* returning the result as a JavaSchemaRDD.
*/
@Experimental
- def jsonFile(path: String, schema: JStructType): JavaSchemaRDD =
+ def jsonFile(path: String, schema: StructType): JavaSchemaRDD =
jsonRDD(sqlContext.sparkContext.textFile(path), schema)
/**
@@ -155,10 +154,10 @@ class JavaSQLContext(val sqlContext: SQLContext) {
* returning the result as a JavaSchemaRDD.
*/
@Experimental
- def jsonRDD(json: JavaRDD[String], schema: JStructType): JavaSchemaRDD = {
+ def jsonRDD(json: JavaRDD[String], schema: StructType): JavaSchemaRDD = {
val appliedScalaSchema =
Option(asScalaDataType(schema)).getOrElse(
- JsonRDD.nullTypeToStringType(JsonRDD.inferSchema(json.rdd, 1.0))).asInstanceOf[StructType]
+ JsonRDD.nullTypeToStringType(JsonRDD.inferSchema(json.rdd, 1.0))).asInstanceOf[SStructType]
val scalaRowRDD = JsonRDD.jsonStringToRow(json.rdd, appliedScalaSchema)
val logicalPlan =
SparkLogicalPlan(ExistingRdd(appliedScalaSchema.toAttributes, scalaRowRDD))(sqlContext)
@@ -181,22 +180,37 @@ class JavaSQLContext(val sqlContext: SQLContext) {
val fields = beanInfo.getPropertyDescriptors.filterNot(_.getName == "class")
fields.map { property =>
val (dataType, nullable) = property.getPropertyType match {
- case c: Class[_] if c == classOf[java.lang.String] => (StringType, true)
- case c: Class[_] if c == java.lang.Short.TYPE => (ShortType, false)
- case c: Class[_] if c == java.lang.Integer.TYPE => (IntegerType, false)
- case c: Class[_] if c == java.lang.Long.TYPE => (LongType, false)
- case c: Class[_] if c == java.lang.Double.TYPE => (DoubleType, false)
- case c: Class[_] if c == java.lang.Byte.TYPE => (ByteType, false)
- case c: Class[_] if c == java.lang.Float.TYPE => (FloatType, false)
- case c: Class[_] if c == java.lang.Boolean.TYPE => (BooleanType, false)
-
- case c: Class[_] if c == classOf[java.lang.Short] => (ShortType, true)
- case c: Class[_] if c == classOf[java.lang.Integer] => (IntegerType, true)
- case c: Class[_] if c == classOf[java.lang.Long] => (LongType, true)
- case c: Class[_] if c == classOf[java.lang.Double] => (DoubleType, true)
- case c: Class[_] if c == classOf[java.lang.Byte] => (ByteType, true)
- case c: Class[_] if c == classOf[java.lang.Float] => (FloatType, true)
- case c: Class[_] if c == classOf[java.lang.Boolean] => (BooleanType, true)
+ case c: Class[_] if c == classOf[java.lang.String] =>
+ (org.apache.spark.sql.StringType, true)
+ case c: Class[_] if c == java.lang.Short.TYPE =>
+ (org.apache.spark.sql.ShortType, false)
+ case c: Class[_] if c == java.lang.Integer.TYPE =>
+ (org.apache.spark.sql.IntegerType, false)
+ case c: Class[_] if c == java.lang.Long.TYPE =>
+ (org.apache.spark.sql.LongType, false)
+ case c: Class[_] if c == java.lang.Double.TYPE =>
+ (org.apache.spark.sql.DoubleType, false)
+ case c: Class[_] if c == java.lang.Byte.TYPE =>
+ (org.apache.spark.sql.ByteType, false)
+ case c: Class[_] if c == java.lang.Float.TYPE =>
+ (org.apache.spark.sql.FloatType, false)
+ case c: Class[_] if c == java.lang.Boolean.TYPE =>
+ (org.apache.spark.sql.BooleanType, false)
+
+ case c: Class[_] if c == classOf[java.lang.Short] =>
+ (org.apache.spark.sql.ShortType, true)
+ case c: Class[_] if c == classOf[java.lang.Integer] =>
+ (org.apache.spark.sql.IntegerType, true)
+ case c: Class[_] if c == classOf[java.lang.Long] =>
+ (org.apache.spark.sql.LongType, true)
+ case c: Class[_] if c == classOf[java.lang.Double] =>
+ (org.apache.spark.sql.DoubleType, true)
+ case c: Class[_] if c == classOf[java.lang.Byte] =>
+ (org.apache.spark.sql.ByteType, true)
+ case c: Class[_] if c == classOf[java.lang.Float] =>
+ (org.apache.spark.sql.FloatType, true)
+ case c: Class[_] if c == classOf[java.lang.Boolean] =>
+ (org.apache.spark.sql.BooleanType, true)
}
AttributeReference(property.getName, dataType, nullable)()
}
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/scala/org/apache/spark/sql/api/java/JavaSchemaRDD.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/api/java/JavaSchemaRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/api/java/JavaSchemaRDD.scala
index 8245741..4d799b4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/api/java/JavaSchemaRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/api/java/JavaSchemaRDD.scala
@@ -22,7 +22,6 @@ import java.util.{List => JList}
import org.apache.spark.Partitioner
import org.apache.spark.api.java.{JavaRDDLike, JavaRDD}
import org.apache.spark.api.java.function.{Function => JFunction}
-import org.apache.spark.sql.api.java.types.StructType
import org.apache.spark.sql.types.util.DataTypeConversions
import org.apache.spark.sql.{SQLContext, SchemaRDD, SchemaRDDLike}
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
http://git-wip-us.apache.org/repos/asf/spark/blob/c41fdf04/sql/core/src/main/scala/org/apache/spark/sql/package-info.java
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/package-info.java b/sql/core/src/main/scala/org/apache/spark/sql/package-info.java
deleted file mode 100644
index 5360361..0000000
--- a/sql/core/src/main/scala/org/apache/spark/sql/package-info.java
+++ /dev/null
@@ -1,21 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one or more
- * contributor license agreements. See the NOTICE file distributed with
- * this work for additional information regarding copyright ownership.
- * The ASF licenses this file to You under the Apache License, Version 2.0
- * (the "License"); you may not use this file except in compliance with
- * the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-/**
- * Allows the execution of relational queries, including those expressed in SQL using Spark.
- */
-package org.apache.spark.sql;
\ No newline at end of file