You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/04/23 10:43:48 UTC
spark git commit: [SPARK-7069][SQL] Rename NativeType -> AtomicType.
Repository: spark
Updated Branches:
refs/heads/master 29163c520 -> f60bece14
[SPARK-7069][SQL] Rename NativeType -> AtomicType.
Also renamed JvmType to InternalType.
Author: Reynold Xin <rx...@databricks.com>
Closes #5651 from rxin/native-to-atomic-type and squashes the following commits:
cbd4028 [Reynold Xin] [SPARK-7069][SQL] Rename NativeType -> AtomicType.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/f60bece1
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/f60bece1
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/f60bece1
Branch: refs/heads/master
Commit: f60bece14f98450b4a71b00d7b58525f06e1f9ed
Parents: 29163c5
Author: Reynold Xin <rx...@databricks.com>
Authored: Thu Apr 23 01:43:40 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Thu Apr 23 01:43:40 2015 -0700
----------------------------------------------------------------------
.../spark/sql/catalyst/ScalaReflection.scala | 24 ++--
.../sql/catalyst/expressions/arithmetic.scala | 4 +-
.../expressions/codegen/CodeGenerator.scala | 18 ++-
.../codegen/GenerateProjection.scala | 4 +-
.../sql/catalyst/expressions/predicates.scala | 10 +-
.../spark/sql/catalyst/expressions/rows.scala | 6 +-
.../org/apache/spark/sql/types/dataTypes.scala | 114 +++++++++----------
.../spark/sql/columnar/ColumnAccessor.scala | 2 +-
.../spark/sql/columnar/ColumnBuilder.scala | 4 +-
.../apache/spark/sql/columnar/ColumnType.scala | 6 +-
.../CompressibleColumnAccessor.scala | 4 +-
.../compression/CompressibleColumnBuilder.scala | 4 +-
.../compression/CompressionScheme.scala | 10 +-
.../compression/compressionSchemes.scala | 42 +++----
.../org/apache/spark/sql/json/JsonRDD.scala | 6 +-
.../spark/sql/parquet/ParquetConverter.scala | 12 +-
.../spark/sql/parquet/ParquetTableSupport.scala | 2 +-
.../spark/sql/columnar/ColumnStatsSuite.scala | 6 +-
.../spark/sql/columnar/ColumnTypeSuite.scala | 8 +-
.../spark/sql/columnar/ColumnarTestUtils.scala | 6 +-
.../compression/DictionaryEncodingSuite.scala | 4 +-
.../compression/IntegralDeltaSuite.scala | 6 +-
.../compression/RunLengthEncodingSuite.scala | 4 +-
.../TestCompressibleColumnBuilder.scala | 6 +-
24 files changed, 159 insertions(+), 153 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
index d952195..c529655 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/ScalaReflection.scala
@@ -17,8 +17,6 @@
package org.apache.spark.sql.catalyst
-import java.sql.Timestamp
-
import org.apache.spark.util.Utils
import org.apache.spark.sql.catalyst.expressions._
import org.apache.spark.sql.catalyst.plans.logical.LocalRelation
@@ -110,7 +108,7 @@ trait ScalaReflection {
StructField(p.name.toString, dataType, nullable)
}), nullable = true)
case t if t <:< typeOf[String] => Schema(StringType, nullable = true)
- case t if t <:< typeOf[Timestamp] => Schema(TimestampType, nullable = true)
+ case t if t <:< typeOf[java.sql.Timestamp] => Schema(TimestampType, nullable = true)
case t if t <:< typeOf[java.sql.Date] => Schema(DateType, nullable = true)
case t if t <:< typeOf[BigDecimal] => Schema(DecimalType.Unlimited, nullable = true)
case t if t <:< typeOf[java.math.BigDecimal] => Schema(DecimalType.Unlimited, nullable = true)
@@ -136,20 +134,20 @@ trait ScalaReflection {
def typeOfObject: PartialFunction[Any, DataType] = {
// The data type can be determined without ambiguity.
- case obj: BooleanType.JvmType => BooleanType
- case obj: BinaryType.JvmType => BinaryType
+ case obj: Boolean => BooleanType
+ case obj: Array[Byte] => BinaryType
case obj: String => StringType
- case obj: StringType.JvmType => StringType
- case obj: ByteType.JvmType => ByteType
- case obj: ShortType.JvmType => ShortType
- case obj: IntegerType.JvmType => IntegerType
- case obj: LongType.JvmType => LongType
- case obj: FloatType.JvmType => FloatType
- case obj: DoubleType.JvmType => DoubleType
+ case obj: UTF8String => StringType
+ case obj: Byte => ByteType
+ case obj: Short => ShortType
+ case obj: Int => IntegerType
+ case obj: Long => LongType
+ case obj: Float => FloatType
+ case obj: Double => DoubleType
case obj: java.sql.Date => DateType
case obj: java.math.BigDecimal => DecimalType.Unlimited
case obj: Decimal => DecimalType.Unlimited
- case obj: TimestampType.JvmType => TimestampType
+ case obj: java.sql.Timestamp => TimestampType
case null => NullType
// For other cases, there is no obvious mapping from the type of the given object to a
// Catalyst data type. A user should provide his/her specific rules
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
index 566b34f..140ccd8 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/arithmetic.scala
@@ -346,7 +346,7 @@ case class MaxOf(left: Expression, right: Expression) extends Expression {
}
lazy val ordering = left.dataType match {
- case i: NativeType => i.ordering.asInstanceOf[Ordering[Any]]
+ case i: AtomicType => i.ordering.asInstanceOf[Ordering[Any]]
case other => sys.error(s"Type $other does not support ordered operations")
}
@@ -391,7 +391,7 @@ case class MinOf(left: Expression, right: Expression) extends Expression {
}
lazy val ordering = left.dataType match {
- case i: NativeType => i.ordering.asInstanceOf[Ordering[Any]]
+ case i: AtomicType => i.ordering.asInstanceOf[Ordering[Any]]
case other => sys.error(s"Type $other does not support ordered operations")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
index eeffedb..cbe5203 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/CodeGenerator.scala
@@ -623,7 +623,7 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
protected def getColumn(inputRow: TermName, dataType: DataType, ordinal: Int) = {
dataType match {
case StringType => q"$inputRow($ordinal).asInstanceOf[org.apache.spark.sql.types.UTF8String]"
- case dt @ NativeType() => q"$inputRow.${accessorForType(dt)}($ordinal)"
+ case dt: DataType if isNativeType(dt) => q"$inputRow.${accessorForType(dt)}($ordinal)"
case _ => q"$inputRow.apply($ordinal).asInstanceOf[${termForType(dataType)}]"
}
}
@@ -635,7 +635,8 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
value: TermName) = {
dataType match {
case StringType => q"$destinationRow.update($ordinal, $value)"
- case dt @ NativeType() => q"$destinationRow.${mutatorForType(dt)}($ordinal, $value)"
+ case dt: DataType if isNativeType(dt) =>
+ q"$destinationRow.${mutatorForType(dt)}($ordinal, $value)"
case _ => q"$destinationRow.update($ordinal, $value)"
}
}
@@ -675,7 +676,18 @@ abstract class CodeGenerator[InType <: AnyRef, OutType <: AnyRef] extends Loggin
}
protected def termForType(dt: DataType) = dt match {
- case n: NativeType => n.tag
+ case n: AtomicType => n.tag
case _ => typeTag[Any]
}
+
+ /**
+ * List of data types that have special accessors and setters in [[Row]].
+ */
+ protected val nativeTypes =
+ Seq(IntegerType, BooleanType, LongType, DoubleType, FloatType, ShortType, ByteType, StringType)
+
+ /**
+ * Returns true if the data type has a special accessor and setter in [[Row]].
+ */
+ protected def isNativeType(dt: DataType) = nativeTypes.contains(dt)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala
index d491bab..584f938 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/codegen/GenerateProjection.scala
@@ -109,7 +109,7 @@ object GenerateProjection extends CodeGenerator[Seq[Expression], Projection] {
q"override def update(i: Int, value: Any): Unit = { ..$cases; $accessorFailure }"
}
- val specificAccessorFunctions = NativeType.all.map { dataType =>
+ val specificAccessorFunctions = nativeTypes.map { dataType =>
val ifStatements = expressions.zipWithIndex.flatMap {
// getString() is not used by expressions
case (e, i) if e.dataType == dataType && dataType != StringType =>
@@ -135,7 +135,7 @@ object GenerateProjection extends CodeGenerator[Seq[Expression], Projection] {
}
}
- val specificMutatorFunctions = NativeType.all.map { dataType =>
+ val specificMutatorFunctions = nativeTypes.map { dataType =>
val ifStatements = expressions.zipWithIndex.flatMap {
// setString() is not used by expressions
case (e, i) if e.dataType == dataType && dataType != StringType =>
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
index 46522eb..9cb00cb 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/predicates.scala
@@ -20,7 +20,7 @@ package org.apache.spark.sql.catalyst.expressions
import org.apache.spark.sql.catalyst.analysis.UnresolvedException
import org.apache.spark.sql.catalyst.errors.TreeNodeException
import org.apache.spark.sql.catalyst.plans.logical.LogicalPlan
-import org.apache.spark.sql.types.{DataType, BinaryType, BooleanType, NativeType}
+import org.apache.spark.sql.types.{DataType, BinaryType, BooleanType, AtomicType}
object InterpretedPredicate {
def create(expression: Expression, inputSchema: Seq[Attribute]): (Row => Boolean) =
@@ -211,7 +211,7 @@ case class LessThan(left: Expression, right: Expression) extends BinaryCompariso
s"Types do not match ${left.dataType} != ${right.dataType}")
}
left.dataType match {
- case i: NativeType => i.ordering.asInstanceOf[Ordering[Any]]
+ case i: AtomicType => i.ordering.asInstanceOf[Ordering[Any]]
case other => sys.error(s"Type $other does not support ordered operations")
}
}
@@ -240,7 +240,7 @@ case class LessThanOrEqual(left: Expression, right: Expression) extends BinaryCo
s"Types do not match ${left.dataType} != ${right.dataType}")
}
left.dataType match {
- case i: NativeType => i.ordering.asInstanceOf[Ordering[Any]]
+ case i: AtomicType => i.ordering.asInstanceOf[Ordering[Any]]
case other => sys.error(s"Type $other does not support ordered operations")
}
}
@@ -269,7 +269,7 @@ case class GreaterThan(left: Expression, right: Expression) extends BinaryCompar
s"Types do not match ${left.dataType} != ${right.dataType}")
}
left.dataType match {
- case i: NativeType => i.ordering.asInstanceOf[Ordering[Any]]
+ case i: AtomicType => i.ordering.asInstanceOf[Ordering[Any]]
case other => sys.error(s"Type $other does not support ordered operations")
}
}
@@ -298,7 +298,7 @@ case class GreaterThanOrEqual(left: Expression, right: Expression) extends Binar
s"Types do not match ${left.dataType} != ${right.dataType}")
}
left.dataType match {
- case i: NativeType => i.ordering.asInstanceOf[Ordering[Any]]
+ case i: AtomicType => i.ordering.asInstanceOf[Ordering[Any]]
case other => sys.error(s"Type $other does not support ordered operations")
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/rows.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/rows.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/rows.scala
index 9813734..5fd892c 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/rows.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/catalyst/expressions/rows.scala
@@ -17,7 +17,7 @@
package org.apache.spark.sql.catalyst.expressions
-import org.apache.spark.sql.types.{UTF8String, DataType, StructType, NativeType}
+import org.apache.spark.sql.types.{UTF8String, DataType, StructType, AtomicType}
/**
* An extended interface to [[Row]] that allows the values for each column to be updated. Setting
@@ -227,9 +227,9 @@ class RowOrdering(ordering: Seq[SortOrder]) extends Ordering[Row] {
return if (order.direction == Ascending) 1 else -1
} else {
val comparison = order.dataType match {
- case n: NativeType if order.direction == Ascending =>
+ case n: AtomicType if order.direction == Ascending =>
n.ordering.asInstanceOf[Ordering[Any]].compare(left, right)
- case n: NativeType if order.direction == Descending =>
+ case n: AtomicType if order.direction == Descending =>
n.ordering.asInstanceOf[Ordering[Any]].reverse.compare(left, right)
case other => sys.error(s"Type $other does not support ordered operations")
}
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/catalyst/src/main/scala/org/apache/spark/sql/types/dataTypes.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/dataTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/dataTypes.scala
index 42e26e0..87c7b75 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/dataTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/dataTypes.scala
@@ -42,7 +42,8 @@ object DataType {
def fromJson(json: String): DataType = parseDataType(parse(json))
private val nonDecimalNameToType = {
- (Seq(NullType, DateType, TimestampType, BinaryType) ++ NativeType.all)
+ Seq(NullType, DateType, TimestampType, BinaryType,
+ IntegerType, BooleanType, LongType, DoubleType, FloatType, ShortType, ByteType, StringType)
.map(t => t.typeName -> t).toMap
}
@@ -309,22 +310,17 @@ class NullType private() extends DataType {
case object NullType extends NullType
-protected[sql] object NativeType {
- val all = Seq(
- IntegerType, BooleanType, LongType, DoubleType, FloatType, ShortType, ByteType, StringType)
-
- def unapply(dt: DataType): Boolean = all.contains(dt)
-}
-
-
-protected[sql] abstract class NativeType extends DataType {
- private[sql] type JvmType
- @transient private[sql] val tag: TypeTag[JvmType]
- private[sql] val ordering: Ordering[JvmType]
+/**
+ * An internal type used to represent everything that is not null, UDTs, arrays, structs, and maps.
+ */
+protected[sql] abstract class AtomicType extends DataType {
+ private[sql] type InternalType
+ @transient private[sql] val tag: TypeTag[InternalType]
+ private[sql] val ordering: Ordering[InternalType]
@transient private[sql] val classTag = ScalaReflectionLock.synchronized {
val mirror = runtimeMirror(Utils.getSparkClassLoader)
- ClassTag[JvmType](mirror.runtimeClass(tag.tpe))
+ ClassTag[InternalType](mirror.runtimeClass(tag.tpe))
}
}
@@ -336,13 +332,13 @@ protected[sql] abstract class NativeType extends DataType {
* @group dataType
*/
@DeveloperApi
-class StringType private() extends NativeType {
+class StringType private() extends AtomicType {
// The companion object and this class is separated so the companion object also subclasses
// this type. Otherwise, the companion object would be of type "StringType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
- private[sql] type JvmType = UTF8String
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[JvmType] }
- private[sql] val ordering = implicitly[Ordering[JvmType]]
+ private[sql] type InternalType = UTF8String
+ @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ private[sql] val ordering = implicitly[Ordering[InternalType]]
/**
* The default size of a value of the StringType is 4096 bytes.
@@ -363,13 +359,13 @@ case object StringType extends StringType
* @group dataType
*/
@DeveloperApi
-class BinaryType private() extends NativeType {
+class BinaryType private() extends AtomicType {
// The companion object and this class is separated so the companion object also subclasses
// this type. Otherwise, the companion object would be of type "BinaryType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
- private[sql] type JvmType = Array[Byte]
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[JvmType] }
- private[sql] val ordering = new Ordering[JvmType] {
+ private[sql] type InternalType = Array[Byte]
+ @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ private[sql] val ordering = new Ordering[InternalType] {
def compare(x: Array[Byte], y: Array[Byte]): Int = {
for (i <- 0 until x.length; if i < y.length) {
val res = x(i).compareTo(y(i))
@@ -397,13 +393,13 @@ case object BinaryType extends BinaryType
*@group dataType
*/
@DeveloperApi
-class BooleanType private() extends NativeType {
+class BooleanType private() extends AtomicType {
// The companion object and this class is separated so the companion object also subclasses
// this type. Otherwise, the companion object would be of type "BooleanType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
- private[sql] type JvmType = Boolean
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[JvmType] }
- private[sql] val ordering = implicitly[Ordering[JvmType]]
+ private[sql] type InternalType = Boolean
+ @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
+ private[sql] val ordering = implicitly[Ordering[InternalType]]
/**
* The default size of a value of the BooleanType is 1 byte.
@@ -424,15 +420,15 @@ case object BooleanType extends BooleanType
* @group dataType
*/
@DeveloperApi
-class TimestampType private() extends NativeType {
+class TimestampType private() extends AtomicType {
// The companion object and this class is separated so the companion object also subclasses
// this type. Otherwise, the companion object would be of type "TimestampType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
- private[sql] type JvmType = Timestamp
+ private[sql] type InternalType = Timestamp
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[JvmType] }
+ @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
- private[sql] val ordering = new Ordering[JvmType] {
+ private[sql] val ordering = new Ordering[InternalType] {
def compare(x: Timestamp, y: Timestamp): Int = x.compareTo(y)
}
@@ -455,15 +451,15 @@ case object TimestampType extends TimestampType
* @group dataType
*/
@DeveloperApi
-class DateType private() extends NativeType {
+class DateType private() extends AtomicType {
// The companion object and this class is separated so the companion object also subclasses
// this type. Otherwise, the companion object would be of type "DateType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
- private[sql] type JvmType = Int
+ private[sql] type InternalType = Int
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[JvmType] }
+ @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
- private[sql] val ordering = implicitly[Ordering[JvmType]]
+ private[sql] val ordering = implicitly[Ordering[InternalType]]
/**
* The default size of a value of the DateType is 4 bytes.
@@ -482,13 +478,13 @@ case object DateType extends DateType
*
* @group dataType
*/
-abstract class NumericType extends NativeType {
+abstract class NumericType extends AtomicType {
// Unfortunately we can't get this implicitly as that breaks Spark Serialization. In order for
// implicitly[Numeric[JvmType]] to be valid, we have to change JvmType from a type variable to a
// type parameter and and add a numeric annotation (i.e., [JvmType : Numeric]). This gets
// desugared by the compiler into an argument to the objects constructor. This means there is no
// longer an no argument constructor and thus the JVM cannot serialize the object anymore.
- private[sql] val numeric: Numeric[JvmType]
+ private[sql] val numeric: Numeric[InternalType]
}
@@ -507,7 +503,7 @@ protected[sql] object IntegralType {
protected[sql] sealed abstract class IntegralType extends NumericType {
- private[sql] val integral: Integral[JvmType]
+ private[sql] val integral: Integral[InternalType]
}
@@ -522,11 +518,11 @@ class LongType private() extends IntegralType {
// The companion object and this class is separated so the companion object also subclasses
// this type. Otherwise, the companion object would be of type "LongType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
- private[sql] type JvmType = Long
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[JvmType] }
+ private[sql] type InternalType = Long
+ @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
private[sql] val numeric = implicitly[Numeric[Long]]
private[sql] val integral = implicitly[Integral[Long]]
- private[sql] val ordering = implicitly[Ordering[JvmType]]
+ private[sql] val ordering = implicitly[Ordering[InternalType]]
/**
* The default size of a value of the LongType is 8 bytes.
@@ -552,11 +548,11 @@ class IntegerType private() extends IntegralType {
// The companion object and this class is separated so the companion object also subclasses
// this type. Otherwise, the companion object would be of type "IntegerType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
- private[sql] type JvmType = Int
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[JvmType] }
+ private[sql] type InternalType = Int
+ @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
private[sql] val numeric = implicitly[Numeric[Int]]
private[sql] val integral = implicitly[Integral[Int]]
- private[sql] val ordering = implicitly[Ordering[JvmType]]
+ private[sql] val ordering = implicitly[Ordering[InternalType]]
/**
* The default size of a value of the IntegerType is 4 bytes.
@@ -582,11 +578,11 @@ class ShortType private() extends IntegralType {
// The companion object and this class is separated so the companion object also subclasses
// this type. Otherwise, the companion object would be of type "ShortType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
- private[sql] type JvmType = Short
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[JvmType] }
+ private[sql] type InternalType = Short
+ @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
private[sql] val numeric = implicitly[Numeric[Short]]
private[sql] val integral = implicitly[Integral[Short]]
- private[sql] val ordering = implicitly[Ordering[JvmType]]
+ private[sql] val ordering = implicitly[Ordering[InternalType]]
/**
* The default size of a value of the ShortType is 2 bytes.
@@ -612,11 +608,11 @@ class ByteType private() extends IntegralType {
// The companion object and this class is separated so the companion object also subclasses
// this type. Otherwise, the companion object would be of type "ByteType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
- private[sql] type JvmType = Byte
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[JvmType] }
+ private[sql] type InternalType = Byte
+ @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
private[sql] val numeric = implicitly[Numeric[Byte]]
private[sql] val integral = implicitly[Integral[Byte]]
- private[sql] val ordering = implicitly[Ordering[JvmType]]
+ private[sql] val ordering = implicitly[Ordering[InternalType]]
/**
* The default size of a value of the ByteType is 1 byte.
@@ -641,8 +637,8 @@ protected[sql] object FractionalType {
protected[sql] sealed abstract class FractionalType extends NumericType {
- private[sql] val fractional: Fractional[JvmType]
- private[sql] val asIntegral: Integral[JvmType]
+ private[sql] val fractional: Fractional[InternalType]
+ private[sql] val asIntegral: Integral[InternalType]
}
@@ -665,8 +661,8 @@ case class DecimalType(precisionInfo: Option[PrecisionInfo]) extends FractionalT
/** No-arg constructor for kryo. */
protected def this() = this(null)
- private[sql] type JvmType = Decimal
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[JvmType] }
+ private[sql] type InternalType = Decimal
+ @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
private[sql] val numeric = Decimal.DecimalIsFractional
private[sql] val fractional = Decimal.DecimalIsFractional
private[sql] val ordering = Decimal.DecimalIsFractional
@@ -743,11 +739,11 @@ class DoubleType private() extends FractionalType {
// The companion object and this class is separated so the companion object also subclasses
// this type. Otherwise, the companion object would be of type "DoubleType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
- private[sql] type JvmType = Double
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[JvmType] }
+ private[sql] type InternalType = Double
+ @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
private[sql] val numeric = implicitly[Numeric[Double]]
private[sql] val fractional = implicitly[Fractional[Double]]
- private[sql] val ordering = implicitly[Ordering[JvmType]]
+ private[sql] val ordering = implicitly[Ordering[InternalType]]
private[sql] val asIntegral = DoubleAsIfIntegral
/**
@@ -772,11 +768,11 @@ class FloatType private() extends FractionalType {
// The companion object and this class is separated so the companion object also subclasses
// this type. Otherwise, the companion object would be of type "FloatType$" in byte code.
// Defined with a private constructor so the companion object is the only possible instantiation.
- private[sql] type JvmType = Float
- @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[JvmType] }
+ private[sql] type InternalType = Float
+ @transient private[sql] lazy val tag = ScalaReflectionLock.synchronized { typeTag[InternalType] }
private[sql] val numeric = implicitly[Numeric[Float]]
private[sql] val fractional = implicitly[Fractional[Float]]
- private[sql] val ordering = implicitly[Ordering[JvmType]]
+ private[sql] val ordering = implicitly[Ordering[InternalType]]
private[sql] val asIntegral = FloatAsIfIntegral
/**
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnAccessor.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnAccessor.scala b/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnAccessor.scala
index f615fb3..64449b2 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnAccessor.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnAccessor.scala
@@ -61,7 +61,7 @@ private[sql] abstract class BasicColumnAccessor[T <: DataType, JvmType](
protected def underlyingBuffer = buffer
}
-private[sql] abstract class NativeColumnAccessor[T <: NativeType](
+private[sql] abstract class NativeColumnAccessor[T <: AtomicType](
override protected val buffer: ByteBuffer,
override protected val columnType: NativeColumnType[T])
extends BasicColumnAccessor(buffer, columnType)
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnBuilder.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnBuilder.scala
index 00ed704..aa10af4 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnBuilder.scala
@@ -84,10 +84,10 @@ private[sql] abstract class ComplexColumnBuilder[T <: DataType, JvmType](
extends BasicColumnBuilder[T, JvmType](columnStats, columnType)
with NullableColumnBuilder
-private[sql] abstract class NativeColumnBuilder[T <: NativeType](
+private[sql] abstract class NativeColumnBuilder[T <: AtomicType](
override val columnStats: ColumnStats,
override val columnType: NativeColumnType[T])
- extends BasicColumnBuilder[T, T#JvmType](columnStats, columnType)
+ extends BasicColumnBuilder[T, T#InternalType](columnStats, columnType)
with NullableColumnBuilder
with AllCompressionSchemes
with CompressibleColumnBuilder[T]
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnType.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnType.scala b/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnType.scala
index 1b9e0df..20be5ca 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnType.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/columnar/ColumnType.scala
@@ -101,16 +101,16 @@ private[sql] sealed abstract class ColumnType[T <: DataType, JvmType](
override def toString: String = getClass.getSimpleName.stripSuffix("$")
}
-private[sql] abstract class NativeColumnType[T <: NativeType](
+private[sql] abstract class NativeColumnType[T <: AtomicType](
val dataType: T,
typeId: Int,
defaultSize: Int)
- extends ColumnType[T, T#JvmType](typeId, defaultSize) {
+ extends ColumnType[T, T#InternalType](typeId, defaultSize) {
/**
* Scala TypeTag. Can be used to create primitive arrays and hash tables.
*/
- def scalaTag: TypeTag[dataType.JvmType] = dataType.tag
+ def scalaTag: TypeTag[dataType.InternalType] = dataType.tag
}
private[sql] object INT extends NativeColumnType(IntegerType, 0, 4) {
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnAccessor.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnAccessor.scala b/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnAccessor.scala
index d0b602a..cb205de 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnAccessor.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnAccessor.scala
@@ -19,9 +19,9 @@ package org.apache.spark.sql.columnar.compression
import org.apache.spark.sql.catalyst.expressions.MutableRow
import org.apache.spark.sql.columnar.{ColumnAccessor, NativeColumnAccessor}
-import org.apache.spark.sql.types.NativeType
+import org.apache.spark.sql.types.AtomicType
-private[sql] trait CompressibleColumnAccessor[T <: NativeType] extends ColumnAccessor {
+private[sql] trait CompressibleColumnAccessor[T <: AtomicType] extends ColumnAccessor {
this: NativeColumnAccessor[T] =>
private var decoder: Decoder[T] = _
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnBuilder.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnBuilder.scala b/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnBuilder.scala
index b9cfc5d..8e2a1af 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnBuilder.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressibleColumnBuilder.scala
@@ -22,7 +22,7 @@ import java.nio.{ByteBuffer, ByteOrder}
import org.apache.spark.Logging
import org.apache.spark.sql.Row
import org.apache.spark.sql.columnar.{ColumnBuilder, NativeColumnBuilder}
-import org.apache.spark.sql.types.NativeType
+import org.apache.spark.sql.types.AtomicType
/**
* A stackable trait that builds optionally compressed byte buffer for a column. Memory layout of
@@ -41,7 +41,7 @@ import org.apache.spark.sql.types.NativeType
* header body
* }}}
*/
-private[sql] trait CompressibleColumnBuilder[T <: NativeType]
+private[sql] trait CompressibleColumnBuilder[T <: AtomicType]
extends ColumnBuilder with Logging {
this: NativeColumnBuilder[T] with WithCompressionSchemes =>
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressionScheme.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressionScheme.scala b/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressionScheme.scala
index 879d29b..17c2d9b 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressionScheme.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/CompressionScheme.scala
@@ -22,9 +22,9 @@ import java.nio.{ByteBuffer, ByteOrder}
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.expressions.MutableRow
import org.apache.spark.sql.columnar.{ColumnType, NativeColumnType}
-import org.apache.spark.sql.types.NativeType
+import org.apache.spark.sql.types.AtomicType
-private[sql] trait Encoder[T <: NativeType] {
+private[sql] trait Encoder[T <: AtomicType] {
def gatherCompressibilityStats(row: Row, ordinal: Int): Unit = {}
def compressedSize: Int
@@ -38,7 +38,7 @@ private[sql] trait Encoder[T <: NativeType] {
def compress(from: ByteBuffer, to: ByteBuffer): ByteBuffer
}
-private[sql] trait Decoder[T <: NativeType] {
+private[sql] trait Decoder[T <: AtomicType] {
def next(row: MutableRow, ordinal: Int): Unit
def hasNext: Boolean
@@ -49,9 +49,9 @@ private[sql] trait CompressionScheme {
def supports(columnType: ColumnType[_, _]): Boolean
- def encoder[T <: NativeType](columnType: NativeColumnType[T]): Encoder[T]
+ def encoder[T <: AtomicType](columnType: NativeColumnType[T]): Encoder[T]
- def decoder[T <: NativeType](buffer: ByteBuffer, columnType: NativeColumnType[T]): Decoder[T]
+ def decoder[T <: AtomicType](buffer: ByteBuffer, columnType: NativeColumnType[T]): Decoder[T]
}
private[sql] trait WithCompressionSchemes {
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/compressionSchemes.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/compressionSchemes.scala b/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/compressionSchemes.scala
index 8727d71..534ae90 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/compressionSchemes.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/columnar/compression/compressionSchemes.scala
@@ -35,16 +35,16 @@ private[sql] case object PassThrough extends CompressionScheme {
override def supports(columnType: ColumnType[_, _]): Boolean = true
- override def encoder[T <: NativeType](columnType: NativeColumnType[T]): Encoder[T] = {
+ override def encoder[T <: AtomicType](columnType: NativeColumnType[T]): Encoder[T] = {
new this.Encoder[T](columnType)
}
- override def decoder[T <: NativeType](
+ override def decoder[T <: AtomicType](
buffer: ByteBuffer, columnType: NativeColumnType[T]): Decoder[T] = {
new this.Decoder(buffer, columnType)
}
- class Encoder[T <: NativeType](columnType: NativeColumnType[T]) extends compression.Encoder[T] {
+ class Encoder[T <: AtomicType](columnType: NativeColumnType[T]) extends compression.Encoder[T] {
override def uncompressedSize: Int = 0
override def compressedSize: Int = 0
@@ -56,7 +56,7 @@ private[sql] case object PassThrough extends CompressionScheme {
}
}
- class Decoder[T <: NativeType](buffer: ByteBuffer, columnType: NativeColumnType[T])
+ class Decoder[T <: AtomicType](buffer: ByteBuffer, columnType: NativeColumnType[T])
extends compression.Decoder[T] {
override def next(row: MutableRow, ordinal: Int): Unit = {
@@ -70,11 +70,11 @@ private[sql] case object PassThrough extends CompressionScheme {
private[sql] case object RunLengthEncoding extends CompressionScheme {
override val typeId = 1
- override def encoder[T <: NativeType](columnType: NativeColumnType[T]): Encoder[T] = {
+ override def encoder[T <: AtomicType](columnType: NativeColumnType[T]): Encoder[T] = {
new this.Encoder[T](columnType)
}
- override def decoder[T <: NativeType](
+ override def decoder[T <: AtomicType](
buffer: ByteBuffer, columnType: NativeColumnType[T]): Decoder[T] = {
new this.Decoder(buffer, columnType)
}
@@ -84,7 +84,7 @@ private[sql] case object RunLengthEncoding extends CompressionScheme {
case _ => false
}
- class Encoder[T <: NativeType](columnType: NativeColumnType[T]) extends compression.Encoder[T] {
+ class Encoder[T <: AtomicType](columnType: NativeColumnType[T]) extends compression.Encoder[T] {
private var _uncompressedSize = 0
private var _compressedSize = 0
@@ -152,12 +152,12 @@ private[sql] case object RunLengthEncoding extends CompressionScheme {
}
}
- class Decoder[T <: NativeType](buffer: ByteBuffer, columnType: NativeColumnType[T])
+ class Decoder[T <: AtomicType](buffer: ByteBuffer, columnType: NativeColumnType[T])
extends compression.Decoder[T] {
private var run = 0
private var valueCount = 0
- private var currentValue: T#JvmType = _
+ private var currentValue: T#InternalType = _
override def next(row: MutableRow, ordinal: Int): Unit = {
if (valueCount == run) {
@@ -181,12 +181,12 @@ private[sql] case object DictionaryEncoding extends CompressionScheme {
// 32K unique values allowed
val MAX_DICT_SIZE = Short.MaxValue
- override def decoder[T <: NativeType](buffer: ByteBuffer, columnType: NativeColumnType[T])
+ override def decoder[T <: AtomicType](buffer: ByteBuffer, columnType: NativeColumnType[T])
: Decoder[T] = {
new this.Decoder(buffer, columnType)
}
- override def encoder[T <: NativeType](columnType: NativeColumnType[T]): Encoder[T] = {
+ override def encoder[T <: AtomicType](columnType: NativeColumnType[T]): Encoder[T] = {
new this.Encoder[T](columnType)
}
@@ -195,7 +195,7 @@ private[sql] case object DictionaryEncoding extends CompressionScheme {
case _ => false
}
- class Encoder[T <: NativeType](columnType: NativeColumnType[T]) extends compression.Encoder[T] {
+ class Encoder[T <: AtomicType](columnType: NativeColumnType[T]) extends compression.Encoder[T] {
// Size of the input, uncompressed, in bytes. Note that we only count until the dictionary
// overflows.
private var _uncompressedSize = 0
@@ -208,7 +208,7 @@ private[sql] case object DictionaryEncoding extends CompressionScheme {
private var count = 0
// The reverse mapping of _dictionary, i.e. mapping encoded integer to the value itself.
- private var values = new mutable.ArrayBuffer[T#JvmType](1024)
+ private var values = new mutable.ArrayBuffer[T#InternalType](1024)
// The dictionary that maps a value to the encoded short integer.
private val dictionary = mutable.HashMap.empty[Any, Short]
@@ -268,14 +268,14 @@ private[sql] case object DictionaryEncoding extends CompressionScheme {
override def compressedSize: Int = if (overflow) Int.MaxValue else dictionarySize + count * 2
}
- class Decoder[T <: NativeType](buffer: ByteBuffer, columnType: NativeColumnType[T])
+ class Decoder[T <: AtomicType](buffer: ByteBuffer, columnType: NativeColumnType[T])
extends compression.Decoder[T] {
private val dictionary = {
// TODO Can we clean up this mess? Maybe move this to `DataType`?
implicit val classTag = {
val mirror = runtimeMirror(Utils.getSparkClassLoader)
- ClassTag[T#JvmType](mirror.runtimeClass(columnType.scalaTag.tpe))
+ ClassTag[T#InternalType](mirror.runtimeClass(columnType.scalaTag.tpe))
}
Array.fill(buffer.getInt()) {
@@ -296,12 +296,12 @@ private[sql] case object BooleanBitSet extends CompressionScheme {
val BITS_PER_LONG = 64
- override def decoder[T <: NativeType](buffer: ByteBuffer, columnType: NativeColumnType[T])
+ override def decoder[T <: AtomicType](buffer: ByteBuffer, columnType: NativeColumnType[T])
: compression.Decoder[T] = {
new this.Decoder(buffer).asInstanceOf[compression.Decoder[T]]
}
- override def encoder[T <: NativeType](columnType: NativeColumnType[T]): compression.Encoder[T] = {
+ override def encoder[T <: AtomicType](columnType: NativeColumnType[T]): compression.Encoder[T] = {
(new this.Encoder).asInstanceOf[compression.Encoder[T]]
}
@@ -384,12 +384,12 @@ private[sql] case object BooleanBitSet extends CompressionScheme {
private[sql] case object IntDelta extends CompressionScheme {
override def typeId: Int = 4
- override def decoder[T <: NativeType](buffer: ByteBuffer, columnType: NativeColumnType[T])
+ override def decoder[T <: AtomicType](buffer: ByteBuffer, columnType: NativeColumnType[T])
: compression.Decoder[T] = {
new Decoder(buffer, INT).asInstanceOf[compression.Decoder[T]]
}
- override def encoder[T <: NativeType](columnType: NativeColumnType[T]): compression.Encoder[T] = {
+ override def encoder[T <: AtomicType](columnType: NativeColumnType[T]): compression.Encoder[T] = {
(new Encoder).asInstanceOf[compression.Encoder[T]]
}
@@ -464,12 +464,12 @@ private[sql] case object IntDelta extends CompressionScheme {
private[sql] case object LongDelta extends CompressionScheme {
override def typeId: Int = 5
- override def decoder[T <: NativeType](buffer: ByteBuffer, columnType: NativeColumnType[T])
+ override def decoder[T <: AtomicType](buffer: ByteBuffer, columnType: NativeColumnType[T])
: compression.Decoder[T] = {
new Decoder(buffer, LONG).asInstanceOf[compression.Decoder[T]]
}
- override def encoder[T <: NativeType](columnType: NativeColumnType[T]): compression.Encoder[T] = {
+ override def encoder[T <: AtomicType](columnType: NativeColumnType[T]): compression.Encoder[T] = {
(new Encoder).asInstanceOf[compression.Encoder[T]]
}
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
index 29de740..6e94e70 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/json/JsonRDD.scala
@@ -183,7 +183,7 @@ private[sql] object JsonRDD extends Logging {
private def typeOfPrimitiveValue: PartialFunction[Any, DataType] = {
// For Integer values, use LongType by default.
val useLongType: PartialFunction[Any, DataType] = {
- case value: IntegerType.JvmType => LongType
+ case value: IntegerType.InternalType => LongType
}
useLongType orElse ScalaReflection.typeOfObject orElse {
@@ -411,11 +411,11 @@ private[sql] object JsonRDD extends Logging {
desiredType match {
case StringType => UTF8String(toString(value))
case _ if value == null || value == "" => null // guard the non string type
- case IntegerType => value.asInstanceOf[IntegerType.JvmType]
+ case IntegerType => value.asInstanceOf[IntegerType.InternalType]
case LongType => toLong(value)
case DoubleType => toDouble(value)
case DecimalType() => toDecimal(value)
- case BooleanType => value.asInstanceOf[BooleanType.JvmType]
+ case BooleanType => value.asInstanceOf[BooleanType.InternalType]
case NullType => null
case ArrayType(elementType, _) =>
value.asInstanceOf[Seq[Any]].map(enforceCorrectType(_, elementType))
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetConverter.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetConverter.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetConverter.scala
index 1164245..36cb5e0 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetConverter.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetConverter.scala
@@ -90,7 +90,7 @@ private[sql] object CatalystConverter {
createConverter(field.copy(dataType = udt.sqlType), fieldIndex, parent)
}
// For native JVM types we use a converter with native arrays
- case ArrayType(elementType: NativeType, false) => {
+ case ArrayType(elementType: AtomicType, false) => {
new CatalystNativeArrayConverter(elementType, fieldIndex, parent)
}
// This is for other types of arrays, including those with nested fields
@@ -118,19 +118,19 @@ private[sql] object CatalystConverter {
case ShortType => {
new CatalystPrimitiveConverter(parent, fieldIndex) {
override def addInt(value: Int): Unit =
- parent.updateShort(fieldIndex, value.asInstanceOf[ShortType.JvmType])
+ parent.updateShort(fieldIndex, value.asInstanceOf[ShortType.InternalType])
}
}
case ByteType => {
new CatalystPrimitiveConverter(parent, fieldIndex) {
override def addInt(value: Int): Unit =
- parent.updateByte(fieldIndex, value.asInstanceOf[ByteType.JvmType])
+ parent.updateByte(fieldIndex, value.asInstanceOf[ByteType.InternalType])
}
}
case DateType => {
new CatalystPrimitiveConverter(parent, fieldIndex) {
override def addInt(value: Int): Unit =
- parent.updateDate(fieldIndex, value.asInstanceOf[DateType.JvmType])
+ parent.updateDate(fieldIndex, value.asInstanceOf[DateType.InternalType])
}
}
case d: DecimalType => {
@@ -637,13 +637,13 @@ private[parquet] class CatalystArrayConverter(
* @param capacity The (initial) capacity of the buffer
*/
private[parquet] class CatalystNativeArrayConverter(
- val elementType: NativeType,
+ val elementType: AtomicType,
val index: Int,
protected[parquet] val parent: CatalystConverter,
protected[parquet] var capacity: Int = CatalystArrayConverter.INITIAL_ARRAY_SIZE)
extends CatalystConverter {
- type NativeType = elementType.JvmType
+ type NativeType = elementType.InternalType
private var buffer: Array[NativeType] = elementType.classTag.newArray(capacity)
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTableSupport.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTableSupport.scala b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTableSupport.scala
index e05a4c2..c45c431 100644
--- a/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTableSupport.scala
+++ b/sql/core/src/main/scala/org/apache/spark/sql/parquet/ParquetTableSupport.scala
@@ -189,7 +189,7 @@ private[parquet] class RowWriteSupport extends WriteSupport[Row] with Logging {
case t @ StructType(_) => writeStruct(
t,
value.asInstanceOf[CatalystConverter.StructScalaType[_]])
- case _ => writePrimitive(schema.asInstanceOf[NativeType], value)
+ case _ => writePrimitive(schema.asInstanceOf[AtomicType], value)
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnStatsSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnStatsSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnStatsSuite.scala
index fec487f..7cefcf4 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnStatsSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnStatsSuite.scala
@@ -34,7 +34,7 @@ class ColumnStatsSuite extends FunSuite {
testColumnStats(classOf[DateColumnStats], DATE, Row(Int.MaxValue, Int.MinValue, 0))
testColumnStats(classOf[TimestampColumnStats], TIMESTAMP, Row(null, null, 0))
- def testColumnStats[T <: NativeType, U <: ColumnStats](
+ def testColumnStats[T <: AtomicType, U <: ColumnStats](
columnStatsClass: Class[U],
columnType: NativeColumnType[T],
initialStatistics: Row): Unit = {
@@ -55,8 +55,8 @@ class ColumnStatsSuite extends FunSuite {
val rows = Seq.fill(10)(makeRandomRow(columnType)) ++ Seq.fill(10)(makeNullRow(1))
rows.foreach(columnStats.gatherStats(_, 0))
- val values = rows.take(10).map(_(0).asInstanceOf[T#JvmType])
- val ordering = columnType.dataType.ordering.asInstanceOf[Ordering[T#JvmType]]
+ val values = rows.take(10).map(_(0).asInstanceOf[T#InternalType])
+ val ordering = columnType.dataType.ordering.asInstanceOf[Ordering[T#InternalType]]
val stats = columnStats.collectedStatistics
assertResult(values.min(ordering), "Wrong lower bound")(stats(0))
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala
index b48bed1..1e105e2 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnTypeSuite.scala
@@ -196,12 +196,12 @@ class ColumnTypeSuite extends FunSuite with Logging {
}
}
- def testNativeColumnType[T <: NativeType](
+ def testNativeColumnType[T <: AtomicType](
columnType: NativeColumnType[T],
- putter: (ByteBuffer, T#JvmType) => Unit,
- getter: (ByteBuffer) => T#JvmType): Unit = {
+ putter: (ByteBuffer, T#InternalType) => Unit,
+ getter: (ByteBuffer) => T#InternalType): Unit = {
- testColumnType[T, T#JvmType](columnType, putter, getter)
+ testColumnType[T, T#InternalType](columnType, putter, getter)
}
def testColumnType[T <: DataType, JvmType](
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnarTestUtils.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnarTestUtils.scala b/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnarTestUtils.scala
index f76314b..75d993e 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnarTestUtils.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/columnar/ColumnarTestUtils.scala
@@ -24,7 +24,7 @@ import scala.util.Random
import org.apache.spark.sql.Row
import org.apache.spark.sql.catalyst.expressions.GenericMutableRow
-import org.apache.spark.sql.types.{UTF8String, DataType, Decimal, NativeType}
+import org.apache.spark.sql.types.{UTF8String, DataType, Decimal, AtomicType}
object ColumnarTestUtils {
def makeNullRow(length: Int): GenericMutableRow = {
@@ -91,9 +91,9 @@ object ColumnarTestUtils {
row
}
- def makeUniqueValuesAndSingleValueRows[T <: NativeType](
+ def makeUniqueValuesAndSingleValueRows[T <: AtomicType](
columnType: NativeColumnType[T],
- count: Int): (Seq[T#JvmType], Seq[GenericMutableRow]) = {
+ count: Int): (Seq[T#InternalType], Seq[GenericMutableRow]) = {
val values = makeUniqueRandomValues(columnType, count)
val rows = values.map { value =>
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/DictionaryEncodingSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/DictionaryEncodingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/DictionaryEncodingSuite.scala
index c82d979..64b7055 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/DictionaryEncodingSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/DictionaryEncodingSuite.scala
@@ -24,14 +24,14 @@ import org.scalatest.FunSuite
import org.apache.spark.sql.catalyst.expressions.GenericMutableRow
import org.apache.spark.sql.columnar._
import org.apache.spark.sql.columnar.ColumnarTestUtils._
-import org.apache.spark.sql.types.NativeType
+import org.apache.spark.sql.types.AtomicType
class DictionaryEncodingSuite extends FunSuite {
testDictionaryEncoding(new IntColumnStats, INT)
testDictionaryEncoding(new LongColumnStats, LONG)
testDictionaryEncoding(new StringColumnStats, STRING)
- def testDictionaryEncoding[T <: NativeType](
+ def testDictionaryEncoding[T <: AtomicType](
columnStats: ColumnStats,
columnType: NativeColumnType[T]) {
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/IntegralDeltaSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/IntegralDeltaSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/IntegralDeltaSuite.scala
index 8801163..bfd99f1 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/IntegralDeltaSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/IntegralDeltaSuite.scala
@@ -33,7 +33,7 @@ class IntegralDeltaSuite extends FunSuite {
columnType: NativeColumnType[I],
scheme: CompressionScheme) {
- def skeleton(input: Seq[I#JvmType]) {
+ def skeleton(input: Seq[I#InternalType]) {
// -------------
// Tests encoder
// -------------
@@ -120,13 +120,13 @@ class IntegralDeltaSuite extends FunSuite {
case LONG => Seq(2: Long, 1: Long, 2: Long, 130: Long)
}
- skeleton(input.map(_.asInstanceOf[I#JvmType]))
+ skeleton(input.map(_.asInstanceOf[I#InternalType]))
}
test(s"$scheme: long random series") {
// Have to workaround with `Any` since no `ClassTag[I#JvmType]` available here.
val input = Array.fill[Any](10000)(makeRandomValue(columnType))
- skeleton(input.map(_.asInstanceOf[I#JvmType]))
+ skeleton(input.map(_.asInstanceOf[I#InternalType]))
}
}
}
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/RunLengthEncodingSuite.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/RunLengthEncodingSuite.scala b/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/RunLengthEncodingSuite.scala
index 08df1db..fde7a45 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/RunLengthEncodingSuite.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/RunLengthEncodingSuite.scala
@@ -22,7 +22,7 @@ import org.scalatest.FunSuite
import org.apache.spark.sql.catalyst.expressions.GenericMutableRow
import org.apache.spark.sql.columnar._
import org.apache.spark.sql.columnar.ColumnarTestUtils._
-import org.apache.spark.sql.types.NativeType
+import org.apache.spark.sql.types.AtomicType
class RunLengthEncodingSuite extends FunSuite {
testRunLengthEncoding(new NoopColumnStats, BOOLEAN)
@@ -32,7 +32,7 @@ class RunLengthEncodingSuite extends FunSuite {
testRunLengthEncoding(new LongColumnStats, LONG)
testRunLengthEncoding(new StringColumnStats, STRING)
- def testRunLengthEncoding[T <: NativeType](
+ def testRunLengthEncoding[T <: AtomicType](
columnStats: ColumnStats,
columnType: NativeColumnType[T]) {
http://git-wip-us.apache.org/repos/asf/spark/blob/f60bece1/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/TestCompressibleColumnBuilder.scala
----------------------------------------------------------------------
diff --git a/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/TestCompressibleColumnBuilder.scala b/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/TestCompressibleColumnBuilder.scala
index fc8ff3b..5268dfe 100644
--- a/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/TestCompressibleColumnBuilder.scala
+++ b/sql/core/src/test/scala/org/apache/spark/sql/columnar/compression/TestCompressibleColumnBuilder.scala
@@ -18,9 +18,9 @@
package org.apache.spark.sql.columnar.compression
import org.apache.spark.sql.columnar._
-import org.apache.spark.sql.types.NativeType
+import org.apache.spark.sql.types.AtomicType
-class TestCompressibleColumnBuilder[T <: NativeType](
+class TestCompressibleColumnBuilder[T <: AtomicType](
override val columnStats: ColumnStats,
override val columnType: NativeColumnType[T],
override val schemes: Seq[CompressionScheme])
@@ -32,7 +32,7 @@ class TestCompressibleColumnBuilder[T <: NativeType](
}
object TestCompressibleColumnBuilder {
- def apply[T <: NativeType](
+ def apply[T <: AtomicType](
columnStats: ColumnStats,
columnType: NativeColumnType[T],
scheme: CompressionScheme): TestCompressibleColumnBuilder[T] = {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org