You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by rx...@apache.org on 2015/04/23 06:35:46 UTC
spark git commit: [SPARK-7066][MLlib] VectorAssembler should use
NumericType not NativeType.
Repository: spark
Updated Branches:
refs/heads/master 1b85e0850 -> d20686066
[SPARK-7066][MLlib] VectorAssembler should use NumericType not NativeType.
Author: Reynold Xin <rx...@databricks.com>
Closes #5642 from rxin/mllib-native-type and squashes the following commits:
e23af5b [Reynold Xin] Remove StringType
7cbb205 [Reynold Xin] [SPARK-7066][MLlib] VectorAssembler should use NumericType and StringType, not NativeType.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/d2068606
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/d2068606
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/d2068606
Branch: refs/heads/master
Commit: d20686066e978dd12e618e3978f109f05bc412fe
Parents: 1b85e08
Author: Reynold Xin <rx...@databricks.com>
Authored: Wed Apr 22 21:35:42 2015 -0700
Committer: Reynold Xin <rx...@databricks.com>
Committed: Wed Apr 22 21:35:42 2015 -0700
----------------------------------------------------------------------
.../scala/org/apache/spark/ml/feature/VectorAssembler.scala | 5 +++--
.../src/main/scala/org/apache/spark/sql/types/dataTypes.scala | 4 ++--
2 files changed, 5 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/d2068606/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
index e567e06..fd16d3d 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/feature/VectorAssembler.scala
@@ -55,7 +55,8 @@ class VectorAssembler extends Transformer with HasInputCols with HasOutputCol {
schema(c).dataType match {
case DoubleType => UnresolvedAttribute(c)
case t if t.isInstanceOf[VectorUDT] => UnresolvedAttribute(c)
- case _: NativeType => Alias(Cast(UnresolvedAttribute(c), DoubleType), s"${c}_double_$uid")()
+ case _: NumericType =>
+ Alias(Cast(UnresolvedAttribute(c), DoubleType), s"${c}_double_$uid")()
}
}
dataset.select(col("*"), assembleFunc(new Column(CreateStruct(args))).as(map(outputCol)))
@@ -67,7 +68,7 @@ class VectorAssembler extends Transformer with HasInputCols with HasOutputCol {
val outputColName = map(outputCol)
val inputDataTypes = inputColNames.map(name => schema(name).dataType)
inputDataTypes.foreach {
- case _: NativeType =>
+ case _: NumericType =>
case t if t.isInstanceOf[VectorUDT] =>
case other =>
throw new IllegalArgumentException(s"Data type $other is not supported.")
http://git-wip-us.apache.org/repos/asf/spark/blob/d2068606/sql/catalyst/src/main/scala/org/apache/spark/sql/types/dataTypes.scala
----------------------------------------------------------------------
diff --git a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/dataTypes.scala b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/dataTypes.scala
index 7cd7bd1..ddf9d66 100644
--- a/sql/catalyst/src/main/scala/org/apache/spark/sql/types/dataTypes.scala
+++ b/sql/catalyst/src/main/scala/org/apache/spark/sql/types/dataTypes.scala
@@ -299,7 +299,7 @@ class NullType private() extends DataType {
case object NullType extends NullType
-protected[spark] object NativeType {
+protected[sql] object NativeType {
val all = Seq(
IntegerType, BooleanType, LongType, DoubleType, FloatType, ShortType, ByteType, StringType)
@@ -327,7 +327,7 @@ protected[sql] object PrimitiveType {
}
}
-protected[spark] abstract class NativeType extends DataType {
+protected[sql] abstract class NativeType extends DataType {
private[sql] type JvmType
@transient private[sql] val tag: TypeTag[JvmType]
private[sql] val ordering: Ordering[JvmType]
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org