You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by db...@apache.org on 2015/07/10 00:49:34 UTC
spark git commit: [SPARK-8963][ML] cleanup tests in linear regression
suite
Repository: spark
Updated Branches:
refs/heads/master 691653303 -> e29ce319f
[SPARK-8963][ML] cleanup tests in linear regression suite
Simplify model weight assertions to use vector comparision, switch to using absTol when comparing with 0.0 intercepts
Author: Holden Karau <ho...@pigscanfly.ca>
Closes #7327 from holdenk/SPARK-8913-cleanup-tests-from-SPARK-8700-logistic-regression and squashes the following commits:
5bac185 [Holden Karau] Simplify model weight assertions to use vector comparision, switch to using absTol when comparing with 0.0 intercepts
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/e29ce319
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/e29ce319
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/e29ce319
Branch: refs/heads/master
Commit: e29ce319fa6ffb9c8e5110814d4923d433aa1b76
Parents: 6916533
Author: Holden Karau <ho...@pigscanfly.ca>
Authored: Thu Jul 9 15:49:30 2015 -0700
Committer: DB Tsai <db...@netflix.com>
Committed: Thu Jul 9 15:49:30 2015 -0700
----------------------------------------------------------------------
.../ml/regression/LinearRegressionSuite.scala | 57 +++++++++-----------
1 file changed, 24 insertions(+), 33 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/e29ce319/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala
index 5f39d44..4f6a577 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/regression/LinearRegressionSuite.scala
@@ -18,7 +18,7 @@
package org.apache.spark.ml.regression
import org.apache.spark.SparkFunSuite
-import org.apache.spark.mllib.linalg.DenseVector
+import org.apache.spark.mllib.linalg.{DenseVector, Vectors}
import org.apache.spark.mllib.util.{LinearDataGenerator, MLlibTestSparkContext}
import org.apache.spark.mllib.util.TestingUtils._
import org.apache.spark.sql.{DataFrame, Row}
@@ -75,11 +75,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V3. 7.198257
*/
val interceptR = 6.298698
- val weightsR = Array(4.700706, 7.199082)
+ val weightsR = Vectors.dense(4.700706, 7.199082)
assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.weights ~= weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
@@ -104,11 +103,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V2. 6.995908
as.numeric.data.V3. 5.275131
*/
- val weightsR = Array(6.995908, 5.275131)
+ val weightsR = Vectors.dense(6.995908, 5.275131)
- assert(model.intercept ~== 0 relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.intercept ~== 0 absTol 1E-3)
+ assert(model.weights ~= weightsR relTol 1E-3)
/*
Then again with the data with no intercept:
> weightsWithoutIntercept
@@ -118,11 +116,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data3.V2. 4.70011
as.numeric.data3.V3. 7.19943
*/
- val weightsWithoutInterceptR = Array(4.70011, 7.19943)
+ val weightsWithoutInterceptR = Vectors.dense(4.70011, 7.19943)
- assert(modelWithoutIntercept.intercept ~== 0 relTol 1E-3)
- assert(modelWithoutIntercept.weights(0) ~== weightsWithoutInterceptR(0) relTol 1E-3)
- assert(modelWithoutIntercept.weights(1) ~== weightsWithoutInterceptR(1) relTol 1E-3)
+ assert(modelWithoutIntercept.intercept ~== 0 absTol 1E-3)
+ assert(modelWithoutIntercept.weights ~= weightsWithoutInterceptR relTol 1E-3)
}
test("linear regression with intercept with L1 regularization") {
@@ -139,11 +136,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V3. 6.679841
*/
val interceptR = 6.24300
- val weightsR = Array(4.024821, 6.679841)
+ val weightsR = Vectors.dense(4.024821, 6.679841)
assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.weights ~= weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
@@ -169,11 +165,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V3. 4.772913
*/
val interceptR = 0.0
- val weightsR = Array(6.299752, 4.772913)
+ val weightsR = Vectors.dense(6.299752, 4.772913)
- assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.intercept ~== interceptR absTol 1E-5)
+ assert(model.weights ~= weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
@@ -197,11 +192,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V3. 4.926260
*/
val interceptR = 5.269376
- val weightsR = Array(3.736216, 5.712356)
+ val weightsR = Vectors.dense(3.736216, 5.712356)
assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.weights ~= weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
@@ -227,11 +221,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V3. 4.214502
*/
val interceptR = 0.0
- val weightsR = Array(5.522875, 4.214502)
+ val weightsR = Vectors.dense(5.522875, 4.214502)
- assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.intercept ~== interceptR absTol 1E-3)
+ assert(model.weights ~== weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
@@ -255,11 +248,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.data.V3. 5.200403
*/
val interceptR = 5.696056
- val weightsR = Array(3.670489, 6.001122)
+ val weightsR = Vectors.dense(3.670489, 6.001122)
assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.weights ~== weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
@@ -285,11 +277,10 @@ class LinearRegressionSuite extends SparkFunSuite with MLlibTestSparkContext {
as.numeric.dataM.V3. 4.322251
*/
val interceptR = 0.0
- val weightsR = Array(5.673348, 4.322251)
+ val weightsR = Vectors.dense(5.673348, 4.322251)
- assert(model.intercept ~== interceptR relTol 1E-3)
- assert(model.weights(0) ~== weightsR(0) relTol 1E-3)
- assert(model.weights(1) ~== weightsR(1) relTol 1E-3)
+ assert(model.intercept ~== interceptR absTol 1E-3)
+ assert(model.weights ~= weightsR relTol 1E-3)
model.transform(dataset).select("features", "prediction").collect().foreach {
case Row(features: DenseVector, prediction1: Double) =>
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org