You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hivemall.apache.org by my...@apache.org on 2016/12/02 08:02:23 UTC
[33/50] [abbrv] incubator-hivemall git commit: change method of
testing for spark
change method of testing for spark
Project: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/commit/ce4a4898
Tree: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/tree/ce4a4898
Diff: http://git-wip-us.apache.org/repos/asf/incubator-hivemall/diff/ce4a4898
Branch: refs/heads/JIRA-22/pr-385
Commit: ce4a48980e33b9f16c74a62fcea6878f28b9c08b
Parents: 8d9f0d4
Author: amaya <gi...@sapphire.in.net>
Authored: Fri Sep 30 17:05:20 2016 +0900
Committer: amaya <gi...@sapphire.in.net>
Committed: Fri Sep 30 17:05:20 2016 +0900
----------------------------------------------------------------------
.../spark/sql/hive/HivemallOpsSuite.scala | 23 ++++++++++----------
.../spark/sql/hive/HivemallOpsSuite.scala | 17 ++++++---------
2 files changed, 18 insertions(+), 22 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/ce4a4898/spark/spark-1.6/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala
----------------------------------------------------------------------
diff --git a/spark/spark-1.6/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala b/spark/spark-1.6/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala
index cce22ce..c7016c0 100644
--- a/spark/spark-1.6/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala
+++ b/spark/spark-1.6/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala
@@ -24,7 +24,6 @@ import org.apache.spark.sql.{Column, Row}
import org.apache.spark.test.HivemallQueryTest
import org.apache.spark.test.TestDoubleWrapper._
import org.apache.spark.test.TestUtils._
-import org.scalatest.Matchers._
final class HivemallOpsSuite extends HivemallQueryTest {
@@ -189,7 +188,6 @@ final class HivemallOpsSuite extends HivemallQueryTest {
test("ftvec.selection - chi2") {
import hiveContext.implicits._
- implicit val doubleEquality = org.scalactic.TolerantNumerics.tolerantDoubleEquality(1e-5)
// see also hivemall.ftvec.selection.ChiSquareUDFTest
val df = Seq(
@@ -204,17 +202,17 @@ final class HivemallOpsSuite extends HivemallQueryTest {
.toDF("arg0", "arg1")
val result = df.select(chi2(df("arg0"), df("arg1"))).collect
- result should have length 1
+ assert(result.length == 1)
val chi2Val = result.head.getAs[Row](0).getAs[Seq[Double]](0)
val pVal = result.head.getAs[Row](0).getAs[Seq[Double]](1)
(chi2Val, Seq(10.81782088, 3.59449902, 116.16984746, 67.24482759))
.zipped
- .foreach((actual, expected) => actual shouldEqual expected)
+ .foreach((actual, expected) => assert(actual ~== expected))
(pVal, Seq(4.47651499e-03, 1.65754167e-01, 5.94344354e-26, 2.50017968e-15))
.zipped
- .foreach((actual, expected) => actual shouldEqual expected)
+ .foreach((actual, expected) => assert(actual ~== expected))
}
test("ftvec.conv - quantify") {
@@ -370,8 +368,9 @@ final class HivemallOpsSuite extends HivemallQueryTest {
val data = Seq(Seq(0, 1, 3), Seq(2, 4, 1), Seq(5, 4, 9))
val df = data.map(d => (d, Seq(3, 1, 2), 2)).toDF("features", "importance_list", "k")
- df.select(select_k_best(df("features"), df("importance_list"), df("k"))).collect shouldEqual
- data.map(s => Row(Seq(s(0).toDouble, s(2).toDouble)))
+ // if use checkAnswer here, fail for some reason, maybe type? but it's okay on spark-2.0
+ assert(df.select(select_k_best(df("features"), df("importance_list"), df("k"))).collect ===
+ data.map(s => Row(Seq(s(0).toDouble, s(2).toDouble))))
}
test("misc - sigmoid") {
@@ -573,7 +572,6 @@ final class HivemallOpsSuite extends HivemallQueryTest {
test("user-defined aggregators for ftvec.selection") {
import hiveContext.implicits._
- implicit val doubleEquality = org.scalactic.TolerantNumerics.tolerantDoubleEquality(1e-5)
// see also hivemall.ftvec.selection.SignalNoiseRatioUDAFTest
// binary class
@@ -595,7 +593,7 @@ final class HivemallOpsSuite extends HivemallQueryTest {
val row0 = df0.groupby($"c0").snr("arg0", "arg1").collect
(row0(0).getAs[Seq[Double]](1), Seq(4.38425236, 0.26390002, 15.83984511, 26.87005769))
.zipped
- .foreach((actual, expected) => actual shouldEqual expected)
+ .foreach((actual, expected) => assert(actual ~== expected))
// multiple class
// +-----------------+-------+
@@ -616,7 +614,7 @@ final class HivemallOpsSuite extends HivemallQueryTest {
val row1 = df1.groupby($"c0").snr("arg0", "arg1").collect
(row1(0).getAs[Seq[Double]](1), Seq(8.43181818, 1.32121212, 42.94949495, 33.80952381))
.zipped
- .foreach((actual, expected) => actual shouldEqual expected)
+ .foreach((actual, expected) => assert(actual ~== expected))
}
test("user-defined aggregators for tools.matrix") {
@@ -627,7 +625,8 @@ final class HivemallOpsSuite extends HivemallQueryTest {
val df0 = Seq((1, Seq(1, 2, 3), Seq(5, 6, 7)), (1, Seq(3, 4, 5), Seq(7, 8, 9)))
.toDF("c0", "arg0", "arg1")
- df0.groupby($"c0").transpose_and_dot("arg0", "arg1").collect() shouldEqual
- Seq(Row(1, Seq(Seq(26.0, 30.0, 34.0), Seq(38.0, 44.0, 50.0), Seq(50.0, 58.0, 66.0))))
+ // if use checkAnswer here, fail for some reason, maybe type? but it's okay on spark-2.0
+ assert(df0.groupby($"c0").transpose_and_dot("arg0", "arg1").collect() ===
+ Seq(Row(1, Seq(Seq(26.0, 30.0, 34.0), Seq(38.0, 44.0, 50.0), Seq(50.0, 58.0, 66.0)))))
}
}
http://git-wip-us.apache.org/repos/asf/incubator-hivemall/blob/ce4a4898/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala
----------------------------------------------------------------------
diff --git a/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala b/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala
index fe73a1b..8446677 100644
--- a/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala
+++ b/spark/spark-2.0/src/test/scala/org/apache/spark/sql/hive/HivemallOpsSuite.scala
@@ -23,7 +23,6 @@ import org.apache.spark.sql.types._
import org.apache.spark.sql.{AnalysisException, Column, Row, functions}
import org.apache.spark.test.TestDoubleWrapper._
import org.apache.spark.test.{HivemallFeatureQueryTest, TestUtils, VectorQueryTest}
-import org.scalatest.Matchers._
final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest {
@@ -188,7 +187,6 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest {
test("ftvec.selection - chi2") {
import hiveContext.implicits._
- implicit val doubleEquality = org.scalactic.TolerantNumerics.tolerantDoubleEquality(1e-5)
// see also hivemall.ftvec.selection.ChiSquareUDFTest
val df = Seq(
@@ -203,17 +201,17 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest {
.toDF("arg0", "arg1")
val result = df.select(chi2(df("arg0"), df("arg1"))).collect
- result should have length 1
+ assert(result.length == 1)
val chi2Val = result.head.getAs[Row](0).getAs[Seq[Double]](0)
val pVal = result.head.getAs[Row](0).getAs[Seq[Double]](1)
(chi2Val, Seq(10.81782088, 3.59449902, 116.16984746, 67.24482759))
.zipped
- .foreach((actual, expected) => actual shouldEqual expected)
+ .foreach((actual, expected) => assert(actual ~== expected))
(pVal, Seq(4.47651499e-03, 1.65754167e-01, 5.94344354e-26, 2.50017968e-15))
.zipped
- .foreach((actual, expected) => actual shouldEqual expected)
+ .foreach((actual, expected) => assert(actual ~== expected))
}
test("ftvec.conv - quantify") {
@@ -393,8 +391,8 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest {
val data = Seq(Seq(0, 1, 3), Seq(2, 4, 1), Seq(5, 4, 9))
val df = data.map(d => (d, Seq(3, 1, 2), 2)).toDF("features", "importance_list", "k")
- df.select(select_k_best(df("features"), df("importance_list"), df("k"))).collect shouldEqual
- data.map(s => Row(Seq(s(0).toDouble, s(2).toDouble)))
+ checkAnswer(df.select(select_k_best(df("features"), df("importance_list"), df("k"))),
+ data.map(s => Row(Seq(s(0).toDouble, s(2).toDouble))))
}
test("misc - sigmoid") {
@@ -689,7 +687,6 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest {
test("user-defined aggregators for ftvec.selection") {
import hiveContext.implicits._
- implicit val doubleEquality = org.scalactic.TolerantNumerics.tolerantDoubleEquality(1e-5)
// see also hivemall.ftvec.selection.SignalNoiseRatioUDAFTest
// binary class
@@ -711,7 +708,7 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest {
val row0 = df0.groupby($"c0").snr("arg0", "arg1").collect
(row0(0).getAs[Seq[Double]](1), Seq(4.38425236, 0.26390002, 15.83984511, 26.87005769))
.zipped
- .foreach((actual, expected) => actual shouldEqual expected)
+ .foreach((actual, expected) => assert(actual ~== expected))
// multiple class
// +-----------------+-------+
@@ -732,7 +729,7 @@ final class HivemallOpsWithFeatureSuite extends HivemallFeatureQueryTest {
val row1 = df1.groupby($"c0").snr("arg0", "arg1").collect
(row1(0).getAs[Seq[Double]](1), Seq(8.43181818, 1.32121212, 42.94949495, 33.80952381))
.zipped
- .foreach((actual, expected) => actual shouldEqual expected)
+ .foreach((actual, expected) => assert(actual ~== expected))
}
test("user-defined aggregators for tools.matrix") {