You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2017/07/26 09:37:53 UTC
spark git commit: [SPARK-21524][ML] unit test fix:
ValidatorParamsSuiteHelpers generates wrong temp files
Repository: spark
Updated Branches:
refs/heads/master 16612638f -> ae4ea5fe2
[SPARK-21524][ML] unit test fix: ValidatorParamsSuiteHelpers generates wrong temp files
## What changes were proposed in this pull request?
jira: https://issues.apache.org/jira/browse/SPARK-21524
ValidatorParamsSuiteHelpers.testFileMove() is generating temp dir in the wrong place and does not delete them.
ValidatorParamsSuiteHelpers.testFileMove() is invoked by TrainValidationSplitSuite and crossValidatorSuite. Currently it uses `tempDir` from `TempDirectory`, which unfortunately is never initialized since the `boforeAll()` of `ValidatorParamsSuiteHelpers` is never invoked.
In my system, it leaves some temp directories in the assembly folder each time I run the TrainValidationSplitSuite and crossValidatorSuite.
## How was this patch tested?
unit test fix
Author: Yuhao Yang <yu...@intel.com>
Closes #18728 from hhbyyh/tempDirFix.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ae4ea5fe
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ae4ea5fe
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ae4ea5fe
Branch: refs/heads/master
Commit: ae4ea5fe253e9083e807bdc769e829f3f37265b4
Parents: 1661263
Author: Yuhao Yang <yu...@intel.com>
Authored: Wed Jul 26 10:37:48 2017 +0100
Committer: Sean Owen <so...@cloudera.com>
Committed: Wed Jul 26 10:37:48 2017 +0100
----------------------------------------------------------------------
.../org/apache/spark/ml/tuning/CrossValidatorSuite.scala | 2 +-
.../apache/spark/ml/tuning/TrainValidationSplitSuite.scala | 2 +-
.../spark/ml/tuning/ValidatorParamsSuiteHelpers.scala | 9 +++++----
3 files changed, 7 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ae4ea5fe/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala
index 2791ea7..dc6043e 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tuning/CrossValidatorSuite.scala
@@ -222,7 +222,7 @@ class CrossValidatorSuite
.setNumFolds(20)
.setEstimatorParamMaps(paramMaps)
- ValidatorParamsSuiteHelpers.testFileMove(cv)
+ ValidatorParamsSuiteHelpers.testFileMove(cv, tempDir)
}
test("read/write: CrossValidator with complex estimator") {
http://git-wip-us.apache.org/repos/asf/spark/blob/ae4ea5fe/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala
index 71a1776..7c97865 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tuning/TrainValidationSplitSuite.scala
@@ -209,7 +209,7 @@ class TrainValidationSplitSuite
.setEstimatorParamMaps(paramMaps)
.setSeed(42L)
- ValidatorParamsSuiteHelpers.testFileMove(tvs)
+ ValidatorParamsSuiteHelpers.testFileMove(tvs, tempDir)
}
test("read/write: TrainValidationSplitModel") {
http://git-wip-us.apache.org/repos/asf/spark/blob/ae4ea5fe/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala b/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala
index 1df673c..eae1f5a 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/tuning/ValidatorParamsSuiteHelpers.scala
@@ -20,11 +20,12 @@ package org.apache.spark.ml.tuning
import java.io.File
import java.nio.file.{Files, StandardCopyOption}
-import org.apache.spark.SparkFunSuite
+import org.scalatest.Assertions
+
import org.apache.spark.ml.param.{ParamMap, ParamPair, Params}
-import org.apache.spark.ml.util.{DefaultReadWriteTest, Identifiable, MLReader, MLWritable}
+import org.apache.spark.ml.util.{Identifiable, MLReader, MLWritable}
-object ValidatorParamsSuiteHelpers extends SparkFunSuite with DefaultReadWriteTest {
+object ValidatorParamsSuiteHelpers extends Assertions {
/**
* Assert sequences of estimatorParamMaps are identical.
* If the values for a parameter are not directly comparable with ===
@@ -62,7 +63,7 @@ object ValidatorParamsSuiteHelpers extends SparkFunSuite with DefaultReadWriteTe
* the path of the estimator so that if the parent directory changes, loading the
* model still works.
*/
- def testFileMove[T <: Params with MLWritable](instance: T): Unit = {
+ def testFileMove[T <: Params with MLWritable](instance: T, tempDir: File): Unit = {
val uid = instance.uid
val subdirName = Identifiable.randomUID("test")
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org