You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2016/02/09 09:47:31 UTC
spark git commit: [SPARK-13201][SPARK-13200] Deprecation warning
cleanups: KMeans & MFDataGenerator
Repository: spark
Updated Branches:
refs/heads/master 159198eff -> ce83fe975
[SPARK-13201][SPARK-13200] Deprecation warning cleanups: KMeans & MFDataGenerator
KMeans:
Make a private non-deprecated version of setRuns API so that we can call it from the PythonAPI without deprecation warnings in our own build. Also use it internally when being called from train. Add a logWarning for non-1 values
MFDataGenerator:
Apparently we are calling round on an integer which now in Scala 2.11 results in a warning (it didn't make any sense before either). Figure out if this is a mistake we can just remove or if we got the types wrong somewhere.
I put these two together since they are both deprecation fixes in MLlib and pretty small, but I can split them up if we would prefer it that way.
Author: Holden Karau <ho...@us.ibm.com>
Closes #11112 from holdenk/SPARK-13201-non-deprecated-setRuns-SPARK-mathround-integer.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ce83fe97
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ce83fe97
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ce83fe97
Branch: refs/heads/master
Commit: ce83fe9756582e73ada21c3741d15aa9bbf385ed
Parents: 159198e
Author: Holden Karau <ho...@us.ibm.com>
Authored: Tue Feb 9 08:47:28 2016 +0000
Committer: Sean Owen <so...@cloudera.com>
Committed: Tue Feb 9 08:47:28 2016 +0000
----------------------------------------------------------------------
.../apache/spark/mllib/api/python/PythonMLLibAPI.scala | 2 +-
.../org/apache/spark/mllib/clustering/KMeans.scala | 13 +++++++++++--
.../org/apache/spark/mllib/util/MFDataGenerator.scala | 3 +--
3 files changed, 13 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ce83fe97/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala b/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala
index 088ec6a..93cf16e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/api/python/PythonMLLibAPI.scala
@@ -357,7 +357,7 @@ private[python] class PythonMLLibAPI extends Serializable {
val kMeansAlg = new KMeans()
.setK(k)
.setMaxIterations(maxIterations)
- .setRuns(runs)
+ .internalSetRuns(runs)
.setInitializationMode(initializationMode)
.setInitializationSteps(initializationSteps)
.setEpsilon(epsilon)
http://git-wip-us.apache.org/repos/asf/spark/blob/ce83fe97/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
index 901164a..67de62b 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/clustering/KMeans.scala
@@ -119,9 +119,18 @@ class KMeans private (
@Since("0.8.0")
@deprecated("Support for runs is deprecated. This param will have no effect in 2.0.0.", "1.6.0")
def setRuns(runs: Int): this.type = {
+ internalSetRuns(runs)
+ }
+
+ // Internal version of setRuns for Python API, this should be removed at the same time as setRuns
+ // this is done to avoid deprecation warnings in our build.
+ private[mllib] def internalSetRuns(runs: Int): this.type = {
if (runs <= 0) {
throw new IllegalArgumentException("Number of runs must be positive")
}
+ if (runs != 1) {
+ logWarning("Setting number of runs is deprecated and will have no effect in 2.0.0")
+ }
this.runs = runs
this
}
@@ -502,7 +511,7 @@ object KMeans {
seed: Long): KMeansModel = {
new KMeans().setK(k)
.setMaxIterations(maxIterations)
- .setRuns(runs)
+ .internalSetRuns(runs)
.setInitializationMode(initializationMode)
.setSeed(seed)
.run(data)
@@ -528,7 +537,7 @@ object KMeans {
initializationMode: String): KMeansModel = {
new KMeans().setK(k)
.setMaxIterations(maxIterations)
- .setRuns(runs)
+ .internalSetRuns(runs)
.setInitializationMode(initializationMode)
.run(data)
}
http://git-wip-us.apache.org/repos/asf/spark/blob/ce83fe97/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala b/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala
index 8af6750..898a09e 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/util/MFDataGenerator.scala
@@ -105,8 +105,7 @@ object MFDataGenerator {
// optionally generate testing data
if (test) {
- val testSampSize = math.min(
- math.round(sampSize * testSampFact), math.round(mn - sampSize)).toInt
+ val testSampSize = math.min(math.round(sampSize * testSampFact).toInt, mn - sampSize)
val testOmega = shuffled.slice(sampSize, sampSize + testSampSize)
val testOrdered = testOmega.sortWith(_ < _).toArray
val testData: RDD[(Int, Int, Double)] = sc.parallelize(testOrdered)
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org