You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by jk...@apache.org on 2016/05/19 17:25:42 UTC
spark git commit: [SPARK-15362][ML] Make spark.ml KMeansModel load
backwards compatible
Repository: spark
Updated Branches:
refs/heads/master 3facca515 -> 1052d3644
[SPARK-15362][ML] Make spark.ml KMeansModel load backwards compatible
## What changes were proposed in this pull request?
[SPARK-14646](https://issues.apache.org/jira/browse/SPARK-14646) makes ```KMeansModel``` store the cluster centers one per row. ```KMeansModel.load()``` method needs to be updated in order to load models saved with Spark 1.6.
## How was this patch tested?
Since ```save/load``` is ```Experimental``` for 1.6, I think offline test for backwards compatibility is enough.
Author: Yanbo Liang <yb...@gmail.com>
Closes #13149 from yanboliang/spark-15362.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/1052d364
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/1052d364
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/1052d364
Branch: refs/heads/master
Commit: 1052d3644d7eb0e784eb883293ce63a352a3b123
Parents: 3facca5
Author: Yanbo Liang <yb...@gmail.com>
Authored: Thu May 19 10:25:33 2016 -0700
Committer: Joseph K. Bradley <jo...@databricks.com>
Committed: Thu May 19 10:25:33 2016 -0700
----------------------------------------------------------------------
.../org/apache/spark/ml/clustering/KMeans.scala | 22 +++++++++++++++-----
1 file changed, 17 insertions(+), 5 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/1052d364/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
index 41c0aec..986f7e0 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/KMeans.scala
@@ -185,6 +185,12 @@ object KMeansModel extends MLReadable[KMeansModel] {
/** Helper class for storing model data */
private case class Data(clusterIdx: Int, clusterCenter: Vector)
+ /**
+ * We store all cluster centers in a single row and use this class to store model data by
+ * Spark 1.6 and earlier. A model can be loaded from such older data for backward compatibility.
+ */
+ private case class OldData(clusterCenters: Array[OldVector])
+
/** [[MLWriter]] instance for [[KMeansModel]] */
private[KMeansModel] class KMeansModelWriter(instance: KMeansModel) extends MLWriter {
@@ -211,13 +217,19 @@ object KMeansModel extends MLReadable[KMeansModel] {
import sqlContext.implicits._
val metadata = DefaultParamsReader.loadMetadata(path, sc, className)
-
val dataPath = new Path(path, "data").toString
- val data: Dataset[Data] = sqlContext.read.parquet(dataPath).as[Data]
- val clusterCenters = data.collect().sortBy(_.clusterIdx).map(_.clusterCenter)
- val model = new KMeansModel(metadata.uid,
- new MLlibKMeansModel(clusterCenters.map(OldVectors.fromML)))
+ val versionRegex = "([0-9]+)\\.(.+)".r
+ val versionRegex(major, _) = metadata.sparkVersion
+
+ val clusterCenters = if (major.toInt >= 2) {
+ val data: Dataset[Data] = sqlContext.read.parquet(dataPath).as[Data]
+ data.collect().sortBy(_.clusterIdx).map(_.clusterCenter).map(OldVectors.fromML)
+ } else {
+ // Loads KMeansModel stored with the old format used by Spark 1.6 and earlier.
+ sqlContext.read.parquet(dataPath).as[OldData].head().clusterCenters
+ }
+ val model = new KMeansModel(metadata.uid, new MLlibKMeansModel(clusterCenters))
DefaultParamsReader.getAndSetParams(model, metadata)
model
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org