You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by do...@apache.org on 2019/06/01 05:39:05 UTC
[spark] branch branch-2.4 updated: Revert "[SPARK-27896][ML] Fix
definition of clustering silhouette coefficient for 1-element clusters"
This is an automated email from the ASF dual-hosted git repository.
dongjoon pushed a commit to branch branch-2.4
in repository https://gitbox.apache.org/repos/asf/spark.git
The following commit(s) were added to refs/heads/branch-2.4 by this push:
new ee46b0f Revert "[SPARK-27896][ML] Fix definition of clustering silhouette coefficient for 1-element clusters"
ee46b0f is described below
commit ee46b0f23c2941f4307d5bc77788661cc879f529
Author: Dongjoon Hyun <dh...@apple.com>
AuthorDate: Fri May 31 22:37:52 2019 -0700
Revert "[SPARK-27896][ML] Fix definition of clustering silhouette coefficient for 1-element clusters"
This reverts commit 16f2ceb0cb8407023e7bd14575221b0a718e50de.
---
.../spark/ml/evaluation/ClusteringEvaluator.scala | 38 +++++++++++-----------
.../ml/evaluation/ClusteringEvaluatorSuite.scala | 11 -------
2 files changed, 19 insertions(+), 30 deletions(-)
diff --git a/mllib/src/main/scala/org/apache/spark/ml/evaluation/ClusteringEvaluator.scala b/mllib/src/main/scala/org/apache/spark/ml/evaluation/ClusteringEvaluator.scala
index 4c915e0..5c1d1ae 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/evaluation/ClusteringEvaluator.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/evaluation/ClusteringEvaluator.scala
@@ -146,27 +146,27 @@ private[evaluation] abstract class Silhouette {
pointClusterId: Double,
pointClusterNumOfPoints: Long,
averageDistanceToCluster: (Double) => Double): Double = {
- if (pointClusterNumOfPoints == 1) {
- // Single-element clusters have silhouette 0
+ // Here we compute the average dissimilarity of the current point to any cluster of which the
+ // point is not a member.
+ // The cluster with the lowest average dissimilarity - i.e. the nearest cluster to the current
+ // point - is said to be the "neighboring cluster".
+ val otherClusterIds = clusterIds.filter(_ != pointClusterId)
+ val neighboringClusterDissimilarity = otherClusterIds.map(averageDistanceToCluster).min
+
+ // adjustment for excluding the node itself from the computation of the average dissimilarity
+ val currentClusterDissimilarity = if (pointClusterNumOfPoints == 1) {
0.0
} else {
- // Here we compute the average dissimilarity of the current point to any cluster of which the
- // point is not a member.
- // The cluster with the lowest average dissimilarity - i.e. the nearest cluster to the current
- // point - is said to be the "neighboring cluster".
- val otherClusterIds = clusterIds.filter(_ != pointClusterId)
- val neighboringClusterDissimilarity = otherClusterIds.map(averageDistanceToCluster).min
- // adjustment for excluding the node itself from the computation of the average dissimilarity
- val currentClusterDissimilarity =
- averageDistanceToCluster(pointClusterId) * pointClusterNumOfPoints /
- (pointClusterNumOfPoints - 1)
- if (currentClusterDissimilarity < neighboringClusterDissimilarity) {
- 1 - (currentClusterDissimilarity / neighboringClusterDissimilarity)
- } else if (currentClusterDissimilarity > neighboringClusterDissimilarity) {
- (neighboringClusterDissimilarity / currentClusterDissimilarity) - 1
- } else {
- 0.0
- }
+ averageDistanceToCluster(pointClusterId) * pointClusterNumOfPoints /
+ (pointClusterNumOfPoints - 1)
+ }
+
+ if (currentClusterDissimilarity < neighboringClusterDissimilarity) {
+ 1 - (currentClusterDissimilarity / neighboringClusterDissimilarity)
+ } else if (currentClusterDissimilarity > neighboringClusterDissimilarity) {
+ (neighboringClusterDissimilarity / currentClusterDissimilarity) - 1
+ } else {
+ 0.0
}
}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/evaluation/ClusteringEvaluatorSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/evaluation/ClusteringEvaluatorSuite.scala
index a47d0c5..e2d7756 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/evaluation/ClusteringEvaluatorSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/evaluation/ClusteringEvaluatorSuite.scala
@@ -134,15 +134,4 @@ class ClusteringEvaluatorSuite
// with wrong metadata the evaluator throws an Exception
intercept[SparkException](evaluator.evaluate(dfWrong))
}
-
- test("SPARK-27896: single-element clusters should have silhouette score of 0") {
- val twoSingleItemClusters =
- irisDataset.where($"label" === 0.0).limit(1).union(
- irisDataset.where($"label" === 1.0).limit(1))
- val evaluator = new ClusteringEvaluator()
- .setFeaturesCol("features")
- .setPredictionCol("label")
- assert(evaluator.evaluate(twoSingleItemClusters) === 0.0)
- }
-
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org