You are viewing a plain text version of this content. The canonical link for it is here.
Posted to reviews@spark.apache.org by GitBox <gi...@apache.org> on 2019/01/07 17:19:46 UTC
[GitHub] asfgit closed pull request #21509: [SPARK-24489][ML]Check for
invalid input type of weight data in ml.PowerIterationClustering
asfgit closed pull request #21509: [SPARK-24489][ML]Check for invalid input type of weight data in ml.PowerIterationClustering
URL: https://github.com/apache/spark/pull/21509
This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:
As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):
diff --git a/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala b/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala
index d9a330f67e8dc..149e99d2f195a 100644
--- a/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala
+++ b/mllib/src/main/scala/org/apache/spark/ml/clustering/PowerIterationClustering.scala
@@ -166,6 +166,7 @@ class PowerIterationClustering private[clustering] (
val w = if (!isDefined(weightCol) || $(weightCol).isEmpty) {
lit(1.0)
} else {
+ SchemaUtils.checkNumericType(dataset.schema, $(weightCol))
col($(weightCol)).cast(DoubleType)
}
diff --git a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala
index 55b460f1a4524..0ba3ffabb75d2 100644
--- a/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/ml/clustering/PowerIterationClusteringSuite.scala
@@ -145,6 +145,21 @@ class PowerIterationClusteringSuite extends SparkFunSuite
assert(msg.contains("Similarity must be nonnegative"))
}
+ test("check for invalid input types of weight") {
+ val invalidWeightData = spark.createDataFrame(Seq(
+ (0L, 1L, "a"),
+ (2L, 3L, "b")
+ )).toDF("src", "dst", "weight")
+
+ val msg = intercept[IllegalArgumentException] {
+ new PowerIterationClustering()
+ .setWeightCol("weight")
+ .assignClusters(invalidWeightData)
+ }.getMessage
+ assert(msg.contains("requirement failed: Column weight must be of type numeric" +
+ " but was actually of type string."))
+ }
+
test("test default weight") {
val dataWithoutWeight = data.sample(0.5, 1L).select('src, 'dst)
----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
For queries about this service, please contact Infrastructure at:
users@infra.apache.org
With regards,
Apache Git Services
---------------------------------------------------------------------
To unsubscribe, e-mail: reviews-unsubscribe@spark.apache.org
For additional commands, e-mail: reviews-help@spark.apache.org