You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by sr...@apache.org on 2018/07/16 14:50:47 UTC
spark git commit: [SPARK-18230][MLLIB] Throw a better exception,
if the user or product doesn't exist
Repository: spark
Updated Branches:
refs/heads/master 9549a2814 -> cf9704534
[SPARK-18230][MLLIB] Throw a better exception, if the user or product doesn't exist
When invoking MatrixFactorizationModel.recommendProducts(Int, Int) with a non-existing user, a java.util.NoSuchElementException is thrown:
> java.util.NoSuchElementException: next on empty iterator
at scala.collection.Iterator$$anon$2.next(Iterator.scala:39)
at scala.collection.Iterator$$anon$2.next(Iterator.scala:37)
at scala.collection.IndexedSeqLike$Elements.next(IndexedSeqLike.scala:63)
at scala.collection.IterableLike$class.head(IterableLike.scala:107)
at scala.collection.mutable.WrappedArray.scala$collection$IndexedSeqOptimized$$super$head(WrappedArray.scala:35)
at scala.collection.IndexedSeqOptimized$class.head(IndexedSeqOptimized.scala:126)
at scala.collection.mutable.WrappedArray.head(WrappedArray.scala:35)
at org.apache.spark.mllib.recommendation.MatrixFactorizationModel.recommendProducts(MatrixFactorizationModel.scala:169)
## What changes were proposed in this pull request?
Throw a better exception, like "user-id/product-id doesn't found in the model", for a non-existent user/product
## How was this patch tested?
Added UT
Author: Shahid <sh...@gmail.com>
Closes #21740 from shahidki31/checkInvalidUserProduct.
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/cf970453
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/cf970453
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/cf970453
Branch: refs/heads/master
Commit: cf9704534903b5bbd9bd4834728c92953e45293e
Parents: 9549a28
Author: Shahid <sh...@gmail.com>
Authored: Mon Jul 16 09:50:43 2018 -0500
Committer: Sean Owen <sr...@gmail.com>
Committed: Mon Jul 16 09:50:43 2018 -0500
----------------------------------------------------------------------
.../MatrixFactorizationModel.scala | 23 +++++++++++++++-----
.../MatrixFactorizationModelSuite.scala | 21 ++++++++++++++++++
2 files changed, 38 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/cf970453/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
index ac709ad..7b49d4d 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModel.scala
@@ -78,8 +78,13 @@ class MatrixFactorizationModel @Since("0.8.0") (
/** Predict the rating of one user for one product. */
@Since("0.8.0")
def predict(user: Int, product: Int): Double = {
- val userVector = userFeatures.lookup(user).head
- val productVector = productFeatures.lookup(product).head
+ val userFeatureSeq = userFeatures.lookup(user)
+ require(userFeatureSeq.nonEmpty, s"userId: $user not found in the model")
+ val productFeatureSeq = productFeatures.lookup(product)
+ require(productFeatureSeq.nonEmpty, s"productId: $product not found in the model")
+
+ val userVector = userFeatureSeq.head
+ val productVector = productFeatureSeq.head
blas.ddot(rank, userVector, 1, productVector, 1)
}
@@ -164,9 +169,12 @@ class MatrixFactorizationModel @Since("0.8.0") (
* recommended the product is.
*/
@Since("1.1.0")
- def recommendProducts(user: Int, num: Int): Array[Rating] =
- MatrixFactorizationModel.recommend(userFeatures.lookup(user).head, productFeatures, num)
+ def recommendProducts(user: Int, num: Int): Array[Rating] = {
+ val userFeatureSeq = userFeatures.lookup(user)
+ require(userFeatureSeq.nonEmpty, s"userId: $user not found in the model")
+ MatrixFactorizationModel.recommend(userFeatureSeq.head, productFeatures, num)
.map(t => Rating(user, t._1, t._2))
+ }
/**
* Recommends users to a product. That is, this returns users who are most likely to be
@@ -181,9 +189,12 @@ class MatrixFactorizationModel @Since("0.8.0") (
* recommended the user is.
*/
@Since("1.1.0")
- def recommendUsers(product: Int, num: Int): Array[Rating] =
- MatrixFactorizationModel.recommend(productFeatures.lookup(product).head, userFeatures, num)
+ def recommendUsers(product: Int, num: Int): Array[Rating] = {
+ val productFeatureSeq = productFeatures.lookup(product)
+ require(productFeatureSeq.nonEmpty, s"productId: $product not found in the model")
+ MatrixFactorizationModel.recommend(productFeatureSeq.head, userFeatures, num)
.map(t => Rating(t._1, product, t._2))
+ }
protected override val formatVersion: String = "1.0"
http://git-wip-us.apache.org/repos/asf/spark/blob/cf970453/mllib/src/test/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModelSuite.scala
----------------------------------------------------------------------
diff --git a/mllib/src/test/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModelSuite.scala b/mllib/src/test/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModelSuite.scala
index 2c8ed05..5ed9d07 100644
--- a/mllib/src/test/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModelSuite.scala
+++ b/mllib/src/test/scala/org/apache/spark/mllib/recommendation/MatrixFactorizationModelSuite.scala
@@ -72,6 +72,27 @@ class MatrixFactorizationModelSuite extends SparkFunSuite with MLlibTestSparkCon
}
}
+ test("invalid user and product") {
+ val model = new MatrixFactorizationModel(rank, userFeatures, prodFeatures)
+
+ intercept[IllegalArgumentException] {
+ // invalid user
+ model.predict(5, 2)
+ }
+ intercept[IllegalArgumentException] {
+ // invalid product
+ model.predict(0, 5)
+ }
+ intercept[IllegalArgumentException] {
+ // invalid user
+ model.recommendProducts(5, 2)
+ }
+ intercept[IllegalArgumentException] {
+ // invalid product
+ model.recommendUsers(5, 2)
+ }
+ }
+
test("batch predict API recommendProductsForUsers") {
val model = new MatrixFactorizationModel(rank, userFeatures, prodFeatures)
val topK = 10
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org