You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by me...@apache.org on 2015/01/19 19:10:18 UTC
spark git commit: [SPARK-5282][mllib]: RowMatrix easily gets int
overflow in the memory size warning
Repository: spark
Updated Branches:
refs/heads/master 1ac1c1dc1 -> 4432568aa
[SPARK-5282][mllib]: RowMatrix easily gets int overflow in the memory size warning
JIRA: https://issues.apache.org/jira/browse/SPARK-5282
fix the possible int overflow in the memory computation warning
Author: Yuhao Yang <hh...@gmail.com>
Closes #4069 from hhbyyh/addscStop and squashes the following commits:
e54e5c8 [Yuhao Yang] change to MB based number
7afac23 [Yuhao Yang] 5282: fix int overflow in the warning
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/4432568a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/4432568a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/4432568a
Branch: refs/heads/master
Commit: 4432568aac1d4a44fa1a7c3469f095eb7a6ce945
Parents: 1ac1c1d
Author: Yuhao Yang <hh...@gmail.com>
Authored: Mon Jan 19 10:10:15 2015 -0800
Committer: Xiangrui Meng <me...@databricks.com>
Committed: Mon Jan 19 10:10:15 2015 -0800
----------------------------------------------------------------------
.../org/apache/spark/mllib/linalg/distributed/RowMatrix.scala | 4 ++--
1 file changed, 2 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/4432568a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
index d5abba6..02075ed 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
@@ -131,8 +131,8 @@ class RowMatrix(
throw new IllegalArgumentException(s"Argument with more than 65535 cols: $cols")
}
if (cols > 10000) {
- val mem = cols * cols * 8
- logWarning(s"$cols columns will require at least $mem bytes of memory!")
+ val memMB = (cols.toLong * cols) / 125000
+ logWarning(s"$cols columns will require at least $memMB megabytes of memory!")
}
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org