You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by me...@apache.org on 2015/03/09 19:44:46 UTC
spark git commit: [branch-1.0][SPARK-4355] ColumnStatisticsAggregator
doesn't merge mean correctly
Repository: spark
Updated Branches:
refs/heads/branch-1.0 e751f8f26 -> 0afb04250
[branch-1.0][SPARK-4355] ColumnStatisticsAggregator doesn't merge mean correctly
This backports the bug fix in #3220.
Author: Xiangrui Meng <me...@databricks.com>
Closes #3850 from mengxr/SPARK-4355-1.0 and squashes the following commits:
ae9b94a [Xiangrui Meng] ColumnStatisticsAggregator doesn't merge mean correctly
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/0afb0425
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/0afb0425
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/0afb0425
Branch: refs/heads/branch-1.0
Commit: 0afb042503b081288d9800fdc45ee4d20062e7c4
Parents: e751f8f
Author: Xiangrui Meng <me...@databricks.com>
Authored: Mon Mar 9 11:44:13 2015 -0700
Committer: Xiangrui Meng <me...@gmail.com>
Committed: Mon Mar 9 11:44:13 2015 -0700
----------------------------------------------------------------------
.../mllib/linalg/distributed/RowMatrix.scala | 20 +++++++++-----------
1 file changed, 9 insertions(+), 11 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/0afb0425/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
----------------------------------------------------------------------
diff --git a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
index a2c8fb3..b8e869b 100644
--- a/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
+++ b/mllib/src/main/scala/org/apache/spark/mllib/linalg/distributed/RowMatrix.scala
@@ -136,21 +136,19 @@ private class ColumnStatisticsAggregator(private val n: Int)
var i = 0
while (i < n) {
- // merge mean together
- if (other.currMean(i) != 0.0) {
+ if (nnz(i) + other.nnz(i) != 0.0) {
+ // merge mean together
currMean(i) = (currMean(i) * nnz(i) + other.currMean(i) * other.nnz(i)) /
(nnz(i) + other.nnz(i))
- }
- // merge m2n together
- if (nnz(i) + other.nnz(i) != 0.0) {
+ // merge m2n together
currM2n(i) += other.currM2n(i) + deltaMean(i) * deltaMean(i) * nnz(i) * other.nnz(i) /
(nnz(i) + other.nnz(i))
- }
- if (currMax(i) < other.currMax(i)) {
- currMax(i) = other.currMax(i)
- }
- if (currMin(i) > other.currMin(i)) {
- currMin(i) = other.currMin(i)
+ if (currMax(i) < other.currMax(i)) {
+ currMax(i) = other.currMax(i)
+ }
+ if (currMin(i) > other.currMin(i)) {
+ currMin(i) = other.currMin(i)
+ }
}
i += 1
}
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org