You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by pw...@apache.org on 2014/08/07 06:22:18 UTC
git commit: [SPARK-2887] fix bug of countApproxDistinct() when have
more than one partition
Repository: spark
Updated Branches:
refs/heads/master a263a7e9f -> ffd1f59a6
[SPARK-2887] fix bug of countApproxDistinct() when have more than one partition
fix bug of countApproxDistinct() when have more than one partition
Author: Davies Liu <da...@gmail.com>
Closes #1812 from davies/approx and squashes the following commits:
bf757ce [Davies Liu] fix bug of countApproxDistinct() when have more than one partition
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/ffd1f59a
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/ffd1f59a
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/ffd1f59a
Branch: refs/heads/master
Commit: ffd1f59a62a9dd9a4d5a7b09490b9d01ff1cd42d
Parents: a263a7e
Author: Davies Liu <da...@gmail.com>
Authored: Wed Aug 6 21:22:13 2014 -0700
Committer: Patrick Wendell <pw...@gmail.com>
Committed: Wed Aug 6 21:22:13 2014 -0700
----------------------------------------------------------------------
core/src/main/scala/org/apache/spark/rdd/RDD.scala | 2 +-
core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala | 10 +++++-----
2 files changed, 6 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/ffd1f59a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
----------------------------------------------------------------------
diff --git a/core/src/main/scala/org/apache/spark/rdd/RDD.scala b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
index e1c49e3..0159003 100644
--- a/core/src/main/scala/org/apache/spark/rdd/RDD.scala
+++ b/core/src/main/scala/org/apache/spark/rdd/RDD.scala
@@ -1004,7 +1004,7 @@ abstract class RDD[T: ClassTag](
},
(h1: HyperLogLogPlus, h2: HyperLogLogPlus) => {
h1.addAll(h2)
- h2
+ h1
}).cardinality()
}
http://git-wip-us.apache.org/repos/asf/spark/blob/ffd1f59a/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala b/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
index b31e3a0..4a7dc8d 100644
--- a/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/RDDSuite.scala
@@ -81,11 +81,11 @@ class RDDSuite extends FunSuite with SharedSparkContext {
def error(est: Long, size: Long) = math.abs(est - size) / size.toDouble
- val size = 100
- val uniformDistro = for (i <- 1 to 100000) yield i % size
- val simpleRdd = sc.makeRDD(uniformDistro)
- assert(error(simpleRdd.countApproxDistinct(4, 0), size) < 0.4)
- assert(error(simpleRdd.countApproxDistinct(8, 0), size) < 0.1)
+ val size = 1000
+ val uniformDistro = for (i <- 1 to 5000) yield i % size
+ val simpleRdd = sc.makeRDD(uniformDistro, 10)
+ assert(error(simpleRdd.countApproxDistinct(8, 0), size) < 0.2)
+ assert(error(simpleRdd.countApproxDistinct(12, 0), size) < 0.1)
}
test("SparkContext.union") {
---------------------------------------------------------------------
To unsubscribe, e-mail: commits-unsubscribe@spark.apache.org
For additional commands, e-mail: commits-help@spark.apache.org