You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@spark.apache.org by ma...@apache.org on 2014/03/04 01:39:02 UTC
git commit: Added a unit test for PairRDDFunctions.lookup
Repository: spark
Updated Branches:
refs/heads/master b55cade85 -> 923dba509
Added a unit test for PairRDDFunctions.lookup
Lookup didn't have a unit test. Added two tests, one for with a partitioner, and one for without.
Author: Bryn Keller <br...@intel.com>
Closes #36 from xoltar/lookup and squashes the following commits:
3bc0d44 [Bryn Keller] Added a unit test for PairRDDFunctions.lookup
Project: http://git-wip-us.apache.org/repos/asf/spark/repo
Commit: http://git-wip-us.apache.org/repos/asf/spark/commit/923dba50
Tree: http://git-wip-us.apache.org/repos/asf/spark/tree/923dba50
Diff: http://git-wip-us.apache.org/repos/asf/spark/diff/923dba50
Branch: refs/heads/master
Commit: 923dba5096d4f7a96d67e4ee243b3b1085984bb9
Parents: b55cade
Author: Bryn Keller <br...@intel.com>
Authored: Mon Mar 3 16:38:57 2014 -0800
Committer: Matei Zaharia <ma...@databricks.com>
Committed: Mon Mar 3 16:38:57 2014 -0800
----------------------------------------------------------------------
.../spark/rdd/PairRDDFunctionsSuite.scala | 26 ++++++++++++++++++++
1 file changed, 26 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/spark/blob/923dba50/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala
----------------------------------------------------------------------
diff --git a/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala b/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala
index e3e2377..85e8eb5 100644
--- a/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala
+++ b/core/src/test/scala/org/apache/spark/rdd/PairRDDFunctionsSuite.scala
@@ -347,6 +347,32 @@ class PairRDDFunctionsSuite extends FunSuite with SharedSparkContext {
*/
pairs.saveAsNewAPIHadoopFile[ConfigTestFormat]("ignored")
}
+
+ test("lookup") {
+ val pairs = sc.parallelize(Array((1,2), (3,4), (5,6), (5,7)))
+
+ assert(pairs.partitioner === None)
+ assert(pairs.lookup(1) === Seq(2))
+ assert(pairs.lookup(5) === Seq(6,7))
+ assert(pairs.lookup(-1) === Seq())
+
+ }
+
+ test("lookup with partitioner") {
+ val pairs = sc.parallelize(Array((1,2), (3,4), (5,6), (5,7)))
+
+ val p = new Partitioner {
+ def numPartitions: Int = 2
+
+ def getPartition(key: Any): Int = Math.abs(key.hashCode() % 2)
+ }
+ val shuffled = pairs.partitionBy(p)
+
+ assert(shuffled.partitioner === Some(p))
+ assert(shuffled.lookup(1) === Seq(2))
+ assert(shuffled.lookup(5) === Seq(6,7))
+ assert(shuffled.lookup(-1) === Seq())
+ }
}
/*