You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mnemonic.apache.org by ga...@apache.org on 2017/04/25 21:54:35 UTC
incubator-mnemonic git commit: MNEMONIC-246: Make test data generated
from Random for unit tests
Repository: incubator-mnemonic
Updated Branches:
refs/heads/master 1e6bf9558 -> 3e628a6bb
MNEMONIC-246: Make test data generated from Random for unit tests
Project: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/commit/3e628a6b
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/tree/3e628a6b
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/diff/3e628a6b
Branch: refs/heads/master
Commit: 3e628a6bbef915c17cf13458e965f5c699d2b767
Parents: 1e6bf95
Author: Wang, Gang(Gary) <ga...@intel.com>
Authored: Tue Apr 25 14:53:50 2017 -0700
Committer: Wang, Gang(Gary) <ga...@intel.com>
Committed: Tue Apr 25 14:53:50 2017 -0700
----------------------------------------------------------------------
.../org/apache/mnemonic/sessions/DurableOutputSession.java | 2 +-
.../org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala | 9 ++++++---
2 files changed, 7 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/3e628a6b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
----------------------------------------------------------------------
diff --git a/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
index e21b5a8..fad491f 100644
--- a/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
+++ b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
@@ -50,7 +50,7 @@ public abstract class DurableOutputSession<V, A extends RestorableAllocator<A>>
/**
* Initialize the next pool, must be called before use
- *
+ * and after configuration
* @return true if success
*/
protected abstract boolean initNextPool();
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/3e628a6b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala
index 6349ce5..ac33e7c 100644
--- a/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala
+++ b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDSpec.scala
@@ -17,6 +17,7 @@
package org.apache.mnemonic.spark.rdd;
+import scala.util._
import org.apache.mnemonic.spark.TestSpec
import org.apache.spark.SparkConf
import org.apache.spark.SparkContext
@@ -33,7 +34,8 @@ class DurableRDDSpec extends TestSpec {
val defaultSlotKeyId = 2L
val defaultPartitionSize = 1024 * 1024 * 1024L
val defaultBaseDirectory = "."
- val defaultNumOfRecordsPerPartition = 5000L
+ val defaultNumOfPartitions = 8
+ val defaultNumOfRecordsPerPartition = 5000
behavior of "A DurableRDD"
@@ -43,8 +45,9 @@ class DurableRDDSpec extends TestSpec {
.setAppName("Test")
val sc = new SparkContext(conf)
// sc.getConf.getAll.foreach(println)
- val ds = Seq(5, 6, 3, 8, 4)
- val data: RDD[Int] = sc.parallelize(ds)
+ val seed: RDD[Int] = sc.parallelize(
+ Seq.fill(defaultNumOfPartitions)(defaultNumOfRecordsPerPartition), defaultNumOfPartitions)
+ val data = seed flatMap (recnum => Seq.fill(recnum)(Random.nextInt)) cache //must be cached to fix rand numbers
val durdd = data.makeDurable[Long](
defaultServiceName,
Array(DurableType.LONG), Array(),