You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mnemonic.apache.org by ga...@apache.org on 2017/04/24 22:42:36 UTC
incubator-mnemonic git commit: MNEMONIC-245: Class Path is not
serializable as a parameter of RDD so fallback to String
Repository: incubator-mnemonic
Updated Branches:
refs/heads/master 0292d5894 -> 3a8c52cef
MNEMONIC-245: Class Path is not serializable as a parameter of RDD so fallback to String
Project: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/commit/3a8c52ce
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/tree/3a8c52ce
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/diff/3a8c52ce
Branch: refs/heads/master
Commit: 3a8c52cefe062897eb95c7b8dd7ffbd2d24a9713
Parents: 0292d58
Author: Wang, Gang(Gary) <ga...@intel.com>
Authored: Mon Apr 24 15:42:06 2017 -0700
Committer: Wang, Gang(Gary) <ga...@intel.com>
Committed: Mon Apr 24 15:42:06 2017 -0700
----------------------------------------------------------------------
.../org/apache/mnemonic/spark/MneDurableOutputSession.scala | 7 +++----
.../main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala | 3 +--
.../org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala | 3 +--
3 files changed, 5 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/3a8c52ce/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
index 9a55150..657cb16 100644
--- a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
+++ b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
@@ -17,7 +17,6 @@
package org.apache.mnemonic.spark
import java.io.File
-import java.nio.file.Path
import scala.reflect.{ classTag, ClassTag }
import scala.collection.mutable.ArrayBuffer
@@ -33,14 +32,14 @@ import org.apache.mnemonic.sessions.DurableOutputSession
class MneDurableOutputSession[V: ClassTag]
extends DurableOutputSession[V, NonVolatileMemAllocator] {
- var baseDir: Path = null
+ var baseDir: String = null
var fileList: ArrayBuffer[File] = new ArrayBuffer[File]
var outputFile: File = null
var outputPrefix: String = null
private var _outidx: Long = 0L
protected def genNextPoolFile(): File = {
- val file = new File(baseDir.toFile(), f"${outputPrefix}_${_outidx}%05d.mne")
+ val file = new File(baseDir, f"${outputPrefix}_${_outidx}%05d.mne")
_outidx = _outidx + 1
fileList += file
file
@@ -71,7 +70,7 @@ object MneDurableOutputSession {
entityFactoryProxies: Array[EntityFactoryProxy],
slotKeyId: Long,
partitionPoolSize: Long,
- baseDir: Path,
+ baseDir: String,
outputPrefix: String): MneDurableOutputSession[V] = {
var ret = new MneDurableOutputSession[V]
ret.setServiceName(serviceName)
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/3a8c52ce/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
index a8462be..4c0adb4 100644
--- a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
+++ b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDD.scala
@@ -18,7 +18,6 @@
package org.apache.mnemonic.spark.rdd;
import java.io.File
-import java.nio.file.Path
import org.apache.spark.rdd.RDD
import org.apache.spark.{ Partition, TaskContext }
@@ -39,7 +38,7 @@ class DurableRDD[D: ClassTag, T: ClassTag](
var prev: RDD[T],
serviceName: String, durableTypes: Array[DurableType],
entityFactoryProxies: Array[EntityFactoryProxy], slotKeyId: Long,
- partitionPoolSize: Long, baseDir: Path,
+ partitionPoolSize: Long, baseDir: String,
f: (T, ObjectCreator[D, NonVolatileMemAllocator]) => Option[D],
preservesPartitioning: Boolean = false)
extends RDD[D](prev) {
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/3a8c52ce/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala
index 3b7800f..7ef8325 100644
--- a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala
+++ b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/rdd/DurableRDDFunctions.scala
@@ -17,7 +17,6 @@
package org.apache.mnemonic.spark.rdd
-import java.nio.file.Path
import org.apache.spark.rdd.RDD
import org.apache.spark.TaskContext
import scala.reflect.ClassTag
@@ -35,7 +34,7 @@ class DurableRDDFunctions[T: ClassTag](rdd: RDD[T]) extends Serializable {
entityFactoryProxies: Array[EntityFactoryProxy],
slotKeyId: Long,
partitionPoolSize: Long,
- baseDir: Path,
+ baseDir: String,
f: (T, ObjectCreator[D, NonVolatileMemAllocator]) => Option[D],
preservesPartitioning: Boolean = false) =
new DurableRDD[D, T](rdd,