You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@mnemonic.apache.org by ga...@apache.org on 2017/04/27 21:25:31 UTC
incubator-mnemonic git commit: MNEMONIC-249: Add a test case to
verify the durable generic object type dataset & bugfixes
Repository: incubator-mnemonic
Updated Branches:
refs/heads/master f2565bfcf -> b862d1241
MNEMONIC-249: Add a test case to verify the durable generic object type dataset & bugfixes
Project: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/commit/b862d124
Tree: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/tree/b862d124
Diff: http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/diff/b862d124
Branch: refs/heads/master
Commit: b862d12410461ad7a9f8c59224dc237a73c39aef
Parents: f2565bf
Author: Wang, Gang(Gary) <ga...@intel.com>
Authored: Thu Apr 27 13:05:54 2017 -0700
Committer: Wang, Gang(Gary) <ga...@intel.com>
Committed: Thu Apr 27 14:08:55 2017 -0700
----------------------------------------------------------------------
build-tools/test.conf | 2 +
.../mnemonic/sessions/DurableOutputSession.java | 14 ++-
.../spark/MneDurableOutputSession.scala | 1 +
.../java/org/apache/mnemonic/spark/Person.java | 108 +++++++++++++++++++
.../mnemonic/spark/PersonListEFProxy.java | 40 +++++++
.../spark/rdd/DurableRDDPersonDataSpec.scala | 72 +++++++++++++
mnemonic-spark/pom.xml | 23 +++-
pom.xml | 4 +-
8 files changed, 260 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/b862d124/build-tools/test.conf
----------------------------------------------------------------------
diff --git a/build-tools/test.conf b/build-tools/test.conf
index f759c9f..98a9318 100644
--- a/build-tools/test.conf
+++ b/build-tools/test.conf
@@ -62,3 +62,5 @@ mvn -Dsuites=org.apache.mnemonic.spark.rdd.DurableRDDLongDataSpec test -pl mnemo
mvn -Dsuites=org.apache.mnemonic.spark.rdd.DurableRDDChunkDataSpec test -pl mnemonic-spark/mnemonic-spark-core -DskipTests=false
+mvn -Dsuites=org.apache.mnemonic.spark.rdd.DurableRDDPersonDataSpec test -pl mnemonic-spark/mnemonic-spark-core -DskipTests=false
+
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/b862d124/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
----------------------------------------------------------------------
diff --git a/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
index fad491f..4f675ed 100644
--- a/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
+++ b/mnemonic-sessions/src/main/java/org/apache/mnemonic/sessions/DurableOutputSession.java
@@ -126,9 +126,11 @@ public abstract class DurableOutputSession<V, A extends RestorableAllocator<A>>
} catch (OutOfHybridMemory e) {
if (nv != null) {
nv.destroy();
+ nv = null;
}
if (ret != null) {
((Durable) ret).destroy();
+ ret = null;
}
if (initNextPool()) {
try { /* retry */
@@ -137,18 +139,25 @@ public abstract class DurableOutputSession<V, A extends RestorableAllocator<A>>
} catch (OutOfHybridMemory ee) {
if (nv != null) {
nv.destroy();
+ nv = null;
}
if (ret != null) {
((Durable) ret).destroy();
+ ret = null;
}
}
}
}
- if (null != ret) {
+ if (null != ret && null != nv) {
m_recordmap.put(ret, nv);
} else {
if (null != nv) {
nv.destroy();
+ nv = null;
+ }
+ if (ret != null) {
+ ((Durable) ret).destroy();
+ ret = null;
}
}
return ret;
@@ -173,7 +182,8 @@ public abstract class DurableOutputSession<V, A extends RestorableAllocator<A>>
if (m_recordmap.containsKey(v)) {
nv = m_recordmap.remove(v);
} else {
- throw new RuntimeException("The record hasn't been created by newDurableObjectRecord()");
+ throw new RuntimeException("The record hasn't been created by newDurableObjectRecord(...) "
+ + "Please make sure the overrides of hashCode() and/or equals() are appropriate.");
}
break;
default:
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/b862d124/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
index 0a5b9c0..fdf456f 100644
--- a/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
+++ b/mnemonic-spark/mnemonic-spark-core/src/main/scala/org/apache/mnemonic/spark/MneDurableOutputSession.scala
@@ -59,6 +59,7 @@ private[spark] class MneDurableOutputSession[V: ClassTag] (
setEntityFactoryProxies(entityFactoryProxies)
setSlotKeyId(slotKeyId)
setPoolSize(partitionPoolSize)
+ m_recparmpair = Utils.shiftDurableParams(getDurableTypes, getEntityFactoryProxies, 1);
if (!initNextPool) {
throw new DurableException("Firstly init next pool failed")
}
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/b862d124/mnemonic-spark/mnemonic-spark-core/src/test/java/org/apache/mnemonic/spark/Person.java
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/test/java/org/apache/mnemonic/spark/Person.java b/mnemonic-spark/mnemonic-spark-core/src/test/java/org/apache/mnemonic/spark/Person.java
new file mode 100644
index 0000000..bee2194
--- /dev/null
+++ b/mnemonic-spark/mnemonic-spark-core/src/test/java/org/apache/mnemonic/spark/Person.java
@@ -0,0 +1,108 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mnemonic.spark;
+
+//import java.util.Objects;
+
+import org.apache.mnemonic.Durable;
+import org.apache.mnemonic.EntityFactoryProxy;
+import org.apache.mnemonic.DurableEntity;
+import org.apache.mnemonic.DurableGetter;
+import org.apache.mnemonic.DurableSetter;
+import org.apache.mnemonic.DurableType;
+import org.apache.mnemonic.OutOfHybridMemory;
+import org.apache.mnemonic.RetrieveDurableEntityError;
+import org.testng.annotations.Test;
+
+/**
+ *
+ *
+ */
+
+@DurableEntity
+public abstract class Person<E> implements Durable, Comparable<Person<E>> {
+ E element;
+
+ @Override
+ public void initializeAfterCreate() {
+ //System.out.println("Initializing After Created");
+ }
+
+ @Override
+ public void initializeAfterRestore() {
+ //System.out.println("Initializing After Restored");
+ }
+
+ @Override
+ public void setupGenericInfo(EntityFactoryProxy[] efproxies, DurableType[] gftypes) {
+
+ }
+
+ @Test
+ public void testOutput() throws RetrieveDurableEntityError {
+ System.out.printf("Person %s, Age: %d ( %s ) \n", getName(), getAge(),
+ null == getMother() ? "No Recorded Mother" : "Has Recorded Mother");
+ }
+
+ public int compareTo(Person<E> anotherPerson) {
+ int ret = 0;
+ if (0 == ret) {
+ ret = getAge().compareTo(anotherPerson.getAge());
+ }
+ if (0 == ret) {
+ ret = getName().compareTo(anotherPerson.getName());
+ }
+ return ret;
+ }
+
+/*
+ public int hashCode() {
+ return Objects.hash(getAge(), getName());
+ }
+
+ @Override
+ public boolean equals(Object anotherPerson) {
+ return (0 == this.compareTo((Person<E>)anotherPerson)) ? true : false;
+ }
+*/
+
+ @DurableGetter(Id = 1L)
+ public abstract Short getAge();
+
+ @DurableSetter
+ public abstract void setAge(Short age);
+
+ @DurableGetter(Id = 2L)
+ public abstract String getName() throws RetrieveDurableEntityError;
+
+ @DurableSetter
+ public abstract void setName(String name, boolean destroy)
+ throws OutOfHybridMemory, RetrieveDurableEntityError;
+
+ @DurableGetter(Id = 3L)
+ public abstract Person<E> getMother() throws RetrieveDurableEntityError;
+
+ @DurableSetter
+ public abstract void setMother(Person<E> mother, boolean destroy) throws RetrieveDurableEntityError;
+
+ @DurableGetter(Id = 4L)
+ public abstract Person<E> getFather() throws RetrieveDurableEntityError;
+
+ @DurableSetter
+ public abstract void setFather(Person<E> mother, boolean destroy) throws RetrieveDurableEntityError;
+}
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/b862d124/mnemonic-spark/mnemonic-spark-core/src/test/java/org/apache/mnemonic/spark/PersonListEFProxy.java
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/test/java/org/apache/mnemonic/spark/PersonListEFProxy.java b/mnemonic-spark/mnemonic-spark-core/src/test/java/org/apache/mnemonic/spark/PersonListEFProxy.java
new file mode 100644
index 0000000..c5f8b20
--- /dev/null
+++ b/mnemonic-spark/mnemonic-spark-core/src/test/java/org/apache/mnemonic/spark/PersonListEFProxy.java
@@ -0,0 +1,40 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mnemonic.spark;
+
+import java.io.Serializable;
+import org.apache.mnemonic.DurableType;
+import org.apache.mnemonic.EntityFactoryProxy;
+import org.apache.mnemonic.RestorableAllocator;
+
+public class PersonListEFProxy implements EntityFactoryProxy, Serializable {
+ @Override
+ public <A extends RestorableAllocator<A>> Person<Long> restore(
+ A allocator, EntityFactoryProxy[] factoryproxys,
+ DurableType[] gfields, long phandler, boolean autoreclaim) {
+ return PersonFactory.restore(allocator, factoryproxys, gfields, phandler, autoreclaim);
+ }
+ @Override
+ public <A extends RestorableAllocator<A>> Person<Long> create(
+ A allocator, EntityFactoryProxy[] factoryproxys,
+ DurableType[] gfields, boolean autoreclaim) {
+ return PersonFactory.create(allocator, factoryproxys, gfields, autoreclaim);
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/b862d124/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDPersonDataSpec.scala
----------------------------------------------------------------------
diff --git a/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDPersonDataSpec.scala b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDPersonDataSpec.scala
new file mode 100644
index 0000000..fff0849
--- /dev/null
+++ b/mnemonic-spark/mnemonic-spark-core/src/test/scala/org/apache/mnemonic/spark/rdd/DurableRDDPersonDataSpec.scala
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.mnemonic.spark.rdd
+
+import scala.util._
+import scala.language.existentials
+import org.apache.mnemonic.spark.TestSpec
+import org.apache.spark.SparkConf
+import org.apache.spark.SparkContext
+import org.apache.spark.rdd.RDD
+import org.apache.mnemonic.spark.rdd.DurableRDDFunctions._
+import org.apache.mnemonic.spark.Person
+import org.apache.mnemonic.spark.PersonListEFProxy
+import org.apache.mnemonic.DurableType
+import org.apache.mnemonic.Utils
+import org.apache.mnemonic.NonVolatileMemAllocator
+import org.apache.mnemonic.EntityFactoryProxy
+import org.apache.mnemonic.sessions.ObjectCreator
+
+class DurableRDDPersonDataSpec extends TestSpec {
+
+ val defaultServiceName = "pmalloc"
+ val defaultSlotKeyId = 2L
+ val defaultPartitionSize = 1024 * 1024 * 1024L
+ val defaultBaseDirectory = "."
+ val defaultNumOfPartitions = 8
+ val defaultNumOfRecordsPerPartition = 600
+
+ behavior of "A DurableRDD with Person Type Data"
+
+ it should "supports durable generic object as its data type" in {
+ val conf = new SparkConf()
+ .setMaster("local[*]")
+ .setAppName("Test")
+ val sc = new SparkContext(conf)
+ val seed: RDD[Int] = sc.parallelize(
+ Seq.fill(defaultNumOfPartitions)(defaultNumOfRecordsPerPartition), defaultNumOfPartitions)
+ val data = seed flatMap (recnum => Seq.fill(recnum)(Random.nextInt(100).toLong)) cache
+ val durdd = data.makeDurable[Person[Long]](
+ defaultServiceName,
+ Array(DurableType.DURABLE), Array(new PersonListEFProxy),
+ defaultSlotKeyId, defaultPartitionSize,
+ (v: Long, oc: ObjectCreator[Person[Long], NonVolatileMemAllocator]) =>
+ {
+ val person = oc.newDurableObjectRecord
+ person.setAge(v.toShort)
+ person.setName(String.format("Name: [%s]", Utils.genRandomString()), true)
+ Option(person)
+ })
+ val (rcnt, ragesum) = (data.count.toLong, data.sum.toLong)
+ val (dcnt, dagesum) = (durdd.count.toLong, (durdd map {p => p.getAge.toLong} sum).toLong)
+ durdd.reset
+ assertResult((rcnt, ragesum)) {
+ (dcnt, dagesum)
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/b862d124/mnemonic-spark/pom.xml
----------------------------------------------------------------------
diff --git a/mnemonic-spark/pom.xml b/mnemonic-spark/pom.xml
index ac67f93..765b3c5 100644
--- a/mnemonic-spark/pom.xml
+++ b/mnemonic-spark/pom.xml
@@ -167,7 +167,7 @@
<argLine>-Djava.ext.dirs=${memory.service.dist.dir}:${computing.service.dist.dir}</argLine>
<systemProperties>
<spark.durable-basedir>
- .
+ ./target
</spark.durable-basedir>
<spark.service-dist-dirs>
${memory.service.dist.dir}:${computing.service.dist.dir}
@@ -227,9 +227,30 @@
<javacArg>${java.version}</javacArg>
<javacArg>-target</javacArg>
<javacArg>${java.version}</javacArg>
+ <javacArg>-XDignore.symbol.file</javacArg>
+ <javacArg>-XDenableSunApiLintControl</javacArg>
</javacArgs>
</configuration>
</plugin>
+ <plugin>
+ <groupId>org.bsc.maven</groupId>
+ <artifactId>maven-processor-plugin</artifactId>
+ <executions>
+ <execution>
+ <id>process-test</id>
+ <goals>
+ <goal>process-test</goal>
+ </goals>
+ <phase>generate-test-sources</phase>
+ <configuration>
+ <compilerArguments>-XDenableSunApiLintControl</compilerArguments>
+ <processors>
+ <processor>${project.groupId}.DurableEntityProcessor</processor>
+ </processors>
+ </configuration>
+ </execution>
+ </executions>
+ </plugin>
</plugins>
</pluginManagement>
</build>
http://git-wip-us.apache.org/repos/asf/incubator-mnemonic/blob/b862d124/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index a9ba680..c8c8967 100644
--- a/pom.xml
+++ b/pom.xml
@@ -99,7 +99,9 @@
</modules>
<properties>
- <java.version>1.7</java.version>
+ <java.version>1.8</java.version>
+ <maven.compiler.source>${java.version}</maven.compiler.source>
+ <maven.compiler.target>${java.version}</maven.compiler.target>
<maven.min-version>3.2.1</maven.min-version>
<project.build.sourceEncoding>UTF-8</project.build.sourceEncoding>
<project.reporting.outputEncoding>UTF-8</project.reporting.outputEncoding>