You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@myriad.apache.org by da...@apache.org on 2016/04/02 13:18:55 UTC
incubator-myriad git commit: [Myriad 171] - updateFile had parameter
change between Hadoop 2.7.0->2.7.1 [Myriad 190] - Tests fail on Hadoop 2.6.2+
Repository: incubator-myriad
Updated Branches:
refs/heads/master e1db4d8b8 -> 1daad6324
[Myriad 171] - updateFile had parameter change between Hadoop 2.7.0->2.7.1
[Myriad 190] - Tests fail on Hadoop 2.6.2+
Corrects behavior for hadoop 2.7.1+ and hadoop 2.6.2+.
Added support to specify hadoopVer from command line as follows:
```
./gradlew -PhadoopVer=X.Y.Z build
```
where `X.Y.Z` is the hadoop version.
JIRA:
[Myriad-171] https://issues.apache.org/jira/browse/MYRIAD-171
[Myriad-190] https://issues.apache.org/jira/browse/MYRIAD-190
Pull Request:
Closes #63
Author:
darinj darinj@apache.org
Project: http://git-wip-us.apache.org/repos/asf/incubator-myriad/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-myriad/commit/1daad632
Tree: http://git-wip-us.apache.org/repos/asf/incubator-myriad/tree/1daad632
Diff: http://git-wip-us.apache.org/repos/asf/incubator-myriad/diff/1daad632
Branch: refs/heads/master
Commit: 1daad6324ce8b74df0f144600785c1db3d7348d9
Parents: e1db4d8
Author: darinj <da...@gmail.com>
Authored: Mon Mar 14 02:04:26 2016 -0400
Committer: darinj <da...@gmail.com>
Committed: Sat Apr 2 06:59:57 2016 -0400
----------------------------------------------------------------------
build.gradle | 5 ++-
gradle.properties | 1 -
.../recovery/MyriadFileSystemRMStateStore.java | 37 ++++++++++++++++++--
.../myriad/scheduler/fgs/FGSTestBaseSpec.groovy | 2 ++
4 files changed, 41 insertions(+), 4 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/build.gradle
----------------------------------------------------------------------
diff --git a/build.gradle b/build.gradle
index 918f84c..067a52d 100644
--- a/build.gradle
+++ b/build.gradle
@@ -68,7 +68,10 @@ subprojects {
ext {
mesosVer = "0.24.1"
- hadoopVer = "2.7.0"
+ //Allows passing -PhadoopVer=2.6.0 from command line
+ if(!project.hasProperty('hadoopVer')) {
+ hadoopVer = "2.7.0"
+ }
metricsVer = "3.1.0"
}
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/gradle.properties
----------------------------------------------------------------------
diff --git a/gradle.properties b/gradle.properties
index 9667034..bca1f77 100644
--- a/gradle.properties
+++ b/gradle.properties
@@ -18,6 +18,5 @@
org.gradle.jvmargs=-Xmx1024m -XX:MaxPermSize=256m -XX:+HeapDumpOnOutOfMemoryError -Dfile.encoding=UTF-8
org.gradle.parallel=true
org.gradle.daemon=true
-
# faster builds: gradle build -x findBugsM
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
index dda9b28..923e29d 100644
--- a/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
+++ b/myriad-scheduler/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/recovery/MyriadFileSystemRMStateStore.java
@@ -20,6 +20,9 @@
package org.apache.hadoop.yarn.server.resourcemanager.recovery;
import java.io.IOException;
+import java.lang.reflect.InvocationTargetException;
+import java.lang.reflect.Method;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
@@ -43,11 +46,28 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore impleme
private Path myriadPathRoot = null;
private byte[] myriadStateBytes = null;
+ private Method updateFileMethod = null; //This is a cache Method so we do fewer reflection calls
+
@Override
public synchronized void initInternal(Configuration conf) throws Exception {
super.initInternal(conf);
Path rootPath = new Path(fsWorkingPath, ROOT_NAME);
myriadPathRoot = new Path(rootPath, RM_MYRIAD_ROOT);
+ updateFileMethod = getUpdateFileMethod();
+ if (updateFileMethod == null) {
+ //something is broken
+ throw new RuntimeException("Could not find valid updateFile Method");
+ }
+ }
+
+ private Method getUpdateFileMethod() {
+ Method[] methods = super.getClass().getSuperclass().getDeclaredMethods();
+ for (Method m : methods) {
+ if (m.getName().equals("updateFile")) {
+ return m;
+ }
+ }
+ return null;
}
@Override
@@ -59,7 +79,6 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore impleme
@Override
public synchronized RMState loadState() throws Exception {
RMState rmState = super.loadState();
-
Path myriadStatePath = new Path(myriadPathRoot, MYRIAD_STATE_FILE);
LOGGER.info("Loading state information for Myriad from: " + myriadStatePath);
@@ -92,12 +111,26 @@ public class MyriadFileSystemRMStateStore extends FileSystemRMStateStore impleme
LOGGER.debug("Storing state information for Myriad at: " + myriadStatePath);
try {
- updateFile(myriadStatePath, sc.toSerializedContext().toByteArray());
+ reflectedUpdateFile(myriadStatePath, sc.toSerializedContext().toByteArray());
} catch (Exception e) {
LOGGER.error("State information for Myriad could not be stored at: " + myriadStatePath, e);
}
}
+
+
+ protected void reflectedUpdateFile(Path outputPath, byte[] data) throws InvocationTargetException, IllegalAccessException {
+ Class [] parameters = updateFileMethod.getParameterTypes();
+ if (parameters.length == 2 && parameters[0].equals(Path.class) && parameters[1].isArray()) {
+ updateFileMethod.invoke(this, outputPath, data);
+ } else if (parameters.length == 3 && parameters[0].equals(Path.class) && parameters[1].isArray() && parameters[2].isPrimitive()) {
+ updateFileMethod.invoke(this, outputPath, data, true);
+ } else {
+ //something is broken
+ throw new RuntimeException("updateFile Method has unexpected parameters");
+ }
+ }
+
@Override
public synchronized void removeMyriadState() throws Exception {
if (fs.exists(myriadPathRoot)) {
http://git-wip-us.apache.org/repos/asf/incubator-myriad/blob/1daad632/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
----------------------------------------------------------------------
diff --git a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
index c769999..b87c1be 100644
--- a/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
+++ b/myriad-scheduler/src/test/java/org/apache/myriad/scheduler/fgs/FGSTestBaseSpec.groovy
@@ -21,6 +21,7 @@ package org.apache.myriad.scheduler.fgs
import com.fasterxml.jackson.databind.ObjectMapper
import com.fasterxml.jackson.dataformat.yaml.YAMLFactory
import org.apache.hadoop.yarn.api.records.*
+import org.apache.hadoop.yarn.conf.YarnConfiguration
import org.apache.hadoop.yarn.event.Dispatcher
import org.apache.hadoop.yarn.event.EventHandler
import org.apache.hadoop.yarn.server.resourcemanager.MockNodes
@@ -108,6 +109,7 @@ class FGSTestBaseSpec extends Specification {
getRMApplicationHistoryWriter() >> writer
getSystemMetricsPublisher() >> publisher
getRMNodes() >> rmNodes
+ getYarnConfiguration() >> new YarnConfiguration()
}
/******************* Offers Related ****************/