You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@ignite.apache.org by nt...@apache.org on 2015/07/16 09:15:29 UTC
[03/50] incubator-ignite git commit: #IGNITE-857 WIP
#IGNITE-857 WIP
Project: http://git-wip-us.apache.org/repos/asf/incubator-ignite/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-ignite/commit/81cde9b7
Tree: http://git-wip-us.apache.org/repos/asf/incubator-ignite/tree/81cde9b7
Diff: http://git-wip-us.apache.org/repos/asf/incubator-ignite/diff/81cde9b7
Branch: refs/heads/master
Commit: 81cde9b71b3ab88023f68fa6f3bee86f20442412
Parents: b569191
Author: nikolay tikhonov <nt...@gridgain.com>
Authored: Wed Jun 3 18:37:02 2015 +0300
Committer: nikolay tikhonov <nt...@gridgain.com>
Committed: Wed Jun 3 18:37:02 2015 +0300
----------------------------------------------------------------------
.../apache/ignite/yarn/ApplicationMaster.java | 51 +++++++++++++++-----
.../apache/ignite/yarn/IgniteYarnClient.java | 15 ++----
2 files changed, 45 insertions(+), 21 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/81cde9b7/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
----------------------------------------------------------------------
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
index 9ab70d4..532830c 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/ApplicationMaster.java
@@ -19,6 +19,9 @@ package org.apache.ignite.yarn;
import com.google.common.collect.Lists;
import org.apache.hadoop.conf.*;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.*;
import org.apache.hadoop.yarn.api.protocolrecords.*;
import org.apache.hadoop.yarn.api.records.*;
@@ -33,27 +36,33 @@ import java.util.*;
* TODO
*/
public class ApplicationMaster implements AMRMClientAsync.CallbackHandler {
- Configuration configuration;
+ YarnConfiguration configuration;
NMClient nmClient;
- int numContainersToWaitFor = 5;
+ int numContainersToWaitFor = 1;
public ApplicationMaster() {
configuration = new YarnConfiguration();
+
nmClient = NMClient.createNMClient();
nmClient.init(configuration);
nmClient.start();
}
+ /** {@inheritDoc} */
public void onContainersAllocated(List<Container> containers) {
for (Container container : containers) {
try {
// Launch container by create ContainerLaunchContext
- // bin/hadoop fs -rm /user/ntikhonov/*.jar && bin/hadoop fs -copyFromLocal ./ignite-yarn.jar /user/ntikhonov
- ContainerLaunchContext ctx =
- Records.newRecord(ContainerLaunchContext.class);
+ ContainerLaunchContext ctx = Records.newRecord(ContainerLaunchContext.class);
+
+ final LocalResource igniteZip = Records.newRecord(LocalResource.class);
+ setupAppMasterJar(new Path("/user/ntikhonov/gridgain-community-fabric-1.0.6.zip"), igniteZip,
+ configuration);
+
+ ctx.setLocalResources(Collections.singletonMap("ignite", igniteZip));
ctx.setCommands(
Lists.newArrayList(
- "ls " +
+ "$LOCAL_DIRS/ignite/*/bin/ignite.sh" +
" 1>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stdout" +
" 2>" + ApplicationConstants.LOG_DIR_EXPANSION_VAR + "/stderr"
));
@@ -65,6 +74,24 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler {
}
}
+ /** {@inheritDoc} */
+ private static void setupAppMasterJar(Path jarPath, LocalResource appMasterJar, YarnConfiguration conf)
+ throws Exception {
+ FileSystem fileSystem = FileSystem.get(conf);
+ jarPath = fileSystem.makeQualified(jarPath);
+
+ FileStatus jarStat = fileSystem.getFileStatus(jarPath);
+
+ appMasterJar.setResource(ConverterUtils.getYarnUrlFromPath(jarPath));
+ appMasterJar.setSize(jarStat.getLen());
+ appMasterJar.setTimestamp(jarStat.getModificationTime());
+ appMasterJar.setType(LocalResourceType.ARCHIVE);
+ appMasterJar.setVisibility(LocalResourceVisibility.APPLICATION);
+
+ System.out.println("Path :" + jarPath);
+ }
+
+ /** {@inheritDoc} */
public void onContainersCompleted(List<ContainerStatus> statuses) {
for (ContainerStatus status : statuses) {
System.out.println("[AM] Completed container " + status.getContainerId());
@@ -74,20 +101,21 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler {
}
}
+ /** {@inheritDoc} */
public void onNodesUpdated(List<NodeReport> updated) {
}
- public void onReboot() {
- }
-
+ /** {@inheritDoc} */
public void onShutdownRequest() {
}
+ /** {@inheritDoc} */
public void onError(Throwable t) {
}
+ /** {@inheritDoc} */
public float getProgress() {
- return 0;
+ return 50;
}
public boolean doneWithContainers() {
@@ -101,7 +129,6 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler {
public static void main(String[] args) throws Exception {
ApplicationMaster master = new ApplicationMaster();
master.runMainLoop();
-
}
public void runMainLoop() throws Exception {
@@ -136,6 +163,8 @@ public class ApplicationMaster implements AMRMClientAsync.CallbackHandler {
Thread.sleep(100);
}
+
+
System.out.println("[AM] unregisterApplicationMaster 0");
// Un-register with ResourceManager
rmClient.unregisterApplicationMaster(
http://git-wip-us.apache.org/repos/asf/incubator-ignite/blob/81cde9b7/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
----------------------------------------------------------------------
diff --git a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
index e020ef4..092aaa9 100644
--- a/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
+++ b/modules/yarn/src/main/java/org/apache/ignite/yarn/IgniteYarnClient.java
@@ -74,20 +74,15 @@ public class IgniteYarnClient {
final LocalResource igniteZip = Records.newRecord(LocalResource.class);
setupAppMasterJar(new Path("/user/ntikhonov/gridgain-community-fabric-1.0.6.zip"), igniteZip, conf);
- FileSystem fileSystem = FileSystem.get(conf);
-
- Path path = fileSystem.makeQualified(new Path("/user/ntikhonov/gridgain-community-fabric-1.0.6/bin/ignite.sh"));
-
- System.out.println("Path: " + path);
- System.out.println("Path URI: " + path.toUri().toString());
-
- amContainer.setLocalResources(new HashMap<String, LocalResource>(){{
+ amContainer.setLocalResources(new HashMap<String, LocalResource>() {{
put("ignite-yarn.jar", appMasterJar);
- put("ignite", igniteZip);
+ put("gridgain-community-fabric-1.0.6.zip", igniteZip);
}});
+
+
// Setup CLASSPATH for ApplicationMaster
- Map<String, String> appMasterEnv = new HashMap<String, String>();
+ Map<String, String> appMasterEnv = new HashMap<>();
setupAppMasterEnv(appMasterEnv, conf);
amContainer.setEnvironment(appMasterEnv);