You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by af...@apache.org on 2015/11/20 20:28:28 UTC
[1/2] incubator-reef git commit: [REEF-946] Implement YARN REST
submission command builder for REEF .NET client
Repository: incubator-reef
Updated Branches:
refs/heads/master 9e322c9f8 -> c91c0c6e7
[REEF-946] Implement YARN REST submission command builder for REEF .NET
client
JIRA:
[REEF-946](https://issues.apache.org/jira/browse/REEF-946)
Pull Request:
Closes #639
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/4daf5564
Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/4daf5564
Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/4daf5564
Branch: refs/heads/master
Commit: 4daf55645de7db5477a05ca19e70d7a7824183ff
Parents: 2e3e40f
Author: Anupam <an...@gmail.com>
Authored: Fri Nov 13 18:03:36 2015 -0800
Committer: Andrew Chung <af...@gmail.com>
Committed: Fri Nov 20 10:55:03 2015 -0800
----------------------------------------------------------------------
.../Org.Apache.REEF.Client.Tests.csproj | 1 +
.../WindowsYarnJobCommandProviderTests.cs | 186 +++++++++++++++++++
.../Org.Apache.REEF.Client.csproj | 6 +
.../YARN/IYarnCommandLineEnvironment.cs | 35 ++++
.../YARN/IYarnJobCommandProvider.cs | 34 ++++
.../DriverMaxMemoryAllicationPoolSizeMB.cs | 28 +++
.../YARN/Parameters/DriverMaxPermSizeMB.cs | 28 +++
.../YARN/Parameters/EnableDebugLogging.cs | 28 +++
.../YARN/WindowsYarnJobCommandProvider.cs | 96 ++++++++++
.../YARN/YarnCommandLineEnvironment.cs | 32 ++--
.../Files/REEFFileNames.cs | 11 ++
11 files changed, 472 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/4daf5564/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj b/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj
index 8094981..8c46586 100644
--- a/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj
+++ b/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj
@@ -54,6 +54,7 @@ under the License.
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="YarnClientTests.cs" />
<Compile Include="YarnConfigurationUrlProviderTests.cs" />
+ <Compile Include="WindowsYarnJobCommandProviderTests.cs" />
</ItemGroup>
<ItemGroup>
<ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Client\Org.Apache.REEF.Client.csproj">
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/4daf5564/lang/cs/Org.Apache.REEF.Client.Tests/WindowsYarnJobCommandProviderTests.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/WindowsYarnJobCommandProviderTests.cs b/lang/cs/Org.Apache.REEF.Client.Tests/WindowsYarnJobCommandProviderTests.cs
new file mode 100644
index 0000000..ca4af15
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client.Tests/WindowsYarnJobCommandProviderTests.cs
@@ -0,0 +1,186 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Collections.Generic;
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using NSubstitute;
+using Org.Apache.REEF.Client.Yarn;
+using Org.Apache.REEF.Client.YARN;
+using Org.Apache.REEF.Client.YARN.Parameters;
+using Org.Apache.REEF.Tang.Implementations.Tang;
+using Org.Apache.REEF.Tang.Util;
+
+namespace Org.Apache.REEF.Client.Tests
+{
+ [TestClass]
+ public class WindowsYarnJobCommandProviderTests
+ {
+ private static readonly List<string> AnyClassPathItems = new List<string>
+ {
+ "%HADOOP_CONF_DIR%",
+ "%HADOOP_HOME%/*",
+ "%HADOOP_HOME%/lib/*",
+ "%HADOOP_COMMON_HOME%/*",
+ "%HADOOP_COMMON_HOME%/lib/*",
+ "%HADOOP_YARN_HOME%/*",
+ "%HADOOP_YARN_HOME%/lib/*",
+ "%HADOOP_HDFS_HOME%/*",
+ "%HADOOP_HDFS_HOME%/lib/*",
+ "%HADOOP_MAPRED_HOME%/*",
+ "%HADOOP_MAPRED_HOME%/lib/*",
+ "%HADOOP_HOME%/etc/hadoop",
+ "%HADOOP_HOME%/share/hadoop/common/*",
+ "%HADOOP_HOME%/share/hadoop/common/lib/*",
+ "%HADOOP_HOME%/share/hadoop/yarn/*",
+ "%HADOOP_HOME%/share/hadoop/yarn/lib/*",
+ "%HADOOP_HOME%/share/hadoop/hdfs/*",
+ "%HADOOP_HOME%/share/hadoop/hdfs/lib/*",
+ "%HADOOP_HOME%/share/hadoop/mapreduce/*",
+ "%HADOOP_HOME%/share/hadoop/mapreduce/lib/*",
+ "reef/local/*",
+ "reef/global/*"
+ };
+
+ [TestMethod]
+ public void YarnJobCommandBuilderIsDefaultImplementationOfIYarnJobCommandBuilder()
+ {
+ Assert.IsInstanceOfType(TangFactory.GetTang().NewInjector().GetInstance<IYarnJobCommandProvider>(), typeof(WindowsYarnJobCommandProvider));
+ }
+
+ [TestMethod]
+ public void GetJobSubmissionCommandGeneratesCorrectCommand()
+ {
+ var testContext = new TestContext();
+ const string expectedCommand = @"reef\Org.Apache.REEF.Bridge.exe" +
+ " %JAVA_HOME%/bin/java -XX:PermSize=128m -XX:MaxPermSize=128m -Xmx512m " +
+ "-classpath %HADOOP_CONF_DIR%;%HADOOP_HOME%/*;%HADOOP_HOME%/lib/*;%HADO" +
+ "OP_COMMON_HOME%/*;%HADOOP_COMMON_HOME%/lib/*;%HADOOP_YARN_HOME%/*;%HAD" +
+ "OOP_YARN_HOME%/lib/*;%HADOOP_HDFS_HOME%/*;%HADOOP_HDFS_HOME%/lib/*;%HA" +
+ "DOOP_MAPRED_HOME%/*;%HADOOP_MAPRED_HOME%/lib/*;%HADOOP_HOME%/etc/hadoo" +
+ "p;%HADOOP_HOME%/share/hadoop/common/*;%HADOOP_HOME%/share/hadoop/commo" +
+ "n/lib/*;%HADOOP_HOME%/share/hadoop/yarn/*;%HADOOP_HOME%/share/hadoop/y" +
+ "arn/lib/*;%HADOOP_HOME%/share/hadoop/hdfs/*;%HADOOP_HOME%/share/hadoop" +
+ "/hdfs/lib/*;%HADOOP_HOME%/share/hadoop/mapreduce/*;%HADOOP_HOME%/share" +
+ "/hadoop/mapreduce/lib/*;reef/local/*;reef/global/* -Dproc_reef org.apa" +
+ "che.reef.bridge.client.YarnBootstrapREEFLauncher reef/local/job-submis" +
+ "sion-params.json 1> <LOG_DIR>/driver.stdout 2> <LOG_DIR>/driver.stderr";
+
+ var commandBuilder = testContext.GetCommandBuilder();
+ var jobSubmissionCommand = commandBuilder.GetJobSubmissionCommand();
+ Assert.AreEqual(expectedCommand, jobSubmissionCommand);
+ }
+
+ [TestMethod]
+ public void GetJobSubmissionCommandLoggingEnabledGeneratesCorrectCommand()
+ {
+ var testContext = new TestContext();
+ const string expectedCommand = @"reef\Org.Apache.REEF.Bridge.exe" +
+ " %JAVA_HOME%/bin/java -XX:PermSize=128m -XX:MaxPermSize=128m -Xmx512m " +
+ "-classpath %HADOOP_CONF_DIR%;%HADOOP_HOME%/*;%HADOOP_HOME%/lib/*;%HADO" +
+ "OP_COMMON_HOME%/*;%HADOOP_COMMON_HOME%/lib/*;%HADOOP_YARN_HOME%/*;%HAD" +
+ "OOP_YARN_HOME%/lib/*;%HADOOP_HDFS_HOME%/*;%HADOOP_HDFS_HOME%/lib/*;%HA" +
+ "DOOP_MAPRED_HOME%/*;%HADOOP_MAPRED_HOME%/lib/*;%HADOOP_HOME%/etc/hadoo" +
+ "p;%HADOOP_HOME%/share/hadoop/common/*;%HADOOP_HOME%/share/hadoop/commo" +
+ "n/lib/*;%HADOOP_HOME%/share/hadoop/yarn/*;%HADOOP_HOME%/share/hadoop/y" +
+ "arn/lib/*;%HADOOP_HOME%/share/hadoop/hdfs/*;%HADOOP_HOME%/share/hadoop" +
+ "/hdfs/lib/*;%HADOOP_HOME%/share/hadoop/mapreduce/*;%HADOOP_HOME%/share" +
+ "/hadoop/mapreduce/lib/*;reef/local/*;reef/global/* -Dproc_reef -Djava." +
+ "util.logging.config.class=org.apache.reef.util.logging.Config org.apac" +
+ "he.reef.bridge.client.YarnBootstrapREEFLauncher reef/local/job-submiss" +
+ "ion-params.json 1> <LOG_DIR>/driver.stdout 2> <LOG_DIR>/driver.stderr";
+ var commandBuilder = testContext.GetCommandBuilder(true);
+ var jobSubmissionCommand = commandBuilder.GetJobSubmissionCommand();
+ Assert.AreEqual(expectedCommand, jobSubmissionCommand);
+ }
+
+ [TestMethod]
+ public void GetJobSubmissionCommandSetsCorrectDriverMemoryAllocationSize()
+ {
+ var testContext = new TestContext();
+ const int sizeMB = 256;
+ const string expectedCommandFormat = @"reef\Org.Apache.REEF.Bridge.exe" +
+ " %JAVA_HOME%/bin/java -XX:PermSize=128m -XX:MaxPermSize=128m -Xmx{0}m " +
+ "-classpath %HADOOP_CONF_DIR%;%HADOOP_HOME%/*;%HADOOP_HOME%/lib/*;%HADO" +
+ "OP_COMMON_HOME%/*;%HADOOP_COMMON_HOME%/lib/*;%HADOOP_YARN_HOME%/*;%HAD" +
+ "OOP_YARN_HOME%/lib/*;%HADOOP_HDFS_HOME%/*;%HADOOP_HDFS_HOME%/lib/*;%HA" +
+ "DOOP_MAPRED_HOME%/*;%HADOOP_MAPRED_HOME%/lib/*;%HADOOP_HOME%/etc/hadoo" +
+ "p;%HADOOP_HOME%/share/hadoop/common/*;%HADOOP_HOME%/share/hadoop/commo" +
+ "n/lib/*;%HADOOP_HOME%/share/hadoop/yarn/*;%HADOOP_HOME%/share/hadoop/y" +
+ "arn/lib/*;%HADOOP_HOME%/share/hadoop/hdfs/*;%HADOOP_HOME%/share/hadoop" +
+ "/hdfs/lib/*;%HADOOP_HOME%/share/hadoop/mapreduce/*;%HADOOP_HOME%/share" +
+ "/hadoop/mapreduce/lib/*;reef/local/*;reef/global/* -Dproc_reef org.apa" +
+ "che.reef.bridge.client.YarnBootstrapREEFLauncher reef/local/job-submis" +
+ "sion-params.json 1> <LOG_DIR>/driver.stdout 2> <LOG_DIR>/driver.stderr";
+ string expectedCommand = string.Format(expectedCommandFormat, sizeMB);
+ var commandBuilder = testContext.GetCommandBuilder(maxMemAllocPoolSize: sizeMB);
+ var jobSubmissionCommand = commandBuilder.GetJobSubmissionCommand();
+ Assert.AreEqual(expectedCommand, jobSubmissionCommand);
+ }
+
+ [TestMethod]
+ public void GetJobSubmissionCommandSetsCorrectMaxPermSize()
+ {
+ var testContext = new TestContext();
+ const int sizeMB = 256;
+ const string expectedCommandFormat = @"reef\Org.Apache.REEF.Bridge.exe" +
+ " %JAVA_HOME%/bin/java -XX:PermSize=128m -XX:MaxPermSize={0}m -Xmx512m " +
+ "-classpath %HADOOP_CONF_DIR%;%HADOOP_HOME%/*;%HADOOP_HOME%/lib/*;%HADO" +
+ "OP_COMMON_HOME%/*;%HADOOP_COMMON_HOME%/lib/*;%HADOOP_YARN_HOME%/*;%HAD" +
+ "OOP_YARN_HOME%/lib/*;%HADOOP_HDFS_HOME%/*;%HADOOP_HDFS_HOME%/lib/*;%HA" +
+ "DOOP_MAPRED_HOME%/*;%HADOOP_MAPRED_HOME%/lib/*;%HADOOP_HOME%/etc/hadoo" +
+ "p;%HADOOP_HOME%/share/hadoop/common/*;%HADOOP_HOME%/share/hadoop/commo" +
+ "n/lib/*;%HADOOP_HOME%/share/hadoop/yarn/*;%HADOOP_HOME%/share/hadoop/y" +
+ "arn/lib/*;%HADOOP_HOME%/share/hadoop/hdfs/*;%HADOOP_HOME%/share/hadoop" +
+ "/hdfs/lib/*;%HADOOP_HOME%/share/hadoop/mapreduce/*;%HADOOP_HOME%/share" +
+ "/hadoop/mapreduce/lib/*;reef/local/*;reef/global/* -Dproc_reef org.apa" +
+ "che.reef.bridge.client.YarnBootstrapREEFLauncher reef/local/job-submis" +
+ "sion-params.json 1> <LOG_DIR>/driver.stdout 2> <LOG_DIR>/driver.stderr";
+ string expectedCommand = string.Format(expectedCommandFormat, sizeMB);
+
+ var commandBuilder = testContext.GetCommandBuilder(maxPermSize: sizeMB);
+ var jobSubmissionCommand = commandBuilder.GetJobSubmissionCommand();
+
+ Assert.AreEqual(expectedCommand, jobSubmissionCommand);
+ }
+
+ private class TestContext
+ {
+ public WindowsYarnJobCommandProvider GetCommandBuilder(bool enableLogging = false,
+ int? maxPermSize = null,
+ int? maxMemAllocPoolSize = null)
+ {
+ var injector = TangFactory.GetTang().NewInjector();
+
+ IYarnCommandLineEnvironment yarnCommandLineEnvironment = Substitute.For<IYarnCommandLineEnvironment>();
+ yarnCommandLineEnvironment.GetYarnClasspathList().Returns(AnyClassPathItems);
+
+ injector.BindVolatileInstance(GenericType<IYarnCommandLineEnvironment>.Class, yarnCommandLineEnvironment);
+ injector.BindVolatileParameter(GenericType<EnableDebugLogging>.Class, enableLogging);
+ if (maxPermSize.HasValue)
+ {
+ injector.BindVolatileParameter<DriverMaxPermSizeMB, int>(maxPermSize.Value);
+ }
+ if (maxMemAllocPoolSize.HasValue)
+ {
+ injector.BindVolatileParameter<DriverMaxMemoryAllicationPoolSizeMB, int>(maxMemAllocPoolSize.Value);
+ }
+
+ return injector.GetInstance<WindowsYarnJobCommandProvider>();
+ }
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/4daf5564/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
index 910ae2d..622f8f1 100644
--- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
+++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
@@ -89,11 +89,17 @@ under the License.
<Compile Include="Local\Parameters\LocalRuntimeDirectory.cs" />
<Compile Include="Local\Parameters\NumberOfEvaluators.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
+ <Compile Include="YARN\IYarnCommandLineEnvironment.cs" />
+ <Compile Include="YARN\IYarnJobCommandProvider.cs" />
<Compile Include="YARN\IJobResourceUploader.cs" />
<Compile Include="YARN\IJobSubmissionDirectoryProvider.cs" />
+ <Compile Include="YARN\Parameters\DriverMaxMemoryAllicationPoolSizeMB.cs" />
+ <Compile Include="YARN\Parameters\DriverMaxPermSizeMB.cs" />
+ <Compile Include="YARN\WindowsYarnJobCommandProvider.cs" />
<Compile Include="YARN\JobResource.cs" />
<Compile Include="YARN\JobSubmissionDirectoryProvider.cs" />
<Compile Include="YARN\LegacyJobResourceUploader.cs" />
+ <Compile Include="YARN\Parameters\EnableDebugLogging.cs" />
<Compile Include="YARN\Parameters\JobSubmissionDirectoryPrefixParameter.cs" />
<Compile Include="YARN\Parameters\SecurityTokenParameters.cs" />
<Compile Include="YARN\RestClient\DataModel\Acls.cs" />
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/4daf5564/lang/cs/Org.Apache.REEF.Client/YARN/IYarnCommandLineEnvironment.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IYarnCommandLineEnvironment.cs b/lang/cs/Org.Apache.REEF.Client/YARN/IYarnCommandLineEnvironment.cs
new file mode 100644
index 0000000..2990155
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/IYarnCommandLineEnvironment.cs
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Collections.Generic;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.Yarn
+{
+ /// <summary>
+ /// Helper methods to interact with the YARN command line.
+ /// </summary>
+ [DefaultImplementation(typeof(YarnCommandLineEnvironment))]
+ internal interface IYarnCommandLineEnvironment
+ {
+ /// <summary>
+ /// Returns the class path returned by `yarn classpath`.
+ /// </summary>
+ /// <returns>The class path returned by `yarn classpath`.</returns>
+ IList<string> GetYarnClasspathList();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/4daf5564/lang/cs/Org.Apache.REEF.Client/YARN/IYarnJobCommandProvider.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/IYarnJobCommandProvider.cs b/lang/cs/Org.Apache.REEF.Client/YARN/IYarnJobCommandProvider.cs
new file mode 100644
index 0000000..1b3c589
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/IYarnJobCommandProvider.cs
@@ -0,0 +1,34 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.YARN
+{
+ /// <summary>
+ /// Provide the command to be submitted to RM for execution of .NET driver.
+ /// </summary>
+ [DefaultImplementation(typeof(WindowsYarnJobCommandProvider))]
+ internal interface IYarnJobCommandProvider
+ {
+ /// <summary>
+ /// Builds the command to be submitted to YARNRM
+ /// </summary>
+ /// <returns>Command</returns>
+ string GetJobSubmissionCommand();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/4daf5564/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/DriverMaxMemoryAllicationPoolSizeMB.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/DriverMaxMemoryAllicationPoolSizeMB.cs b/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/DriverMaxMemoryAllicationPoolSizeMB.cs
new file mode 100644
index 0000000..0469576
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/DriverMaxMemoryAllicationPoolSizeMB.cs
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.YARN.Parameters
+{
+ [NamedParameter(
+ "Maximum memory allocation pool size for Java driver in MegaBytes.",
+ defaultValue: "512")]
+ public sealed class DriverMaxMemoryAllicationPoolSizeMB : Name<int>
+ {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/4daf5564/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/DriverMaxPermSizeMB.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/DriverMaxPermSizeMB.cs b/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/DriverMaxPermSizeMB.cs
new file mode 100644
index 0000000..dd68ee8
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/DriverMaxPermSizeMB.cs
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.YARN.Parameters
+{
+ [NamedParameter(
+ "Maximum permgen size for Java driver in MegaBytes",
+ defaultValue: "128")]
+ public sealed class DriverMaxPermSizeMB : Name<int>
+ {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/4daf5564/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/EnableDebugLogging.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/EnableDebugLogging.cs b/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/EnableDebugLogging.cs
new file mode 100644
index 0000000..bebd791
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/Parameters/EnableDebugLogging.cs
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.YARN.Parameters
+{
+ [NamedParameter(
+ "Boolean indicating whether debug logging should be enabled in Java part of runtime.",
+ defaultValue: "false")]
+ public sealed class EnableDebugLogging : Name<bool>
+ {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/4daf5564/lang/cs/Org.Apache.REEF.Client/YARN/WindowsYarnJobCommandProvider.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/WindowsYarnJobCommandProvider.cs b/lang/cs/Org.Apache.REEF.Client/YARN/WindowsYarnJobCommandProvider.cs
new file mode 100644
index 0000000..5900504
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/WindowsYarnJobCommandProvider.cs
@@ -0,0 +1,96 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Text;
+using Org.Apache.REEF.Client.Yarn;
+using Org.Apache.REEF.Client.YARN.Parameters;
+using Org.Apache.REEF.Common.Files;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.YARN
+{
+ /// <summary>
+ /// WindowsYarnJobCommandProvider is .NET implementation of `org.apache.reef.runtime.common.launch.JavaLaunchCommandBuilder`
+ /// This class provides the command to be submitted to RM for execution of .NET driver running on Windows environment.
+ /// </summary>
+ internal sealed class WindowsYarnJobCommandProvider : IYarnJobCommandProvider
+ {
+ private static readonly string JavaExe = @"%JAVA_HOME%/bin/java";
+
+ private static readonly string JvmOptionsPermSize = @"-XX:PermSize=128m";
+ private static readonly string JvmOptionsMaxPermSizeFormat = @"-XX:MaxPermSize={0}m";
+ private static readonly string JvmOptionsMaxMemoryAllocationPoolSizeFormat = @"-Xmx{0}m";
+ private static readonly string ClassPathToken = @"-classpath";
+
+ private static readonly string ProcReefProperty = @"-Dproc_reef";
+ private static readonly string JavaLoggingProperty =
+ @"-Djava.util.logging.config.class=org.apache.reef.util.logging.Config";
+ private static readonly string LauncherClassName = @"org.apache.reef.bridge.client.YarnBootstrapREEFLauncher";
+ private readonly REEFFileNames _fileNames;
+ private readonly bool _enableDebugLogging;
+ private readonly IYarnCommandLineEnvironment _yarnCommandLineEnvironment;
+ private readonly int _driverMaxMemoryAllocationPoolSizeMB;
+ private readonly int _driverMaxPermSizeMB;
+
+
+ [Inject]
+ private WindowsYarnJobCommandProvider(
+ [Parameter(typeof(EnableDebugLogging))] bool enableDebugLogging,
+ [Parameter(typeof(DriverMaxMemoryAllicationPoolSizeMB))] int driverMaxMemoryAllocationPoolSizeMB,
+ [Parameter(typeof(DriverMaxPermSizeMB))] int driverMaxPermSizeMB,
+ IYarnCommandLineEnvironment yarnCommandLineEnvironment,
+ REEFFileNames fileNames)
+ {
+ _yarnCommandLineEnvironment = yarnCommandLineEnvironment;
+ _enableDebugLogging = enableDebugLogging;
+ _fileNames = fileNames;
+ _driverMaxMemoryAllocationPoolSizeMB = driverMaxMemoryAllocationPoolSizeMB;
+ _driverMaxPermSizeMB = driverMaxPermSizeMB;
+ }
+
+ /// <summary>
+ /// Builds the command to be submitted to YARNRM
+ /// </summary>
+ /// <returns>Command string</returns>
+ public string GetJobSubmissionCommand()
+ {
+ var sb = new StringBuilder();
+ sb.Append(_fileNames.GetBridgeExePath());
+ sb.Append(" " + JavaExe);
+ sb.Append(" " + JvmOptionsPermSize);
+ sb.Append(" " + string.Format(JvmOptionsMaxPermSizeFormat, _driverMaxPermSizeMB));
+ sb.Append(" " +
+ string.Format(JvmOptionsMaxMemoryAllocationPoolSizeFormat, _driverMaxMemoryAllocationPoolSizeMB));
+ sb.Append(" " + ClassPathToken);
+ sb.Append(" " + string.Join(";", _yarnCommandLineEnvironment.GetYarnClasspathList()));
+ sb.Append(" " + ProcReefProperty);
+ if (_enableDebugLogging)
+ {
+ sb.Append(" " + JavaLoggingProperty);
+ }
+
+ sb.Append(" " + LauncherClassName);
+ sb.Append(" " +
+ string.Format("{0}/{1}/{2}",
+ _fileNames.GetReefFolderName(),
+ _fileNames.GetLocalFolderName(),
+ _fileNames.GetJobSubmissionParametersFile()));
+ sb.Append(" " + _fileNames.GetDriverLoggingConfigCommand());
+ return sb.ToString();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/4daf5564/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs
index 4f590d7..dd2fe92 100644
--- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnCommandLineEnvironment.cs
@@ -32,7 +32,7 @@ namespace Org.Apache.REEF.Client.Yarn
/// <summary>
/// Helper class to interact with the YARN command line.
/// </summary>
- internal sealed class YarnCommandLineEnvironment
+ internal sealed class YarnCommandLineEnvironment : IYarnCommandLineEnvironment
{
private static readonly Logger Logger = Logger.GetLogger(typeof(YarnCommandLineEnvironment));
@@ -42,10 +42,19 @@ namespace Org.Apache.REEF.Client.Yarn
}
/// <summary>
+ /// Returns the class path returned by `yarn classpath`.
+ /// </summary>
+ /// <returns>The class path returned by `yarn classpath`.</returns>
+ public IList<string> GetYarnClasspathList()
+ {
+ return Yarn("classpath").Split(';').Distinct().ToList();
+ }
+
+ /// <summary>
/// Returns the full Path to HADOOP_HOME
/// </summary>
/// <returns>The full Path to HADOOP_HOME</returns>
- internal string GetHadoopHomePath()
+ private string GetHadoopHomePath()
{
var path = Environment.GetEnvironmentVariable("HADOOP_HOME");
if (string.IsNullOrWhiteSpace(path))
@@ -70,7 +79,7 @@ namespace Org.Apache.REEF.Client.Yarn
/// Returns the full Path to the `yarn.cmd` file.
/// </summary>
/// <returns>The full Path to the `yarn.cmd` file.</returns>
- internal string GetYarnCommandPath()
+ private string GetYarnCommandPath()
{
var result = Path.Combine(GetHadoopHomePath(), "bin", "yarn.cmd");
if (!File.Exists(result))
@@ -87,7 +96,7 @@ namespace Org.Apache.REEF.Client.Yarn
/// </summary>
/// <param name="arguments"></param>
/// <returns>Whatever was printed to stdout by YARN.</returns>
- internal string Yarn(params string[] arguments)
+ private string Yarn(params string[] arguments)
{
var startInfo = new ProcessStartInfo
{
@@ -110,6 +119,12 @@ namespace Org.Apache.REEF.Client.Yarn
};
process.BeginOutputReadLine();
process.WaitForExit();
+
+ if (process.ExitCode != 0)
+ {
+ var ex = new Exception("YARN process exited with non-zero error code.");
+ Exceptions.Throw(ex, Logger);
+ }
}
else
{
@@ -119,14 +134,5 @@ namespace Org.Apache.REEF.Client.Yarn
}
return output.ToString();
}
-
- /// <summary>
- /// Returns the class path returned by `yarn classpath`.
- /// </summary>
- /// <returns>The class path returned by `yarn classpath`.</returns>
- internal IList<string> GetYarnClasspathList()
- {
- return Yarn("classpath").Split(';').Distinct().ToList();
- }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/4daf5564/lang/cs/Org.Apache.REEF.Common/Files/REEFFileNames.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Common/Files/REEFFileNames.cs b/lang/cs/Org.Apache.REEF.Common/Files/REEFFileNames.cs
index 4cfba30..d8f5126 100644
--- a/lang/cs/Org.Apache.REEF.Common/Files/REEFFileNames.cs
+++ b/lang/cs/Org.Apache.REEF.Common/Files/REEFFileNames.cs
@@ -55,6 +55,7 @@ namespace Org.Apache.REEF.Common.Files
private const string SECURITY_TOKEN_IDENTIFIER_FILE = "SecurityTokenId";
private const string SECURITY_TOKEN_PASSWORD_FILE = "SecurityTokenPwd";
private const string JOB_SUBMISSION_PARAMETERS_FILE = "job-submission-params.json";
+ private const string DRIVER_COMMAND_LOGGING_CONFIG = "1> <LOG_DIR>/driver.stdout 2> <LOG_DIR>/driver.stderr";
[Inject]
public REEFFileNames()
@@ -244,6 +245,16 @@ namespace Org.Apache.REEF.Common.Files
}
/// <summary>
+ /// The command that allows redirecting Driver stdout and stderr logs
+ /// to appropriate files
+ /// </summary>
+ /// <returns></returns>
+ public string GetDriverLoggingConfigCommand()
+ {
+ return DRIVER_COMMAND_LOGGING_CONFIG;
+ }
+
+ /// <summary>
/// The filename for security token identifier
/// </summary>
/// <returns>filename which contains raw bytes of security token identifier</returns>
[2/2] incubator-reef git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/incubator-reef
Posted by af...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/incubator-reef
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/c91c0c6e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/c91c0c6e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/c91c0c6e
Branch: refs/heads/master
Commit: c91c0c6e7480144397c07ed806277bfd699501ae
Parents: 4daf556 9e322c9
Author: Andrew Chung <af...@gmail.com>
Authored: Fri Nov 20 11:28:16 2015 -0800
Committer: Andrew Chung <af...@gmail.com>
Committed: Fri Nov 20 11:28:16 2015 -0800
----------------------------------------------------------------------
.../common/driver/resourcemanager/ResourceManagerStatus.java | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------