You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by we...@apache.org on 2015/11/18 21:20:01 UTC
incubator-reef git commit: [REEF-945] Add aliases for Named
Parameters in C# and use them in the Evaluator
Repository: incubator-reef
Updated Branches:
refs/heads/master 2c0acac86 -> d169b2591
[REEF-945] Add aliases for Named Parameters in C# and use them in the Evaluator
* added NamedparameterAlias
* Updated evaluator.config file for tests
* Added test cases to deserialize evaluator config
* Fixed an issue in ClassHierarchy merge that is touched in the test scenario
* Added overload method for Bind to accept language for alias
* Added overload methods in AvroSerializer to accept ClassHierarchy
JIRA:
[REEF-945](https://issues.apache.org/jira/browse/REEF-945)
Pull Request:
This closes #640
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/d169b259
Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/d169b259
Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/d169b259
Branch: refs/heads/master
Commit: d169b259117a2dbe00fa36529623280e64b71339
Parents: 2c0acac
Author: Julia Wang <ju...@apache.org>
Authored: Fri Nov 13 17:59:54 2015 -0800
Committer: Markus Weimer <we...@apache.org>
Committed: Wed Nov 18 12:16:45 2015 -0800
----------------------------------------------------------------------
.../Org.Apache.REEF.Common.csproj | 1 +
.../Evaluator/Utils/EvaluatorConfigurations.cs | 11 ++
.../Evaluator/Utils/NamedparameterAlias.cs | 64 +++++++++
.../EvaluatorConfigurationsTests.cs | 140 ++++++++++++++++++-
.../EvaluatorTests.cs | 1 -
.../Org.Apache.REEF.Evaluator.Tests.csproj | 19 +++
.../ConfigFiles/evaluator.conf | Bin 3189 -> 1862 bytes
.../ConfigFiles/evaluatorWithService.conf | Bin 0 -> 6975 bytes
.../Org.Apache.REEF.Examples.csproj | 3 +
.../Formats/AvroConfigurationSerializer.cs | 15 +-
.../Formats/ConfigurationFile.cs | 24 ++++
.../Formats/IConfigurationSerializer.cs | 4 +
.../ClassHierarchy/ClassHierarchyImpl.cs | 2 +-
.../Configuration/ConfigurationBuilderImpl.cs | 19 ++-
.../Interface/IConfigurationBuilder.cs | 9 ++
.../parameters/ApplicationIdentifier.java | 2 +
.../parameters/DriverRemoteIdentifier.java | 2 +
.../parameters/EvaluatorIdentifier.java | 2 +
.../parameters/InitialTaskConfiguration.java | 2 +
.../parameters/RootContextConfiguration.java | 2 +
.../launch/parameters/ErrorHandlerRID.java | 2 +
.../common/launch/parameters/LaunchID.java | 2 +
pom.xml | 2 +-
23 files changed, 311 insertions(+), 17 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj b/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj
index f198e47..eb303ed 100644
--- a/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj
+++ b/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.csproj
@@ -131,6 +131,7 @@ under the License.
<Compile Include="Runtime\Evaluator\Task\TaskStatus.cs" />
<Compile Include="Runtime\Evaluator\Task\TaskStopImpl.cs" />
<Compile Include="Runtime\Evaluator\Utils\EvaluatorConfigurations.cs" />
+ <Compile Include="Runtime\Evaluator\Utils\NamedparameterAlias.cs" />
<Compile Include="Runtime\Evaluator\Utils\RemoteManager.cs" />
<Compile Include="runtime\MachineStatus.cs" />
<Compile Include="Services\IService.cs" />
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Common/Runtime/Evaluator/Utils/EvaluatorConfigurations.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Common/Runtime/Evaluator/Utils/EvaluatorConfigurations.cs b/lang/cs/Org.Apache.REEF.Common/Runtime/Evaluator/Utils/EvaluatorConfigurations.cs
index 1711475..8f60042 100644
--- a/lang/cs/Org.Apache.REEF.Common/Runtime/Evaluator/Utils/EvaluatorConfigurations.cs
+++ b/lang/cs/Org.Apache.REEF.Common/Runtime/Evaluator/Utils/EvaluatorConfigurations.cs
@@ -25,6 +25,8 @@ using Org.Apache.REEF.Common.Services;
using Org.Apache.REEF.Common.Tasks;
using Org.Apache.REEF.Tang.Formats;
using Org.Apache.REEF.Tang.Formats.AvroConfigurationDataContract;
+using Org.Apache.REEF.Tang.Implementations.Tang;
+using Org.Apache.REEF.Tang.Interface;
using Org.Apache.REEF.Utilities;
using Org.Apache.REEF.Utilities.Logging;
@@ -36,6 +38,8 @@ namespace Org.Apache.REEF.Common.Runtime.Evaluator.Utils
private readonly AvroConfiguration _avroConfiguration;
+ private readonly IConfiguration _evaluatorConfiguration;
+
private readonly string _configFile;
private string _applicationId;
@@ -65,6 +69,13 @@ namespace Org.Apache.REEF.Common.Runtime.Evaluator.Utils
_configFile = configFile;
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
_avroConfiguration = serializer.AvroDeserializeFromFile(_configFile);
+
+ var language = _avroConfiguration.language;
+ LOGGER.Log(Level.Info, "The language that created the configFile is " + language);
+
+ var classHierarchy = TangFactory.GetTang()
+ .GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
+ _evaluatorConfiguration = serializer.FromAvro(_avroConfiguration, classHierarchy);
}
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Common/Runtime/Evaluator/Utils/NamedparameterAlias.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Common/Runtime/Evaluator/Utils/NamedparameterAlias.cs b/lang/cs/Org.Apache.REEF.Common/Runtime/Evaluator/Utils/NamedparameterAlias.cs
new file mode 100644
index 0000000..4fe524f
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Common/Runtime/Evaluator/Utils/NamedparameterAlias.cs
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+using Org.Apache.REEF.Tang.Annotations;
+using Org.Apache.REEF.Tang.Formats;
+
+namespace Org.Apache.REEF.Common.Runtime.Evaluator.Utils
+{
+ [NamedParameter(alias: "org.apache.reef.runtime.common.evaluator.parameters.ApplicationIdentifier", aliasLanguage: AvroConfigurationSerializer.Java)]
+ public sealed class ApplicationIdentifier : Name<string>
+ {
+ }
+
+ [NamedParameter(alias: "org.apache.reef.runtime.common.evaluator.parameters.DriverRemoteIdentifier", aliasLanguage: AvroConfigurationSerializer.Java)]
+ public sealed class DriverRemoteIdentifier : Name<string>
+ {
+ }
+
+ [NamedParameter(alias: "org.apache.reef.runtime.common.evaluator.parameters.EvaluatorIdentifier", aliasLanguage: AvroConfigurationSerializer.Java)]
+ public sealed class EvaluatorIdentifier : Name<string>
+ {
+ }
+
+ [NamedParameter(alias: "org.apache.reef.runtime.common.evaluator.parameters.InitialTaskConfiguration", aliasLanguage: AvroConfigurationSerializer.Java)]
+ public sealed class InitialTaskConfiguration : Name<string>
+ {
+ }
+
+ [NamedParameter(alias: "org.apache.reef.runtime.common.evaluator.parameters.RootContextConfiguration", aliasLanguage: AvroConfigurationSerializer.Java)]
+ public sealed class RootContextConfiguration : Name<string>
+ {
+ }
+
+ [NamedParameter(alias: "org.apache.reef.runtime.common.evaluator.parameters.RootServiceConfiguration", aliasLanguage: AvroConfigurationSerializer.Java)]
+ public sealed class RootServiceConfiguration : Name<string>
+ {
+ }
+
+ [NamedParameter(alias: "org.apache.reef.runtime.common.launch.parameters.ErrorHandlerRID", aliasLanguage: AvroConfigurationSerializer.Java)]
+ public sealed class ErrorHandlerRid : Name<string>
+ {
+ }
+
+ [NamedParameter(alias: "org.apache.reef.runtime.common.launch.parameters.LaunchID", aliasLanguage: AvroConfigurationSerializer.Java)]
+ public sealed class LaunchId : Name<string>
+ {
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs b/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs
index aa9864c..d90eacd 100644
--- a/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs
+++ b/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorConfigurationsTests.cs
@@ -19,8 +19,16 @@
using Microsoft.VisualStudio.TestTools.UnitTesting;
using Org.Apache.REEF.Common.Runtime.Evaluator.Utils;
+using Org.Apache.REEF.Common.Services;
+using Org.Apache.REEF.Common.Tasks;
+using Org.Apache.REEF.Driver.Context;
+using Org.Apache.REEF.Examples.HelloREEF;
using Org.Apache.REEF.Tang.Formats;
+using Org.Apache.REEF.Tang.Implementations.Tang;
+using Org.Apache.REEF.Tang.Interface;
using Org.Apache.REEF.Utilities.Logging;
+using Org.Apache.REEF.Wake.Remote.Parameters;
+using Org.Apache.REEF.Wake.StreamingCodec;
namespace Org.Apache.REEF.Evaluator.Tests
{
@@ -28,10 +36,13 @@ namespace Org.Apache.REEF.Evaluator.Tests
public class EvaluatorConfigurationsTests
{
private static readonly Logger Logger = Logger.GetLogger(typeof(EvaluatorConfigurationsTests));
+ private const string EvaluatorIdPrefix = "Node-";
+ private const string ContextIdPrefix = "RootContext_";
+ private const string RemoteIdPrefix = "socket://";
+ private const string AppIdForTest = "REEF_LOCAL_RUNTIME";
[TestMethod, Priority(0), TestCategory("Unit")]
- [DeploymentItem(@"ConfigFiles")]
public void TestEvaluatorConfigurations()
{
EvaluatorConfigurations evaluatorConfigurations = new EvaluatorConfigurations("evaluator.conf");
@@ -44,9 +55,9 @@ namespace Org.Apache.REEF.Evaluator.Tests
Logger.Log(Level.Info, "ApplicationId = " + aId);
Logger.Log(Level.Info, "ErrorHandlerRID = " + rId);
- Assert.IsTrue(eId.Equals("Node-1-1440635067509"));
- Assert.IsTrue(aId.Equals("REEF_LOCAL_RUNTIME"));
- Assert.IsTrue(rId.Equals("socket://10.130.68.76:9316"));
+ Assert.IsTrue(eId.StartsWith(EvaluatorIdPrefix));
+ Assert.IsTrue(aId.Equals(AppIdForTest));
+ Assert.IsTrue(rId.StartsWith(RemoteIdPrefix));
var contextConfigString = evaluatorConfigurations.RootContextConfigurationString;
var serviceConfigString = evaluatorConfigurations.RootServiceConfigurationString;
@@ -57,7 +68,6 @@ namespace Org.Apache.REEF.Evaluator.Tests
}
[TestMethod, Priority(0), TestCategory("Unit")]
- [DeploymentItem(@"ConfigFiles")]
public void TestEvaluatorConfigurationFile()
{
AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
@@ -71,5 +81,125 @@ namespace Org.Apache.REEF.Evaluator.Tests
Logger.Log(Level.Info, "Key = " + b.key + " Value = " + b.value);
}
}
+
+ [TestMethod, Priority(0), TestCategory("Unit")]
+ public void TestDeserializationWithAlias()
+ {
+ AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
+ var avroConfiguration = serializer.AvroDeserializeFromFile("evaluator.conf");
+ var language = avroConfiguration.language;
+ Assert.IsTrue(language.Equals(AvroConfigurationSerializer.Java));
+
+ var classHierarchy = TangFactory.GetTang()
+ .GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
+ var config = serializer.FromAvro(avroConfiguration, classHierarchy);
+
+ IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(config);
+ string appid = evaluatorInjector.GetNamedInstance<ApplicationIdentifier, string>();
+ string remoteId = evaluatorInjector.GetNamedInstance<DriverRemoteIdentifier, string>();
+
+ string evaluatorIdentifier = evaluatorInjector.GetNamedInstance<EvaluatorIdentifier, string>();
+ string rid = evaluatorInjector.GetNamedInstance<ErrorHandlerRid, string>();
+ string launchId = evaluatorInjector.GetNamedInstance<LaunchId, string>();
+
+ Assert.IsTrue(remoteId.StartsWith(RemoteIdPrefix));
+ Assert.IsTrue(appid.Equals(AppIdForTest));
+ Assert.IsTrue(evaluatorIdentifier.StartsWith(EvaluatorIdPrefix));
+ Assert.IsTrue(rid.StartsWith(RemoteIdPrefix));
+ Assert.IsTrue(launchId.Equals(AppIdForTest));
+ }
+
+ /// <summary>
+ /// This test is to deserialize a evaluator configuration file using alias if the parameter cannot be
+ /// found in the class hierarchy. The config file used in the test was generated when running HelloRREEF.
+ /// It contains task and context configuration strings.
+ /// </summary>
+ [TestMethod, Priority(0), TestCategory("Unit")]
+ public void TestDeserializationForContextAndTask()
+ {
+ AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
+
+ var classHierarchy = TangFactory.GetTang()
+ .GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
+ var config = serializer.FromFile("evaluator.conf", classHierarchy);
+
+ IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(config);
+
+ string taskConfigString = evaluatorInjector.GetNamedInstance<InitialTaskConfiguration, string>();
+ string contextConfigString = evaluatorInjector.GetNamedInstance<RootContextConfiguration, string>();
+
+ var contextClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
+ {
+ typeof(ContextConfigurationOptions.ContextIdentifier).Assembly.GetName().Name
+ });
+ var contextConfig = serializer.FromString(contextConfigString, contextClassHierarchy);
+
+ var taskClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
+ {
+ typeof(ITask).Assembly.GetName().Name,
+ typeof(HelloTask).Assembly.GetName().Name
+ });
+ var taskConfig = serializer.FromString(taskConfigString, taskClassHierarchy);
+
+ var contextInjector = evaluatorInjector.ForkInjector(contextConfig);
+ string contextId = contextInjector.GetNamedInstance<ContextConfigurationOptions.ContextIdentifier, string>();
+ Assert.IsTrue(contextId.StartsWith(ContextIdPrefix));
+
+ var taskInjector = contextInjector.ForkInjector(taskConfig);
+
+ string taskId = taskInjector.GetNamedInstance<TaskConfigurationOptions.Identifier, string>();
+ ITask task = taskInjector.GetInstance<ITask>();
+ Assert.IsTrue(taskId.StartsWith("HelloTask"));
+ Assert.IsTrue(task is HelloTask);
+ }
+
+ /// <summary>
+ /// This test is to deserialize a evaluator configuration file using alias if the parameter cannot be
+ /// found in the class hierarchy. The config file used in the test was generated when running TestBroadCastReduceOperators.
+ /// It contains service and context configuration strings.
+ /// </summary>
+ [TestMethod, Priority(0), TestCategory("Unit")]
+ public void TestDeserializationForServiceAndContext()
+ {
+ AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
+
+ var classHierarchy = TangFactory.GetTang()
+ .GetClassHierarchy(new string[] { typeof(ApplicationIdentifier).Assembly.GetName().Name });
+ var config = serializer.FromFile("evaluatorWithService.conf", classHierarchy);
+
+ IInjector evaluatorInjector = TangFactory.GetTang().NewInjector(config);
+
+ string contextConfigString = evaluatorInjector.GetNamedInstance<RootContextConfiguration, string>();
+ string rootServiceConfigString = evaluatorInjector.GetNamedInstance<RootServiceConfiguration, string>();
+
+ var contextClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
+ {
+ typeof(ContextConfigurationOptions.ContextIdentifier).Assembly.GetName().Name
+ });
+
+ var contextConfig = serializer.FromString(contextConfigString, contextClassHierarchy);
+
+ var serviceClassHierarchy = TangFactory.GetTang().GetClassHierarchy(new string[]
+ {
+ typeof(ServicesConfigurationOptions).Assembly.GetName().Name,
+ typeof(IStreamingCodec<>).Assembly.GetName().Name
+ });
+ var rootServiceConfig = serializer.FromString(rootServiceConfigString, serviceClassHierarchy);
+
+ var contextInjector = evaluatorInjector.ForkInjector(contextConfig);
+ string contextId = contextInjector.GetNamedInstance<ContextConfigurationOptions.ContextIdentifier, string>();
+ Assert.IsTrue(contextId.StartsWith("MasterTaskContext"));
+
+ string serviceConfigString = TangFactory.GetTang().NewInjector(rootServiceConfig)
+ .GetNamedInstance<ServicesConfigurationOptions.ServiceConfigString, string>();
+
+ var serviceConfig = serializer.FromString(serviceConfigString, serviceClassHierarchy);
+
+ var serviceInjector = contextInjector.ForkInjector(serviceConfig);
+ var tcpCountRange = serviceInjector.GetNamedInstance<TcpPortRangeStart, int>();
+ var tcpCountCount = serviceInjector.GetNamedInstance<TcpPortRangeCount, int>();
+ Assert.IsTrue(tcpCountRange > 0);
+ Assert.IsTrue(tcpCountCount > 0);
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorTests.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorTests.cs b/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorTests.cs
index 32fc095..a6c55fb 100644
--- a/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorTests.cs
+++ b/lang/cs/Org.Apache.REEF.Evaluator.Tests/EvaluatorTests.cs
@@ -37,7 +37,6 @@ namespace Org.Apache.REEF.Evaluator.Tests
{
[TestMethod, Priority(0), TestCategory("Functional")]
[Description("Parse Evaluator configuration from Java, inject and execute Shell task with DIR command based on the configuration")]
- [DeploymentItem(@"ConfigFiles")]
public void CanInjectAndExecuteTask()
{
//To enforce that shell task dll be copied to output directory.
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.csproj b/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.csproj
index 8ef6d85..93ae56f 100644
--- a/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.csproj
+++ b/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.csproj
@@ -63,11 +63,30 @@ under the License.
<Project>{79e7f89a-1dfb-45e1-8d43-d71a954aeb98}</Project>
<Name>Org.Apache.REEF.Utilities</Name>
</ProjectReference>
+ <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Driver\Org.Apache.REEF.Driver.csproj">
+ <Project>{a6baa2a7-f52f-4329-884e-1bcf711d6805}</Project>
+ <Name>Org.Apache.REEF.Driver</Name>
+ </ProjectReference>
+ <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Examples.HelloREEF\Org.Apache.REEF.Examples.HelloREEF.csproj">
+ <Project>{0ff8cee9-b0b6-4a14-9a52-44441be048fe}</Project>
+ <Name>Org.Apache.REEF.Examples.HelloREEF</Name>
+ </ProjectReference>
+ <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Network\Org.Apache.REEF.Network.csproj">
+ <Project>{883ce800-6a6a-4e0a-b7fe-c054f4f2c1dc}</Project>
+ <Name>Org.Apache.REEF.Network</Name>
+ </ProjectReference>
+ <ProjectReference Include="$(SolutionDir)\Org.Apache.REEF.Wake\Org.Apache.REEF.Wake.csproj">
+ <Project>{cdfb3464-4041-42b1-9271-83af24cd5008}</Project>
+ <Name>Org.Apache.REEF.Wake</Name>
+ </ProjectReference>
</ItemGroup>
<ItemGroup>
<None Include="$(SolutionDir)\Org.Apache.REEF.Examples\ConfigFiles\evaluator.conf">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
</None>
+ <None Include="$(SolutionDir)\Org.Apache.REEF.Examples\ConfigFiles\evaluatorWithService.conf">
+ <CopyToOutputDirectory>Always</CopyToOutputDirectory>
+ </None>
<None Include="packages.config" />
</ItemGroup>
<Import Project="$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets" Condition="Exists('$(VSToolsPath)\TeamTest\Microsoft.TestTools.targets')" />
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf b/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf
index 53a751d..20c2016 100644
Binary files a/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf and b/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf differ
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluatorWithService.conf
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluatorWithService.conf b/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluatorWithService.conf
new file mode 100644
index 0000000..8b91c06
Binary files /dev/null and b/lang/cs/Org.Apache.REEF.Examples/ConfigFiles/evaluatorWithService.conf differ
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.csproj b/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.csproj
index f4bd67f..32d8fe3 100644
--- a/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.csproj
+++ b/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.csproj
@@ -94,6 +94,9 @@ under the License.
<None Include="ConfigFiles\evaluator.conf">
<CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
</None>
+ <None Include="ConfigFiles\evaluatorWithService.conf">
+ <CopyToOutputDirectory>PreserveNewest</CopyToOutputDirectory>
+ </None>
</ItemGroup>
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationSerializer.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationSerializer.cs b/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationSerializer.cs
index 9d2aa56..985570f 100644
--- a/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationSerializer.cs
+++ b/lang/cs/Org.Apache.REEF.Tang/Formats/AvroConfigurationSerializer.cs
@@ -51,7 +51,6 @@ namespace Org.Apache.REEF.Tang.Formats
}
}
- // TODO[REEF-842] Act on the obsoletes
public class AvroConfigurationSerializer : IConfigurationSerializer
{
public const string Java = "Java";
@@ -132,6 +131,12 @@ namespace Org.Apache.REEF.Tang.Formats
return FromAvro(avroConf);
}
+ public IConfiguration FromFile(string fileName, IClassHierarchy classHierarchy)
+ {
+ AvroConfiguration avroConf = AvroDeserializeFromFile(fileName);
+ return FromAvro(avroConf, classHierarchy);
+ }
+
public string ToBase64String(IConfiguration c)
{
return Convert.ToBase64String(ToByteArray(c));
@@ -157,10 +162,10 @@ namespace Org.Apache.REEF.Tang.Formats
return FromAvro(avroConf);
}
- [Obsolete("Deprecated in 0.14, please use AvroDeserializeFromFile instead.")]
- public AvroConfiguration AvroDeseriaizeFromFile(string fileName)
+ public IConfiguration FromString(string josonString, IClassHierarchy ch)
{
- return AvroDeserializeFromFile(fileName);
+ AvroConfiguration avroConf = JsonConvert.DeserializeObject<AvroConfiguration>(josonString);
+ return FromAvro(avroConf, ch);
}
public AvroConfiguration AvroDeserializeFromFile(string fileName)
@@ -329,7 +334,7 @@ namespace Org.Apache.REEF.Tang.Formats
{
settings.Add(new KeyValuePair<string, string>(e.key, e.value));
}
- ConfigurationFile.ProcessConfigData(cb, settings); //TODO
+ ConfigurationFile.ProcessConfigData(cb, settings, avroConfiguration.language);
return cb.Build();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Tang/Formats/ConfigurationFile.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tang/Formats/ConfigurationFile.cs b/lang/cs/Org.Apache.REEF.Tang/Formats/ConfigurationFile.cs
index e61f80a..f0c0157 100644
--- a/lang/cs/Org.Apache.REEF.Tang/Formats/ConfigurationFile.cs
+++ b/lang/cs/Org.Apache.REEF.Tang/Formats/ConfigurationFile.cs
@@ -223,6 +223,30 @@ namespace Org.Apache.REEF.Tang.Formats
return property;
}
+ public static void ProcessConfigData(IConfigurationBuilder conf, IList<KeyValuePair<string, string>> settings, string language)
+ {
+ foreach (KeyValuePair<string, string> kv in settings)
+ {
+ try
+ {
+ conf.Bind(kv.Key, kv.Value, language);
+ }
+ catch (BindException ex)
+ {
+ Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Caught(ex, Level.Error, LOGGER);
+ var e = new BindException("Failed to process configuration tuple: [" + kv.Key + "=" + kv.Value + "]", ex);
+ Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
+ }
+ catch (ClassHierarchyException ex)
+ {
+ Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Caught(ex, Level.Error, LOGGER);
+ var e = new ClassHierarchyException("Failed to process configuration tuple: [" + kv.Key + "=" + kv.Value + "]", ex);
+ Org.Apache.REEF.Utilities.Diagnostics.Exceptions.Throw(e, LOGGER);
+ }
+ }
+ }
+
+ [Obsolete("Will be changed as internal after 0.14")]
public static void ProcessConfigData(IConfigurationBuilder conf, IList<KeyValuePair<string, string>> settings)
{
foreach (KeyValuePair<string, string> kv in settings)
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Tang/Formats/IConfigurationSerializer.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tang/Formats/IConfigurationSerializer.cs b/lang/cs/Org.Apache.REEF.Tang/Formats/IConfigurationSerializer.cs
index e498413..5088d77 100644
--- a/lang/cs/Org.Apache.REEF.Tang/Formats/IConfigurationSerializer.cs
+++ b/lang/cs/Org.Apache.REEF.Tang/Formats/IConfigurationSerializer.cs
@@ -39,10 +39,14 @@ namespace Org.Apache.REEF.Tang.Formats
IConfiguration FromFile(string fileName);
+ IConfiguration FromFile(string fileName, IClassHierarchy classHierarchy);
+
IConfiguration FromByteArray(byte[] bytes);
IConfiguration FromBase64String(string serializedConfig);
IConfiguration FromString(string jsonString);
+
+ IConfiguration FromString(string josonString, IClassHierarchy ch);
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Tang/Implementations/ClassHierarchy/ClassHierarchyImpl.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tang/Implementations/ClassHierarchy/ClassHierarchyImpl.cs b/lang/cs/Org.Apache.REEF.Tang/Implementations/ClassHierarchy/ClassHierarchyImpl.cs
index dccc6b2..afc25b2 100644
--- a/lang/cs/Org.Apache.REEF.Tang/Implementations/ClassHierarchy/ClassHierarchyImpl.cs
+++ b/lang/cs/Org.Apache.REEF.Tang/Implementations/ClassHierarchy/ClassHierarchyImpl.cs
@@ -507,7 +507,7 @@ namespace Org.Apache.REEF.Tang.Implementations.ClassHierarchy
{
return ch;
}
- myJars.AddAll(otherJars);
+ myJars.Union(otherJars);
return new ClassHierarchyImpl(myJars.ToArray());
}
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Tang/Implementations/Configuration/ConfigurationBuilderImpl.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tang/Implementations/Configuration/ConfigurationBuilderImpl.cs b/lang/cs/Org.Apache.REEF.Tang/Implementations/Configuration/ConfigurationBuilderImpl.cs
index da55253..7b148cd 100644
--- a/lang/cs/Org.Apache.REEF.Tang/Implementations/Configuration/ConfigurationBuilderImpl.cs
+++ b/lang/cs/Org.Apache.REEF.Tang/Implementations/Configuration/ConfigurationBuilderImpl.cs
@@ -178,7 +178,7 @@ namespace Org.Apache.REEF.Tang.Implementations.Configuration
public void RegisterLegacyConstructor(string s, IList<string> args)
{
- IClassNode cn = (IClassNode) this.ClassHierarchy.GetNode(s);
+ IClassNode cn = (IClassNode)this.ClassHierarchy.GetNode(s);
IList<IClassNode> cnArgs = new List<IClassNode>();
for (int i = 0; i < args.Count; i++)
{
@@ -206,6 +206,11 @@ namespace Org.Apache.REEF.Tang.Implementations.Configuration
public void Bind(string key, string value)
{
INode n = this.ClassHierarchy.GetNode(key);
+ BindNode(n, value);
+ }
+
+ private void BindNode(INode n, string value)
+ {
if (n is INamedParameterNode)
{
BindParameter((INamedParameterNode)n, value);
@@ -222,6 +227,12 @@ namespace Org.Apache.REEF.Tang.Implementations.Configuration
}
}
+ public void Bind(string key, string value, string aliasLanguage)
+ {
+ INode n = ClassHierarchy.GetNode(key, aliasLanguage);
+ BindNode(n, value);
+ }
+
public void Bind(Types.INode key, Types.INode value)
{
if (key is INamedParameterNode)
@@ -236,7 +247,7 @@ namespace Org.Apache.REEF.Tang.Implementations.Configuration
IClassNode val = (IClassNode)value;
if (val.IsExternalConstructor() && !k.IsExternalConstructor())
{
- BindConstructor(k, (IClassNode) val);
+ BindConstructor(k, (IClassNode)val);
}
else
{
@@ -334,7 +345,7 @@ namespace Org.Apache.REEF.Tang.Implementations.Configuration
public string ClassPrettyDefaultString(string longName)
{
- INamedParameterNode param = (INamedParameterNode) this.ClassHierarchy.GetNode(longName);
+ INamedParameterNode param = (INamedParameterNode)this.ClassHierarchy.GetNode(longName);
return param.GetSimpleArgName() + "=" + Join(",", param.GetDefaultInstanceAsStrings());
}
@@ -358,7 +369,7 @@ namespace Org.Apache.REEF.Tang.Implementations.Configuration
public string ClassPrettyDescriptionString(string fullName)
{
- INamedParameterNode param = (INamedParameterNode) this.ClassHierarchy.GetNode(fullName);
+ INamedParameterNode param = (INamedParameterNode)this.ClassHierarchy.GetNode(fullName);
return param.GetDocumentation() + "\n" + param.GetFullName();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/cs/Org.Apache.REEF.Tang/Interface/IConfigurationBuilder.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tang/Interface/IConfigurationBuilder.cs b/lang/cs/Org.Apache.REEF.Tang/Interface/IConfigurationBuilder.cs
index f5ae58b..96c023a 100644
--- a/lang/cs/Org.Apache.REEF.Tang/Interface/IConfigurationBuilder.cs
+++ b/lang/cs/Org.Apache.REEF.Tang/Interface/IConfigurationBuilder.cs
@@ -28,6 +28,15 @@ namespace Org.Apache.REEF.Tang.Interface
IClassHierarchy GetClassHierarchy();
IConfiguration Build();
void Bind(string iface, string impl);
+
+ /// <summary>
+ /// Bind an implementation to an interface with alias language
+ /// </summary>
+ /// <param name="iface"></param>
+ /// <param name="impl"></param>
+ /// <param name="language"></param>
+ void Bind(string iface, string impl, string language);
+
void Bind(INode key, INode value);
void BindConstructor(IClassNode k, IClassNode v); //v extended from ExternalConstructor
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/ApplicationIdentifier.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/ApplicationIdentifier.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/ApplicationIdentifier.java
index 7571403..070a08a 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/ApplicationIdentifier.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/ApplicationIdentifier.java
@@ -23,6 +23,8 @@ import org.apache.reef.tang.annotations.NamedParameter;
/**
* The RM application/job identifier.
+ * An alias has been defined for this named parameter in .Net REEF as
+ * Org.Apache.REEF.Common.Runtime.Evaluator.Utils.ApplicationIdentifier
* <p>
* In YARN, this is the applicationID assigned by the resource manager.
*/
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/DriverRemoteIdentifier.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/DriverRemoteIdentifier.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/DriverRemoteIdentifier.java
index 0aeac0b..52404e9 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/DriverRemoteIdentifier.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/DriverRemoteIdentifier.java
@@ -23,6 +23,8 @@ import org.apache.reef.tang.annotations.NamedParameter;
/**
* The identifier used by the Evaluator to connect back to the Driver.
+ * An alias has been defined for this named parameter in .Net REEF as
+ * Org.Apache.REEF.Common.Runtime.Evaluator.Utils.DriverRemoteIdentifier
*/
@NamedParameter(doc = "The identifier used by the Evaluator to connect back to the Driver.")
public final class DriverRemoteIdentifier implements Name<String> {
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/EvaluatorIdentifier.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/EvaluatorIdentifier.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/EvaluatorIdentifier.java
index a8308af..7e1cbdb 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/EvaluatorIdentifier.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/EvaluatorIdentifier.java
@@ -23,6 +23,8 @@ import org.apache.reef.tang.annotations.NamedParameter;
/**
* The evaluator identifier.
+ * An alias has been defined for this named parameter in .Net REEF as
+ * Org.Apache.REEF.Common.Runtime.Evaluator.Utils.EvaluatorIdentifier
*/
@NamedParameter(doc = "The evaluator identifier.")
public final class EvaluatorIdentifier implements Name<String> {
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/InitialTaskConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/InitialTaskConfiguration.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/InitialTaskConfiguration.java
index 80cd280..2bcd752 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/InitialTaskConfiguration.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/InitialTaskConfiguration.java
@@ -23,6 +23,8 @@ import org.apache.reef.tang.annotations.NamedParameter;
/**
* An initial task to launch on startup.
+ * An alias has been defined for this named parameter in .Net REEF as
+ * Org.Apache.REEF.Common.Runtime.Evaluator.Utils.InitialTaskConfiguration
*/
@NamedParameter(doc = "An initial task to launch on startup.")
public final class InitialTaskConfiguration implements Name<String> {
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/RootContextConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/RootContextConfiguration.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/RootContextConfiguration.java
index ac87594..d500244 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/RootContextConfiguration.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/evaluator/parameters/RootContextConfiguration.java
@@ -23,6 +23,8 @@ import org.apache.reef.tang.annotations.NamedParameter;
/**
* The evaluator root context configuration.
+ * An alias has been defined for this named parameter in .Net REEF as
+ * Org.Apache.REEF.Common.Runtime.Evaluator.Utils.RootContextConfiguration
*/
@NamedParameter(doc = "The evaluator root context configuration.")
public final class RootContextConfiguration implements Name<String> {
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/launch/parameters/ErrorHandlerRID.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/launch/parameters/ErrorHandlerRID.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/launch/parameters/ErrorHandlerRID.java
index e394734..7fb92c8 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/launch/parameters/ErrorHandlerRID.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/launch/parameters/ErrorHandlerRID.java
@@ -23,6 +23,8 @@ import org.apache.reef.tang.annotations.NamedParameter;
/**
* The error handler remote identifier.
+ * An alias has been defined for this named parameter in .Net REEF as
+ * Org.Apache.REEF.Common.Runtime.Evaluator.Utils.ErrorHandlerRID
*/
@NamedParameter(doc = "The error handler remote identifier.", short_name = ErrorHandlerRID.SHORT_NAME,
default_value = ErrorHandlerRID.NONE)
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/launch/parameters/LaunchID.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/launch/parameters/LaunchID.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/launch/parameters/LaunchID.java
index 096fa34..d594c4d 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/launch/parameters/LaunchID.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/launch/parameters/LaunchID.java
@@ -23,6 +23,8 @@ import org.apache.reef.tang.annotations.NamedParameter;
/**
* The launch identifier.
+ * An alias has been defined for this named parameter in .Net REEF as
+ * Org.Apache.REEF.Common.Runtime.Evaluator.Utils.LaunchID
*/
@NamedParameter(doc = "The launch identifier.", short_name = LaunchID.SHORT_NAME)
public final class LaunchID implements Name<String> {
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d169b259/pom.xml
----------------------------------------------------------------------
diff --git a/pom.xml b/pom.xml
index f9c22e8..bcd3e2c 100644
--- a/pom.xml
+++ b/pom.xml
@@ -297,7 +297,7 @@ under the License.
<exclude>Org.Apache.REEF.Common/Protobuf/ReefProtocol/*</exclude>
<exclude>Org.Apache.REEF.Common/Avro/*</exclude>
<!-- The below are binary data files used in tests -->
- <exclude>Org.Apache.REEF.Examples/ConfigFiles/evaluator.conf</exclude>
+ <exclude>Org.Apache.REEF.Examples/ConfigFiles/*</exclude>
<exclude>Org.Apache.REEF.Tang.Tests/simpleConstructorJavaProto.bin</exclude>
</excludes>
</configuration>