You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by we...@apache.org on 2015/06/19 03:12:46 UTC
incubator-reef git commit: [REEF-390] Pass Service Configuration to
Evaluator as a string (.NET)
Repository: incubator-reef
Updated Branches:
refs/heads/master 9f19b13b8 -> d79f4ece2
[REEF-390] Pass Service Configuration to Evaluator as a string (.NET)
Currently, some of the Nodes were dropped by proto buffer libary when
Java deserializes a class hierarchy created in C#, resulting in Java
driver not be able to deserialize service configurations passed from C#
which use those classes. A typical example is nested generic types for
ICodec which client would like to pass to the evaluator as part of the
service config.
This PR:
* wrap the entire service config as a serialized string in service
config when serializing it cross the bridge
* Updated ServiceConfiguration to inject the named parameter to get
the string config and deserialze it
* Test cases are added for the change
JIRA:
[REEF-390](https://issues.apache.org/jira/browse/REEF-390)
Pull Request:
This closes #228
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/d79f4ece
Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/d79f4ece
Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/d79f4ece
Branch: refs/heads/master
Commit: d79f4ece2414652e627b54c9f569bcd36ea02d2c
Parents: 9f19b13
Author: Julia Wang <jw...@yahoo.com>
Authored: Wed Jun 17 15:18:23 2015 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Thu Jun 18 18:09:52 2015 -0700
----------------------------------------------------------------------
.../Services/ServiceConfiguration.cs | 12 +++-
.../Services/ServicesConfigurationOptions.cs | 5 ++
.../Bridge/Events/AllocatedEvaluator.cs | 25 ++++++-
.../GroupCommunication/GroupCommuDriverTests.cs | 74 ++++++++++++++++++++
.../Org.Apache.REEF.Network.Tests.csproj | 1 +
.../Group/Driver/Impl/GroupCommDriver.cs | 5 +-
6 files changed, 116 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d79f4ece/lang/cs/Org.Apache.REEF.Common/Services/ServiceConfiguration.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Common/Services/ServiceConfiguration.cs b/lang/cs/Org.Apache.REEF.Common/Services/ServiceConfiguration.cs
index af5070b..15ef64e 100644
--- a/lang/cs/Org.Apache.REEF.Common/Services/ServiceConfiguration.cs
+++ b/lang/cs/Org.Apache.REEF.Common/Services/ServiceConfiguration.cs
@@ -17,10 +17,14 @@
* under the License.
*/
+using System;
using System.Collections.Generic;
using System.Diagnostics.CodeAnalysis;
using Org.Apache.REEF.Tang.Annotations;
+using Org.Apache.REEF.Tang.Exceptions;
using Org.Apache.REEF.Tang.Formats;
+using Org.Apache.REEF.Tang.Implementations.Configuration;
+using Org.Apache.REEF.Tang.Implementations.Tang;
using Org.Apache.REEF.Tang.Interface;
using Org.Apache.REEF.Tang.Util;
@@ -49,7 +53,13 @@ namespace Org.Apache.REEF.Common.Services
public ServiceConfiguration(string config)
{
- TangConfig = new AvroConfigurationSerializer().FromString(config);
+ AvroConfigurationSerializer serializer = new AvroConfigurationSerializer();
+
+ string serviceConfigString = (string)TangFactory.GetTang()
+ .NewInjector(serializer.FromString(config))
+ .GetNamedInstance(typeof(ServicesConfigurationOptions.ServiceConfigString));
+
+ TangConfig = serializer.FromString(serviceConfigString);
}
public static ConfigurationModule ConfigurationModule
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d79f4ece/lang/cs/Org.Apache.REEF.Common/Services/ServicesConfigurationOptions.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Common/Services/ServicesConfigurationOptions.cs b/lang/cs/Org.Apache.REEF.Common/Services/ServicesConfigurationOptions.cs
index f4afd7b..7fb77a8 100644
--- a/lang/cs/Org.Apache.REEF.Common/Services/ServicesConfigurationOptions.cs
+++ b/lang/cs/Org.Apache.REEF.Common/Services/ServicesConfigurationOptions.cs
@@ -27,5 +27,10 @@ namespace Org.Apache.REEF.Common.Services
public class Services : Name<string>
{
}
+
+ [NamedParameter]
+ public class ServiceConfigString : Name<string>
+ {
+ }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d79f4ece/lang/cs/Org.Apache.REEF.Driver/Bridge/Events/AllocatedEvaluator.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Bridge/Events/AllocatedEvaluator.cs b/lang/cs/Org.Apache.REEF.Driver/Bridge/Events/AllocatedEvaluator.cs
index 8d544f6..c14960f 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Bridge/Events/AllocatedEvaluator.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Bridge/Events/AllocatedEvaluator.cs
@@ -23,12 +23,15 @@ using System.Globalization;
using System.Runtime.Serialization;
using Org.Apache.REEF.Common.Catalog;
using Org.Apache.REEF.Common.Evaluator;
+using Org.Apache.REEF.Common.Services;
using Org.Apache.REEF.Driver.Bridge.Clr2java;
using Org.Apache.REEF.Driver.Context;
using Org.Apache.REEF.Driver.Evaluator;
using Org.Apache.REEF.Tang.Formats;
using Org.Apache.REEF.Tang.Implementations.Configuration;
+using Org.Apache.REEF.Tang.Implementations.Tang;
using Org.Apache.REEF.Tang.Interface;
+using Org.Apache.REEF.Tang.Util;
using Org.Apache.REEF.Utilities.Logging;
namespace Org.Apache.REEF.Driver.Bridge.Events
@@ -104,9 +107,9 @@ namespace Org.Apache.REEF.Driver.Bridge.Events
public void SubmitContextAndService(IConfiguration contextConfiguration, IConfiguration serviceConfiguration)
{
LOGGER.Log(Level.Info, "AllocatedEvaluator::SubmitContextAndService");
-
+
string context = _serializer.ToString(contextConfiguration);
- string service = _serializer.ToString(serviceConfiguration);
+ string service = _serializer.ToString(WrapServiceConfigAsString(serviceConfiguration));
LOGGER.Log(Level.Verbose, "serialized contextConfiguration: " + context);
LOGGER.Log(Level.Verbose, "serialized serviceConfiguration: " + service);
@@ -121,7 +124,7 @@ namespace Org.Apache.REEF.Driver.Bridge.Events
//TODO: Change this to service configuration when REEF-289(https://issues.apache.org/jira/browse/REEF-289) is fixed.
taskConfiguration = MergeWithConfigurationProviders(taskConfiguration);
string context = _serializer.ToString(contextConfiguration);
- string service = _serializer.ToString(serviceConfiguration);
+ string service = _serializer.ToString(WrapServiceConfigAsString(serviceConfiguration));
string task = _serializer.ToString(taskConfiguration);
LOGGER.Log(Level.Verbose, "serialized contextConfiguration: " + context);
@@ -201,5 +204,21 @@ namespace Org.Apache.REEF.Driver.Bridge.Events
}
return config;
}
+
+ /// <summary>
+ /// This is to wrap entire service configuration in to a serialized string
+ /// At evaluator side, we will unwrap it to get the original Service Configuration string
+ /// It is to avoid issues that some C# class Node are dropped in the deserialized ClassHierarchy by Goold ProtoBuffer deserializer
+ /// </summary>
+ /// <param name="serviceConfiguration"></param>
+ /// <returns></returns>
+ private IConfiguration WrapServiceConfigAsString(IConfiguration serviceConfiguration)
+ {
+ return TangFactory.GetTang().NewConfigurationBuilder()
+ .BindNamedParameter<ServicesConfigurationOptions.ServiceConfigString, string>(
+ GenericType<ServicesConfigurationOptions.ServiceConfigString>.Class,
+ new AvroConfigurationSerializer().ToString(serviceConfiguration))
+ .Build();
+ }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d79f4ece/lang/cs/Org.Apache.REEF.Network.Tests/GroupCommunication/GroupCommuDriverTests.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Network.Tests/GroupCommunication/GroupCommuDriverTests.cs b/lang/cs/Org.Apache.REEF.Network.Tests/GroupCommunication/GroupCommuDriverTests.cs
new file mode 100644
index 0000000..b02f8d5
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Network.Tests/GroupCommunication/GroupCommuDriverTests.cs
@@ -0,0 +1,74 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+using Microsoft.VisualStudio.TestTools.UnitTesting;
+using Org.Apache.REEF.Common.Services;
+using Org.Apache.REEF.Network.Group.Config;
+using Org.Apache.REEF.Tang.Exceptions;
+using Org.Apache.REEF.Tang.Formats;
+using Org.Apache.REEF.Tang.Implementations.Configuration;
+using Org.Apache.REEF.Tang.Implementations.Tang;
+using Org.Apache.REEF.Tang.Interface;
+using Org.Apache.REEF.Tang.Util;
+using Org.Apache.REEF.Wake.Remote.Impl;
+
+namespace Org.Apache.REEF.Network.Tests.GroupCommunication
+{
+ [TestClass]
+ public class GroupCommuDriverTests
+ {
+ [TestMethod]
+ public void TestServiceConfiguration()
+ {
+ string groupName = "group1";
+ string broadcastOperatorName = "broadcast";
+ string reduceOperatorName = "reduce";
+ string masterTaskId = "task0";
+ string driverId = "Driver Id";
+ int numTasks = 3;
+ int fanOut = 2;
+
+ var serializer = new AvroConfigurationSerializer();
+
+ var groupCommunicationDriver = GroupCommunicationTests.GetInstanceOfGroupCommDriver(driverId, masterTaskId,
+ groupName, fanOut,
+ numTasks);
+
+ //driver side to prepar for service config
+ var codecConfig = CodecConfiguration<int>.Conf
+ .Set(CodecConfiguration<int>.Codec, GenericType<IntCodec>.Class)
+ .Build();
+ var driverServiceConfig = groupCommunicationDriver.GetServiceConfiguration();
+ var serviceConfig = Configurations.Merge(driverServiceConfig, codecConfig);
+
+ //wrap it before serializing
+ var wrappedSeriveConfig = TangFactory.GetTang().NewConfigurationBuilder()
+ .BindNamedParameter<ServicesConfigurationOptions.ServiceConfigString, string>(
+ GenericType<ServicesConfigurationOptions.ServiceConfigString>.Class,
+ new AvroConfigurationSerializer().ToString(serviceConfig))
+ .Build();
+ var serviceConfigString = serializer.ToString(wrappedSeriveConfig);
+
+ //the configuration string is received at Evaluator side
+ var serviceConfig2 = new ServiceConfiguration(serviceConfigString);
+
+ Assert.AreEqual(serializer.ToString(serviceConfig), serializer.ToString(serviceConfig2.TangConfig));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d79f4ece/lang/cs/Org.Apache.REEF.Network.Tests/Org.Apache.REEF.Network.Tests.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Network.Tests/Org.Apache.REEF.Network.Tests.csproj b/lang/cs/Org.Apache.REEF.Network.Tests/Org.Apache.REEF.Network.Tests.csproj
index eebfdaf..6cd5c05 100644
--- a/lang/cs/Org.Apache.REEF.Network.Tests/Org.Apache.REEF.Network.Tests.csproj
+++ b/lang/cs/Org.Apache.REEF.Network.Tests/Org.Apache.REEF.Network.Tests.csproj
@@ -50,6 +50,7 @@ under the License.
</ItemGroup>
<ItemGroup>
<Compile Include="BlockingCollectionExtensionTests.cs" />
+ <Compile Include="GroupCommunication\GroupCommuDriverTests.cs" />
<Compile Include="GroupCommunication\GroupCommunicationTests.cs" />
<Compile Include="GroupCommunication\GroupCommunicationTreeTopologyTests.cs" />
<Compile Include="NamingService\NameServerTests.cs" />
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/d79f4ece/lang/cs/Org.Apache.REEF.Network/Group/Driver/Impl/GroupCommDriver.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Network/Group/Driver/Impl/GroupCommDriver.cs b/lang/cs/Org.Apache.REEF.Network/Group/Driver/Impl/GroupCommDriver.cs
index a1a2548..3ccf31f 100644
--- a/lang/cs/Org.Apache.REEF.Network/Group/Driver/Impl/GroupCommDriver.cs
+++ b/lang/cs/Org.Apache.REEF.Network/Group/Driver/Impl/GroupCommDriver.cs
@@ -32,6 +32,7 @@ using Org.Apache.REEF.Network.Naming;
using Org.Apache.REEF.Network.NetworkService;
using Org.Apache.REEF.Tang.Annotations;
using Org.Apache.REEF.Tang.Formats;
+using Org.Apache.REEF.Tang.Implementations.Configuration;
using Org.Apache.REEF.Tang.Implementations.Tang;
using Org.Apache.REEF.Tang.Interface;
using Org.Apache.REEF.Tang.Util;
@@ -170,10 +171,10 @@ namespace Org.Apache.REEF.Network.Group.Driver.Impl
GenericType<ICodec<GroupCommunicationMessage>>.Class,
GenericType<GroupCommunicationMessageCodec>.Class)
.BindNamedParameter<NamingConfigurationOptions.NameServerAddress, string>(
- GenericType<NamingConfigurationOptions.NameServerAddress>.Class,
+ GenericType<NamingConfigurationOptions.NameServerAddress>.Class,
_nameServerAddr)
.BindNamedParameter<NamingConfigurationOptions.NameServerPort, int>(
- GenericType<NamingConfigurationOptions.NameServerPort>.Class,
+ GenericType<NamingConfigurationOptions.NameServerPort>.Class,
_nameServerPort.ToString(CultureInfo.InvariantCulture))
.BindImplementation(GenericType<INameClient>.Class,
GenericType<NameClient>.Class)