You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by mo...@apache.org on 2017/11/03 18:37:25 UTC
reef git commit: [REEF-1948] Add IREEFClient.NewJobRequestBuilder
Repository: reef
Updated Branches:
refs/heads/master fece62949 -> 7a2aa5cc6
[REEF-1948] Add IREEFClient.NewJobRequestBuilder
This adds the new API to `IREEFClient`.
To ease implementation and foster re-use, this also adds the new class
`JobRequestBuilderFactory`. It is used on all implementations of
`IREEFClient` to implement the new method.
Lastly, this changes `HelloREEF` to use the new API to demonstrate it to
work.
JIRA:
[REEF-1948](https://issues.apache.org/jira/browse/REEF-1948)
Pull Request:
This closes #1411
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/7a2aa5cc
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/7a2aa5cc
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/7a2aa5cc
Branch: refs/heads/master
Commit: 7a2aa5cc6e2f6133976c60f00b6146c383c3d1ab
Parents: fece629
Author: Markus Weimer <we...@apache.org>
Authored: Fri Nov 3 09:46:18 2017 -0700
Committer: Sergiy Matusevych <mo...@apache.org>
Committed: Fri Nov 3 11:07:24 2017 -0700
----------------------------------------------------------------------
.../Org.Apache.REEF.Client/API/IREEFClient.cs | 6 +++
.../API/JobRequestBuilder.cs | 7 +--
.../API/JobRequestBuilderFactory.cs | 48 ++++++++++++++++++++
.../Org.Apache.REEF.Client/Local/LocalClient.cs | 24 +++++++---
.../Org.Apache.REEF.Client.csproj | 1 +
.../YARN/YARNREEFClient.cs | 10 +++-
.../YARN/YarnREEFDotNetClient.cs | 10 +++-
.../HelloREEF.cs | 6 +--
8 files changed, 94 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs b/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs
index a4734f5..bc04186 100644
--- a/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs
+++ b/lang/cs/Org.Apache.REEF.Client/API/IREEFClient.cs
@@ -35,6 +35,12 @@ namespace Org.Apache.REEF.Client.API
void Submit(JobRequest jobRequest);
/// <summary>
+ /// Instantiate a new JobRequestBuilder.
+ /// </summary>
+ /// <returns>A new JobRequestBuilder</returns>
+ JobRequestBuilder NewJobRequestBuilder();
+
+ /// <summary>
/// Submit the job described in jobRequest to the cluster.
/// Expect IJobSubmissionResult returned after the call.
/// </summary>
http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs
index 234cd7a..7760b7f 100644
--- a/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs
+++ b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs
@@ -34,16 +34,11 @@ namespace Org.Apache.REEF.Client.API
}
[Inject]
- private JobRequestBuilder([Parameter(typeof(DriverConfigurationProviders))] ISet<IConfigurationProvider> configurationProviders)
+ internal JobRequestBuilder([Parameter(typeof(DriverConfigurationProviders))] ISet<IConfigurationProvider> configurationProviders)
{
AddDriverConfigurationProviders(configurationProviders);
}
- public static JobRequestBuilder NewBuilder()
- {
- return new JobRequestBuilder();
- }
-
/// <summary>
/// Bake the information provided so far and return a IJobSubmission
/// </summary>
http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilderFactory.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilderFactory.cs b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilderFactory.cs
new file mode 100644
index 0000000..6986c34
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilderFactory.cs
@@ -0,0 +1,48 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Collections.Generic;
+using Org.Apache.REEF.Common.Client.Parameters;
+using Org.Apache.REEF.Tang.Annotations;
+using Org.Apache.REEF.Tang.Interface;
+
+namespace Org.Apache.REEF.Client.API
+{
+ /// <summary>
+ /// Helper class to create instances of JobRequestBuilder.
+ /// </summary>
+ internal sealed class JobRequestBuilderFactory
+ {
+ private readonly ISet<IConfigurationProvider> _configurationProviders;
+
+ [Inject]
+ private JobRequestBuilderFactory(
+ [Parameter(typeof(DriverConfigurationProviders))] ISet<IConfigurationProvider> configurationProviders)
+ {
+ _configurationProviders = configurationProviders;
+ }
+
+ /// <summary>
+ /// Instantiates a new JobRequestBuilder.
+ /// </summary>
+ /// <returns>JobRequestBuilder</returns>
+ public JobRequestBuilder NewInstance()
+ {
+ return new JobRequestBuilder(_configurationProviders);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs b/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs
index 0f1ef1d..722bf9b 100644
--- a/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs
+++ b/lang/cs/Org.Apache.REEF.Client/Local/LocalClient.cs
@@ -31,7 +31,6 @@ using Org.Apache.REEF.Common.Files;
using Org.Apache.REEF.Tang.Annotations;
using Org.Apache.REEF.Tang.Implementations.Tang;
using Org.Apache.REEF.Tang.Interface;
-using Org.Apache.REEF.Tang.Util;
using Org.Apache.REEF.Utilities.AsyncUtils;
using Org.Apache.REEF.Utilities.Attributes;
using Org.Apache.REEF.Utilities.Logging;
@@ -63,20 +62,25 @@ namespace Org.Apache.REEF.Client.Local
private readonly string _runtimeFolder;
private readonly REEFFileNames _fileNames;
private readonly IConfiguration _localConfigurationOnDriver;
+ private readonly JobRequestBuilderFactory _jobRequestBuilderFactory;
[Inject]
private LocalClient(DriverFolderPreparationHelper driverFolderPreparationHelper,
[Parameter(typeof(LocalRuntimeDirectory))] string runtimeFolder,
[Parameter(typeof(NumberOfEvaluators))] int maxNumberOfConcurrentEvaluators,
IJavaClientLauncher javaClientLauncher,
- REEFFileNames fileNames)
+ REEFFileNames fileNames,
+ JobRequestBuilderFactory jobRequestBuilderFactory)
{
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_runtimeFolder = runtimeFolder;
_maxNumberOfConcurrentEvaluators = maxNumberOfConcurrentEvaluators;
_javaClientLauncher = javaClientLauncher;
_fileNames = fileNames;
- _localConfigurationOnDriver = TangFactory.GetTang().NewConfigurationBuilder().BindImplementation(GenericType<ILocalAddressProvider>.Class, GenericType<LoopbackLocalAddressProvider>.Class).Build();
+ _jobRequestBuilderFactory = jobRequestBuilderFactory;
+ _localConfigurationOnDriver = TangFactory.GetTang().NewConfigurationBuilder()
+ .BindImplementation<ILocalAddressProvider, LoopbackLocalAddressProvider>()
+ .Build();
}
/// <summary>
@@ -86,13 +90,16 @@ namespace Org.Apache.REEF.Client.Local
/// <param name="numberOfEvaluators"></param>
/// <param name="javaClientLauncher"></param>
/// <param name="fileNames"></param>
+ /// <param name="jobRequestBuilderFactory">The helper used to instantiate JobRequestBuilder instances.</param>
[Inject]
private LocalClient(
DriverFolderPreparationHelper driverFolderPreparationHelper,
[Parameter(typeof(NumberOfEvaluators))] int numberOfEvaluators,
IJavaClientLauncher javaClientLauncher,
- REEFFileNames fileNames)
- : this(driverFolderPreparationHelper, Path.GetTempPath(), numberOfEvaluators, javaClientLauncher, fileNames)
+ REEFFileNames fileNames,
+ JobRequestBuilderFactory jobRequestBuilderFactory)
+ : this(driverFolderPreparationHelper, Path.GetTempPath(),
+ numberOfEvaluators, javaClientLauncher, fileNames, jobRequestBuilderFactory)
{
// Intentionally left blank.
}
@@ -171,6 +178,11 @@ namespace Org.Apache.REEF.Client.Local
Logger.Log(Level.Info, "Submitted the Driver for execution.");
}
+ public JobRequestBuilder NewJobRequestBuilder()
+ {
+ return _jobRequestBuilderFactory.NewInstance();
+ }
+
public IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest)
{
var driverFolder = PrepareDriverFolder(jobRequest);
@@ -211,4 +223,4 @@ namespace Org.Apache.REEF.Client.Local
return Path.Combine(_runtimeFolder, string.Join("-", "reef", jobId, timestamp));
}
}
-}
\ No newline at end of file
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
index 00c6d87..80b8d90 100644
--- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
+++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
@@ -73,6 +73,7 @@ under the License.
<Compile Include="API\JobParametersBuilder.cs" />
<Compile Include="API\JobRequest.cs" />
<Compile Include="API\JobRequestBuilder.cs" />
+ <Compile Include="API\JobRequestBuilderFactory.cs" />
<Compile Include="API\TcpPortConfigurationModule.cs" />
<Compile Include="Avro\AvroAppSubmissionParameters.cs" />
<Compile Include="Avro\AvroJobSubmissionParameters.cs" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs
index b4e5ad7..09434f0 100644
--- a/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/YARNREEFClient.cs
@@ -47,6 +47,7 @@ namespace Org.Apache.REEF.Client.Yarn
private readonly REEFFileNames _fileNames;
private readonly IYarnRMClient _yarnClient;
private readonly YarnREEFParamSerializer _paramSerializer;
+ private readonly JobRequestBuilderFactory _jobRequestBuilderFactory;
[Inject]
internal YarnREEFClient(IJavaClientLauncher javaClientLauncher,
@@ -54,7 +55,8 @@ namespace Org.Apache.REEF.Client.Yarn
REEFFileNames fileNames,
YarnCommandLineEnvironment yarn,
IYarnRMClient yarnClient,
- YarnREEFParamSerializer paramSerializer)
+ YarnREEFParamSerializer paramSerializer,
+ JobRequestBuilderFactory jobRequestBuilderFactory)
{
_javaClientLauncher = javaClientLauncher;
_javaClientLauncher.AddToClassPath(yarn.GetYarnClasspathList());
@@ -62,6 +64,7 @@ namespace Org.Apache.REEF.Client.Yarn
_fileNames = fileNames;
_yarnClient = yarnClient;
_paramSerializer = paramSerializer;
+ _jobRequestBuilderFactory = jobRequestBuilderFactory;
}
public void Submit(JobRequest jobRequest)
@@ -73,6 +76,11 @@ namespace Org.Apache.REEF.Client.Yarn
Launch(jobRequest, driverFolderPath);
}
+ public JobRequestBuilder NewJobRequestBuilder()
+ {
+ return _jobRequestBuilderFactory.NewInstance();
+ }
+
public IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest)
{
// Prepare the job submission folder
http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs
index babb613..65c6c87 100644
--- a/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs
+++ b/lang/cs/Org.Apache.REEF.Client/YARN/YarnREEFDotNetClient.cs
@@ -51,6 +51,7 @@ namespace Org.Apache.REEF.Client.YARN
private readonly REEFFileNames _fileNames;
private readonly IJobSubmissionDirectoryProvider _jobSubmissionDirectoryProvider;
private readonly YarnREEFDotNetParamSerializer _paramSerializer;
+ private readonly JobRequestBuilderFactory _jobRequestBuilderFactory;
[Inject]
private YarnREEFDotNetClient(
@@ -60,7 +61,8 @@ namespace Org.Apache.REEF.Client.YARN
IJobResourceUploader jobResourceUploader,
REEFFileNames fileNames,
IJobSubmissionDirectoryProvider jobSubmissionDirectoryProvider,
- YarnREEFDotNetParamSerializer paramSerializer)
+ YarnREEFDotNetParamSerializer paramSerializer,
+ JobRequestBuilderFactory jobRequestBuilderFactory)
{
_injector = injector;
_jobSubmissionDirectoryProvider = jobSubmissionDirectoryProvider;
@@ -69,6 +71,7 @@ namespace Org.Apache.REEF.Client.YARN
_driverFolderPreparationHelper = driverFolderPreparationHelper;
_yarnRMClient = yarnRMClient;
_paramSerializer = paramSerializer;
+ _jobRequestBuilderFactory = jobRequestBuilderFactory;
}
public void Submit(JobRequest jobRequest)
@@ -135,6 +138,11 @@ namespace Org.Apache.REEF.Client.YARN
}
}
+ public JobRequestBuilder NewJobRequestBuilder()
+ {
+ return _jobRequestBuilderFactory.NewInstance();
+ }
+
public IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest)
{
throw new NotSupportedException();
http://git-wip-us.apache.org/repos/asf/reef/blob/7a2aa5cc/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
index b1d0cd3..e29ea0c 100644
--- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
+++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
@@ -41,13 +41,11 @@ namespace Org.Apache.REEF.Examples.HelloREEF
private const string YARNRest = "yarnrest";
private const string HDInsight = "hdi";
private readonly IREEFClient _reefClient;
- private readonly JobRequestBuilder _jobRequestBuilder;
[Inject]
- private HelloREEF(IREEFClient reefClient, JobRequestBuilder jobRequestBuilder)
+ private HelloREEF(IREEFClient reefClient)
{
_reefClient = reefClient;
- _jobRequestBuilder = jobRequestBuilder;
}
/// <summary>
@@ -62,7 +60,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF
.Build();
// The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
- var helloJobRequest = _jobRequestBuilder
+ var helloJobRequest = _reefClient.NewJobRequestBuilder()
.AddDriverConfiguration(helloDriverConfiguration)
.AddGlobalAssemblyForType(typeof(HelloDriver))
.SetJobIdentifier("HelloREEF")