You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by do...@apache.org on 2018/03/10 01:33:28 UTC
[19/28] reef git commit: [REEF-1826] Clarify the API to add
assemblies to a Job submission
[REEF-1826] Clarify the API to add assemblies to a Job submission
This addressed the issue by
* Moved all functionality for assigning global assemblies to the
`JobRequestBuilder`; removed from `DriverFolderPreparationHelper`
* Job request builder adds the assemblies of the local directory if no
assemblies were added (local or global) and logs a warning.
* Added a helper function to add assemblies from the Executing
Assembly's directory
* Added a helper function to use the directory of the currently-running
executable
* Updated HelloREEF to use the executable directory instead of the
current working directory
JIRA:
[REEF-1826](https://issues.apache.org/jira/browse/REEF-1826)
Pull request:
This closes #1826
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/4296e20f
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/4296e20f
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/4296e20f
Branch: refs/heads/REEF-335
Commit: 4296e20f24baded5143f1f53c29b2deb2f5e90a0
Parents: cc1073f
Author: roganc <ro...@microsoft.com>
Authored: Sat Nov 4 16:35:45 2017 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Thu Nov 16 17:35:29 2017 -0800
----------------------------------------------------------------------
.../API/JobRequestBuilder.cs | 82 +++++++++++++++++++-
.../Common/DriverFolderPreparationHelper.cs | 31 --------
.../HelloREEF.cs | 11 ++-
.../ClientUtilities.cs | 38 +++++++++
.../Org.Apache.Reef.Utilities.csproj | 1 +
5 files changed, 129 insertions(+), 34 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/4296e20f/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs
index 7760b7f..29d7a66 100644
--- a/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs
+++ b/lang/cs/Org.Apache.REEF.Client/API/JobRequestBuilder.cs
@@ -18,23 +18,32 @@
using System;
using System.Collections.Generic;
using Org.Apache.REEF.Common.Client.Parameters;
+using Org.Apache.REEF.Client.Common;
using Org.Apache.REEF.Tang.Annotations;
using Org.Apache.REEF.Tang.Interface;
using Org.Apache.REEF.Utilities.Logging;
+using System.IO;
+using System.Linq;
+using Org.Apache.REEF.Utilities;
namespace Org.Apache.REEF.Client.API
{
public sealed class JobRequestBuilder
{
+ private static readonly Logger Logger = Logger.GetLogger(typeof(JobRequestBuilder));
private readonly AppParametersBuilder _appParametersBuilder = AppParametersBuilder.NewBuilder();
private readonly JobParametersBuilder _jobParametersBuilder = JobParametersBuilder.NewBuilder();
+ private const string DLLFileNameExtension = ".dll";
+ private const string EXEFileNameExtension = ".exe";
+ private bool _assembliesAdded = false;
private JobRequestBuilder()
{
}
[Inject]
- internal JobRequestBuilder([Parameter(typeof(DriverConfigurationProviders))] ISet<IConfigurationProvider> configurationProviders)
+ internal JobRequestBuilder(
+ [Parameter(typeof(DriverConfigurationProviders))] ISet<IConfigurationProvider> configurationProviders)
{
AddDriverConfigurationProviders(configurationProviders);
}
@@ -44,6 +53,13 @@ namespace Org.Apache.REEF.Client.API
/// </summary>
public JobRequest Build()
{
+ // If no assemblies have been added, default to those in the working directory
+ if (!_assembliesAdded)
+ {
+ Logger.Log(Level.Warning, "No assemlies added to the job; Adding assemblies from the current working directory.");
+ AddGlobalAssembliesInDirectory(Directory.GetCurrentDirectory());
+ }
+
return new JobRequest(_jobParametersBuilder.Build(), _appParametersBuilder.Build());
}
@@ -52,6 +68,7 @@ namespace Org.Apache.REEF.Client.API
/// </summary>
public JobRequestBuilder AddGlobalFile(string fileName)
{
+ ThrowIfFileDoesnotExist(fileName, "Global File");
_appParametersBuilder.AddGlobalFile(fileName);
return this;
}
@@ -61,6 +78,7 @@ namespace Org.Apache.REEF.Client.API
/// </summary>
public JobRequestBuilder AddLocalFile(string fileName)
{
+ ThrowIfFileDoesnotExist(fileName, "Local File");
_appParametersBuilder.AddLocalFile(fileName);
return this;
}
@@ -72,18 +90,54 @@ namespace Org.Apache.REEF.Client.API
/// <returns></returns>
public JobRequestBuilder AddGlobalAssembly(string fileName)
{
+ ThrowIfFileDoesnotExist(fileName, "Global Assembly");
_appParametersBuilder.AddGlobalAssembly(fileName);
+ _assembliesAdded = true;
return this;
}
/// <summary>
+ /// Add all the assemblies in a directory of type EXE, DLL, or with the Client-JarFile Prefix to the global path
+ /// </summary>
+ /// <param name="globalAssemblyDirectory">The directory to search for assemblies</param>
+ public JobRequestBuilder AddGlobalAssembliesInDirectory(string globalAssemblyDirectory)
+ {
+ if (Directory.Exists(globalAssemblyDirectory))
+ {
+ // For input paths that are directories, extract only files of a predetermined type
+ foreach (var assembly in Directory.GetFiles(globalAssemblyDirectory).Where(IsAssemblyToCopy))
+ {
+ AddGlobalAssembly(assembly);
+ }
+ }
+ else
+ {
+ // Throw if a path input was not a file or a directory
+ throw new FileNotFoundException(string.Format("Global Assembly Directory not Found: {0}", globalAssemblyDirectory));
+ }
+ return this;
+ }
+
+ /// <summary>
+ /// Add any assemblies of type EXE, DLL, or with the Client-JarFile Prefix to the global path
+ /// found in the same directory as the executing assembly's directory
+ /// </summary>
+ public JobRequestBuilder AddGlobalAssembliesInDirectoryOfExecutingAssembly()
+ {
+ var directory = ClientUtilities.GetPathToExecutingAssembly();
+ return AddGlobalAssembliesInDirectory(directory);
+ }
+
+ /// <summary>
/// Add an assembly to the driver only.
/// </summary>
/// <param name="fileName"></param>
/// <returns></returns>
public JobRequestBuilder AddLocalAssembly(string fileName)
{
+ ThrowIfFileDoesnotExist(fileName, "Local Assembly");
_appParametersBuilder.AddLocalAssembly(fileName);
+ _assembliesAdded = true;
return this;
}
@@ -204,5 +258,31 @@ namespace Org.Apache.REEF.Client.API
_jobParametersBuilder.SetJavaLogLevel(javaLogLevel);
return this;
}
+
+ /// <summary>
+ /// Returns true, if the given file path references a DLL or EXE or JAR.
+ /// </summary>
+ /// <param name="filePath"></param>
+ /// <returns></returns>
+ private static bool IsAssemblyToCopy(string filePath)
+ {
+ var fileName = Path.GetFileName(filePath);
+ if (string.IsNullOrWhiteSpace(fileName))
+ {
+ return false;
+ }
+ var lowerCasePath = fileName.ToLower();
+ return lowerCasePath.EndsWith(DLLFileNameExtension) ||
+ lowerCasePath.EndsWith(EXEFileNameExtension) ||
+ lowerCasePath.StartsWith(ClientConstants.ClientJarFilePrefix);
+ }
+
+ private static void ThrowIfFileDoesnotExist(string path, string fileDescription)
+ {
+ if (!File.Exists(path))
+ {
+ throw new FileNotFoundException(string.Format("{0} not Found: {1}", fileDescription, path));
+ }
+ }
}
}
http://git-wip-us.apache.org/repos/asf/reef/blob/4296e20f/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs b/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs
index 7401235..dfab211 100644
--- a/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs
+++ b/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs
@@ -87,9 +87,6 @@ namespace Org.Apache.REEF.Client.Common
// Create the driver configuration
CreateDriverConfiguration(appParameters, driverFolderPath);
- // Add the REEF assemblies
- AddAssemblies();
-
// Initiate the final copy
_fileSets.CopyToDriverFolder(driverFolderPath);
@@ -149,33 +146,5 @@ namespace Org.Apache.REEF.Client.Common
File.WriteAllText(Path.Combine(driverFolderPath, _fileNames.GetGlobalFolderPath(), EvaluatorExecutable),
DefaultDriverConfigurationFileContents);
}
-
- /// <summary>
- /// Adds all Assemlies to the Global folder in the Driver.
- /// </summary>
- private void AddAssemblies()
- {
- // TODO: Be more precise, e.g. copy the JAR only to the driver.
- var assemblies = Directory.GetFiles(@".\").Where(IsAssemblyToCopy);
- _fileSets.AddToGlobalFiles(assemblies);
- }
-
- /// <summary>
- /// Returns true, if the given file path references a DLL or EXE or JAR.
- /// </summary>
- /// <param name="filePath"></param>
- /// <returns></returns>
- private static bool IsAssemblyToCopy(string filePath)
- {
- var fileName = Path.GetFileName(filePath);
- if (string.IsNullOrWhiteSpace(fileName))
- {
- return false;
- }
- var lowerCasePath = fileName.ToLower();
- return lowerCasePath.EndsWith(DLLFileNameExtension) ||
- lowerCasePath.EndsWith(EXEFileNameExtension) ||
- lowerCasePath.StartsWith(ClientConstants.ClientJarFilePrefix);
- }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/4296e20f/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
index 8f9053d..33f5aa7 100644
--- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
+++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
@@ -17,6 +17,7 @@
using System;
using Org.Apache.REEF.Client.API;
+using Org.Apache.REEF.Client.Common;
using Org.Apache.REEF.Client.Local;
using Org.Apache.REEF.Client.Yarn;
using Org.Apache.REEF.Client.YARN.HDI;
@@ -27,7 +28,6 @@ using Org.Apache.REEF.Tang.Implementations.Tang;
using Org.Apache.REEF.Tang.Interface;
using Org.Apache.REEF.Tang.Util;
using Org.Apache.REEF.Utilities.Logging;
-using Org.Apache.REEF.Client.Common;
namespace Org.Apache.REEF.Examples.HelloREEF
{
@@ -64,6 +64,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF
var helloJobRequest = _reefClient.NewJobRequestBuilder()
.AddDriverConfiguration(helloDriverConfiguration)
.AddGlobalAssemblyForType(typeof(HelloDriver))
+ .AddGlobalAssembliesInDirectoryOfExecutingAssembly()
.SetJobIdentifier("HelloREEF")
.SetJavaLogLevel(JavaLoggingSetting.Verbose)
.Build();
@@ -110,7 +111,13 @@ namespace Org.Apache.REEF.Examples.HelloREEF
public static void MainSimple(string[] args)
{
- TangFactory.GetTang().NewInjector(GetRuntimeConfiguration(args.Length > 0 ? args[0] : Local)).GetInstance<HelloREEF>().Run();
+ var runtime = args.Length > 0 ? args[0] : Local;
+
+ // Execute the HelloREEF, with these parameters injected
+ TangFactory.GetTang()
+ .NewInjector(GetRuntimeConfiguration(runtime))
+ .GetInstance<HelloREEF>()
+ .Run();
}
}
}
http://git-wip-us.apache.org/repos/asf/reef/blob/4296e20f/lang/cs/Org.Apache.REEF.Utilities/ClientUtilities.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Utilities/ClientUtilities.cs b/lang/cs/Org.Apache.REEF.Utilities/ClientUtilities.cs
new file mode 100644
index 0000000..c70ab39
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Utilities/ClientUtilities.cs
@@ -0,0 +1,38 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System;
+using System.IO;
+using System.Reflection;
+
+namespace Org.Apache.REEF.Utilities
+{
+ /// <summary>
+ /// A library of helper functions for REEF clients
+ /// </summary>
+ public static class ClientUtilities
+ {
+ /// <summary>
+ /// Gets the path of the executing assembly as a string.
+ /// </summary>
+ /// <returns>A string path to the executing assembly</returns>
+ public static string GetPathToExecutingAssembly()
+ {
+ return Path.GetDirectoryName(new Uri(Assembly.GetExecutingAssembly().CodeBase).AbsolutePath);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/4296e20f/lang/cs/Org.Apache.REEF.Utilities/Org.Apache.Reef.Utilities.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Utilities/Org.Apache.Reef.Utilities.csproj b/lang/cs/Org.Apache.REEF.Utilities/Org.Apache.Reef.Utilities.csproj
index 0e5ce4e..660a280 100644
--- a/lang/cs/Org.Apache.REEF.Utilities/Org.Apache.Reef.Utilities.csproj
+++ b/lang/cs/Org.Apache.REEF.Utilities/Org.Apache.Reef.Utilities.csproj
@@ -54,6 +54,7 @@ under the License.
<Compile Include="Attributes\ThreadSafeAttribute.cs" />
<Compile Include="Attributes\UnstableAttribute.cs" />
<Compile Include="ByteUtilities.cs" />
+ <Compile Include="ClientUtilities.cs" />
<Compile Include="Collections\PriorityQueue.cs" />
<Compile Include="Collections\ReadOnlySet.cs" />
<Compile Include="Diagnostics\DiagnosticsMessages.cs" />