You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by do...@apache.org on 2018/05/01 01:14:07 UTC
[01/16] reef git commit: [REEF-2005] Update DotNet projects to target
.net461
Repository: reef
Updated Branches:
refs/heads/REEF-335 133c064f9 -> 386069e4e
[REEF-2005] Update DotNet projects to target .net461
This updates all projects to target .net461, including the Bridge
project.
- Removes the conditional library references in the csproj files
for.net452 as this is no longer required.
- Additional updates were done to get the build and tests passing.
JIRA:
[REEF-2005](https://issues.apache.org/jira/browse/REEF-2005)
Pull Request:
This closes #1448
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/7343e62c
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/7343e62c
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/7343e62c
Branch: refs/heads/REEF-335
Commit: 7343e62c408dede8c46d2e72b4c58dda785e24b0
Parents: 30abddb
Author: Scott Inglis <si...@microsoft.com>
Authored: Fri Apr 20 13:02:39 2018 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Mon Apr 23 14:35:16 2018 -0700
----------------------------------------------------------------------
.../Org.Apache.REEF.Bridge.CLR.DotNet.csproj | 13 -----------
.../Org.Apache.REEF.Bridge.JAR.DotNet.csproj | 2 +-
.../Org.Apache.REEF.Bridge.DotNet.vcxproj | 13 ++++++++---
.../Org.Apache.REEF.Client.Tests.DotNet.csproj | 9 --------
.../Org.Apache.REEF.Client.DotNet.csproj | 23 ++++----------------
.../Org.Apache.REEF.Common.Tests.DotNet.csproj | 10 ---------
.../Org.Apache.REEF.Common.DotNet.csproj | 7 ------
.../Org.Apache.REEF.Driver.DotNet.csproj | 6 -----
...rg.Apache.REEF.Evaluator.Tests.DotNet.csproj | 4 ----
.../Org.Apache.REEF.Evaluator.DotNet.csproj | 6 -----
...ache.REEF.Examples.AllHandlers.DotNet.csproj | 10 ---------
...he.REEF.Examples.DriverRestart.DotNet.csproj | 10 ---------
...Apache.REEF.Examples.HelloREEF.DotNet.csproj | 10 ---------
.../Org.Apache.REEF.Examples.DotNet.csproj | 6 -----
.../Org.Apache.REEF.FatNuGet.DotNet.csproj | 12 ++++------
.../Org.Apache.REEF.IMRU.Examples.DotNet.csproj | 10 ---------
.../Org.Apache.REEF.IMRU.Tests.DotNet.csproj | 10 ---------
.../Org.Apache.REEF.IMRU.DotNet.csproj | 10 ---------
.../Org.Apache.REEF.IO.TestClient.DotNet.csproj | 10 ---------
.../Org.Apache.REEF.IO.Tests.DotNet.csproj | 13 -----------
.../TestAzureBlockBlobFileSystemE2E.cs | 4 +++-
.../Org.Apache.REEF.IO.DotNet.csproj | 13 -----------
...e.REEF.Network.Examples.Client.DotNet.csproj | 10 ---------
...g.Apache.REEF.Network.Examples.DotNet.csproj | 10 ---------
.../Org.Apache.REEF.Network.Tests.DotNet.csproj | 4 ----
.../Org.Apache.REEF.Network.DotNet.csproj | 7 ------
.../Org.Apache.REEF.Tang.Examples.DotNet.csproj | 5 -----
.../Org.Apache.REEF.Tang.Tests.DotNet.csproj | 5 -----
.../Org.Apache.REEF.Tang.DotNet.csproj | 7 ------
.../Org.Apache.REEF.Tests.DotNet.csproj | 6 -----
.../Org.Apache.REEF.Wake.DotNet.csproj | 6 -----
lang/cs/TestRunner.DotNet.proj | 2 +-
lang/cs/build.DotNetApp.props | 7 ++++--
lang/cs/build.DotNetLibrary.props | 16 +++++---------
lang/cs/build.DotNetTest.props | 10 ++++++++-
lang/cs/build.DotNetTest.targets | 2 +-
lang/cs/xunit.DotNet.props | 13 +----------
37 files changed, 45 insertions(+), 276 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Bridge.CLR/Org.Apache.REEF.Bridge.CLR.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Bridge.CLR/Org.Apache.REEF.Bridge.CLR.DotNet.csproj b/lang/cs/Org.Apache.REEF.Bridge.CLR/Org.Apache.REEF.Bridge.CLR.DotNet.csproj
index 7d149d9..8648713 100644
--- a/lang/cs/Org.Apache.REEF.Bridge.CLR/Org.Apache.REEF.Bridge.CLR.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Bridge.CLR/Org.Apache.REEF.Bridge.CLR.DotNet.csproj
@@ -35,19 +35,6 @@ under the License.
<PackageReference Include="Microsoft.Avro.Tools" Version="$(AvroVersion)" />
<PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetGramework)' == 'net46'">
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Data" />
- <Reference Include="System.Drawing" />
- <Reference Include="System.IO.Compression.FileSystem" />
- <Reference Include="System.Numerics" />
- <Reference Include="System.Runtime.Serialization" />
- <Reference Include="System.Xml" />
- <Reference Include="System.Xml.Linq" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Bridge.JAR/Org.Apache.REEF.Bridge.JAR.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Bridge.JAR/Org.Apache.REEF.Bridge.JAR.DotNet.csproj b/lang/cs/Org.Apache.REEF.Bridge.JAR/Org.Apache.REEF.Bridge.JAR.DotNet.csproj
index 58ba70a..0c700bd 100644
--- a/lang/cs/Org.Apache.REEF.Bridge.JAR/Org.Apache.REEF.Bridge.JAR.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Bridge.JAR/Org.Apache.REEF.Bridge.JAR.DotNet.csproj
@@ -18,7 +18,7 @@ under the License.
<PropertyGroup>
<AssemblyName>Org.Apache.REEF.Bridge.JAR</AssemblyName>
<Description>Helper project that builds the Bridge JAR </Description>
- <TargetFramework>net452</TargetFramework>
+ <TargetFramework>net461</TargetFramework>
</PropertyGroup>
<Import Project="..\build.DotNet.props" />
<PropertyGroup>
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Bridge/Org.Apache.REEF.Bridge.DotNet.vcxproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Bridge/Org.Apache.REEF.Bridge.DotNet.vcxproj b/lang/cs/Org.Apache.REEF.Bridge/Org.Apache.REEF.Bridge.DotNet.vcxproj
index 23dc0cb..49cff7d 100644
--- a/lang/cs/Org.Apache.REEF.Bridge/Org.Apache.REEF.Bridge.DotNet.vcxproj
+++ b/lang/cs/Org.Apache.REEF.Bridge/Org.Apache.REEF.Bridge.DotNet.vcxproj
@@ -23,6 +23,10 @@ under the License.
<Platform Condition="'$(Platform)' == ''">x64</Platform>
<SolutionDir Condition="'$(SolutionDir)' == ''">..</SolutionDir>
<ConfigurationType>Application</ConfigurationType>
+
+ <!-- Added for when the .net core build is enabled. This prevents assemblies for the bridge
+ to be copied to the bin folder. -->
+ <ImplicitlyExpandNETStandardFacades>false</ImplicitlyExpandNETStandardFacades>
</PropertyGroup>
<Import Project="..\build.Common.DotNet.props" />
<ItemGroup Label="ProjectConfigurations">
@@ -37,7 +41,7 @@ under the License.
</ItemGroup>
<PropertyGroup Label="Globals">
<ProjectGuid>{6C7325D1-EBB6-4642-B34F-B66F46152230}</ProjectGuid>
- <TargetFrameworkVersion>v4.5.2</TargetFrameworkVersion>
+ <TargetFrameworkVersion>v4.6.1</TargetFrameworkVersion>
<Keyword>ManagedCProj</Keyword>
<RootNamespace>Org.Apache.REEF.Bridge</RootNamespace>
<WindowsTargetPlatformVersion Condition="'$(MSBuildAssemblyVersion)' >= '15.0'">10.0.15063.0</WindowsTargetPlatformVersion>
@@ -108,7 +112,8 @@ under the License.
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<AdditionalIncludeDirectories>..\..\java\reef-bridge-java\target\classes;$(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
- <AdditionalUsingDirectories>$(BinDir)$(Configuration)\Org.Apache.REEF.Driver\net452</AdditionalUsingDirectories>
+ <AdditionalUsingDirectories Condition="'$(EnableDotNetBuild)' == 'False'">$(AdditionalUsingDirectories);$(BinDir)$(Configuration)\Org.Apache.REEF.Driver\net461</AdditionalUsingDirectories>
+ <AdditionalUsingDirectories Condition="'$(EnableDotNetBuild)' == 'True'">$(AdditionalUsingDirectories);$(BinDir)$(Configuration)\Org.Apache.REEF.Driver\netstandard2.0</AdditionalUsingDirectories>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -123,7 +128,9 @@ under the License.
<PreprocessorDefinitions>WIN32;NDEBUG;%(PreprocessorDefinitions)</PreprocessorDefinitions>
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<AdditionalIncludeDirectories>..\..\java\reef-bridge-java\target\classes;$(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
- <AdditionalUsingDirectories>$(BinDir)$(Configuration)\Org.Apache.REEF.Driver\net452</AdditionalUsingDirectories>
+ <RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
+ <AdditionalUsingDirectories Condition="'$(EnableDotNetBuild)' == 'False'">$(AdditionalUsingDirectories);$(BinDir)$(Configuration)\Org.Apache.REEF.Driver\net461</AdditionalUsingDirectories>
+ <AdditionalUsingDirectories Condition="'$(EnableDotNetBuild)' == 'True'">$(AdditionalUsingDirectories);$(BinDir)$(Configuration)\Org.Apache.REEF.Driver\netstandard2.0</AdditionalUsingDirectories>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.DotNet.csproj b/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.DotNet.csproj
index 182dbb0..7803aa3 100644
--- a/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.DotNet.csproj
@@ -26,17 +26,8 @@ under the License.
<ItemGroup>
<PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
<PackageReference Include="NSubstitute" Version="$(NSubstituteVersion)" />
- </ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'netcoreapp2.0'">
<PackageReference Include="System.ServiceProcess.ServiceController" Version="4.4.0" />
</ItemGroup>
- <!-- TODO: REEF-1888 Remove ItemGroup when removing net451 and net46 targets -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Net.Http" />
- <Reference Include="System.ServiceProcess" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Client\Org.Apache.REEF.Client.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.IO\Org.Apache.REEF.IO.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj
index b568e1d..a7cc2e8 100644
--- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj
@@ -37,31 +37,16 @@ under the License.
<PackageReference Include="Microsoft.Avro.Core" Version="$(AvroVersion)" />
<PackageReference Include="System.Reactive.Interfaces" Version="$(SystemReactiveVersion)" />
<PackageReference Include="EnterpriseLibrary.TransientFaultHandling.Core" Version="$(TransientFaultHandlingVersion)" />
- </ItemGroup>
- <!-- TODO[JIRA REEF-1888] Remove the conditional, but keep the item group -->
- <ItemGroup Condition="'$(TargetFramework)' == 'netcoreapp2.0'">
<PackageReference Include="System.Net.NameResolution" Version="4.3.0" />
- </ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.IO.Compression" />
- <Reference Include="System.IO.Compression.FileSystem" />
- <Reference Include="System.Net.Http" />
- <Reference Include="System.Net.Http.WebRequest" />
- <Reference Include="System.Xml" />
- <Reference Include="System.Xml.Linq" />
- <Reference Include="System.Runtime.Serialization" />
+ <PackageReference Include="System.Diagnostics.DiagnosticSource" Version="4.3.0" />
</ItemGroup>
<!-- Project Reference conditionals allow for targeting specific projects based upon the target framework. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452'">
+ <ItemGroup Condition="'$(TargetFramework)' == 'net461'">
<ProjectReference Include="..\Org.Apache.REEF.Bridge\Org.Apache.REEF.Bridge.DotNet.vcxproj" PrivateAssets="All" />
</ItemGroup>
<ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
- <ProjectReference Include="..\Org.Apache.REEF.Bridge.CLR\Org.Apache.REEF.Bridge.CLR.DotNet.csproj" PrivateAssets="All"/>
+ <ProjectReference Include="..\Org.Apache.REEF.Bridge.CLR\Org.Apache.REEF.Bridge.CLR.DotNet.csproj" PrivateAssets="All" />
</ItemGroup>
<ItemGroup>
@@ -107,7 +92,7 @@ under the License.
var srcDir = @"lang\cs\Org.Apache.REEF.Client\Properties";
var binDir = @"lang\cs\bin\.netcore";
var resxInputPath = Path.Combine(ProjectFolder, srcDir, "Resources.DotNet.xml");
- var resourceDir = Path.Combine(ProjectFolder, binDir, DebugOrRelease, "Org.Apache.REEF.Bridge.JAR", "net452");
+ var resourceDir = Path.Combine(ProjectFolder, binDir, DebugOrRelease, "Org.Apache.REEF.Bridge.JAR", "net461");
var clrDriverDir = Path.Combine(ProjectFolder, binDir, DebugOrRelease, "Org.Apache.REEF.Bridge");
var byteArrayType = ";System.Byte[], mscorlib, Version=4.0.0.0, Culture=neutral, PublicKeyToken=b77a5c561934e089";
var jarRest = reefVersion + "-shaded.jar" + byteArrayType;
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Common.Tests/Org.Apache.REEF.Common.Tests.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Common.Tests/Org.Apache.REEF.Common.Tests.DotNet.csproj b/lang/cs/Org.Apache.REEF.Common.Tests/Org.Apache.REEF.Common.Tests.DotNet.csproj
index b7cce9d..bd2b3f9 100644
--- a/lang/cs/Org.Apache.REEF.Common.Tests/Org.Apache.REEF.Common.Tests.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Common.Tests/Org.Apache.REEF.Common.Tests.DotNet.csproj
@@ -23,16 +23,6 @@ under the License.
<PackageReference Include="NSubstitute" Version="$(NSubstituteVersion)" />
<PackageReference Include="WindowsAzure.Storage" Version="$(WindowsAzureStorageVersion)" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Xml.Linq" />
- <Reference Include="System.Data.DataSetExtensions" />
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System.Data" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Utilities\Org.Apache.REEF.Utilities.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.DotNet.csproj b/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.DotNet.csproj
index 1fd5366..8f13175 100644
--- a/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Common/Org.Apache.REEF.Common.DotNet.csproj
@@ -30,13 +30,6 @@ under the License.
<PackageReference Include="System.Reactive.Core" Version="$(SystemReactiveVersion)" />
<PackageReference Include="System.Reactive.Interfaces" Version="$(SystemReactiveVersion)" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Runtime.Serialization" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Utilities\Org.Apache.REEF.Utilities.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.DotNet.csproj b/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.DotNet.csproj
index 8be3f62..8c95b88 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.DotNet.csproj
@@ -27,12 +27,6 @@ under the License.
<PackageReference Include="protobuf-net" Version="$(ProtobufVersion)" />
<PackageReference Include="Microsoft.Avro.Core" Version="$(AvroVersion)" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetGramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Runtime.Serialization" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Utilities\Org.Apache.REEF.Utilities.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.DotNet.csproj b/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.DotNet.csproj
index 07f6136..8462277 100644
--- a/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Evaluator.Tests/Org.Apache.REEF.Evaluator.Tests.DotNet.csproj
@@ -23,10 +23,6 @@ under the License.
<PackageReference Include="NSubstitute" Version="$(NSubstituteVersion)" />
<PackageReference Include="protobuf-net" Version="$(ProtobufVersion)" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Examples\Org.Apache.REEF.Examples.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Evaluator/Org.Apache.REEF.Evaluator.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Evaluator/Org.Apache.REEF.Evaluator.DotNet.csproj b/lang/cs/Org.Apache.REEF.Evaluator/Org.Apache.REEF.Evaluator.DotNet.csproj
index 7de529f..965ae8e 100644
--- a/lang/cs/Org.Apache.REEF.Evaluator/Org.Apache.REEF.Evaluator.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Evaluator/Org.Apache.REEF.Evaluator.DotNet.csproj
@@ -26,12 +26,6 @@ under the License.
<ItemGroup>
<PackageReference Include="protobuf-net" Version="$(ProtobufVersion)" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Configuration" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Utilities\Org.Apache.REEF.Utilities.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Examples.AllHandlers/Org.Apache.REEF.Examples.AllHandlers.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Examples.AllHandlers/Org.Apache.REEF.Examples.AllHandlers.DotNet.csproj b/lang/cs/Org.Apache.REEF.Examples.AllHandlers/Org.Apache.REEF.Examples.AllHandlers.DotNet.csproj
index 3f6c259..cfeab13 100644
--- a/lang/cs/Org.Apache.REEF.Examples.AllHandlers/Org.Apache.REEF.Examples.AllHandlers.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Examples.AllHandlers/Org.Apache.REEF.Examples.AllHandlers.DotNet.csproj
@@ -23,16 +23,6 @@ under the License.
<PackageTags>REEF Examples</PackageTags>
</PropertyGroup>
<Import Project="..\build.DotNetApp.props" />
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetGramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Xml.Linq" />
- <Reference Include="System.Data.DataSetExtensions" />
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System.Data" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Client\Org.Apache.REEF.Client.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Examples.DriverRestart/Org.Apache.REEF.Examples.DriverRestart.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Examples.DriverRestart/Org.Apache.REEF.Examples.DriverRestart.DotNet.csproj b/lang/cs/Org.Apache.REEF.Examples.DriverRestart/Org.Apache.REEF.Examples.DriverRestart.DotNet.csproj
index c35c031..50cbc49 100644
--- a/lang/cs/Org.Apache.REEF.Examples.DriverRestart/Org.Apache.REEF.Examples.DriverRestart.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Examples.DriverRestart/Org.Apache.REEF.Examples.DriverRestart.DotNet.csproj
@@ -23,16 +23,6 @@ under the License.
<PackageTags>REEF Examples</PackageTags>
</PropertyGroup>
<Import Project="..\build.DotNetApp.props" />
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetGramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Xml.Linq" />
- <Reference Include="System.Data.DataSetExtensions" />
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System.Data" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Client\Org.Apache.REEF.Client.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.DotNet.csproj b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.DotNet.csproj
index 49628fe..47ee42c 100644
--- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Org.Apache.REEF.Examples.HelloREEF.DotNet.csproj
@@ -23,16 +23,6 @@ under the License.
<PackageTags>REEF Examples</PackageTags>
</PropertyGroup>
<Import Project="..\build.DotNetApp.props" />
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetGramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Xml.Linq" />
- <Reference Include="System.Data.DataSetExtensions" />
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System.Data" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Client\Org.Apache.REEF.Client.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.DotNet.csproj b/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.DotNet.csproj
index 8772120..94fd89a 100644
--- a/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Examples/Org.Apache.REEF.Examples.DotNet.csproj
@@ -27,12 +27,6 @@ under the License.
<PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
<PackageReference Include="Microsoft.Avro.Core" Version="$(AvroVersion)" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Runtime.Serialization" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Utilities\Org.Apache.REEF.Utilities.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.FatNuGet/Org.Apache.REEF.FatNuGet.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.FatNuGet/Org.Apache.REEF.FatNuGet.DotNet.csproj b/lang/cs/Org.Apache.REEF.FatNuGet/Org.Apache.REEF.FatNuGet.DotNet.csproj
index dd72889..c0c4b2a 100644
--- a/lang/cs/Org.Apache.REEF.FatNuGet/Org.Apache.REEF.FatNuGet.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.FatNuGet/Org.Apache.REEF.FatNuGet.DotNet.csproj
@@ -28,16 +28,13 @@ under the License.
<PackageReference Include="protobuf-net" Version="$(ProtobufVersion)" />
<PackageReference Include="Microsoft.Avro.Core" Version="$(AvroVersion)" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetGramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- </ItemGroup>
<!-- Project Reference conditionals allow for targeting specific projects based upon the target framework. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452'">
- <!-- Evaluator as a .netcoreapp can not be linked by a .netstandard library, therefore this is only linked for net45 -->
+ <ItemGroup Condition="'$(TargetFramework)' == 'net461'">
+ <!-- Evaluator as a .netcoreapp can not be linked by a .netstandard library, therefore this is only linked for net461 -->
<ProjectReference Include="..\Org.Apache.REEF.Evaluator\Org.Apache.REEF.Evaluator.DotNet.csproj" PrivateAssets="All"/>
+ <!-- IMRU Examples as a .netcoreapp can not be linked by a .netstandard library, therefore this is only linked for net461 -->
+ <ProjectReference Include="..\Org.Apache.REEF.IMRU.Examples\Org.Apache.REEF.IMRU.Examples.DotNet.csproj" PrivateAssets="All"/>
</ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Client\Org.Apache.REEF.Client.DotNet.csproj" PrivateAssets="All"/>
@@ -50,7 +47,6 @@ under the License.
<ProjectReference Include="..\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.DotNet.csproj" PrivateAssets="All"/>
<ProjectReference Include="..\Org.Apache.REEF.Utilities\Org.Apache.REEF.Utilities.DotNet.csproj" PrivateAssets="All"/>
<ProjectReference Include="..\Org.Apache.REEF.Wake\Org.Apache.REEF.Wake.DotNet.csproj" PrivateAssets="All"/>
- <ProjectReference Include="..\Org.Apache.REEF.IMRU.Examples\Org.Apache.REEF.IMRU.Examples.DotNet.csproj" PrivateAssets="All"/>
</ItemGroup>
<Import Project="..\build.DotNet.targets" />
<Target Name="CopyProjectReferencesToPackage" DependsOnTargets="ResolveProjectReferences">
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.IMRU.Examples/Org.Apache.REEF.IMRU.Examples.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IMRU.Examples/Org.Apache.REEF.IMRU.Examples.DotNet.csproj b/lang/cs/Org.Apache.REEF.IMRU.Examples/Org.Apache.REEF.IMRU.Examples.DotNet.csproj
index c615303..b99319f 100644
--- a/lang/cs/Org.Apache.REEF.IMRU.Examples/Org.Apache.REEF.IMRU.Examples.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.IMRU.Examples/Org.Apache.REEF.IMRU.Examples.DotNet.csproj
@@ -23,16 +23,6 @@ under the License.
<PackageTags>REEF Examples IMRU</PackageTags>
</PropertyGroup>
<Import Project="..\build.DotNetApp.props" />
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Xml.Linq" />
- <Reference Include="System.Data.DataSetExtensions" />
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System.Data" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Client\Org.Apache.REEF.Client.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.IMRU.Tests/Org.Apache.REEF.IMRU.Tests.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IMRU.Tests/Org.Apache.REEF.IMRU.Tests.DotNet.csproj b/lang/cs/Org.Apache.REEF.IMRU.Tests/Org.Apache.REEF.IMRU.Tests.DotNet.csproj
index d8c435a..f464933 100644
--- a/lang/cs/Org.Apache.REEF.IMRU.Tests/Org.Apache.REEF.IMRU.Tests.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.IMRU.Tests/Org.Apache.REEF.IMRU.Tests.DotNet.csproj
@@ -24,16 +24,6 @@ under the License.
<ItemGroup>
<PackageReference Include="NSubstitute" Version="$(NSubstituteVersion)" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Xml.Linq" />
- <Reference Include="System.Data.DataSetExtensions" />
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System.Data" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.IMRU\Org.Apache.REEF.IMRU.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Network\Org.Apache.REEF.Network.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.IMRU/Org.Apache.REEF.IMRU.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IMRU/Org.Apache.REEF.IMRU.DotNet.csproj b/lang/cs/Org.Apache.REEF.IMRU/Org.Apache.REEF.IMRU.DotNet.csproj
index b002a72..5dfd5c2 100644
--- a/lang/cs/Org.Apache.REEF.IMRU/Org.Apache.REEF.IMRU.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.IMRU/Org.Apache.REEF.IMRU.DotNet.csproj
@@ -23,16 +23,6 @@ under the License.
<PackageTags>REEF IMRU</PackageTags>
</PropertyGroup>
<Import Project="..\build.DotNetLibrary.props" />
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Xml.Linq" />
- <Reference Include="System.Data.DataSetExtensions" />
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System.Data" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Driver\Org.Apache.REEF.Driver.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.IO.TestClient/Org.Apache.REEF.IO.TestClient.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IO.TestClient/Org.Apache.REEF.IO.TestClient.DotNet.csproj b/lang/cs/Org.Apache.REEF.IO.TestClient/Org.Apache.REEF.IO.TestClient.DotNet.csproj
index e02e7ba..c6fee1b 100644
--- a/lang/cs/Org.Apache.REEF.IO.TestClient/Org.Apache.REEF.IO.TestClient.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.IO.TestClient/Org.Apache.REEF.IO.TestClient.DotNet.csproj
@@ -23,16 +23,6 @@ under the License.
<PackageTags>REEF IO TestClient</PackageTags>
</PropertyGroup>
<Import Project="..\build.DotNetApp.props" />
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Xml.Linq" />
- <Reference Include="System.Data.DataSetExtensions" />
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System.Data" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Client\Org.Apache.REEF.Client.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.DotNet.csproj b/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.DotNet.csproj
index bc5c3f9..15f0d18 100644
--- a/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.DotNet.csproj
@@ -23,22 +23,9 @@ under the License.
<PackageReference Include="Microsoft.Azure.DataLake.Store" Version="$(MicrosoftDataLakeStoreVersion)" />
<PackageReference Include="NSubstitute" Version="$(NSubstituteVersion)" />
<PackageReference Include="WindowsAzure.Storage" Version="$(WindowsAzureStorageVersion)" />
- </ItemGroup>
- <!-- TODO[JIRA REEF-1888] Remove the conditional, but keep the item group -->
- <ItemGroup Condition="'$(TargetFramework)' == 'netcoreapp2.0'">
<PackageReference Include="System.Net.NameResolution" Version="4.3.0" />
<PackageReference Include="System.Net.Sockets" Version="4.3.0" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Xml.Linq" />
- <Reference Include="System.Data.DataSetExtensions" />
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System.Data" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.IO\Org.Apache.REEF.IO.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs
index 90a4746..4e47d6b 100644
--- a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs
+++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs
@@ -118,7 +118,9 @@ namespace Org.Apache.REEF.IO.Tests
}
blob = container.GetBlockBlobReference(HelloFile);
Assert.True(CheckBlobExists(blob));
- using (var reader = new StreamReader(blob.OpenRead()))
+ var readTask = blob.OpenReadAsync();
+ readTask.Wait();
+ using (var reader = new StreamReader(readTask.Result))
{
string streamText = reader.ReadToEnd();
Assert.Equal(Text, streamText);
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.DotNet.csproj b/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.DotNet.csproj
index d9f1801..3dc419e 100644
--- a/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.DotNet.csproj
@@ -37,23 +37,10 @@ under the License.
<PackageReference Include="System.Spatial" Version="5.8.2" />
<PackageReference Include="WindowsAzure.Storage" Version="$(WindowsAzureStorageVersion)" />
<PackageReference Include="Newtonsoft.Json" Version="$(NewtonsoftJsonVersion)" />
- </ItemGroup>
- <!-- TODO[JIRA REEF-1888] Remove the conditional, but keep the item group -->
- <ItemGroup Condition="'$(TargetFramework)' == 'netstandard2.0'">
<PackageReference Include="System.IO.FileSystem.AccessControl" Version="4.4.0" />
<PackageReference Include="System.Net.NameResolution" Version="4.3.0" />
<PackageReference Include="System.Net.Sockets" Version="4.3.0" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Xml.Linq" />
- <Reference Include="System.Data.DataSetExtensions" />
- <Reference Include="System.Data" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Utilities\Org.Apache.REEF.Utilities.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Network.Examples.Client/Org.Apache.REEF.Network.Examples.Client.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Network.Examples.Client/Org.Apache.REEF.Network.Examples.Client.DotNet.csproj b/lang/cs/Org.Apache.REEF.Network.Examples.Client/Org.Apache.REEF.Network.Examples.Client.DotNet.csproj
index c8cffbe..0bbdb48 100644
--- a/lang/cs/Org.Apache.REEF.Network.Examples.Client/Org.Apache.REEF.Network.Examples.Client.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Network.Examples.Client/Org.Apache.REEF.Network.Examples.Client.DotNet.csproj
@@ -23,16 +23,6 @@ under the License.
<PackageTags>REEF Examples Network Client</PackageTags>
</PropertyGroup>
<Import Project="..\build.DotNetApp.props" />
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Xml.Linq" />
- <Reference Include="System.Data.DataSetExtensions" />
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System.Data" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Client\Org.Apache.REEF.Client.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Network.Examples/Org.Apache.REEF.Network.Examples.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Network.Examples/Org.Apache.REEF.Network.Examples.DotNet.csproj b/lang/cs/Org.Apache.REEF.Network.Examples/Org.Apache.REEF.Network.Examples.DotNet.csproj
index 9dffe68..cf65931 100644
--- a/lang/cs/Org.Apache.REEF.Network.Examples/Org.Apache.REEF.Network.Examples.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Network.Examples/Org.Apache.REEF.Network.Examples.DotNet.csproj
@@ -23,16 +23,6 @@ under the License.
<PackageTags>REEF Network services examples</PackageTags>
</PropertyGroup>
<Import Project="..\build.DotNetLibrary.props" />
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Xml.Linq" />
- <Reference Include="System.Data.DataSetExtensions" />
- <Reference Include="Microsoft.CSharp" />
- <Reference Include="System.Data" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Driver\Org.Apache.REEF.Driver.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Network.Tests/Org.Apache.REEF.Network.Tests.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Network.Tests/Org.Apache.REEF.Network.Tests.DotNet.csproj b/lang/cs/Org.Apache.REEF.Network.Tests/Org.Apache.REEF.Network.Tests.DotNet.csproj
index 5b286e9..3ee6b09 100644
--- a/lang/cs/Org.Apache.REEF.Network.Tests/Org.Apache.REEF.Network.Tests.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Network.Tests/Org.Apache.REEF.Network.Tests.DotNet.csproj
@@ -23,10 +23,6 @@ under the License.
<PackageReference Include="System.Reactive.Core" Version="$(SystemReactiveVersion)" />
<PackageReference Include="System.Reactive.Interfaces" Version="$(SystemReactiveVersion)" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Examples\Org.Apache.REEF.Examples.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Network/Org.Apache.REEF.Network.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Network/Org.Apache.REEF.Network.DotNet.csproj b/lang/cs/Org.Apache.REEF.Network/Org.Apache.REEF.Network.DotNet.csproj
index 230cf8c..ff49cb8 100644
--- a/lang/cs/Org.Apache.REEF.Network/Org.Apache.REEF.Network.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Network/Org.Apache.REEF.Network.DotNet.csproj
@@ -30,13 +30,6 @@ under the License.
<PackageReference Include="System.Reactive.Interfaces" Version="$(SystemReactiveVersion)" />
<PackageReference Include="Microsoft.Extensions.Caching.Memory" Version="1.1.2" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Runtime.Serialization" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Utilities\Org.Apache.REEF.Utilities.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Tang.Examples/Org.Apache.REEF.Tang.Examples.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tang.Examples/Org.Apache.REEF.Tang.Examples.DotNet.csproj b/lang/cs/Org.Apache.REEF.Tang.Examples/Org.Apache.REEF.Tang.Examples.DotNet.csproj
index 6160d31..c48bcfd 100644
--- a/lang/cs/Org.Apache.REEF.Tang.Examples/Org.Apache.REEF.Tang.Examples.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Tang.Examples/Org.Apache.REEF.Tang.Examples.DotNet.csproj
@@ -23,11 +23,6 @@ under the License.
<PackageTags> Apache REEF tang dependency injection examples</PackageTags>
</PropertyGroup>
<Import Project="..\build.DotNetLibrary.props" />
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Wake\Org.Apache.REEF.Wake.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Tang.Tests/Org.Apache.REEF.Tang.Tests.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tang.Tests/Org.Apache.REEF.Tang.Tests.DotNet.csproj b/lang/cs/Org.Apache.REEF.Tang.Tests/Org.Apache.REEF.Tang.Tests.DotNet.csproj
index b3b5d68..60cd8dd 100644
--- a/lang/cs/Org.Apache.REEF.Tang.Tests/Org.Apache.REEF.Tang.Tests.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Tang.Tests/Org.Apache.REEF.Tang.Tests.DotNet.csproj
@@ -24,11 +24,6 @@ under the License.
<PackageReference Include="protobuf-net" Version="$(ProtobufVersion)" />
<PackageReference Include="Microsoft.Avro.Core" Version="$(AvroVersion)" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- </ItemGroup>
<ItemGroup>
<None Include="simpleConstructorJavaProto.bin">
<CopyToOutputDirectory>Always</CopyToOutputDirectory>
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Tang/Org.Apache.REEF.Tang.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tang/Org.Apache.REEF.Tang.DotNet.csproj b/lang/cs/Org.Apache.REEF.Tang/Org.Apache.REEF.Tang.DotNet.csproj
index 1d40e8e..7305a5d 100644
--- a/lang/cs/Org.Apache.REEF.Tang/Org.Apache.REEF.Tang.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Tang/Org.Apache.REEF.Tang.DotNet.csproj
@@ -28,13 +28,6 @@ under the License.
<PackageReference Include="protobuf-net" Version="$(ProtobufVersion)" />
<PackageReference Include="Microsoft.Avro.Core" Version="$(AvroVersion)" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Runtime.Serialization" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Utilities\Org.Apache.REEF.Utilities.DotNet.csproj" />
</ItemGroup>
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.DotNet.csproj b/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.DotNet.csproj
index f03a11f..06b94f4 100644
--- a/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.DotNet.csproj
@@ -30,12 +30,6 @@ under the License.
<PackageReference Include="System.Reactive.Interfaces" Version="$(SystemReactiveVersion)" />
<PackageReference Include="System.Spatial" Version="5.8.2" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Data" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Driver\Org.Apache.REEF.Driver.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/Org.Apache.REEF.Wake/Org.Apache.REEF.Wake.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Wake/Org.Apache.REEF.Wake.DotNet.csproj b/lang/cs/Org.Apache.REEF.Wake/Org.Apache.REEF.Wake.DotNet.csproj
index 34ad138..777d41e 100644
--- a/lang/cs/Org.Apache.REEF.Wake/Org.Apache.REEF.Wake.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Wake/Org.Apache.REEF.Wake.DotNet.csproj
@@ -27,12 +27,6 @@ under the License.
<PackageReference Include="EnterpriseLibrary.TransientFaultHandling.Core" Version="$(TransientFaultHandlingVersion)" />
<PackageReference Include="System.Reactive.Interfaces" Version="$(SystemReactiveVersion)" />
</ItemGroup>
- <!-- TODO[JIRA REEF-1888] This item group will not be needed when only .netcore is targeted. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net452' Or '$(TargetFramework)' == 'net46'">
- <Reference Include="System" />
- <Reference Include="System.Core" />
- <Reference Include="System.Xml" />
- </ItemGroup>
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Tang\Org.Apache.REEF.Tang.DotNet.csproj" />
<ProjectReference Include="..\Org.Apache.REEF.Utilities\Org.Apache.REEF.Utilities.DotNet.csproj" />
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/TestRunner.DotNet.proj
----------------------------------------------------------------------
diff --git a/lang/cs/TestRunner.DotNet.proj b/lang/cs/TestRunner.DotNet.proj
index c141813..5e853fe 100644
--- a/lang/cs/TestRunner.DotNet.proj
+++ b/lang/cs/TestRunner.DotNet.proj
@@ -27,7 +27,7 @@ under the License.
<Platform>x64</Platform>
</PropertyGroup>
<PropertyGroup Condition=" '$(TargetFramework)' == '' ">
- <TargetFramework>net452</TargetFramework>
+ <TargetFramework>net461</TargetFramework>
</PropertyGroup>
<Target Name="Test">
<MSBuild Projects="@(ProjectReferences)"
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/build.DotNetApp.props
----------------------------------------------------------------------
diff --git a/lang/cs/build.DotNetApp.props b/lang/cs/build.DotNetApp.props
index aa1a6fc..81ff791 100644
--- a/lang/cs/build.DotNetApp.props
+++ b/lang/cs/build.DotNetApp.props
@@ -27,10 +27,13 @@ under the License.
<!-- Settings for Windows platforms -->
<PropertyGroup Condition="'$(OS)' == 'Windows_NT'">
- <TargetFrameworks>net452</TargetFrameworks>
+ <TargetFrameworks>net461</TargetFrameworks>
<!-- If EnableDotNetBuild is true, override the target frameworks to include dot net core -->
- <TargetFrameworks Condition="'$(EnableDotNetBuild)' == 'true'">net452;netcoreapp2.0</TargetFrameworks>
+ <TargetFrameworks Condition="'$(EnableDotNetBuild)' == 'true'">netcoreapp2.0;net461</TargetFrameworks>
<OutputType>Exe</OutputType>
+
+ <!-- This is required to copy all referenced assemblies to the bin folder -->
+ <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
<!-- REEF-1888: win7-x64 is needed for .net46 and .net451. Target a newer version of windows once net46 and 451 are removed" -->
<RuntimeIdentifiers>win7-x64</RuntimeIdentifiers>
</PropertyGroup>
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/build.DotNetLibrary.props
----------------------------------------------------------------------
diff --git a/lang/cs/build.DotNetLibrary.props b/lang/cs/build.DotNetLibrary.props
index 1bee8c7..16ad7a8 100644
--- a/lang/cs/build.DotNetLibrary.props
+++ b/lang/cs/build.DotNetLibrary.props
@@ -17,17 +17,13 @@ under the License.
-->
<Import Project="build.DotNet.props"/>
+<PropertyGroup>
+ <TargetFrameworks>net461</TargetFrameworks>
-<!-- Settings for Linux platform -->
-<PropertyGroup Condition="'$(OS)' != 'Windows_NT'">
- <TargetFrameworks>netstandard2.0</TargetFrameworks>
-</PropertyGroup>
-
-<!-- Settings for Windows platform -->
-<PropertyGroup Condition="'$(OS)' == 'Windows_NT'">
- <TargetFrameworks>net452</TargetFrameworks>
<!-- If EnableDotNetBuild is true, override the target frameworks to include dot net core -->
- <TargetFrameworks Condition="'$(EnableDotNetBuild)' == 'true'">net452;netstandard2.0</TargetFrameworks>
-</PropertyGroup>
+ <TargetFrameworks Condition="'$(EnableDotNetBuild)' == 'true'">netstandard2.0</TargetFrameworks>
+ <!-- If on Linux, override the target frameworks to netcoreapp2.0 -->
+ <TargetFrameworks Condition="'$(OS)' != 'Windows_NT'">netcoreapp2.0</TargetFrameworks>
+</PropertyGroup>
</Project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/build.DotNetTest.props
----------------------------------------------------------------------
diff --git a/lang/cs/build.DotNetTest.props b/lang/cs/build.DotNetTest.props
index 467d73a..444ed1a 100644
--- a/lang/cs/build.DotNetTest.props
+++ b/lang/cs/build.DotNetTest.props
@@ -19,9 +19,17 @@ under the License.
<Import Project="build.DotNet.props"/>
<Import Project="xunit.DotNet.props"/>
<PropertyGroup>
- <TargetFrameworks>net452</TargetFrameworks>
+ <TargetFrameworks>net461</TargetFrameworks>
+
+ <!-- If EnableDotNetBuild is true, override the target frameworks to include dot net core -->
+ <TargetFrameworks Condition="'$(EnableDotNetBuild)' == 'true'">netcoreapp2.0;net461</TargetFrameworks>
+
+ <!-- If on Linux, override the target frameworks to netcoreapp2.0 -->
<TargetFrameworks Condition="'$(OS)' != 'Windows_NT'">netcoreapp2.0</TargetFrameworks>
+ <!-- This is required to copy all referenced assemblies to the bin folder -->
+ <CopyLocalLockFileAssemblies>true</CopyLocalLockFileAssemblies>
+
<!-- Set this attribute to prevent test projects from being packaged -->
<IsTestProject>true</IsTestProject>
</PropertyGroup>
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/build.DotNetTest.targets
----------------------------------------------------------------------
diff --git a/lang/cs/build.DotNetTest.targets b/lang/cs/build.DotNetTest.targets
index 3787bbc..4be3d8c 100644
--- a/lang/cs/build.DotNetTest.targets
+++ b/lang/cs/build.DotNetTest.targets
@@ -18,7 +18,7 @@ under the License.
<Import Project="build.DotNet.targets"/>
<Target Name="Test">
- <Exec Command="$(PackagesDir)\xunit.runner.console\2.1.0\tools\xunit.console.exe $(OutputPath)\$(AssemblyName).dll -notrait Environment=Yarn -html $(OutputPath)\xunit_report.html"
+ <Exec Command="$(PackagesDir)\xunit.runner.console\2.2.0\tools\xunit.console.exe $(OutputPath)\$(AssemblyName).dll -notrait Environment=Yarn -html $(OutputPath)\xunit_report.html"
IgnoreStandardErrorWarningFormat="true"/>
</Target>
http://git-wip-us.apache.org/repos/asf/reef/blob/7343e62c/lang/cs/xunit.DotNet.props
----------------------------------------------------------------------
diff --git a/lang/cs/xunit.DotNet.props b/lang/cs/xunit.DotNet.props
index 2fb4fab..8bb0a93 100644
--- a/lang/cs/xunit.DotNet.props
+++ b/lang/cs/xunit.DotNet.props
@@ -15,22 +15,11 @@ KIND, either express or implied. See the License for the
specific language governing permissions and limitations
under the License.
-->
- <PropertyGroup>
- <!-- Required check for .net451 test projects to work appropriately -->
- <DebugType Condition="'$(TargetFramework)' != '' AND !$(TargetFramework.StartsWith('netcoreapp'))">Full</DebugType>
- </PropertyGroup>
-
- <ItemGroup Condition="'$(TargetFramework)' == 'netcoreapp2.0' Or '$(TargetFramework)' == 'net46'">
+ <ItemGroup>
<PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.0.0" NoWarn="NU1701"/>
<PackageReference Include="xunit" Version="2.2.0" NoWarn="NU1701"/>
<PackageReference Include="xunit.runner.visualstudio" Version="2.2.0" NoWarn="NU1701"/>
<PackageReference Include="xunit.runner.console" Version="2.2.0" NoWarn="NU1701"/>
</ItemGroup>
- <ItemGroup Condition="'$(TargetFramework)' == 'net452'">
- <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.0.0" NoWarn="NU1701"/>
- <PackageReference Include="xunit" Version="2.1.0" NoWarn="NU1701"/>
- <PackageReference Include="xunit.runner.visualstudio" Version="2.1.0" NoWarn="NU1701"/>
- <PackageReference Include="xunit.runner.console" Version="2.1.0" NoWarn="NU1701"/>
- </ItemGroup>
</Project>
[05/16] reef git commit: [REEF-1965] Implement .NET REEF runtime for
Azure Batch
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/packages.config
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/packages.config b/lang/cs/Org.Apache.REEF.Client/packages.config
index d6832d2..094733d 100644
--- a/lang/cs/Org.Apache.REEF.Client/packages.config
+++ b/lang/cs/Org.Apache.REEF.Client/packages.config
@@ -18,7 +18,20 @@ specific language governing permissions and limitations
under the License.
-->
<packages>
+ <package id="Azure.Batch" version="8.0.1" targetFramework="net452" />
+ <package id="Microsoft.Azure.KeyVault.Core" version="1.0.0" targetFramework="net452" />
+ <package id="Microsoft.Data.Edm" version="5.8.2" targetFramework="net452" />
+ <package id="Microsoft.Data.OData" version="5.8.2" targetFramework="net452" />
+ <package id="Microsoft.Data.Services.Client" version="5.8.2" targetFramework="net452" />
+ <package id="Microsoft.Rest.ClientRuntime" version="2.3.10" targetFramework="net452" />
+ <package id="Microsoft.Rest.ClientRuntime.Azure" version="3.3.10" targetFramework="net452" />
<package id="Newtonsoft.Json" version="10.0.3" targetFramework="net451" />
<package id="StyleCop.MSBuild" version="5.0.0" targetFramework="net45" developmentDependency="true" />
+ <package id="System.ComponentModel.EventBasedAsync" version="4.0.11" targetFramework="net452" />
+ <package id="System.Dynamic.Runtime" version="4.0.0" targetFramework="net452" />
+ <package id="System.Linq.Queryable" version="4.0.0" targetFramework="net452" />
+ <package id="System.Net.Requests" version="4.0.11" targetFramework="net452" />
+ <package id="System.Spatial" version="5.8.2" targetFramework="net452" />
<package id="TransientFaultHandling.Core" version="5.1.1209.1" targetFramework="net45" />
+ <package id="WindowsAzure.Storage" version="8.1.3" targetFramework="net452" />
</packages>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Common/Runtime/RuntimeName.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Common/Runtime/RuntimeName.cs b/lang/cs/Org.Apache.REEF.Common/Runtime/RuntimeName.cs
index 59ea062..2328289 100644
--- a/lang/cs/Org.Apache.REEF.Common/Runtime/RuntimeName.cs
+++ b/lang/cs/Org.Apache.REEF.Common/Runtime/RuntimeName.cs
@@ -38,6 +38,11 @@ namespace Org.Apache.REEF.Common.Runtime
Mesos,
/// <summary>
+ /// Same value as org.apache.reef.runtime.mesos.driver.RuntimeIdentifier.RUNTIME_NAME
+ /// </summary>
+ AzBatch,
+
+ /// <summary>
/// Default value for the enum
/// </summary>
Default
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
index 33f5aa7..406fb36 100644
--- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
+++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/HelloREEF.cs
@@ -16,7 +16,9 @@
// under the License.
using System;
+using System.Globalization;
using Org.Apache.REEF.Client.API;
+using Org.Apache.REEF.Client.AzureBatch;
using Org.Apache.REEF.Client.Common;
using Org.Apache.REEF.Client.Local;
using Org.Apache.REEF.Client.Yarn;
@@ -40,6 +42,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF
private const string YARN = "yarn";
private const string YARNRest = "yarnrest";
private const string HDInsight = "hdi";
+ private const string AzureBatch = "azurebatch";
private readonly IREEFClient _reefClient;
[Inject]
@@ -60,19 +63,26 @@ namespace Org.Apache.REEF.Examples.HelloREEF
.Set(DriverConfiguration.CustomTraceLevel, Level.Verbose.ToString())
.Build();
+ string applicationId = GetApplicationId();
+
// The JobSubmission contains the Driver configuration as well as the files needed on the Driver.
var helloJobRequest = _reefClient.NewJobRequestBuilder()
.AddDriverConfiguration(helloDriverConfiguration)
.AddGlobalAssemblyForType(typeof(HelloDriver))
.AddGlobalAssembliesInDirectoryOfExecutingAssembly()
- .SetJobIdentifier("HelloREEF")
+ .SetJobIdentifier(applicationId)
.SetJavaLogLevel(JavaLoggingSetting.Verbose)
.Build();
IJobSubmissionResult jobSubmissionResult = _reefClient.SubmitAndGetJobStatus(helloJobRequest);
// Wait for the Driver to complete.
- jobSubmissionResult.WaitForDriverToFinish();
+ jobSubmissionResult?.WaitForDriverToFinish();
+ }
+
+ private string GetApplicationId()
+ {
+ return "HelloWorldJob-" + DateTime.Now.ToString("ddd-MMM-d-HH-mm-ss-yyyy", CultureInfo.CreateSpecificCulture("en-US"));
}
/// <summary>
@@ -104,6 +114,17 @@ namespace Org.Apache.REEF.Examples.HelloREEF
.Set(HDInsightClientConfiguration.JobSubmissionDirectoryPrefix, string.Format(@"/{0}/tmp", continerName))
.Set(AzureBlockBlobFileSystemConfiguration.ConnectionString, connectionString)
.Build();
+ case AzureBatch:
+ return AzureBatchRuntimeClientConfiguration.ConfigurationModule
+ .Set(AzureBatchRuntimeClientConfiguration.AzureBatchAccountKey, @"##########################################")
+ .Set(AzureBatchRuntimeClientConfiguration.AzureBatchAccountName, @"######")
+ .Set(AzureBatchRuntimeClientConfiguration.AzureBatchAccountUri, @"######################")
+ .Set(AzureBatchRuntimeClientConfiguration.AzureBatchPoolId, @"######")
+ .Set(AzureBatchRuntimeClientConfiguration.AzureStorageAccountKey, @"##########################################")
+ .Set(AzureBatchRuntimeClientConfiguration.AzureStorageAccountName, @"############")
+ .Set(AzureBatchRuntimeClientConfiguration.AzureStorageContainerName, @"###########")
+ .Build();
+
default:
throw new Exception("Unknown runtime: " + name);
}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Run.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Run.cs b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Run.cs
index f8aa715..e88892e 100644
--- a/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Run.cs
+++ b/lang/cs/Org.Apache.REEF.Examples.HelloREEF/Run.cs
@@ -25,14 +25,7 @@ namespace Org.Apache.REEF.Examples.HelloREEF
/// <param name="args"></param>
public static void Main(string[] args)
{
- if (args.Length < 2)
- {
- HelloREEF.MainSimple(args);
- }
- else
- {
- HelloREEFYarn.MainYarn(args);
- }
+ HelloREEF.MainSimple(args);
}
}
}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.sln
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.sln b/lang/cs/Org.Apache.REEF.sln
index b4cbccd..95fb5d0 100644
Binary files a/lang/cs/Org.Apache.REEF.sln and b/lang/cs/Org.Apache.REEF.sln differ
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/java/reef-bridge-client/pom.xml
----------------------------------------------------------------------
diff --git a/lang/java/reef-bridge-client/pom.xml b/lang/java/reef-bridge-client/pom.xml
index 7bc6a28..4c1b279 100644
--- a/lang/java/reef-bridge-client/pom.xml
+++ b/lang/java/reef-bridge-client/pom.xml
@@ -58,6 +58,11 @@ under the License.
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
+ <artifactId>reef-runtime-azbatch</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
<artifactId>reef-io</artifactId>
<version>${project.version}</version>
</dependency>
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc
----------------------------------------------------------------------
diff --git a/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc b/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc
index ecf0043..e2a30f3 100644
--- a/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc
+++ b/lang/java/reef-bridge-client/src/main/avro/JobSubmissionParameters.avsc
@@ -65,5 +65,22 @@
{ "name": "driverStdoutFilePath", "type": "string" },
{ "name": "driverStderrFilePath", "type": "string" }
]
+ },
+ {
+ "namespace": "org.apache.reef.reef.bridge.client.avro",
+ "type": "record",
+ "name": "AvroAzureBatchJobSubmissionParameters",
+ "doc": "Cross-language submission parameters to the Azure Batch runtime",
+ "fields": [
+ { "name": "sharedJobSubmissionParameters", "type": "AvroJobSubmissionParameters" },
+ { "name": "AzureBatchAccountKey", "type": "string" },
+ { "name": "AzureBatchAccountName", "type": "string" },
+ { "name": "AzureBatchAccountUri", "type": "string" },
+ { "name": "AzureBatchPoolId", "type": "string" },
+ { "name": "AzureStorageAccountKey", "type": "string" },
+ { "name": "AzureStorageAccountName", "type": "string" },
+ { "name": "AzureStorageContainerName", "type": "string" },
+ { "name": "AzureBatchIsWindows", "type": "boolean" }
+ ]
}
]
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java
new file mode 100644
index 0000000..c054722
--- /dev/null
+++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapDriverConfigGenerator.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.bridge.client;
+
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.io.JsonDecoder;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters;
+import org.apache.reef.runtime.common.client.DriverConfigurationProvider;
+import org.apache.reef.runtime.common.driver.parameters.ClientRemoteIdentifier;
+import org.apache.reef.tang.Configuration;
+
+import javax.inject.Inject;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * This is the Java Driver configuration generator for .NET Drivers that generates
+ * the Driver configuration at runtime. Called by {@link AzureBatchBootstrapREEFLauncher}.
+ */
+final class AzureBatchBootstrapDriverConfigGenerator {
+
+ private static final Logger LOG = Logger.getLogger(AzureBatchBootstrapDriverConfigGenerator.class.getName());
+
+ private final DriverConfigurationProvider driverConfigurationProvider;
+
+ @Inject
+ private AzureBatchBootstrapDriverConfigGenerator(final DriverConfigurationProvider driverConfigurationProvider) {
+ this.driverConfigurationProvider = driverConfigurationProvider;
+ }
+
+ Configuration getDriverConfigurationFromParams(final String bootstrapJobArgsLocation) throws IOException {
+
+ final File bootstrapJobArgsFile = new File(bootstrapJobArgsLocation).getCanonicalFile();
+
+ final AvroAzureBatchJobSubmissionParameters azureBatchBootstrapJobArgs =
+ readAzureBatchJobSubmissionParametersFromFile(bootstrapJobArgsFile);
+
+ final String jobId = azureBatchBootstrapJobArgs.getSharedJobSubmissionParameters().getJobId().toString();
+ final File jobFolder = new File(azureBatchBootstrapJobArgs
+ .getSharedJobSubmissionParameters().getJobSubmissionFolder().toString());
+
+ LOG.log(Level.INFO, "jobFolder {0} jobId {1}.", new Object[]{jobFolder.toURI(), jobId});
+
+ return this.driverConfigurationProvider.getDriverConfiguration(
+ jobFolder.toURI(), ClientRemoteIdentifier.NONE, jobId,
+ Constants.DRIVER_CONFIGURATION_WITH_HTTP_AND_NAMESERVER);
+ }
+
+ private AvroAzureBatchJobSubmissionParameters readAzureBatchJobSubmissionParametersFromFile(final File file)
+ throws IOException {
+ try (final FileInputStream fileInputStream = new FileInputStream(file)) {
+ return readAzureBatchSubmissionParametersFromInputStream(fileInputStream);
+ }
+ }
+
+ private static AvroAzureBatchJobSubmissionParameters readAzureBatchSubmissionParametersFromInputStream(
+ final InputStream inputStream) throws IOException {
+ final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(
+ AvroAzureBatchJobSubmissionParameters.getClassSchema(), inputStream);
+ final SpecificDatumReader<AvroAzureBatchJobSubmissionParameters> reader = new SpecificDatumReader<>(
+ AvroAzureBatchJobSubmissionParameters.class);
+ return reader.read(null, decoder);
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java
new file mode 100644
index 0000000..481ae53
--- /dev/null
+++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java
@@ -0,0 +1,135 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.bridge.client;
+
+import org.apache.avro.io.DecoderFactory;
+import org.apache.avro.io.JsonDecoder;
+import org.apache.avro.specific.SpecificDatumReader;
+import org.apache.reef.annotations.audience.Interop;
+import org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters;
+import org.apache.reef.runtime.azbatch.client.AzureBatchRuntimeConfiguration;
+import org.apache.reef.runtime.azbatch.client.AzureBatchRuntimeConfigurationCreator;
+import org.apache.reef.runtime.common.REEFEnvironment;
+import org.apache.reef.runtime.common.evaluator.PIDStoreStartHandler;
+import org.apache.reef.runtime.common.launch.REEFErrorHandler;
+import org.apache.reef.runtime.common.launch.REEFMessageCodec;
+import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.Tang;
+import org.apache.reef.tang.exceptions.InjectionException;
+import org.apache.reef.wake.remote.RemoteConfiguration;
+import org.apache.reef.wake.time.Clock;
+
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * This is a bootstrap launcher for Azure Batch for submission from C#. It allows for Java Driver
+ * configuration generation directly on the Driver without need of Java dependency if REST
+ * submission is used. Note that the name of the class must contain "REEFLauncher" for the time
+ * being in order for the Interop code to discover the class.
+ */
+@Interop(CppFiles = "DriverLauncher.cpp")
+public final class AzureBatchBootstrapREEFLauncher {
+
+ private static final Logger LOG = Logger.getLogger(AzureBatchBootstrapREEFLauncher.class.getName());
+ private static final Tang TANG = Tang.Factory.getTang();
+
+ public static void main(final String[] args) throws IOException, InjectionException {
+
+ LOG.log(Level.INFO, "Entering BootstrapLauncher.main(). {0}", args[0]);
+
+ if (args.length != 1) {
+
+ final StringBuilder sb = new StringBuilder(
+ "Bootstrap launcher should have one configuration file input," +
+ " specifying the job submission parameters to be deserialized" +
+ " to create the Azure Batch DriverConfiguration on the fly." +
+ " Current args are [ ");
+ for (String arg : args) {
+ sb.append(arg).append(" ");
+ }
+ sb.append("]");
+
+ final String message = sb.toString();
+ throw fatal(message, new IllegalArgumentException(message));
+ }
+
+ final File partialConfigFile = new File(args[0]);
+ final AzureBatchBootstrapDriverConfigGenerator azureBatchBootstrapDriverConfigGenerator =
+ TANG.newInjector(generateConfigurationFromJobSubmissionParameters(partialConfigFile))
+ .getInstance(AzureBatchBootstrapDriverConfigGenerator.class);
+
+ final Configuration launcherConfig =
+ TANG.newConfigurationBuilder()
+ .bindNamedParameter(RemoteConfiguration.ManagerName.class, "AzureBatchBootstrapREEFLauncher")
+ .bindNamedParameter(RemoteConfiguration.ErrorHandler.class, REEFErrorHandler.class)
+ .bindNamedParameter(RemoteConfiguration.MessageCodec.class, REEFMessageCodec.class)
+ .bindSetEntry(Clock.RuntimeStartHandler.class, PIDStoreStartHandler.class)
+ .build();
+
+ try (final REEFEnvironment reef = REEFEnvironment.fromConfiguration(
+ azureBatchBootstrapDriverConfigGenerator.getDriverConfigurationFromParams(args[0]), launcherConfig)) {
+ reef.run();
+ } catch (final InjectionException ex) {
+ throw fatal("Unable to configure and start REEFEnvironment.", ex);
+ }
+ }
+
+ private static Configuration generateConfigurationFromJobSubmissionParameters(final File params) throws IOException {
+
+ final AvroAzureBatchJobSubmissionParameters avroAzureBatchJobSubmissionParameters;
+
+ try (final FileInputStream fileInputStream = new FileInputStream(params)) {
+ final JsonDecoder decoder = DecoderFactory.get().jsonDecoder(
+ AvroAzureBatchJobSubmissionParameters.getClassSchema(), fileInputStream);
+ final SpecificDatumReader<AvroAzureBatchJobSubmissionParameters> reader =
+ new SpecificDatumReader<>(AvroAzureBatchJobSubmissionParameters.class);
+ avroAzureBatchJobSubmissionParameters = reader.read(null, decoder);
+ }
+
+ return AzureBatchRuntimeConfigurationCreator
+ .getOrCreateAzureBatchRuntimeConfiguration(avroAzureBatchJobSubmissionParameters.getAzureBatchIsWindows())
+ .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_ACCOUNT_NAME,
+ avroAzureBatchJobSubmissionParameters.getAzureBatchAccountName().toString())
+ .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_ACCOUNT_KEY,
+ avroAzureBatchJobSubmissionParameters.getAzureBatchAccountKey().toString())
+ .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_ACCOUNT_URI,
+ avroAzureBatchJobSubmissionParameters.getAzureBatchAccountUri().toString())
+ .set(AzureBatchRuntimeConfiguration.AZURE_BATCH_POOL_ID,
+ avroAzureBatchJobSubmissionParameters.getAzureBatchPoolId().toString())
+ .set(AzureBatchRuntimeConfiguration.AZURE_STORAGE_ACCOUNT_NAME,
+ avroAzureBatchJobSubmissionParameters.getAzureStorageAccountName().toString())
+ .set(AzureBatchRuntimeConfiguration.AZURE_STORAGE_ACCOUNT_KEY,
+ avroAzureBatchJobSubmissionParameters.getAzureStorageAccountKey().toString())
+ .set(AzureBatchRuntimeConfiguration.AZURE_STORAGE_CONTAINER_NAME,
+ avroAzureBatchJobSubmissionParameters.getAzureStorageContainerName().toString())
+ .build();
+ }
+
+ private static RuntimeException fatal(final String msg, final Throwable t) {
+ LOG.log(Level.SEVERE, msg, t);
+ return new RuntimeException(msg, t);
+ }
+
+ private AzureBatchBootstrapREEFLauncher() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/java/reef-bridge-java/pom.xml
----------------------------------------------------------------------
diff --git a/lang/java/reef-bridge-java/pom.xml b/lang/java/reef-bridge-java/pom.xml
index 2728e44..3b4ff86 100644
--- a/lang/java/reef-bridge-java/pom.xml
+++ b/lang/java/reef-bridge-java/pom.xml
@@ -53,6 +53,11 @@ under the License.
</dependency>
<dependency>
<groupId>${project.groupId}</groupId>
+ <artifactId>reef-runtime-azbatch</artifactId>
+ <version>${project.version}</version>
+ </dependency>
+ <dependency>
+ <groupId>${project.groupId}</groupId>
<artifactId>reef-io</artifactId>
<version>${project.version}</version>
</dependency>
[10/16] reef git commit: [REEF-2013] Running on Azure Batch fails
with a Null Exception error
Posted by do...@apache.org.
[REEF-2013] Running on Azure Batch fails with a Null Exception error
- This was due to the node label not being set correctly resulting a
null value when being passed down to Java. This is now set to the
correct value.
JIRA:
[REEF-2013](https://issues.apache.org/jira/browse/REEF-2013)
Pull Request:
This closes #1454
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/c0ddb8d3
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/c0ddb8d3
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/c0ddb8d3
Branch: refs/heads/REEF-335
Commit: c0ddb8d3f1042ba92c4170b675ddd02a7525c082
Parents: 49255b0
Author: Scott Inglis <si...@microsoft.com>
Authored: Fri Apr 27 17:50:27 2018 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Sun Apr 29 16:15:00 2018 -0700
----------------------------------------------------------------------
lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs | 2 +-
1 file changed, 1 insertion(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/c0ddb8d3/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs b/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs
index f4e827f..0538dfd 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs
@@ -75,7 +75,7 @@ namespace Org.Apache.REEF.Driver.Evaluator
RuntimeName = runtimeName;
NodeNames = nodeNames;
RelaxLocality = relaxLocality;
- NodeLabelExpression = NodeLabelExpression;
+ NodeLabelExpression = nodeLabelExpression;
}
[DataMember]
[15/16] reef git commit: [REEF-2012] Add driver restart capabilities
to reef runtime mock
Posted by do...@apache.org.
[REEF-2012] Add driver restart capabilities to reef runtime mock
JIRA: [REEF-2012](https://issues.apache.org/jira/browse/REEF-2012)
Pull Request:
Closes #1452
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/5ed56eba
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/5ed56eba
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/5ed56eba
Branch: refs/heads/REEF-335
Commit: 5ed56eba3b0b438dc0fdf10a37a58fcfdeb81223
Parents: 35df820
Author: Tyson Condie <tc...@apache.org>
Authored: Thu Apr 26 12:52:51 2018 -0700
Committer: Sergiy Matusevych <mo...@apache.org>
Committed: Mon Apr 30 11:09:13 2018 -0700
----------------------------------------------------------------------
.../org/apache/reef/mock/AutoCompletable.java | 45 --
.../mock/DefaultTaskReturnValueProvider.java | 41 --
.../org/apache/reef/mock/MockConfiguration.java | 173 ------
.../java/org/apache/reef/mock/MockFailure.java | 67 ---
.../java/org/apache/reef/mock/MockRuntime.java | 76 ---
.../reef/mock/MockTaskReturnValueProvider.java | 44 --
.../org/apache/reef/mock/ProcessRequest.java | 54 --
.../reef/mock/driver/AutoCompletable.java | 45 ++
.../driver/DefaultTaskReturnValueProvider.java | 41 ++
.../reef/mock/driver/MockConfiguration.java | 208 ++++++++
.../mock/driver/MockDriverRestartContext.java | 172 ++++++
.../apache/reef/mock/driver/MockFailure.java | 74 +++
.../apache/reef/mock/driver/MockRuntime.java | 92 ++++
.../driver/MockTaskReturnValueProvider.java | 44 ++
.../apache/reef/mock/driver/ProcessRequest.java | 54 ++
.../apache/reef/mock/driver/package-info.java | 40 ++
.../mock/driver/request/AllocateEvaluator.java | 72 +++
.../reef/mock/driver/request/CloseContext.java | 76 +++
.../mock/driver/request/CloseEvaluator.java | 78 +++
.../reef/mock/driver/request/CloseTask.java | 89 ++++
.../reef/mock/driver/request/CompleteTask.java | 82 +++
.../reef/mock/driver/request/CreateContext.java | 76 +++
.../driver/request/CreateContextAndTask.java | 98 ++++
.../reef/mock/driver/request/CreateTask.java | 89 ++++
.../driver/request/ProcessRequestInternal.java | 44 ++
.../request/SendMessageDriverToContext.java | 81 +++
.../driver/request/SendMessageDriverToTask.java | 81 +++
.../reef/mock/driver/request/SuspendTask.java | 90 ++++
.../reef/mock/driver/request/package-info.java | 23 +
.../mock/driver/runtime/MockActiveContext.java | 139 +++++
.../driver/runtime/MockAllocatedEvaluator.java | 172 ++++++
.../reef/mock/driver/runtime/MockClock.java | 120 +++++
.../mock/driver/runtime/MockClosedContext.java | 71 +++
.../mock/driver/runtime/MockCompletedTask.java | 57 ++
.../driver/runtime/MockEvaluatorDescriptor.java | 64 +++
.../driver/runtime/MockEvaluatorRequestor.java | 85 +++
.../mock/driver/runtime/MockFailedContext.java | 93 ++++
.../driver/runtime/MockFailedEvaluator.java | 79 +++
.../mock/driver/runtime/MockNodeDescriptor.java | 68 +++
.../mock/driver/runtime/MockRunningTask.java | 97 ++++
.../mock/driver/runtime/MockRuntimeDriver.java | 522 +++++++++++++++++++
.../mock/driver/runtime/MockSuspendedTask.java | 54 ++
.../reef/mock/driver/runtime/MockUtils.java | 48 ++
.../reef/mock/driver/runtime/package-info.java | 23 +
.../java/org/apache/reef/mock/package-info.java | 40 --
.../reef/mock/request/AllocateEvaluator.java | 72 ---
.../apache/reef/mock/request/CloseContext.java | 76 ---
.../reef/mock/request/CloseEvaluator.java | 78 ---
.../org/apache/reef/mock/request/CloseTask.java | 89 ----
.../apache/reef/mock/request/CompleteTask.java | 82 ---
.../apache/reef/mock/request/CreateContext.java | 76 ---
.../reef/mock/request/CreateContextAndTask.java | 98 ----
.../apache/reef/mock/request/CreateTask.java | 89 ----
.../mock/request/ProcessRequestInternal.java | 44 --
.../request/SendMessageDriverToContext.java | 81 ---
.../mock/request/SendMessageDriverToTask.java | 81 ---
.../apache/reef/mock/request/SuspendTask.java | 90 ----
.../apache/reef/mock/request/package-info.java | 23 -
.../reef/mock/runtime/MockActiveContext.java | 139 -----
.../mock/runtime/MockAllocatedEvalautor.java | 149 ------
.../org/apache/reef/mock/runtime/MockClock.java | 120 -----
.../reef/mock/runtime/MockClosedContext.java | 71 ---
.../reef/mock/runtime/MockCompletedTask.java | 57 --
.../mock/runtime/MockEvaluatorDescriptor.java | 64 ---
.../mock/runtime/MockEvaluatorRequestor.java | 85 ---
.../reef/mock/runtime/MockFailedContext.java | 93 ----
.../reef/mock/runtime/MockFailedEvaluator.java | 79 ---
.../reef/mock/runtime/MockNodeDescriptor.java | 68 ---
.../reef/mock/runtime/MockRunningTask.java | 97 ----
.../reef/mock/runtime/MockRuntimeDriver.java | 454 ----------------
.../reef/mock/runtime/MockSuspendedTask.java | 54 --
.../org/apache/reef/mock/runtime/MockUtils.java | 48 --
.../apache/reef/mock/runtime/package-info.java | 23 -
.../org/apache/reef/mock/BasicMockTests.java | 205 --------
.../org/apache/reef/mock/MockApplication.java | 275 ----------
.../apache/reef/mock/driver/BasicMockTests.java | 163 ++++++
.../reef/mock/driver/MockApplication.java | 275 ++++++++++
.../apache/reef/mock/driver/package-info.java | 23 +
.../java/org/apache/reef/mock/package-info.java | 23 -
.../driver/YarnDriverRestartConfiguration.java | 2 +-
.../driver/YarnDriverRuntimeRestartManager.java | 2 +-
81 files changed, 3904 insertions(+), 3625 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/AutoCompletable.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/AutoCompletable.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/AutoCompletable.java
deleted file mode 100644
index 173b410..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/AutoCompletable.java
+++ /dev/null
@@ -1,45 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock;
-
-import org.apache.reef.annotations.Unstable;
-
-/**
- * Indicates that a process request should auto complete.
- */
-@Unstable
-public interface AutoCompletable {
-
- /**
- * @return true if should auto complete
- */
- boolean doAutoComplete();
-
- /**
- * Set auto complete.
- * @param value to set
- */
- void setAutoComplete(final boolean value);
-
- /**
- * @return auto complete process request
- */
- ProcessRequest getCompletionProcessRequest();
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/DefaultTaskReturnValueProvider.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/DefaultTaskReturnValueProvider.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/DefaultTaskReturnValueProvider.java
deleted file mode 100644
index 2ae81d4..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/DefaultTaskReturnValueProvider.java
+++ /dev/null
@@ -1,41 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.reef.mock;
-
-import org.apache.reef.mock.runtime.MockRunningTask;
-
-import javax.inject.Inject;
-
-/**
- * A default task return value provider.
- */
-final class DefaultTaskReturnValueProvider implements MockTaskReturnValueProvider {
-
- @Inject
- DefaultTaskReturnValueProvider() {
-
- }
-
- @Override
- public byte[] getReturnValue(final MockRunningTask task) {
- return new byte[0];
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockConfiguration.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockConfiguration.java
deleted file mode 100644
index beec5c4..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockConfiguration.java
+++ /dev/null
@@ -1,173 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.reef.mock;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.driver.client.JobMessageObserver;
-import org.apache.reef.driver.context.ActiveContext;
-import org.apache.reef.driver.context.ClosedContext;
-import org.apache.reef.driver.context.ContextMessage;
-import org.apache.reef.driver.context.FailedContext;
-import org.apache.reef.driver.evaluator.AllocatedEvaluator;
-import org.apache.reef.driver.evaluator.CompletedEvaluator;
-import org.apache.reef.driver.evaluator.EvaluatorRequestor;
-import org.apache.reef.driver.evaluator.FailedEvaluator;
-import org.apache.reef.driver.parameters.*;
-import org.apache.reef.driver.task.*;
-import org.apache.reef.mock.runtime.MockClock;
-import org.apache.reef.mock.runtime.MockEvaluatorRequestor;
-import org.apache.reef.mock.runtime.MockRuntimeDriver;
-import org.apache.reef.tang.formats.ConfigurationModule;
-import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
-import org.apache.reef.tang.formats.OptionalImpl;
-import org.apache.reef.tang.formats.RequiredImpl;
-import org.apache.reef.wake.EventHandler;
-import org.apache.reef.wake.time.Clock;
-import org.apache.reef.wake.time.event.StartTime;
-import org.apache.reef.wake.time.event.StopTime;
-
-/**
- * Configure a mock runtime.
- */
-@Unstable
-public class MockConfiguration extends ConfigurationModuleBuilder {
-
- /**
- * The event handler invoked right after the driver boots up.
- */
- public static final RequiredImpl<EventHandler<StartTime>> ON_DRIVER_STARTED = new RequiredImpl<>();
-
- /**
- * The event handler invoked right before the driver shuts down. Defaults to ignore.
- */
- public static final OptionalImpl<EventHandler<StopTime>> ON_DRIVER_STOP = new OptionalImpl<>();
-
- // ***** EVALUATOR HANDLER BINDINGS:
-
- /**
- * Event handler for allocated evaluators. Defaults to returning the evaluator if not bound.
- */
- public static final OptionalImpl<EventHandler<AllocatedEvaluator>> ON_EVALUATOR_ALLOCATED = new OptionalImpl<>();
-
- /**
- * Event handler for completed evaluators. Defaults to logging if not bound.
- */
- public static final OptionalImpl<EventHandler<CompletedEvaluator>> ON_EVALUATOR_COMPLETED = new OptionalImpl<>();
-
- /**
- * Event handler for failed evaluators. Defaults to job failure if not bound.
- */
- public static final OptionalImpl<EventHandler<FailedEvaluator>> ON_EVALUATOR_FAILED = new OptionalImpl<>();
-
- // ***** TASK HANDLER BINDINGS:
-
- /**
- * Event handler for task messages. Defaults to logging if not bound.
- */
- public static final OptionalImpl<EventHandler<TaskMessage>> ON_TASK_MESSAGE = new OptionalImpl<>();
-
- /**
- * Event handler for completed tasks. Defaults to closing the context the task ran on if not bound.
- */
- public static final OptionalImpl<EventHandler<CompletedTask>> ON_TASK_COMPLETED = new OptionalImpl<>();
-
- /**
- * Event handler for failed tasks. Defaults to job failure if not bound.
- */
- public static final OptionalImpl<EventHandler<FailedTask>> ON_TASK_FAILED = new OptionalImpl<>();
-
- /**
- * Event handler for running tasks. Defaults to logging if not bound.
- */
- public static final OptionalImpl<EventHandler<RunningTask>> ON_TASK_RUNNING = new OptionalImpl<>();
-
- /**
- * Event handler for suspended tasks. Defaults to job failure if not bound. Rationale: many jobs don't support
- * task suspension. Hence, this parameter should be optional. The only sane default is to crash the job, then.
- */
- public static final OptionalImpl<EventHandler<SuspendedTask>> ON_TASK_SUSPENDED = new OptionalImpl<>();
-
- // ***** CONTEXT HANDLER BINDINGS:
-
- /**
- * Event handler for active context. Defaults to closing the context if not bound.
- */
- public static final OptionalImpl<EventHandler<ActiveContext>> ON_CONTEXT_ACTIVE = new OptionalImpl<>();
-
- /**
- * Event handler for closed context. Defaults to logging if not bound.
- */
- public static final OptionalImpl<EventHandler<ClosedContext>> ON_CONTEXT_CLOSED = new OptionalImpl<>();
-
- /**
- * Event handler for closed context. Defaults to job failure if not bound.
- */
- public static final OptionalImpl<EventHandler<FailedContext>> ON_CONTEXT_FAILED = new OptionalImpl<>();
-
- /**
- * Event handler for context messages. Defaults to logging if not bound.
- */
- public static final OptionalImpl<EventHandler<ContextMessage>> ON_CONTEXT_MESSAGE = new OptionalImpl<>();
-
-
- /**
- * Receiver of messages sent by the Driver to the client.
- */
- public static final OptionalImpl<JobMessageObserver> ON_JOB_MESSAGE = new OptionalImpl<>();
-
- /**
- * An implementation of a task return value provider.
- */
- public static final OptionalImpl<MockTaskReturnValueProvider> TASK_RETURN_VALUE_PROVIDER = new OptionalImpl<>();
-
- public static final ConfigurationModule CONF = new MockConfiguration()
- .bindImplementation(EvaluatorRequestor.class, MockEvaluatorRequestor.class) // requesting evaluators
- .bindImplementation(MockRuntime.class, MockRuntimeDriver.class)
- .bindImplementation(MockFailure.class, MockRuntimeDriver.class)
- .bindImplementation(Clock.class, MockClock.class)
- .bindImplementation(MockTaskReturnValueProvider.class, TASK_RETURN_VALUE_PROVIDER)
-
- // client handlers
- .bindImplementation(JobMessageObserver.class, ON_JOB_MESSAGE) // sending message to job client
-
- // Driver start/stop handlers
- .bindSetEntry(DriverStartHandler.class, ON_DRIVER_STARTED)
- .bindSetEntry(Clock.StopHandler.class, ON_DRIVER_STOP)
-
- // Evaluator handlers
- .bindSetEntry(EvaluatorAllocatedHandlers.class, ON_EVALUATOR_ALLOCATED)
- .bindSetEntry(EvaluatorCompletedHandlers.class, ON_EVALUATOR_COMPLETED)
- .bindSetEntry(EvaluatorFailedHandlers.class, ON_EVALUATOR_FAILED)
-
- // Task handlers
- .bindSetEntry(TaskRunningHandlers.class, ON_TASK_RUNNING)
- .bindSetEntry(TaskFailedHandlers.class, ON_TASK_FAILED)
- .bindSetEntry(TaskMessageHandlers.class, ON_TASK_MESSAGE)
- .bindSetEntry(TaskCompletedHandlers.class, ON_TASK_COMPLETED)
- .bindSetEntry(TaskSuspendedHandlers.class, ON_TASK_SUSPENDED)
-
- // Context handlers
- .bindSetEntry(ContextActiveHandlers.class, ON_CONTEXT_ACTIVE)
- .bindSetEntry(ContextClosedHandlers.class, ON_CONTEXT_CLOSED)
- .bindSetEntry(ContextMessageHandlers.class, ON_CONTEXT_MESSAGE)
- .bindSetEntry(ContextFailedHandlers.class, ON_CONTEXT_FAILED)
-
- .build();
-
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockFailure.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockFailure.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockFailure.java
deleted file mode 100644
index f8822a2..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockFailure.java
+++ /dev/null
@@ -1,67 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-package org.apache.reef.mock;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.driver.context.ActiveContext;
-import org.apache.reef.driver.evaluator.AllocatedEvaluator;
-import org.apache.reef.driver.task.RunningTask;
-
-import java.util.Collection;
-
-/**
- * Used to fail running REEF entities i.e., Evaluators, Contexts, Tasks.
- */
-@Unstable
-public interface MockFailure {
-
- /**
- * @return current Collection of allocated evaluators.
- */
- Collection<AllocatedEvaluator> getCurrentAllocatedEvaluators();
-
- /**
- * Fail an allocated evaluator.
- * @param evaluator to be failed
- */
- void fail(final AllocatedEvaluator evaluator);
-
- /**
- * @return current Collection of active contexts
- */
- Collection<ActiveContext> getCurrentActiveContexts();
-
- /**
- * Fail an ActiveContext.
- * @param context to be failed
- */
- void fail(final ActiveContext context);
-
- /**
- * @return current Collection of running tasks
- */
- Collection<RunningTask> getCurrentRunningTasks();
-
- /**
- * Fail a running task.
- * @param task to be failed
- */
- void fail(final RunningTask task);
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockRuntime.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockRuntime.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockRuntime.java
deleted file mode 100644
index 0e09f5d..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockRuntime.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock;
-
-import org.apache.reef.annotations.Unstable;
-
-/**
- * Mock API used to drive the evaluation of ProcessRequest
- * events, which are triggered by the Application driver.
- * Clients used this to determine whether a particular ProcessRequest
- * event should succeed or fail.
- */
-@Unstable
-public interface MockRuntime extends MockFailure {
-
- /**
- * Initiate the start time event to the application driver.
- */
- void start();
-
- /**
- * Initiate the stop time event to the application driver.
- */
- void stop();
-
- /**
- * @return true if there is an outstanding ProcessRequest
- */
- boolean hasProcessRequest();
-
- /**
- * The client (caller) is responsible for determining what
- * to do with a ProcessRequest event. There are three options:
- * 1. Pass to the succeed method, which signals success to the driver.
- * 2. Pass to the fail method, signaling failure to the driver.
- * 3. Drop it on the floor (e.g., network failure).
- *
- * @return the next ProcessRequest object to be processed.
- */
- ProcessRequest getNextProcessRequest();
-
- /**
- * The driver will be informed that the operation corresponding
- * to the ProcessRequest succeeded, and will be given any relevant
- * data structures e.g., AllocatedEvaluator, RunningTask, etc.
- *
- * @param request to be processed successfully
- */
- void succeed(final ProcessRequest request);
-
- /**
- * The driver will be informed that the operation corresponding
- * to the PRocessRequest failed, and will be given any relevant
- * data structures e.g., FailedEvaluator, FailedTask, etc.
- *
- * @param request to be failed.
- */
- void fail(final ProcessRequest request);
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockTaskReturnValueProvider.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockTaskReturnValueProvider.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockTaskReturnValueProvider.java
deleted file mode 100644
index a0e794b..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/MockTaskReturnValueProvider.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.reef.mock;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.driver.task.CompletedTask;
-import org.apache.reef.mock.runtime.MockRunningTask;
-import org.apache.reef.tang.annotations.DefaultImplementation;
-
-/**
- * Clients bind an implementation of this interface, which
- * will be used to create a mock return value for a mock
- * task execution. This return value will be returned by
- * the {@link CompletedTask#get()}} method.
- */
-@Unstable
-@DefaultImplementation(DefaultTaskReturnValueProvider.class)
-public interface MockTaskReturnValueProvider {
-
- /**
- * Provide a valid return value for the {@link CompletedTask#get()} method.
- * @param task that is to be provided with a return value
- * @return {@link org.apache.reef.task.Task#call(byte[])} return value
- */
- byte[] getReturnValue(final MockRunningTask task);
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/ProcessRequest.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/ProcessRequest.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/ProcessRequest.java
deleted file mode 100644
index 09e9691..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/ProcessRequest.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock;
-
-import org.apache.reef.annotations.Unstable;
-
-/**
- * A ProcessRequest refers to an outstanding event that is
- * waiting to be processed by the REEF mock runtime. Clients
- * are responsible for deciding how a ProcessRequest should be
- * handled, by either:
- * 1. successfully processing the request
- * 2. unsucessfully processing the request
- * 3. dropping the processing request (i.e., loosing it)
- * These decisions are conveyed through the {MockRuntime} API.
- */
-@Unstable
-public interface ProcessRequest extends AutoCompletable {
- /**
- * process request type.
- */
- enum Type {
- ALLOCATE_EVALUATOR,
- CLOSE_EVALUATOR,
- CREATE_CONTEXT,
- CLOSE_CONTEXT,
- CREATE_TASK,
- SUSPEND_TASK,
- CLOSE_TASK,
- COMPLETE_TASK,
- CREATE_CONTEXT_AND_TASK,
- SEND_MESSAGE_DRIVER_TO_TASK,
- SEND_MESSAGE_DRIVER_TO_CONTEXT
- }
-
- Type getType();
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/AutoCompletable.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/AutoCompletable.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/AutoCompletable.java
new file mode 100644
index 0000000..3ecb0b3
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/AutoCompletable.java
@@ -0,0 +1,45 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver;
+
+import org.apache.reef.annotations.Unstable;
+
+/**
+ * Indicates that a process request should auto complete.
+ */
+@Unstable
+public interface AutoCompletable {
+
+ /**
+ * @return true if should auto complete
+ */
+ boolean doAutoComplete();
+
+ /**
+ * Set auto complete.
+ * @param value to set
+ */
+ void setAutoComplete(final boolean value);
+
+ /**
+ * @return auto complete process request
+ */
+ ProcessRequest getCompletionProcessRequest();
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/DefaultTaskReturnValueProvider.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/DefaultTaskReturnValueProvider.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/DefaultTaskReturnValueProvider.java
new file mode 100644
index 0000000..d3a6be7
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/DefaultTaskReturnValueProvider.java
@@ -0,0 +1,41 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.reef.mock.driver;
+
+import org.apache.reef.mock.driver.runtime.MockRunningTask;
+
+import javax.inject.Inject;
+
+/**
+ * A default task return value provider.
+ */
+final class DefaultTaskReturnValueProvider implements MockTaskReturnValueProvider {
+
+ @Inject
+ DefaultTaskReturnValueProvider() {
+
+ }
+
+ @Override
+ public byte[] getReturnValue(final MockRunningTask task) {
+ return new byte[0];
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockConfiguration.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockConfiguration.java
new file mode 100644
index 0000000..3366220
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockConfiguration.java
@@ -0,0 +1,208 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.mock.driver;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.driver.client.JobMessageObserver;
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.context.ClosedContext;
+import org.apache.reef.driver.context.ContextMessage;
+import org.apache.reef.driver.context.FailedContext;
+import org.apache.reef.driver.evaluator.AllocatedEvaluator;
+import org.apache.reef.driver.evaluator.CompletedEvaluator;
+import org.apache.reef.driver.evaluator.EvaluatorRequestor;
+import org.apache.reef.driver.evaluator.FailedEvaluator;
+import org.apache.reef.driver.parameters.*;
+import org.apache.reef.driver.restart.DriverRestartCompleted;
+import org.apache.reef.driver.restart.DriverRestarted;
+import org.apache.reef.driver.task.*;
+import org.apache.reef.mock.driver.runtime.MockClock;
+import org.apache.reef.mock.driver.runtime.MockEvaluatorRequestor;
+import org.apache.reef.mock.driver.runtime.MockRuntimeDriver;
+import org.apache.reef.tang.formats.ConfigurationModule;
+import org.apache.reef.tang.formats.ConfigurationModuleBuilder;
+import org.apache.reef.tang.formats.OptionalImpl;
+import org.apache.reef.tang.formats.RequiredImpl;
+import org.apache.reef.wake.EventHandler;
+import org.apache.reef.wake.time.Clock;
+import org.apache.reef.wake.time.event.StartTime;
+import org.apache.reef.wake.time.event.StopTime;
+
+/**
+ * Configure a mock runtime.
+ */
+@Unstable
+public class MockConfiguration extends ConfigurationModuleBuilder {
+
+ /**
+ * The event handler invoked right after the driver boots up.
+ */
+ public static final RequiredImpl<EventHandler<StartTime>> ON_DRIVER_STARTED = new RequiredImpl<>();
+
+ /**
+ * The event handler invoked right before the driver shuts down. Defaults to ignore.
+ */
+ public static final OptionalImpl<EventHandler<StopTime>> ON_DRIVER_STOP = new OptionalImpl<>();
+
+ // ***** EVALUATOR HANDLER BINDINGS:
+
+ /**
+ * Event handler for allocated evaluators. Defaults to returning the evaluator if not bound.
+ */
+ public static final OptionalImpl<EventHandler<AllocatedEvaluator>> ON_EVALUATOR_ALLOCATED = new OptionalImpl<>();
+
+ /**
+ * Event handler for completed evaluators. Defaults to logging if not bound.
+ */
+ public static final OptionalImpl<EventHandler<CompletedEvaluator>> ON_EVALUATOR_COMPLETED = new OptionalImpl<>();
+
+ /**
+ * Event handler for failed evaluators. Defaults to job failure if not bound.
+ */
+ public static final OptionalImpl<EventHandler<FailedEvaluator>> ON_EVALUATOR_FAILED = new OptionalImpl<>();
+
+ // ***** TASK HANDLER BINDINGS:
+
+ /**
+ * Event handler for task messages. Defaults to logging if not bound.
+ */
+ public static final OptionalImpl<EventHandler<TaskMessage>> ON_TASK_MESSAGE = new OptionalImpl<>();
+
+ /**
+ * Event handler for completed tasks. Defaults to closing the context the task ran on if not bound.
+ */
+ public static final OptionalImpl<EventHandler<CompletedTask>> ON_TASK_COMPLETED = new OptionalImpl<>();
+
+ /**
+ * Event handler for failed tasks. Defaults to job failure if not bound.
+ */
+ public static final OptionalImpl<EventHandler<FailedTask>> ON_TASK_FAILED = new OptionalImpl<>();
+
+ /**
+ * Event handler for running tasks. Defaults to logging if not bound.
+ */
+ public static final OptionalImpl<EventHandler<RunningTask>> ON_TASK_RUNNING = new OptionalImpl<>();
+
+ /**
+ * Event handler for suspended tasks. Defaults to job failure if not bound. Rationale: many jobs don't support
+ * task suspension. Hence, this parameter should be optional. The only sane default is to crash the job, then.
+ */
+ public static final OptionalImpl<EventHandler<SuspendedTask>> ON_TASK_SUSPENDED = new OptionalImpl<>();
+
+ // ***** CONTEXT HANDLER BINDINGS:
+
+ /**
+ * Event handler for active context. Defaults to closing the context if not bound.
+ */
+ public static final OptionalImpl<EventHandler<ActiveContext>> ON_CONTEXT_ACTIVE = new OptionalImpl<>();
+
+ /**
+ * Event handler for closed context. Defaults to logging if not bound.
+ */
+ public static final OptionalImpl<EventHandler<ClosedContext>> ON_CONTEXT_CLOSED = new OptionalImpl<>();
+
+ /**
+ * Event handler for closed context. Defaults to job failure if not bound.
+ */
+ public static final OptionalImpl<EventHandler<FailedContext>> ON_CONTEXT_FAILED = new OptionalImpl<>();
+
+ /**
+ * Event handler for context messages. Defaults to logging if not bound.
+ */
+ public static final OptionalImpl<EventHandler<ContextMessage>> ON_CONTEXT_MESSAGE = new OptionalImpl<>();
+
+ /**
+ * This event is fired in place of the ON_DRIVER_STARTED when the Driver is in fact restarted after failure.
+ */
+ public static final OptionalImpl<EventHandler<DriverRestarted>> ON_DRIVER_RESTARTED = new OptionalImpl<>();
+
+ /**
+ * Event handler for running tasks in previous evaluator, when driver restarted. Defaults to crash if not bound.
+ */
+ public static final OptionalImpl<EventHandler<RunningTask>> ON_DRIVER_RESTART_TASK_RUNNING = new OptionalImpl<>();
+
+ /**
+ * Event handler for active context when driver restart. Defaults to closing the context if not bound.
+ */
+ public static final OptionalImpl<EventHandler<ActiveContext>> ON_DRIVER_RESTART_CONTEXT_ACTIVE = new OptionalImpl<>();
+
+ /**
+ * Event handler for the event of driver restart completion, default to logging if not bound.
+ */
+ public static final OptionalImpl<EventHandler<DriverRestartCompleted>> ON_DRIVER_RESTART_COMPLETED =
+ new OptionalImpl<>();
+
+ /**
+ * Event handler for the event of driver restart completion, default to logging if not bound.
+ */
+ public static final OptionalImpl<EventHandler<FailedEvaluator>> ON_DRIVER_RESTART_EVALUATOR_FAILED =
+ new OptionalImpl<>();
+
+ /**
+ * Receiver of messages sent by the Driver to the client.
+ */
+ public static final OptionalImpl<JobMessageObserver> ON_JOB_MESSAGE = new OptionalImpl<>();
+
+ /**
+ * An implementation of a task return value provider.
+ */
+ public static final OptionalImpl<MockTaskReturnValueProvider> TASK_RETURN_VALUE_PROVIDER = new OptionalImpl<>();
+
+ public static final ConfigurationModule CONF = new MockConfiguration()
+ .bindImplementation(EvaluatorRequestor.class, MockEvaluatorRequestor.class) // requesting evaluators
+ .bindImplementation(MockRuntime.class, MockRuntimeDriver.class)
+ .bindImplementation(MockFailure.class, MockRuntimeDriver.class)
+ .bindImplementation(Clock.class, MockClock.class)
+ .bindImplementation(MockTaskReturnValueProvider.class, TASK_RETURN_VALUE_PROVIDER)
+
+ // recovery handlers
+ .bindSetEntry(DriverRestartFailedEvaluatorHandlers.class, ON_DRIVER_RESTART_EVALUATOR_FAILED)
+ .bindSetEntry(DriverRestartCompletedHandlers.class, ON_DRIVER_RESTART_COMPLETED)
+ .bindSetEntry(DriverRestartContextActiveHandlers.class, ON_DRIVER_RESTART_CONTEXT_ACTIVE)
+ .bindSetEntry(DriverRestartTaskRunningHandlers.class, ON_DRIVER_RESTART_TASK_RUNNING)
+ .bindSetEntry(DriverRestartHandler.class, ON_DRIVER_RESTARTED)
+
+ // client handlers
+ .bindImplementation(JobMessageObserver.class, ON_JOB_MESSAGE) // sending message to job client
+
+ // Driver start/stop handlers
+ .bindSetEntry(DriverStartHandler.class, ON_DRIVER_STARTED)
+ .bindSetEntry(Clock.StopHandler.class, ON_DRIVER_STOP)
+
+ // Evaluator handlers
+ .bindSetEntry(EvaluatorAllocatedHandlers.class, ON_EVALUATOR_ALLOCATED)
+ .bindSetEntry(EvaluatorCompletedHandlers.class, ON_EVALUATOR_COMPLETED)
+ .bindSetEntry(EvaluatorFailedHandlers.class, ON_EVALUATOR_FAILED)
+
+ // Task handlers
+ .bindSetEntry(TaskRunningHandlers.class, ON_TASK_RUNNING)
+ .bindSetEntry(TaskFailedHandlers.class, ON_TASK_FAILED)
+ .bindSetEntry(TaskMessageHandlers.class, ON_TASK_MESSAGE)
+ .bindSetEntry(TaskCompletedHandlers.class, ON_TASK_COMPLETED)
+ .bindSetEntry(TaskSuspendedHandlers.class, ON_TASK_SUSPENDED)
+
+ // Context handlers
+ .bindSetEntry(ContextActiveHandlers.class, ON_CONTEXT_ACTIVE)
+ .bindSetEntry(ContextClosedHandlers.class, ON_CONTEXT_CLOSED)
+ .bindSetEntry(ContextMessageHandlers.class, ON_CONTEXT_MESSAGE)
+ .bindSetEntry(ContextFailedHandlers.class, ON_CONTEXT_FAILED)
+
+ .build();
+
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockDriverRestartContext.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockDriverRestartContext.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockDriverRestartContext.java
new file mode 100644
index 0000000..4c0ec28
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockDriverRestartContext.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.mock.driver;
+
+import org.apache.reef.driver.restart.DriverRestartCompleted;
+import org.apache.reef.driver.restart.DriverRestarted;
+import org.apache.reef.mock.driver.runtime.MockActiveContext;
+import org.apache.reef.mock.driver.runtime.MockAllocatedEvaluator;
+import org.apache.reef.mock.driver.runtime.MockFailedEvaluator;
+import org.apache.reef.mock.driver.runtime.MockRunningTask;
+import org.apache.reef.wake.time.Time;
+import org.apache.reef.wake.time.event.StartTime;
+import org.apache.reef.wake.time.event.StopTime;
+
+import java.util.*;
+
+/**
+ * Contains the runtime driver state at the time of a driver
+ * failure, triggered by {@link MockFailure}.
+ */
+public final class MockDriverRestartContext {
+
+ private final int restartAttemps;
+
+ private final StartTime startTime;
+
+ private final List<MockAllocatedEvaluator> allocatedEvaluators;
+
+ private final List<MockActiveContext> activeContexts;
+
+ private final List<MockRunningTask> runningTasks;
+
+ private final List<MockFailedEvaluator> failedEvaluators;
+
+ public MockDriverRestartContext(
+ final int restartAttemps,
+ final StartTime startTime,
+ final List<MockAllocatedEvaluator> allocatedEvaluators,
+ final List<MockActiveContext> activeContexts,
+ final List<MockRunningTask> runningTasks) {
+ this.restartAttemps = restartAttemps;
+ this.startTime = startTime;
+ this.allocatedEvaluators = allocatedEvaluators;
+ this.activeContexts = activeContexts;
+ this.runningTasks = runningTasks;
+ this.failedEvaluators = new ArrayList<>();
+ }
+
+ /**
+ * Generate a DriverRestarted event to be passed to the
+ * {@link org.apache.reef.driver.parameters.DriverRestartHandler}.
+ * @return DriverRestarted event based on the state at the time of driver failure
+ */
+ public DriverRestarted getDriverRestarted() {
+ final Set<String> expectedEvaluatorIds = new HashSet<>();
+ for (final MockAllocatedEvaluator allocatedEvaluator : this.allocatedEvaluators) {
+ expectedEvaluatorIds.add(allocatedEvaluator.getId());
+ }
+ return new DriverRestarted() {
+ @Override
+ public int getResubmissionAttempts() {
+ return restartAttemps;
+ }
+
+ @Override
+ public StartTime getStartTime() {
+ return startTime;
+ }
+
+ @Override
+ public Set<String> getExpectedEvaluatorIds() {
+ return expectedEvaluatorIds;
+ }
+ };
+ }
+
+ public DriverRestartCompleted getDriverRestartCompleted(final boolean isTimeout, final long restartDuration) {
+ return new DriverRestartCompleted() {
+ @Override
+ public Time getCompletedTime() {
+ return new StopTime(startTime.getTimestamp() + restartDuration);
+ }
+
+ @Override
+ public boolean isTimedOut() {
+ return isTimeout;
+ }
+ };
+ }
+
+ /**
+ * Pass these tasks to the {@link org.apache.reef.driver.parameters.DriverRestartTaskRunningHandlers}.
+ * @return MockRunningTasks at the time of driver failure
+ */
+ public List<MockRunningTask> getRunningTasks() {
+ return this.runningTasks;
+ }
+
+ /**
+ * Pass these active contexts to the {@link org.apache.reef.driver.parameters.DriverRestartContextActiveHandlers}.
+ * These active contexts have no tasks running.
+ * @return
+ */
+ public List<MockActiveContext> getIdleActiveContexts() {
+ final List<MockActiveContext> idleActiveContexts = new ArrayList<>();
+ final Set<String> activeContextsWithRunningTasks = new HashSet<>();
+ for (final MockRunningTask task : this.runningTasks) {
+ activeContextsWithRunningTasks.add(task.getActiveContext().getEvaluatorId());
+ }
+ for (final MockActiveContext context : this.activeContexts) {
+ if (!activeContextsWithRunningTasks.contains(context.getEvaluatorId())) {
+ idleActiveContexts.add(context);
+ }
+ }
+ return idleActiveContexts;
+ }
+
+ public List<MockFailedEvaluator> getFailedEvaluators() {
+ return this.failedEvaluators;
+ }
+
+ /**
+ * Fail a task.
+ * @param task to fail
+ */
+ public void failTask(final MockRunningTask task) {
+ this.runningTasks.remove(task);
+ }
+
+ /**
+ * Fail an evaluator; automatically cleans up state i.e., running tasks and contexts
+ * pertaining to the evaluator, and adds the evaluator to {@link this#getFailedEvaluators()}, which
+ * can be passed to the {@link org.apache.reef.driver.parameters.DriverRestartFailedEvaluatorHandlers}.
+ * @param evaluator to fail
+ */
+ public void failEvaluator(final MockAllocatedEvaluator evaluator) {
+ if (this.allocatedEvaluators.remove(evaluator)) {
+ this.failedEvaluators.add(new MockFailedEvaluator(evaluator.getId()));
+ // cleanup
+ final Iterator<MockRunningTask> taskIter = this.runningTasks.iterator();
+ while (taskIter.hasNext()) {
+ final MockRunningTask task = taskIter.next();
+ if (task.evaluatorID().equals(evaluator.getId())) {
+ taskIter.remove();
+ }
+ }
+ final Iterator<MockActiveContext> contextIter = this.activeContexts.iterator();
+ while (contextIter.hasNext()) {
+ final MockActiveContext context = contextIter.next();
+ if (context.getEvaluatorId().equals(evaluator.getId())) {
+ contextIter.remove();
+ }
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockFailure.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockFailure.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockFailure.java
new file mode 100644
index 0000000..09ad186
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockFailure.java
@@ -0,0 +1,74 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+package org.apache.reef.mock.driver;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.evaluator.AllocatedEvaluator;
+import org.apache.reef.driver.task.RunningTask;
+import org.apache.reef.wake.time.event.StartTime;
+
+import java.util.Collection;
+
+/**
+ * Used to fail running REEF entities i.e., Evaluators, Contexts, Tasks.
+ */
+@Unstable
+public interface MockFailure {
+
+ /**
+ * @return current Collection of allocated evaluators.
+ */
+ Collection<AllocatedEvaluator> getCurrentAllocatedEvaluators();
+
+ /**
+ * Fail an allocated evaluator.
+ * @param evaluator to be failed
+ */
+ void fail(final AllocatedEvaluator evaluator);
+
+ /**
+ * @return current Collection of active contexts
+ */
+ Collection<ActiveContext> getCurrentActiveContexts();
+
+ /**
+ * Fail an ActiveContext.
+ * @param context to be failed
+ */
+ void fail(final ActiveContext context);
+
+ /**
+ * @return current Collection of running tasks
+ */
+ Collection<RunningTask> getCurrentRunningTasks();
+
+ /**
+ * Fail a running task.
+ * @param task to be failed
+ */
+ void fail(final RunningTask task);
+
+ /**
+ * Fail the driver.
+ * @return the state of the driver at the time of the failure
+ */
+ MockDriverRestartContext failDriver(final int attempt, final StartTime startTime);
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockRuntime.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockRuntime.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockRuntime.java
new file mode 100644
index 0000000..a110ef7
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockRuntime.java
@@ -0,0 +1,92 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.driver.context.ContextMessage;
+import org.apache.reef.driver.restart.DriverRestartCompleted;
+
+/**
+ * Mock API used to drive the evaluation of ProcessRequest
+ * events, which are triggered by the Application driver.
+ * Clients used this to determine whether a particular ProcessRequest
+ * event should succeed or fail.
+ */
+@Unstable
+public interface MockRuntime extends MockFailure {
+
+ /**
+ * Initiate the start time event to the application driver.
+ */
+ void start();
+
+ /**
+ * Initiate the stop time event to the application driver.
+ */
+ void stop();
+
+ /**
+ * Initiate a driver restart.
+ * @param restartContext contains the state of the driver at the time of failure
+ * @param isTimeout used to fill in {@link DriverRestartCompleted#isTimedOut()}
+ * @param duration recover time duration (added to start time)
+ */
+ void restart(final MockDriverRestartContext restartContext, final boolean isTimeout, final long duration);
+
+ /**
+ * @return true if there is an outstanding ProcessRequest
+ */
+ boolean hasProcessRequest();
+
+ /**
+ * The client (caller) is responsible for determining what
+ * to do with a ProcessRequest event. There are three options:
+ * 1. Pass to the succeed method, which signals success to the driver.
+ * 2. Pass to the fail method, signaling failure to the driver.
+ * 3. Drop it on the floor (e.g., network failure).
+ *
+ * @return the next ProcessRequest object to be processed.
+ */
+ ProcessRequest getNextProcessRequest();
+
+ /**
+ * The driver will be informed that the operation corresponding
+ * to the ProcessRequest succeeded, and will be given any relevant
+ * data structures e.g., AllocatedEvaluator, RunningTask, etc.
+ *
+ * @param request to be processed successfully
+ */
+ void succeed(final ProcessRequest request);
+
+ /**
+ * The driver will be informed that the operation corresponding
+ * to the PRocessRequest failed, and will be given any relevant
+ * data structures e.g., FailedEvaluator, FailedTask, etc.
+ *
+ * @param request to be failed.
+ */
+ void fail(final ProcessRequest request);
+
+ /**
+ * Publish a context message to the application event handlers.
+ * @param contextMessage to be published
+ */
+ void publish(final ContextMessage contextMessage);
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockTaskReturnValueProvider.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockTaskReturnValueProvider.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockTaskReturnValueProvider.java
new file mode 100644
index 0000000..6972a18
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/MockTaskReturnValueProvider.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.reef.mock.driver;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.driver.task.CompletedTask;
+import org.apache.reef.mock.driver.runtime.MockRunningTask;
+import org.apache.reef.tang.annotations.DefaultImplementation;
+
+/**
+ * Clients bind an implementation of this interface, which
+ * will be used to create a mock return value for a mock
+ * task execution. This return value will be returned by
+ * the {@link CompletedTask#get()}} method.
+ */
+@Unstable
+@DefaultImplementation(DefaultTaskReturnValueProvider.class)
+public interface MockTaskReturnValueProvider {
+
+ /**
+ * Provide a valid return value for the {@link CompletedTask#get()} method.
+ * @param task that is to be provided with a return value
+ * @return {@link org.apache.reef.task.Task#call(byte[])} return value
+ */
+ byte[] getReturnValue(final MockRunningTask task);
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/ProcessRequest.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/ProcessRequest.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/ProcessRequest.java
new file mode 100644
index 0000000..cf4429e
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/ProcessRequest.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver;
+
+import org.apache.reef.annotations.Unstable;
+
+/**
+ * A ProcessRequest refers to an outstanding event that is
+ * waiting to be processed by the REEF mock runtime. Clients
+ * are responsible for deciding how a ProcessRequest should be
+ * handled, by either:
+ * 1. successfully processing the request
+ * 2. unsucessfully processing the request
+ * 3. dropping the processing request (i.e., loosing it)
+ * These decisions are conveyed through the {MockRuntime} API.
+ */
+@Unstable
+public interface ProcessRequest extends AutoCompletable {
+ /**
+ * process request type.
+ */
+ enum Type {
+ ALLOCATE_EVALUATOR,
+ CLOSE_EVALUATOR,
+ CREATE_CONTEXT,
+ CLOSE_CONTEXT,
+ CREATE_TASK,
+ SUSPEND_TASK,
+ CLOSE_TASK,
+ COMPLETE_TASK,
+ CREATE_CONTEXT_AND_TASK,
+ SEND_MESSAGE_DRIVER_TO_TASK,
+ SEND_MESSAGE_DRIVER_TO_CONTEXT
+ }
+
+ Type getType();
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/package-info.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/package-info.java
new file mode 100644
index 0000000..f9d73f5
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/package-info.java
@@ -0,0 +1,40 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+/**
+ * Mock runtime API.
+ *
+ * Mock runtime is meant to mimic the semantics of the REEF runtime and
+ * allow:
+ * 1. Applications to driver the forward progress of processing REEF events.
+ * See {@link org.apache.reef.mock.driver.MockRuntime} API
+ * 2. Control the advancement of the Clock and Alarm callbacks.
+ * See {@link org.apache.reef.mock.driver.runtime.MockClock}
+ * 3. Inject failures into the REEF applications.
+ * See {@link org.apache.reef.mock.driver.MockFailure}
+ *
+ * Use {@link org.apache.reef.mock.driver.MockConfiguration} to bind your REEF
+ * driver application event handlers.
+ *
+ * Use {@link org.apache.reef.mock.driver.MockRuntime#start()} to trigger the
+ * driver start event and {@link org.apache.reef.mock.driver.MockRuntime#stop()}}
+ * or {@link org.apache.reef.mock.driver.runtime.MockClock#close()} to trigger the driver
+ * stop event.
+ */
+package org.apache.reef.mock.driver;
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/AllocateEvaluator.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/AllocateEvaluator.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/AllocateEvaluator.java
new file mode 100644
index 0000000..d717fc6
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/AllocateEvaluator.java
@@ -0,0 +1,72 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.request;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.evaluator.FailedEvaluator;
+import org.apache.reef.mock.driver.ProcessRequest;
+import org.apache.reef.mock.driver.runtime.MockAllocatedEvaluator;
+import org.apache.reef.mock.driver.runtime.MockFailedEvaluator;
+
+/**
+ * Allocate Evaluator process request.
+ */
+@Unstable
+@Private
+public final class AllocateEvaluator implements
+ ProcessRequestInternal<MockAllocatedEvaluator, FailedEvaluator> {
+
+ private final MockAllocatedEvaluator evaluator;
+
+ public AllocateEvaluator(final MockAllocatedEvaluator evaluator) {
+ this.evaluator = evaluator;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.ALLOCATE_EVALUATOR;
+ }
+
+ @Override
+ public MockAllocatedEvaluator getSuccessEvent() {
+ return this.evaluator;
+ }
+
+ @Override
+ public FailedEvaluator getFailureEvent() {
+ return new MockFailedEvaluator(evaluator.getId());
+ }
+
+ @Override
+ public boolean doAutoComplete() {
+ return false;
+ }
+
+ @Override
+ public void setAutoComplete(final boolean value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ProcessRequest getCompletionProcessRequest() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CloseContext.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CloseContext.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CloseContext.java
new file mode 100644
index 0000000..d6408d7
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CloseContext.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.request;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.context.ClosedContext;
+import org.apache.reef.driver.context.FailedContext;
+import org.apache.reef.mock.driver.AutoCompletable;
+import org.apache.reef.mock.driver.ProcessRequest;
+import org.apache.reef.mock.driver.runtime.MockActiveContext;
+import org.apache.reef.mock.driver.runtime.MockClosedContext;
+import org.apache.reef.mock.driver.runtime.MockFailedContext;
+
+/**
+ * close context process request.
+ */
+@Unstable
+@Private
+public final class CloseContext implements
+ ProcessRequestInternal<ClosedContext, FailedContext>,
+ AutoCompletable {
+
+ private final MockActiveContext context;
+
+ public CloseContext(final MockActiveContext context) {
+ this.context = context;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.CLOSE_CONTEXT;
+ }
+
+ @Override
+ public MockClosedContext getSuccessEvent() {
+ return new MockClosedContext(this.context);
+ }
+
+ @Override
+ public FailedContext getFailureEvent() {
+ return new MockFailedContext(this.context);
+ }
+
+ @Override
+ public boolean doAutoComplete() {
+ return !this.context.getParentContext().isPresent();
+ }
+
+ @Override
+ public void setAutoComplete(final boolean value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ProcessRequest getCompletionProcessRequest() {
+ return new CloseEvaluator(this.context.getEvaluator());
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CloseEvaluator.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CloseEvaluator.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CloseEvaluator.java
new file mode 100644
index 0000000..c329873
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CloseEvaluator.java
@@ -0,0 +1,78 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.request;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.evaluator.CompletedEvaluator;
+import org.apache.reef.driver.evaluator.FailedEvaluator;
+import org.apache.reef.mock.driver.ProcessRequest;
+import org.apache.reef.mock.driver.runtime.MockAllocatedEvaluator;
+import org.apache.reef.mock.driver.runtime.MockFailedEvaluator;
+
+/**
+ * close evaluator request.
+ */
+@Unstable
+@Private
+public final class CloseEvaluator implements ProcessRequestInternal<CompletedEvaluator, FailedEvaluator> {
+
+ private final MockAllocatedEvaluator evaluator;
+
+ public CloseEvaluator(final MockAllocatedEvaluator evaluator) {
+ this.evaluator = evaluator;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.CLOSE_EVALUATOR;
+ }
+
+ @Override
+ public CompletedEvaluator getSuccessEvent() {
+ return new CompletedEvaluator() {
+ @Override
+ public String getId() {
+ return evaluator.getId();
+ }
+ };
+ }
+
+ @Override
+ public FailedEvaluator getFailureEvent() {
+ // TODO[initialize remaining failed contstructer fields]
+ return new MockFailedEvaluator(evaluator.getId());
+ }
+
+ @Override
+ public boolean doAutoComplete() {
+ return false;
+ }
+
+ @Override
+ public void setAutoComplete(final boolean value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ProcessRequest getCompletionProcessRequest() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CloseTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CloseTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CloseTask.java
new file mode 100644
index 0000000..c726d94
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CloseTask.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.request;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.task.CompletedTask;
+import org.apache.reef.driver.task.FailedTask;
+import org.apache.reef.mock.driver.MockTaskReturnValueProvider;
+import org.apache.reef.mock.driver.ProcessRequest;
+import org.apache.reef.mock.driver.runtime.MockCompletedTask;
+import org.apache.reef.mock.driver.runtime.MockRunningTask;
+import org.apache.reef.util.Optional;
+
+/**
+ * close task process request.
+ */
+@Unstable
+@Private
+public final class CloseTask implements ProcessRequestInternal<CompletedTask, FailedTask> {
+
+ private final MockRunningTask task;
+
+ private final MockTaskReturnValueProvider taskReturnValueProvider;
+
+ public CloseTask(
+ final MockRunningTask task,
+ final MockTaskReturnValueProvider taskReturnValueProvider) {
+ this.task = task;
+ this.taskReturnValueProvider = taskReturnValueProvider;
+ }
+
+ public MockRunningTask getTask() {
+ return task;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.CLOSE_TASK;
+ }
+
+ @Override
+ public MockCompletedTask getSuccessEvent() {
+ return new MockCompletedTask(this.task, this.taskReturnValueProvider.getReturnValue(task));
+ }
+
+ @Override
+ public FailedTask getFailureEvent() {
+ return new FailedTask(
+ task.getId(),
+ "mock",
+ Optional.<String>empty(),
+ Optional.<Throwable>empty(),
+ Optional.<byte[]>empty(),
+ Optional.of(this.task.getActiveContext()));
+ }
+
+ @Override
+ public boolean doAutoComplete() {
+ return false;
+ }
+
+ @Override
+ public void setAutoComplete(final boolean value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ProcessRequest getCompletionProcessRequest() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CompleteTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CompleteTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CompleteTask.java
new file mode 100644
index 0000000..c38c10b
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CompleteTask.java
@@ -0,0 +1,82 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.request;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.task.CompletedTask;
+import org.apache.reef.driver.task.FailedTask;
+import org.apache.reef.mock.driver.MockTaskReturnValueProvider;
+import org.apache.reef.mock.driver.ProcessRequest;
+import org.apache.reef.mock.driver.runtime.MockCompletedTask;
+import org.apache.reef.mock.driver.runtime.MockRunningTask;
+
+/**
+ * close task process request.
+ */
+@Unstable
+@Private
+public final class CompleteTask implements ProcessRequestInternal<CompletedTask, FailedTask> {
+
+ private final MockRunningTask task;
+
+ private final MockTaskReturnValueProvider returnValueProvider;
+
+ public CompleteTask(
+ final MockRunningTask task,
+ final MockTaskReturnValueProvider returnValueProvider) {
+ this.task = task;
+ this.returnValueProvider = returnValueProvider;
+ }
+
+ public MockRunningTask getTask() {
+ return task;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.COMPLETE_TASK;
+ }
+
+ @Override
+ public CompletedTask getSuccessEvent() {
+ return new MockCompletedTask(this.task, this.returnValueProvider.getReturnValue(task));
+ }
+
+ @Override
+ public FailedTask getFailureEvent() {
+ return null;
+ }
+
+ @Override
+ public boolean doAutoComplete() {
+ return false;
+ }
+
+ @Override
+ public void setAutoComplete(final boolean value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ProcessRequest getCompletionProcessRequest() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CreateContext.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CreateContext.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CreateContext.java
new file mode 100644
index 0000000..138d070
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CreateContext.java
@@ -0,0 +1,76 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.request;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.context.FailedContext;
+import org.apache.reef.mock.driver.AutoCompletable;
+import org.apache.reef.mock.driver.ProcessRequest;
+import org.apache.reef.mock.driver.runtime.MockActiveContext;
+import org.apache.reef.mock.driver.runtime.MockFailedContext;
+
+/**
+ * create context process request.
+ */
+@Unstable
+@Private
+public final class CreateContext implements
+ ProcessRequestInternal<MockActiveContext, FailedContext>,
+ AutoCompletable {
+
+ private final MockActiveContext context;
+
+ private boolean autoComplete = false;
+
+ public CreateContext(final MockActiveContext context) {
+ this.context = context;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.CREATE_CONTEXT;
+ }
+
+ @Override
+ public MockActiveContext getSuccessEvent() {
+ return this.context;
+ }
+
+ @Override
+ public FailedContext getFailureEvent() {
+ return new MockFailedContext(this.context);
+ }
+
+ @Override
+ public boolean doAutoComplete() {
+ return this.autoComplete;
+ }
+
+ @Override
+ public void setAutoComplete(final boolean value) {
+ this.autoComplete = value;
+ }
+
+ @Override
+ public ProcessRequest getCompletionProcessRequest() {
+ return new CloseContext(this.context);
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CreateContextAndTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CreateContextAndTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CreateContextAndTask.java
new file mode 100644
index 0000000..7d724a0
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CreateContextAndTask.java
@@ -0,0 +1,98 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.request;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.task.FailedTask;
+import org.apache.reef.io.Tuple;
+import org.apache.reef.mock.driver.AutoCompletable;
+import org.apache.reef.mock.driver.MockTaskReturnValueProvider;
+import org.apache.reef.mock.driver.ProcessRequest;
+import org.apache.reef.mock.driver.runtime.MockActiveContext;
+import org.apache.reef.mock.driver.runtime.MockFailedContext;
+import org.apache.reef.mock.driver.runtime.MockRunningTask;
+import org.apache.reef.util.Optional;
+
+/**
+ * create context and task process request.
+ */
+@Unstable
+@Private
+public final class CreateContextAndTask implements
+ ProcessRequestInternal<Tuple<MockActiveContext, MockRunningTask>, Tuple<MockFailedContext, FailedTask>>,
+ AutoCompletable {
+
+ private final MockActiveContext context;
+
+ private final MockRunningTask task;
+
+ private final MockTaskReturnValueProvider taskReturnValueProvider;
+
+ private boolean autoComplete = true;
+
+ public CreateContextAndTask(
+ final MockActiveContext context,
+ final MockRunningTask task,
+ final MockTaskReturnValueProvider taskReturnValueProvider) {
+ this.context = context;
+ this.task = task;
+ this.taskReturnValueProvider = taskReturnValueProvider;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.CREATE_CONTEXT_AND_TASK;
+ }
+
+ @Override
+ public Tuple<MockActiveContext, MockRunningTask> getSuccessEvent() {
+ return new Tuple<>(this.context, this.task);
+ }
+
+ @Override
+ public Tuple<MockFailedContext, FailedTask> getFailureEvent() {
+ return new Tuple<>(
+ new MockFailedContext(this.context),
+ new FailedTask(
+ this.task.getId(),
+ "mock",
+ Optional.<String>empty(),
+ Optional.<Throwable>empty(),
+ Optional.<byte[]>empty(),
+ Optional.of((ActiveContext)this.context)));
+ }
+
+ @Override
+ public boolean doAutoComplete() {
+ return this.autoComplete;
+ }
+
+ @Override
+ public void setAutoComplete(final boolean value) {
+ this.autoComplete = value;
+ }
+
+ @Override
+ public ProcessRequest getCompletionProcessRequest() {
+ return new CompleteTask(this.task, this.taskReturnValueProvider);
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CreateTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CreateTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CreateTask.java
new file mode 100644
index 0000000..699b0ae
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/CreateTask.java
@@ -0,0 +1,89 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.request;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.task.FailedTask;
+import org.apache.reef.driver.task.RunningTask;
+import org.apache.reef.mock.driver.AutoCompletable;
+import org.apache.reef.mock.driver.MockTaskReturnValueProvider;
+import org.apache.reef.mock.driver.ProcessRequest;
+import org.apache.reef.mock.driver.runtime.MockRunningTask;
+import org.apache.reef.util.Optional;
+
+/**
+ * create task process request.
+ */
+@Unstable
+@Private
+public final class CreateTask implements
+ ProcessRequestInternal<RunningTask, FailedTask>,
+ AutoCompletable {
+
+ private final MockRunningTask task;
+
+ private final MockTaskReturnValueProvider returnValueProvider;
+
+ private boolean autoComplete = true;
+
+ public CreateTask(
+ final MockRunningTask task,
+ final MockTaskReturnValueProvider returnValueProvider) {
+ this.task = task;
+ this.returnValueProvider = returnValueProvider;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.CREATE_TASK;
+ }
+
+ @Override
+ public boolean doAutoComplete() {
+ return this.autoComplete;
+ }
+
+ @Override
+ public ProcessRequest getCompletionProcessRequest() {
+ return new CompleteTask(this.task, this.returnValueProvider);
+ }
+
+ @Override
+ public void setAutoComplete(final boolean value) {
+ this.autoComplete = value;
+ }
+
+ @Override
+ public MockRunningTask getSuccessEvent() {
+ return this.task;
+ }
+
+ @Override
+ public FailedTask getFailureEvent() {
+ return new FailedTask(
+ this.task.getId(),
+ "mock",
+ Optional.<String>empty(),
+ Optional.<Throwable>empty(),
+ Optional.<byte[]>empty(),
+ Optional.of(this.task.getActiveContext()));
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/ProcessRequestInternal.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/ProcessRequestInternal.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/ProcessRequestInternal.java
new file mode 100644
index 0000000..d8509d6
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/ProcessRequestInternal.java
@@ -0,0 +1,44 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.request;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.mock.driver.ProcessRequest;
+
+/**
+ * internal process request API.
+ * @param <S> successful event
+ * @param <F> failure event
+ */
+@Unstable
+@Private
+public interface ProcessRequestInternal<S, F> extends ProcessRequest {
+
+ /**
+ * @return the outcome of a successful processing of this request
+ */
+ S getSuccessEvent();
+
+ /**
+ * @return the outcome of an unsuccessful processing of this request
+ */
+ F getFailureEvent();
+}
[03/16] reef git commit: [REEF-2009] Replace Version tag with
VersionPrefix and VersionSuffix
Posted by do...@apache.org.
[REEF-2009] Replace Version tag with VersionPrefix and VersionSuffix
This changes the Version tag in the props file to now be specified with
VersionPrefix and VersionSuffix. VersionPrefix is now the main version
of REEF while VersionSuffix is the SNAPSHOT information if set as a
SNAPSHOT build.
JIRA:
[REEF-2009](https://issues.apache.org/jira/browse/REEF-2009)
Pull Request:
This closes #1449
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/8dc33313
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/8dc33313
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/8dc33313
Branch: refs/heads/REEF-335
Commit: 8dc333136811f22eaac943da8d1d69b6bedcada0
Parents: c93e121
Author: Scott Inglis <si...@microsoft.com>
Authored: Tue Apr 24 10:47:19 2018 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Wed Apr 25 13:17:38 2018 -0700
----------------------------------------------------------------------
lang/cs/build.Common.DotNet.props | 8 +++++---
1 file changed, 5 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/8dc33313/lang/cs/build.Common.DotNet.props
----------------------------------------------------------------------
diff --git a/lang/cs/build.Common.DotNet.props b/lang/cs/build.Common.DotNet.props
index e713cbe..7664c72 100644
--- a/lang/cs/build.Common.DotNet.props
+++ b/lang/cs/build.Common.DotNet.props
@@ -17,7 +17,11 @@ under the License.
-->
<PropertyGroup>
- <Version>0.17.0</Version>
+ <SnapshotNumber>01</SnapshotNumber>
+ <IsSnapshot>true</IsSnapshot>
+ <VersionPrefix>0.17.0</VersionPrefix>
+ <VersionSuffix Condition="'$(IsSnapshot)' == true">SNAPSHOT-$(SnapshotNumber)</VersionSuffix>
+
<Authors>Apache Software Foundation</Authors>
<Owners>The Apache REEF project</Owners>
<Product>Apache REEF</Product>
@@ -32,8 +36,6 @@ under the License.
<RepositoryUrl>https://github.com/apache/reef.git</RepositoryUrl>
<PackageReleaseNotes>Contact the Apache REEF development alias dev@reef.apache.org for questions or issues.</PackageReleaseNotes>
<DefaultItemExcludes>$(DefaultItemExcludes);**/AssemblyInfo.cs;**/Resources.xml;packages.config;*.nuspec</DefaultItemExcludes>
- <IsSnapshot>true</IsSnapshot>
- <SnapshotNumber>01</SnapshotNumber>
</PropertyGroup>
<PropertyGroup>
[11/16] reef git commit: [REEF-2014] Force-exit
AzureBatchBootstrapREEFLauncher
Posted by do...@apache.org.
[REEF-2014] Force-exit AzureBatchBootstrapREEFLauncher
This adds `System.exit(0)` in `AzureBatchBootstrapREEFLauncher`, which
applies the same logic in REEFLauncher.
JIRA:
[REEF-2014](https://issues.apache.org/jira/browse/REEF-2014)
Pull request:
This closes #1455
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/35df8205
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/35df8205
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/35df8205
Branch: refs/heads/REEF-335
Commit: 35df8205febfb565f73e3cf502639d9d17731acd
Parents: c0ddb8d
Author: Chenxi Zhao <33...@users.noreply.github.com>
Authored: Fri Apr 27 19:39:58 2018 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Sun Apr 29 16:18:31 2018 -0700
----------------------------------------------------------------------
.../reef/bridge/client/AzureBatchBootstrapREEFLauncher.java | 4 ++++
1 file changed, 4 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/35df8205/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java
index 481ae53..35a1e59 100644
--- a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java
+++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/AzureBatchBootstrapREEFLauncher.java
@@ -92,6 +92,10 @@ public final class AzureBatchBootstrapREEFLauncher {
} catch (final InjectionException ex) {
throw fatal("Unable to configure and start REEFEnvironment.", ex);
}
+
+ LOG.log(Level.INFO, "Exiting BootstrapLauncher.main()");
+
+ System.exit(0); // TODO[REEF-1715]: Should be able to exit cleanly at the end of main()
}
private static Configuration generateConfigurationFromJobSubmissionParameters(final File params) throws IOException {
[02/16] reef git commit: [REEF-1992] Upgrade Azure Storage from 6.1
to 8.1.3
Posted by do...@apache.org.
[REEF-1992] Upgrade Azure Storage from 6.1 to 8.1.3
Upgrades all Azure Storage dependencies to version 8.1.3. Change is
necessary for external consumers of REEF as well as matching the
Org.Apache.Reef.DotNet.sln.
JIRA:
[REEF-1992](https://issues.apache.org/jira/browse/REEF-1992)
Pull Request:
This closes #1439
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/c93e1219
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/c93e1219
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/c93e1219
Branch: refs/heads/REEF-335
Commit: c93e1219c0783371b5e31e00d4c0d812c8f3d98a
Parents: 7343e62
Author: Markus Weimer <we...@apache.org>
Authored: Tue Apr 24 17:50:15 2018 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Tue Apr 24 17:52:02 2018 -0700
----------------------------------------------------------------------
lang/cs/App.config | 14 +++++++++++-
.../Org.Apache.REEF.IO.Tests.csproj | 24 ++++++++++++++++----
.../TestAzureBlockBlobFileSystemE2E.cs | 1 +
.../cs/Org.Apache.REEF.IO.Tests/packages.config | 10 ++++++++
.../FileSystem/AzureBlob/AzureCloudBlockBlob.cs | 6 +----
.../Org.Apache.REEF.IO.csproj | 24 ++++++++++----------
lang/cs/Org.Apache.REEF.IO/packages.config | 16 ++++++++-----
.../Org.Apache.REEF.Tests.csproj | 22 +++++++++---------
lang/cs/Org.Apache.REEF.Tests/packages.config | 16 ++++++++-----
9 files changed, 87 insertions(+), 46 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/c93e1219/lang/cs/App.config
----------------------------------------------------------------------
diff --git a/lang/cs/App.config b/lang/cs/App.config
index ccdbecf..51d8c4f 100644
--- a/lang/cs/App.config
+++ b/lang/cs/App.config
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8" ?>
+<?xml version="1.0" encoding="utf-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -27,6 +27,18 @@ under the License.
<assemblyIdentity name="Newtonsoft.Json" publicKeyToken="30ad4fe6b2a6aeed" culture="neutral" />
<bindingRedirect oldVersion="0.0.0.0-10.0.0.0" newVersion="10.0.0.0" />
</dependentAssembly>
+ <dependentAssembly>
+ <assemblyIdentity name="Microsoft.Data.Services.Client" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+ <bindingRedirect oldVersion="0.0.0.0-5.8.1.0" newVersion="5.8.1.0" />
+ </dependentAssembly>
+ <dependentAssembly>
+ <assemblyIdentity name="Microsoft.Data.OData" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+ <bindingRedirect oldVersion="0.0.0.0-5.8.1.0" newVersion="5.8.1.0" />
+ </dependentAssembly>
+ <dependentAssembly>
+ <assemblyIdentity name="Microsoft.Data.Edm" publicKeyToken="31bf3856ad364e35" culture="neutral" />
+ <bindingRedirect oldVersion="0.0.0.0-5.8.1.0" newVersion="5.8.1.0" />
+ </dependentAssembly>
</assemblyBinding>
</runtime>
</configuration>
http://git-wip-us.apache.org/repos/asf/reef/blob/c93e1219/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj b/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj
index 5d688cc..da352b4 100644
--- a/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj
+++ b/lang/cs/Org.Apache.REEF.IO.Tests/Org.Apache.REEF.IO.Tests.csproj
@@ -38,6 +38,18 @@ under the License.
<HintPath>$(PackagesDir)\Microsoft.Azure.DataLake.Store.1.0.4\lib\net452\Microsoft.Azure.DataLake.Store.dll</HintPath>
<Private>True</Private>
</Reference>
+ <Reference Include="Microsoft.Azure.KeyVault.Core, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll</HintPath>
+ </Reference>
+ <Reference Include="Microsoft.Data.Edm, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
+ </Reference>
+ <Reference Include="Microsoft.Data.OData, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
+ </Reference>
+ <Reference Include="Microsoft.Data.Services.Client, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
+ </Reference>
<Reference Include="Microsoft.IdentityModel.Clients.ActiveDirectory, Version=2.28.3.860, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>$(PackagesDir)\Microsoft.IdentityModel.Clients.ActiveDirectory.2.28.3\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll</HintPath>
</Reference>
@@ -51,6 +63,10 @@ under the License.
<Reference Include="Microsoft.Rest.ClientRuntime.Azure.Authentication, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>$(PackagesDir)\Microsoft.Rest.ClientRuntime.Azure.Authentication.2.3.1\lib\net452\Microsoft.Rest.ClientRuntime.Azure.Authentication.dll</HintPath>
</Reference>
+ <Reference Include="Microsoft.WindowsAzure.Storage, Version=8.1.3.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\WindowsAzure.Storage.8.1.3\lib\net45\Microsoft.WindowsAzure.Storage.dll</HintPath>
+ <Private>True</Private>
+ </Reference>
<Reference Include="Newtonsoft.Json">
<HintPath>$(PackagesDir)\Newtonsoft.Json.$(NewtonsoftJsonVersion)\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
@@ -62,6 +78,9 @@ under the License.
<Reference Include="System.Core" />
<Reference Include="System.Management" />
<Reference Include="System.Net.Http" />
+ <Reference Include="System.Spatial, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\System.Spatial.5.8.2\lib\net40\System.Spatial.dll</HintPath>
+ </Reference>
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Data.DataSetExtensions" />
<Reference Include="Microsoft.CSharp" />
@@ -71,11 +90,6 @@ under the License.
<HintPath>$(PackagesDir)\NSubstitute.$(NSubstituteVersion)\lib\net45\NSubstitute.dll</HintPath>
<Private>True</Private>
</Reference>
- <Reference Include="Microsoft.WindowsAzure.Storage, Version=6.1.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
- <SpecificVersion>True</SpecificVersion>
- <HintPath>$(PackagesDir)\WindowsAzure.Storage.6.1.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
- <Private>True</Private>
- </Reference>
</ItemGroup>
<ItemGroup>
<Compile Include="$(SolutionDir)\SharedAssemblyInfo.cs">
http://git-wip-us.apache.org/repos/asf/reef/blob/c93e1219/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs
index 4e47d6b..bbe2b9e 100644
--- a/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs
+++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestAzureBlockBlobFileSystemE2E.cs
@@ -178,6 +178,7 @@ namespace Org.Apache.REEF.IO.Tests
var blob = _container.GetBlockBlobReference(HelloFile);
var tempFilePath = Path.GetTempFileName();
File.Delete(tempFilePath); // Delete the file as CopyToLocal will create it
+
const string Text = "hello";
try
http://git-wip-us.apache.org/repos/asf/reef/blob/c93e1219/lang/cs/Org.Apache.REEF.IO.Tests/packages.config
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/packages.config b/lang/cs/Org.Apache.REEF.IO.Tests/packages.config
index cc3ad3b..285e0b9 100644
--- a/lang/cs/Org.Apache.REEF.IO.Tests/packages.config
+++ b/lang/cs/Org.Apache.REEF.IO.Tests/packages.config
@@ -19,6 +19,10 @@ under the License.
-->
<packages>
<package id="Microsoft.Azure.DataLake.Store" version="1.0.4" targetFramework="net452" />
+ <package id="Microsoft.Azure.KeyVault.Core" version="1.0.0" targetFramework="net452" />
+ <package id="Microsoft.Data.Edm" version="5.8.2" targetFramework="net452" />
+ <package id="Microsoft.Data.OData" version="5.8.2" targetFramework="net452" />
+ <package id="Microsoft.Data.Services.Client" version="5.8.2" targetFramework="net452" />
<package id="Microsoft.IdentityModel.Clients.ActiveDirectory" version="2.28.3" targetFramework="net452" />
<package id="Microsoft.Rest.ClientRuntime" version="2.3.10" targetFramework="net452" />
<package id="Microsoft.Rest.ClientRuntime.Azure.Authentication" version="2.3.1" targetFramework="net452" />
@@ -26,6 +30,12 @@ under the License.
<package id="NLog" version="4.4.12" targetFramework="net452" />
<package id="NSubstitute" version="1.8.2.0" targetFramework="net45" />
<package id="StyleCop.MSBuild" version="5.0.0" targetFramework="net45" developmentDependency="true" />
+ <package id="System.ComponentModel.EventBasedAsync" version="4.0.11" targetFramework="net452" />
+ <package id="System.Dynamic.Runtime" version="4.0.0" targetFramework="net452" />
+ <package id="System.Linq.Queryable" version="4.0.0" targetFramework="net452" />
+ <package id="System.Net.Requests" version="4.0.11" targetFramework="net452" />
+ <package id="System.Spatial" version="5.8.2" targetFramework="net452" />
+ <package id="WindowsAzure.Storage" version="8.1.1" targetFramework="net452" />
<package id="xunit" version="2.1.0" targetFramework="net45" />
<package id="xunit.abstractions" version="2.0.0" targetFramework="net45" />
<package id="xunit.assert" version="2.1.0" targetFramework="net45" />
http://git-wip-us.apache.org/repos/asf/reef/blob/c93e1219/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlockBlob.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlockBlob.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlockBlob.cs
index 3305094..5610230 100644
--- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlockBlob.cs
+++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlockBlob.cs
@@ -112,11 +112,7 @@ namespace Org.Apache.REEF.IO.FileSystem.AzureBlob
public void UploadFromFile(string path, FileMode mode)
{
- #if REEF_DOTNET_BUILD
- _blob.UploadFromFileAsync(path).Wait();
- #else
- _blob.UploadFromFileAsync(path, mode).Wait();
- #endif
+ _blob.UploadFromFileAsync(path).Wait();
}
public void FetchAttributes()
http://git-wip-us.apache.org/repos/asf/reef/blob/c93e1219/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj b/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj
index 055c139..f651c84 100644
--- a/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj
+++ b/lang/cs/Org.Apache.REEF.IO/Org.Apache.REEF.IO.csproj
@@ -44,16 +44,16 @@ under the License.
<Reference Include="Microsoft.Azure.Management.DataLake.Store, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>$(PackagesDir)\Microsoft.Azure.Management.DataLake.Store.2.2.1\lib\net452\Microsoft.Azure.Management.DataLake.Store.dll</HintPath>
</Reference>
- <Reference Include="Microsoft.Data.Edm, Version=5.6.4.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
- <HintPath>$(PackagesDir)\Microsoft.Data.Edm.5.6.4\lib\net40\Microsoft.Data.Edm.dll</HintPath>
+ <Reference Include="Microsoft.Data.Edm, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
<Private>True</Private>
</Reference>
- <Reference Include="Microsoft.Data.OData, Version=5.6.4.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
- <HintPath>$(PackagesDir)\Microsoft.Data.OData.5.6.4\lib\net40\Microsoft.Data.OData.dll</HintPath>
+ <Reference Include="Microsoft.Data.OData, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
<Private>True</Private>
</Reference>
- <Reference Include="Microsoft.Data.Services.Client, Version=5.6.4.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
- <HintPath>$(PackagesDir)\Microsoft.Data.Services.Client.5.6.4\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
+ <Reference Include="Microsoft.Data.Services.Client, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Microsoft.Rest.ClientRuntime, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
@@ -63,10 +63,6 @@ under the License.
<Reference Include="Microsoft.Rest.ClientRuntime.Azure, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>$(PackagesDir)\Microsoft.Rest.ClientRuntime.Azure.3.3.7\lib\net452\Microsoft.Rest.ClientRuntime.Azure.dll</HintPath>
</Reference>
- <Reference Include="Microsoft.WindowsAzure.Storage, Version=6.1.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
- <HintPath>$(PackagesDir)\WindowsAzure.Storage.6.1.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
- <Private>True</Private>
- </Reference>
<Reference Include="Microsoft.IdentityModel.Clients.ActiveDirectory, Version=2.28.3.860, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>$(PackagesDir)\Microsoft.IdentityModel.Clients.ActiveDirectory.2.28.3\lib\net45\Microsoft.IdentityModel.Clients.ActiveDirectory.dll</HintPath>
</Reference>
@@ -76,6 +72,10 @@ under the License.
<Reference Include="Microsoft.Rest.ClientRuntime.Azure.Authentication, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>$(PackagesDir)\Microsoft.Rest.ClientRuntime.Azure.Authentication.2.3.1\lib\net452\Microsoft.Rest.ClientRuntime.Azure.Authentication.dll</HintPath>
</Reference>
+ <Reference Include="Microsoft.WindowsAzure.Storage, Version=8.1.3.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\WindowsAzure.Storage.8.1.3\lib\net45\Microsoft.WindowsAzure.Storage.dll</HintPath>
+ <Private>True</Private>
+ </Reference>
<Reference Include="Newtonsoft.Json">
<HintPath>$(PackagesDir)\Newtonsoft.Json.$(NewtonsoftJsonVersion)\lib\net45\Newtonsoft.Json.dll</HintPath>
<Private>True</Private>
@@ -88,8 +88,8 @@ under the License.
<Reference Include="System.Core" />
<Reference Include="System.Management" />
<Reference Include="System.Net.Http" />
- <Reference Include="System.Spatial, Version=5.6.4.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
- <HintPath>$(PackagesDir)\System.Spatial.5.6.4\lib\net40\System.Spatial.dll</HintPath>
+ <Reference Include="System.Spatial, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\System.Spatial.5.8.2\lib\net40\System.Spatial.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="System.Xml.Linq" />
http://git-wip-us.apache.org/repos/asf/reef/blob/c93e1219/lang/cs/Org.Apache.REEF.IO/packages.config
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IO/packages.config b/lang/cs/Org.Apache.REEF.IO/packages.config
index f53a362..8473e55 100644
--- a/lang/cs/Org.Apache.REEF.IO/packages.config
+++ b/lang/cs/Org.Apache.REEF.IO/packages.config
@@ -21,9 +21,9 @@ under the License.
<package id="Microsoft.Azure.DataLake.Store" version="1.0.4" targetFramework="net452" />
<package id="Microsoft.Azure.KeyVault.Core" version="1.0.0" targetFramework="net45" />
<package id="Microsoft.Azure.Management.DataLake.Store" version="2.2.1" targetFramework="net452" />
- <package id="Microsoft.Data.Edm" version="5.6.4" targetFramework="net45" />
- <package id="Microsoft.Data.OData" version="5.6.4" targetFramework="net45" />
- <package id="Microsoft.Data.Services.Client" version="5.6.4" targetFramework="net45" />
+ <package id="Microsoft.Data.Edm" version="5.8.2" targetFramework="net452" />
+ <package id="Microsoft.Data.OData" version="5.8.2" targetFramework="net452" />
+ <package id="Microsoft.Data.Services.Client" version="5.8.2" targetFramework="net452" />
<package id="Microsoft.IdentityModel.Clients.ActiveDirectory" version="2.28.3" targetFramework="net452" />
<package id="Microsoft.Rest.ClientRuntime" version="2.3.10" targetFramework="net452" />
<package id="Microsoft.Rest.ClientRuntime.Azure" version="3.3.7" targetFramework="net452" />
@@ -31,6 +31,10 @@ under the License.
<package id="Newtonsoft.Json" version="10.0.3" targetFramework="net451" />
<package id="NLog" version="4.4.12" targetFramework="net452" />
<package id="StyleCop.MSBuild" version="5.0.0" targetFramework="net45" developmentDependency="true" />
- <package id="System.Spatial" version="5.6.4" targetFramework="net45" />
- <package id="WindowsAzure.Storage" version="6.1.0" targetFramework="net45" />
-</packages>
+ <package id="System.ComponentModel.EventBasedAsync" version="4.0.11" targetFramework="net452" />
+ <package id="System.Dynamic.Runtime" version="4.0.0" targetFramework="net452" />
+ <package id="System.Linq.Queryable" version="4.0.0" targetFramework="net452" />
+ <package id="System.Net.Requests" version="4.0.11" targetFramework="net452" />
+ <package id="System.Spatial" version="5.8.2" targetFramework="net452" />
+ <package id="WindowsAzure.Storage" version="8.1.3" targetFramework="net452" />
+</packages>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/c93e1219/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj b/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj
index 7436b09..1df8499 100644
--- a/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj
+++ b/lang/cs/Org.Apache.REEF.Tests/Org.Apache.REEF.Tests.csproj
@@ -1,4 +1,4 @@
-<?xml version="1.0" encoding="utf-8"?>
+<?xml version="1.0" encoding="utf-8"?>
<!--
Licensed to the Apache Software Foundation (ASF) under one
or more contributor license agreements. See the NOTICE file
@@ -38,20 +38,20 @@ under the License.
<HintPath>$(PackagesDir)\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll</HintPath>
<Private>True</Private>
</Reference>
- <Reference Include="Microsoft.Data.Edm, Version=5.6.4.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
- <HintPath>$(PackagesDir)\Microsoft.Data.Edm.5.6.4\lib\net40\Microsoft.Data.Edm.dll</HintPath>
+ <Reference Include="Microsoft.Data.Edm, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
<Private>True</Private>
</Reference>
- <Reference Include="Microsoft.Data.OData, Version=5.6.4.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
- <HintPath>$(PackagesDir)\Microsoft.Data.OData.5.6.4\lib\net40\Microsoft.Data.OData.dll</HintPath>
+ <Reference Include="Microsoft.Data.OData, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
<Private>True</Private>
</Reference>
- <Reference Include="Microsoft.Data.Services.Client, Version=5.6.4.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
- <HintPath>$(PackagesDir)\Microsoft.Data.Services.Client.5.6.4\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
+ <Reference Include="Microsoft.Data.Services.Client, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
<Private>True</Private>
</Reference>
- <Reference Include="Microsoft.WindowsAzure.Storage, Version=6.1.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
- <HintPath>$(PackagesDir)\WindowsAzure.Storage.6.1.0\lib\net40\Microsoft.WindowsAzure.Storage.dll</HintPath>
+ <Reference Include="Microsoft.WindowsAzure.Storage, Version=8.1.3.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\WindowsAzure.Storage.8.1.3\lib\net45\Microsoft.WindowsAzure.Storage.dll</HintPath>
<Private>True</Private>
</Reference>
<Reference Include="Newtonsoft.Json">
@@ -67,8 +67,8 @@ under the License.
<Reference Include="System.Reactive.Interfaces">
<HintPath>$(PackagesDir)\System.Reactive.Interfaces.$(SystemReactiveVersion)\lib\net45\System.Reactive.Interfaces.dll</HintPath>
</Reference>
- <Reference Include="System.Spatial, Version=5.6.4.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
- <HintPath>$(PackagesDir)\System.Spatial.5.6.4\lib\net40\System.Spatial.dll</HintPath>
+ <Reference Include="System.Spatial, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>$(PackagesDir)\System.Spatial.5.8.2\lib\net40\System.Spatial.dll</HintPath>
<Private>True</Private>
</Reference>
</ItemGroup>
http://git-wip-us.apache.org/repos/asf/reef/blob/c93e1219/lang/cs/Org.Apache.REEF.Tests/packages.config
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tests/packages.config b/lang/cs/Org.Apache.REEF.Tests/packages.config
index 933ef87..9df6ffc 100644
--- a/lang/cs/Org.Apache.REEF.Tests/packages.config
+++ b/lang/cs/Org.Apache.REEF.Tests/packages.config
@@ -19,16 +19,20 @@ under the License.
-->
<packages>
<package id="Microsoft.Azure.KeyVault.Core" version="1.0.0" targetFramework="net45" />
- <package id="Microsoft.Data.Edm" version="5.6.4" targetFramework="net45" />
- <package id="Microsoft.Data.OData" version="5.6.4" targetFramework="net45" />
- <package id="Microsoft.Data.Services.Client" version="5.6.4" targetFramework="net45" />
+ <package id="Microsoft.Data.Edm" version="5.8.2" targetFramework="net452" />
+ <package id="Microsoft.Data.OData" version="5.8.2" targetFramework="net452" />
+ <package id="Microsoft.Data.Services.Client" version="5.8.2" targetFramework="net452" />
<package id="Newtonsoft.Json" version="10.0.3" targetFramework="net451" />
<package id="protobuf-net" version="2.1.0" targetFramework="net45" />
<package id="StyleCop.MSBuild" version="5.0.0" targetFramework="net45" developmentDependency="true" />
+ <package id="System.ComponentModel.EventBasedAsync" version="4.0.11" targetFramework="net452" />
+ <package id="System.Dynamic.Runtime" version="4.0.0" targetFramework="net452" />
+ <package id="System.Linq.Queryable" version="4.0.0" targetFramework="net452" />
+ <package id="System.Net.Requests" version="4.0.11" targetFramework="net452" />
<package id="System.Reactive.Core" version="3.1.1" targetFramework="net451" />
<package id="System.Reactive.Interfaces" version="3.1.1" targetFramework="net451" />
- <package id="System.Spatial" version="5.6.4" targetFramework="net45" />
- <package id="WindowsAzure.Storage" version="6.1.0" targetFramework="net45" />
+ <package id="System.Spatial" version="5.8.2" targetFramework="net452" />
+ <package id="WindowsAzure.Storage" version="8.1.1" targetFramework="net452" />
<package id="xunit" version="2.1.0" targetFramework="net45" />
<package id="xunit.abstractions" version="2.0.0" targetFramework="net45" />
<package id="xunit.assert" version="2.1.0" targetFramework="net45" />
@@ -36,4 +40,4 @@ under the License.
<package id="xunit.extensibility.core" version="2.1.0" targetFramework="net45" />
<package id="xunit.extensibility.execution" version="2.1.0" targetFramework="net45" />
<package id="xunit.runner.visualstudio" version="2.1.0" targetFramework="net45" />
-</packages>
+</packages>
\ No newline at end of file
[09/16] reef git commit: [Trivial] Pull request close
Posted by do...@apache.org.
[Trivial] Pull request close
I forgot to close the PR in the prior commit. Doing that now.
Pull Request:
This closes #1453
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/49255b0d
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/49255b0d
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/49255b0d
Branch: refs/heads/REEF-335
Commit: 49255b0d93d0483adf3ddb8b819512a9edbc97ca
Parents: 4784959
Author: Markus Weimer <we...@apache.org>
Authored: Fri Apr 27 14:40:23 2018 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Fri Apr 27 14:40:23 2018 -0700
----------------------------------------------------------------------
dev/ClosedPullRequests.md | 1 +
1 file changed, 1 insertion(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/49255b0d/dev/ClosedPullRequests.md
----------------------------------------------------------------------
diff --git a/dev/ClosedPullRequests.md b/dev/ClosedPullRequests.md
index 335b7f1..1aaea49 100644
--- a/dev/ClosedPullRequests.md
+++ b/dev/ClosedPullRequests.md
@@ -10,3 +10,4 @@ process.
|-----------|-------------|-------------------|
| 1434 | 2018-04-25 | weimer@apache.org |
| 1404 | 2018-04-25 | weimer@apache.org |
+| 1453 | 2018-04-27 | weimer@apache.org|
\ No newline at end of file
[08/16] reef git commit: [REEF-2010] Update DotNet libraries to
target .netstandard2.0
Posted by do...@apache.org.
[REEF-2010] Update DotNet libraries to target .netstandard2.0
This change updates all libraries to target .net standard 2.0 while
leaving the executables and tests targeting .net461.
- Update the NSubstitute library and Microsoft.Net.Test.Sdk libraries
as the older versions were causing an issue with the tests.
- Changed a Blob call to use the Async method.
This also moves the client `.resx` file generated during build to the
`bin` folder:
- Moves the resx generation to now happen in
bin/.netcore/Org.Apache.REEF.Client
- Also deletes the file post build process. This is to mirror the
behavior from the old build
JIRA:
[REEF-2010](https://issues.apache.org/jira/browse/REEF-2010)
[REEF-1996](https://issues.apache.org/jira/browse/REEF-1996)
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/4784959c
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/4784959c
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/4784959c
Branch: refs/heads/REEF-335
Commit: 4784959c080dd9e1145569b40dee098d504f9bf1
Parents: 24bfea4
Author: Scott Inglis <si...@microsoft.com>
Authored: Fri Apr 27 11:47:49 2018 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Fri Apr 27 14:37:00 2018 -0700
----------------------------------------------------------------------
.../Org.Apache.REEF.Bridge.DotNet.vcxproj | 12 ++++++------
.../AzureBatch/Storage/AzureStorageClient.cs | 2 +-
.../Org.Apache.REEF.Client.DotNet.csproj | 10 +++-------
.../Org.Apache.REEF.FatNuGet.DotNet.csproj | 17 +++++++++++------
.../AzureBlob/AzureCloudBlobContainer.cs | 4 +++-
lang/cs/build.DotNet.props | 3 +--
lang/cs/build.DotNetLibrary.props | 8 +-------
lang/cs/xunit.DotNet.props | 2 +-
8 files changed, 27 insertions(+), 31 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/4784959c/lang/cs/Org.Apache.REEF.Bridge/Org.Apache.REEF.Bridge.DotNet.vcxproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Bridge/Org.Apache.REEF.Bridge.DotNet.vcxproj b/lang/cs/Org.Apache.REEF.Bridge/Org.Apache.REEF.Bridge.DotNet.vcxproj
index 49cff7d..d713437 100644
--- a/lang/cs/Org.Apache.REEF.Bridge/Org.Apache.REEF.Bridge.DotNet.vcxproj
+++ b/lang/cs/Org.Apache.REEF.Bridge/Org.Apache.REEF.Bridge.DotNet.vcxproj
@@ -23,7 +23,6 @@ under the License.
<Platform Condition="'$(Platform)' == ''">x64</Platform>
<SolutionDir Condition="'$(SolutionDir)' == ''">..</SolutionDir>
<ConfigurationType>Application</ConfigurationType>
-
<!-- Added for when the .net core build is enabled. This prevents assemblies for the bridge
to be copied to the bin folder. -->
<ImplicitlyExpandNETStandardFacades>false</ImplicitlyExpandNETStandardFacades>
@@ -112,8 +111,9 @@ under the License.
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<AdditionalIncludeDirectories>..\..\java\reef-bridge-java\target\classes;$(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
- <AdditionalUsingDirectories Condition="'$(EnableDotNetBuild)' == 'False'">$(AdditionalUsingDirectories);$(BinDir)$(Configuration)\Org.Apache.REEF.Driver\net461</AdditionalUsingDirectories>
- <AdditionalUsingDirectories Condition="'$(EnableDotNetBuild)' == 'True'">$(AdditionalUsingDirectories);$(BinDir)$(Configuration)\Org.Apache.REEF.Driver\netstandard2.0</AdditionalUsingDirectories>
+ <AdditionalUsingDirectories>$(BinDir)$(Configuration)\Org.Apache.REEF.Driver\netstandard2.0;C:\Program Files\dotnet\sdk\2.1.4\Microsoft\Microsoft.NET.Build.Extensions\net461\lib;$(AdditionalUsingDirectories)</AdditionalUsingDirectories>
+ <!-- Disable C4691 about missing references. This is benign as the warning reports using a default implementation. -->
+ <DisableSpecificWarnings>4691;%(DisableSpecificWarnings)</DisableSpecificWarnings>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -129,8 +129,9 @@ under the License.
<PrecompiledHeader>NotUsing</PrecompiledHeader>
<AdditionalIncludeDirectories>..\..\java\reef-bridge-java\target\classes;$(JAVA_HOME)\include;$(JAVA_HOME)\include\win32;%(AdditionalIncludeDirectories)</AdditionalIncludeDirectories>
<RuntimeLibrary>MultiThreadedDLL</RuntimeLibrary>
- <AdditionalUsingDirectories Condition="'$(EnableDotNetBuild)' == 'False'">$(AdditionalUsingDirectories);$(BinDir)$(Configuration)\Org.Apache.REEF.Driver\net461</AdditionalUsingDirectories>
- <AdditionalUsingDirectories Condition="'$(EnableDotNetBuild)' == 'True'">$(AdditionalUsingDirectories);$(BinDir)$(Configuration)\Org.Apache.REEF.Driver\netstandard2.0</AdditionalUsingDirectories>
+ <AdditionalUsingDirectories>$(BinDir)$(Configuration)\Org.Apache.REEF.Driver\netstandard2.0;$(AdditionalUsingDirectories)</AdditionalUsingDirectories>
+ <!-- Disable C4691 about missing references. This is benign as the warning reports using a default implementation. -->
+ <DisableSpecificWarnings>4691;%(DisableSpecificWarnings)</DisableSpecificWarnings>
</ClCompile>
<Link>
<GenerateDebugInformation>true</GenerateDebugInformation>
@@ -209,7 +210,6 @@ under the License.
<Import Project="$(VCTargetsPath)\Microsoft.Cpp.targets" />
<ImportGroup Label="ExtensionTargets">
</ImportGroup>
-
<!-- Adds an empty pack target so that this project is skipped when packing. -->
<Target Name="Pack">
</Target>
http://git-wip-us.apache.org/repos/asf/reef/blob/4784959c/lang/cs/Org.Apache.REEF.Client/AzureBatch/Storage/AzureStorageClient.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Storage/AzureStorageClient.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Storage/AzureStorageClient.cs
index ce39f1e..10bcb37 100644
--- a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Storage/AzureStorageClient.cs
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Storage/AzureStorageClient.cs
@@ -78,7 +78,7 @@ namespace Org.Apache.REEF.Client.AzureBatch.Storage
{
CloudBlobClient cloudBlobClient = CloudStorageAccount.Parse(this._storageConnectionString).CreateCloudBlobClient();
CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(this._storageContainerName);
- cloudBlobContainer.CreateIfNotExists();
+ cloudBlobContainer.CreateIfNotExistsAsync().Wait();
return cloudBlobContainer.GetSharedAccessSignature(CreateSASPolicy());
}
http://git-wip-us.apache.org/repos/asf/reef/blob/4784959c/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj
index 835faf9..ea13564 100644
--- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj
@@ -26,7 +26,7 @@ under the License.
</PropertyGroup>
<Import Project="..\build.DotNetLibrary.props" />
<PropertyGroup>
- <TempResxFile Condition="$(TempResxFile) == ''">Resources.DotNet.resx</TempResxFile>
+ <TempResxFile Condition="$(TempResxFile) == ''">$(OutputPath)\$(TargetFramework)\Resources.DotNet.resx</TempResxFile>
</PropertyGroup>
<ItemGroup>
<EmbeddedResource Include="$(TempResxFile)">
@@ -153,12 +153,8 @@ under the License.
</ItemGroup>
<Copy SourceFiles="@(MySourceFiles)" DestinationFiles="@(MySourceFiles->'$(TargetDir)%(Filename)%(Extension)')" />
</Target>
- <!--
- <Target Name="AfterBuild">
+ <Target Name="RemovingTempResx" AfterTargets="AfterBuild">
+ <Message Text="Removing temporary resx file" Importance="high"/>
<Delete Files="$(TempResxFile)" />
- <RemoveDir Directories="$(BaseIntermediateOutputPath)" />
- </Target>
- <Target Name="BeforeBuild" DependsOnTargets="RewriteClientResources">
</Target>
- -->
</Project>
http://git-wip-us.apache.org/repos/asf/reef/blob/4784959c/lang/cs/Org.Apache.REEF.FatNuGet/Org.Apache.REEF.FatNuGet.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.FatNuGet/Org.Apache.REEF.FatNuGet.DotNet.csproj b/lang/cs/Org.Apache.REEF.FatNuGet/Org.Apache.REEF.FatNuGet.DotNet.csproj
index c0c4b2a..ba30acb 100644
--- a/lang/cs/Org.Apache.REEF.FatNuGet/Org.Apache.REEF.FatNuGet.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.FatNuGet/Org.Apache.REEF.FatNuGet.DotNet.csproj
@@ -29,13 +29,18 @@ under the License.
<PackageReference Include="Microsoft.Avro.Core" Version="$(AvroVersion)" />
</ItemGroup>
- <!-- Project Reference conditionals allow for targeting specific projects based upon the target framework. -->
- <ItemGroup Condition="'$(TargetFramework)' == 'net461'">
- <!-- Evaluator as a .netcoreapp can not be linked by a .netstandard library, therefore this is only linked for net461 -->
- <ProjectReference Include="..\Org.Apache.REEF.Evaluator\Org.Apache.REEF.Evaluator.DotNet.csproj" PrivateAssets="All"/>
- <!-- IMRU Examples as a .netcoreapp can not be linked by a .netstandard library, therefore this is only linked for net461 -->
- <ProjectReference Include="..\Org.Apache.REEF.IMRU.Examples\Org.Apache.REEF.IMRU.Examples.DotNet.csproj" PrivateAssets="All"/>
+ <ItemGroup>
+ <Content Include="$(BinDir)\$(Configuration)\Org.Apache.REEF.Evaluator\net461\Org.Apache.REEF.Evaluator.exe">
+ <Pack>true</Pack>
+ <PackagePath>lib\netstandard2.0</PackagePath>
+ </Content>
+
+ <Content Include="$(BinDir)\$(Configuration)\Org.Apache.REEF.IMRU.Examples\net461\Org.Apache.REEF.IMRU.Examples.exe">
+ <Pack>true</Pack>
+ <PackagePath>lib\netstandard2.0</PackagePath>
+ </Content>
</ItemGroup>
+
<ItemGroup>
<ProjectReference Include="..\Org.Apache.REEF.Client\Org.Apache.REEF.Client.DotNet.csproj" PrivateAssets="All"/>
<ProjectReference Include="..\Org.Apache.REEF.Common\Org.Apache.REEF.Common.DotNet.csproj" PrivateAssets="All"/>
http://git-wip-us.apache.org/repos/asf/reef/blob/4784959c/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobContainer.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobContainer.cs b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobContainer.cs
index c71aa7e..d252fc7 100644
--- a/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobContainer.cs
+++ b/lang/cs/Org.Apache.REEF.IO/FileSystem/AzureBlob/AzureCloudBlobContainer.cs
@@ -33,7 +33,9 @@ namespace Org.Apache.REEF.IO.FileSystem.AzureBlob
public bool CreateIfNotExists()
{
- return _container.CreateIfNotExists();
+ var task = _container.CreateIfNotExistsAsync();
+ task.Wait();
+ return task.Result;
}
public void DeleteIfExists()
http://git-wip-us.apache.org/repos/asf/reef/blob/4784959c/lang/cs/build.DotNet.props
----------------------------------------------------------------------
diff --git a/lang/cs/build.DotNet.props b/lang/cs/build.DotNet.props
index ed01b8d..0946942 100644
--- a/lang/cs/build.DotNet.props
+++ b/lang/cs/build.DotNet.props
@@ -27,12 +27,11 @@ under the License.
<AvroVersion>0.1.0</AvroVersion>
<MicrosoftDataLakeStoreVersion>1.1.3</MicrosoftDataLakeStoreVersion>
<NewtonsoftJsonVersion>10.0.3</NewtonsoftJsonVersion>
- <NSubstituteVersion>1.8.2.0</NSubstituteVersion>
<ProtobufVersion>2.1.0</ProtobufVersion>
<SystemReactiveVersion>3.1.1</SystemReactiveVersion>
<StyleCopVersion>5.0.0</StyleCopVersion>
<TransientFaultHandlingVersion>1.0.0</TransientFaultHandlingVersion>
- <NSubstituteVersion>2.0.3</NSubstituteVersion>
+ <NSubstituteVersion>3.1.0</NSubstituteVersion>
<WindowsAzureStorageVersion>8.1.3</WindowsAzureStorageVersion>
</PropertyGroup>
</Project>
http://git-wip-us.apache.org/repos/asf/reef/blob/4784959c/lang/cs/build.DotNetLibrary.props
----------------------------------------------------------------------
diff --git a/lang/cs/build.DotNetLibrary.props b/lang/cs/build.DotNetLibrary.props
index 16ad7a8..7122375 100644
--- a/lang/cs/build.DotNetLibrary.props
+++ b/lang/cs/build.DotNetLibrary.props
@@ -18,12 +18,6 @@ under the License.
<Import Project="build.DotNet.props"/>
<PropertyGroup>
- <TargetFrameworks>net461</TargetFrameworks>
-
- <!-- If EnableDotNetBuild is true, override the target frameworks to include dot net core -->
- <TargetFrameworks Condition="'$(EnableDotNetBuild)' == 'true'">netstandard2.0</TargetFrameworks>
-
- <!-- If on Linux, override the target frameworks to netcoreapp2.0 -->
- <TargetFrameworks Condition="'$(OS)' != 'Windows_NT'">netcoreapp2.0</TargetFrameworks>
+ <TargetFrameworks>netstandard2.0</TargetFrameworks>
</PropertyGroup>
</Project>
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/4784959c/lang/cs/xunit.DotNet.props
----------------------------------------------------------------------
diff --git a/lang/cs/xunit.DotNet.props b/lang/cs/xunit.DotNet.props
index 8bb0a93..230bd60 100644
--- a/lang/cs/xunit.DotNet.props
+++ b/lang/cs/xunit.DotNet.props
@@ -16,7 +16,7 @@ specific language governing permissions and limitations
under the License.
-->
<ItemGroup>
- <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.0.0" NoWarn="NU1701"/>
+ <PackageReference Include="Microsoft.NET.Test.Sdk" Version="15.5.0" NoWarn="NU1701"/>
<PackageReference Include="xunit" Version="2.2.0" NoWarn="NU1701"/>
<PackageReference Include="xunit.runner.visualstudio" Version="2.2.0" NoWarn="NU1701"/>
<PackageReference Include="xunit.runner.console" Version="2.2.0" NoWarn="NU1701"/>
[13/16] reef git commit: [REEF-2012] Add driver restart capabilities
to reef runtime mock
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CloseTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CloseTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CloseTask.java
deleted file mode 100644
index 504161e..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CloseTask.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.request;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.task.CompletedTask;
-import org.apache.reef.driver.task.FailedTask;
-import org.apache.reef.mock.MockTaskReturnValueProvider;
-import org.apache.reef.mock.ProcessRequest;
-import org.apache.reef.mock.runtime.MockCompletedTask;
-import org.apache.reef.mock.runtime.MockRunningTask;
-import org.apache.reef.util.Optional;
-
-/**
- * close task process request.
- */
-@Unstable
-@Private
-public final class CloseTask implements ProcessRequestInternal<CompletedTask, FailedTask> {
-
- private final MockRunningTask task;
-
- private final MockTaskReturnValueProvider taskReturnValueProvider;
-
- public CloseTask(
- final MockRunningTask task,
- final MockTaskReturnValueProvider taskReturnValueProvider) {
- this.task = task;
- this.taskReturnValueProvider = taskReturnValueProvider;
- }
-
- public MockRunningTask getTask() {
- return task;
- }
-
- @Override
- public Type getType() {
- return Type.CLOSE_TASK;
- }
-
- @Override
- public MockCompletedTask getSuccessEvent() {
- return new MockCompletedTask(this.task, this.taskReturnValueProvider.getReturnValue(task));
- }
-
- @Override
- public FailedTask getFailureEvent() {
- return new FailedTask(
- task.getId(),
- "mock",
- Optional.<String>empty(),
- Optional.<Throwable>empty(),
- Optional.<byte[]>empty(),
- Optional.of(this.task.getActiveContext()));
- }
-
- @Override
- public boolean doAutoComplete() {
- return false;
- }
-
- @Override
- public void setAutoComplete(final boolean value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ProcessRequest getCompletionProcessRequest() {
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CompleteTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CompleteTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CompleteTask.java
deleted file mode 100644
index 25e6df8..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CompleteTask.java
+++ /dev/null
@@ -1,82 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.request;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.task.CompletedTask;
-import org.apache.reef.driver.task.FailedTask;
-import org.apache.reef.mock.MockTaskReturnValueProvider;
-import org.apache.reef.mock.ProcessRequest;
-import org.apache.reef.mock.runtime.MockCompletedTask;
-import org.apache.reef.mock.runtime.MockRunningTask;
-
-/**
- * close task process request.
- */
-@Unstable
-@Private
-public final class CompleteTask implements ProcessRequestInternal<CompletedTask, FailedTask> {
-
- private final MockRunningTask task;
-
- private final MockTaskReturnValueProvider returnValueProvider;
-
- public CompleteTask(
- final MockRunningTask task,
- final MockTaskReturnValueProvider returnValueProvider) {
- this.task = task;
- this.returnValueProvider = returnValueProvider;
- }
-
- public MockRunningTask getTask() {
- return task;
- }
-
- @Override
- public Type getType() {
- return Type.COMPLETE_TASK;
- }
-
- @Override
- public CompletedTask getSuccessEvent() {
- return new MockCompletedTask(this.task, this.returnValueProvider.getReturnValue(task));
- }
-
- @Override
- public FailedTask getFailureEvent() {
- return null;
- }
-
- @Override
- public boolean doAutoComplete() {
- return false;
- }
-
- @Override
- public void setAutoComplete(final boolean value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ProcessRequest getCompletionProcessRequest() {
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CreateContext.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CreateContext.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CreateContext.java
deleted file mode 100644
index e9d533b..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CreateContext.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.request;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.context.FailedContext;
-import org.apache.reef.mock.AutoCompletable;
-import org.apache.reef.mock.ProcessRequest;
-import org.apache.reef.mock.runtime.MockActiveContext;
-import org.apache.reef.mock.runtime.MockFailedContext;
-
-/**
- * create context process request.
- */
-@Unstable
-@Private
-public final class CreateContext implements
- ProcessRequestInternal<MockActiveContext, FailedContext>,
- AutoCompletable {
-
- private final MockActiveContext context;
-
- private boolean autoComplete = false;
-
- public CreateContext(final MockActiveContext context) {
- this.context = context;
- }
-
- @Override
- public Type getType() {
- return Type.CREATE_CONTEXT;
- }
-
- @Override
- public MockActiveContext getSuccessEvent() {
- return this.context;
- }
-
- @Override
- public FailedContext getFailureEvent() {
- return new MockFailedContext(this.context);
- }
-
- @Override
- public boolean doAutoComplete() {
- return this.autoComplete;
- }
-
- @Override
- public void setAutoComplete(final boolean value) {
- this.autoComplete = value;
- }
-
- @Override
- public ProcessRequest getCompletionProcessRequest() {
- return new CloseContext(this.context);
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CreateContextAndTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CreateContextAndTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CreateContextAndTask.java
deleted file mode 100644
index 2169bfd..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CreateContextAndTask.java
+++ /dev/null
@@ -1,98 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.request;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.context.ActiveContext;
-import org.apache.reef.driver.task.FailedTask;
-import org.apache.reef.io.Tuple;
-import org.apache.reef.mock.AutoCompletable;
-import org.apache.reef.mock.MockTaskReturnValueProvider;
-import org.apache.reef.mock.ProcessRequest;
-import org.apache.reef.mock.runtime.MockActiveContext;
-import org.apache.reef.mock.runtime.MockFailedContext;
-import org.apache.reef.mock.runtime.MockRunningTask;
-import org.apache.reef.util.Optional;
-
-/**
- * create context and task process request.
- */
-@Unstable
-@Private
-public final class CreateContextAndTask implements
- ProcessRequestInternal<Tuple<MockActiveContext, MockRunningTask>, Tuple<MockFailedContext, FailedTask>>,
- AutoCompletable {
-
- private final MockActiveContext context;
-
- private final MockRunningTask task;
-
- private final MockTaskReturnValueProvider taskReturnValueProvider;
-
- private boolean autoComplete = true;
-
- public CreateContextAndTask(
- final MockActiveContext context,
- final MockRunningTask task,
- final MockTaskReturnValueProvider taskReturnValueProvider) {
- this.context = context;
- this.task = task;
- this.taskReturnValueProvider = taskReturnValueProvider;
- }
-
- @Override
- public Type getType() {
- return Type.CREATE_CONTEXT_AND_TASK;
- }
-
- @Override
- public Tuple<MockActiveContext, MockRunningTask> getSuccessEvent() {
- return new Tuple<>(this.context, this.task);
- }
-
- @Override
- public Tuple<MockFailedContext, FailedTask> getFailureEvent() {
- return new Tuple<>(
- new MockFailedContext(this.context),
- new FailedTask(
- this.task.getId(),
- "mock",
- Optional.<String>empty(),
- Optional.<Throwable>empty(),
- Optional.<byte[]>empty(),
- Optional.of((ActiveContext)this.context)));
- }
-
- @Override
- public boolean doAutoComplete() {
- return this.autoComplete;
- }
-
- @Override
- public void setAutoComplete(final boolean value) {
- this.autoComplete = value;
- }
-
- @Override
- public ProcessRequest getCompletionProcessRequest() {
- return new CompleteTask(this.task, this.taskReturnValueProvider);
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CreateTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CreateTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CreateTask.java
deleted file mode 100644
index a5eed49..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CreateTask.java
+++ /dev/null
@@ -1,89 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.request;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.task.FailedTask;
-import org.apache.reef.driver.task.RunningTask;
-import org.apache.reef.mock.AutoCompletable;
-import org.apache.reef.mock.MockTaskReturnValueProvider;
-import org.apache.reef.mock.ProcessRequest;
-import org.apache.reef.mock.runtime.MockRunningTask;
-import org.apache.reef.util.Optional;
-
-/**
- * create task process request.
- */
-@Unstable
-@Private
-public final class CreateTask implements
- ProcessRequestInternal<RunningTask, FailedTask>,
- AutoCompletable {
-
- private final MockRunningTask task;
-
- private final MockTaskReturnValueProvider returnValueProvider;
-
- private boolean autoComplete = true;
-
- public CreateTask(
- final MockRunningTask task,
- final MockTaskReturnValueProvider returnValueProvider) {
- this.task = task;
- this.returnValueProvider = returnValueProvider;
- }
-
- @Override
- public Type getType() {
- return Type.CREATE_TASK;
- }
-
- @Override
- public boolean doAutoComplete() {
- return this.autoComplete;
- }
-
- @Override
- public ProcessRequest getCompletionProcessRequest() {
- return new CompleteTask(this.task, this.returnValueProvider);
- }
-
- @Override
- public void setAutoComplete(final boolean value) {
- this.autoComplete = value;
- }
-
- @Override
- public MockRunningTask getSuccessEvent() {
- return this.task;
- }
-
- @Override
- public FailedTask getFailureEvent() {
- return new FailedTask(
- this.task.getId(),
- "mock",
- Optional.<String>empty(),
- Optional.<Throwable>empty(),
- Optional.<byte[]>empty(),
- Optional.of(this.task.getActiveContext()));
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/ProcessRequestInternal.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/ProcessRequestInternal.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/ProcessRequestInternal.java
deleted file mode 100644
index b9dcd2b..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/ProcessRequestInternal.java
+++ /dev/null
@@ -1,44 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.request;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.mock.ProcessRequest;
-
-/**
- * internal process request API.
- * @param <S> successful event
- * @param <F> failure event
- */
-@Unstable
-@Private
-public interface ProcessRequestInternal<S, F> extends ProcessRequest {
-
- /**
- * @return the outcome of a successful processing of this request
- */
- S getSuccessEvent();
-
- /**
- * @return the outcome of an unsuccessful processing of this request
- */
- F getFailureEvent();
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/SendMessageDriverToContext.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/SendMessageDriverToContext.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/SendMessageDriverToContext.java
deleted file mode 100644
index 74faa60..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/SendMessageDriverToContext.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.request;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.context.ActiveContext;
-import org.apache.reef.mock.ProcessRequest;
-
-/**
- * send message from driver to context process request.
- */
-@Unstable
-@Private
-public final class SendMessageDriverToContext implements
- ProcessRequestInternal<Object, Object> {
-
- private final ActiveContext context;
-
- private final byte[] message;
-
- public SendMessageDriverToContext(final ActiveContext context, final byte[] message) {
- this.context = context;
- this.message = message;
- }
-
- @Override
- public Type getType() {
- return Type.SEND_MESSAGE_DRIVER_TO_CONTEXT;
- }
-
- public ActiveContext getContext() {
- return this.context;
- }
-
- public byte[] getMessage() {
- return this.message;
- }
-
- @Override
- public Object getSuccessEvent() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Object getFailureEvent() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean doAutoComplete() {
- return false;
- }
-
- @Override
- public void setAutoComplete(final boolean value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ProcessRequest getCompletionProcessRequest() {
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/SendMessageDriverToTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/SendMessageDriverToTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/SendMessageDriverToTask.java
deleted file mode 100644
index d6dfaaa..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/SendMessageDriverToTask.java
+++ /dev/null
@@ -1,81 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.request;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.task.RunningTask;
-import org.apache.reef.mock.ProcessRequest;
-
-/**
- * send message from driver to task process request.
- */
-@Unstable
-@Private
-public final class SendMessageDriverToTask implements
- ProcessRequestInternal<Object, Object> {
-
- private RunningTask task;
-
- private final byte[] message;
-
- public SendMessageDriverToTask(final RunningTask task, final byte[] message) {
- this.task = task;
- this.message = message;
- }
-
- @Override
- public Type getType() {
- return Type.SEND_MESSAGE_DRIVER_TO_TASK;
- }
-
- public RunningTask getTask() {
- return task;
- }
-
- public byte[] getMessage() {
- return message;
- }
-
- @Override
- public Object getSuccessEvent() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Object getFailureEvent() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public boolean doAutoComplete() {
- return false;
- }
-
- @Override
- public void setAutoComplete(final boolean value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ProcessRequest getCompletionProcessRequest() {
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/SuspendTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/SuspendTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/SuspendTask.java
deleted file mode 100644
index 74a28f6..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/SuspendTask.java
+++ /dev/null
@@ -1,90 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.request;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.task.FailedTask;
-import org.apache.reef.driver.task.SuspendedTask;
-import org.apache.reef.mock.ProcessRequest;
-import org.apache.reef.mock.runtime.MockRunningTask;
-import org.apache.reef.mock.runtime.MockSuspendedTask;
-import org.apache.reef.util.Optional;
-
-/**
- * suspend task process request.
- */
-@Unstable
-@Private
-public final class SuspendTask implements ProcessRequestInternal<SuspendedTask, FailedTask> {
-
- private final MockRunningTask task;
-
- private final Optional<byte[]> message;
-
- public SuspendTask(final MockRunningTask task, final Optional<byte[]> message) {
- this.task = task;
- this.message = message;
- }
-
- public MockRunningTask getTask() {
- return task;
- }
-
- @Override
- public Type getType() {
- return Type.SUSPEND_TASK;
- }
-
- public Optional<byte[]> getMessage() {
- return message;
- }
-
- @Override
- public MockSuspendedTask getSuccessEvent() {
- return new MockSuspendedTask(this.task);
- }
-
- @Override
- public FailedTask getFailureEvent() {
- return new FailedTask(
- this.task.getId(),
- "mock",
- Optional.<String>empty(),
- Optional.<Throwable>empty(),
- Optional.<byte[]>empty(),
- Optional.of(this.task.getActiveContext()));
- }
-
- @Override
- public boolean doAutoComplete() {
- return false;
- }
-
- @Override
- public void setAutoComplete(final boolean value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ProcessRequest getCompletionProcessRequest() {
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/package-info.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/package-info.java
deleted file mode 100644
index 56bc6f0..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-/**
- * process request implementations.
- */
-package org.apache.reef.mock.request;
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockActiveContext.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockActiveContext.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockActiveContext.java
deleted file mode 100644
index ebde788..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockActiveContext.java
+++ /dev/null
@@ -1,139 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.context.ActiveContext;
-import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
-import org.apache.reef.driver.task.TaskConfigurationOptions;
-import org.apache.reef.evaluator.context.parameters.ContextIdentifier;
-import org.apache.reef.mock.request.CloseContext;
-import org.apache.reef.mock.request.CreateContext;
-import org.apache.reef.mock.request.CreateTask;
-import org.apache.reef.mock.request.SendMessageDriverToContext;
-import org.apache.reef.tang.Configuration;
-import org.apache.reef.util.Optional;
-
-/**
- * mock active context.
- */
-@Unstable
-@Private
-public final class MockActiveContext implements ActiveContext {
-
- private final MockRuntimeDriver mockRuntimeDriver;
-
- private final MockAllocatedEvalautor evaluator;
-
- private final Optional<MockActiveContext> parentContext;
-
- private final String contextID;
-
- MockActiveContext(
- final MockRuntimeDriver mockRuntimeDriver,
- final MockAllocatedEvalautor evalautor,
- final Optional<MockActiveContext> parentContext,
- final String contextID) {
- this.mockRuntimeDriver = mockRuntimeDriver;
- this.evaluator = evalautor;
- this.parentContext = parentContext;
- this.contextID = contextID;
- }
-
- @Override
- public int hashCode() {
- final String id = this.getEvaluatorId() + ":" + contextID;
- return id.hashCode();
- }
-
- public boolean equals(final Object that) {
- if (that instanceof MockActiveContext) {
- return this.getEvaluatorId().equals(((MockActiveContext)that).getEvaluatorId()) &&
- this.contextID.equals(((MockActiveContext)that).contextID);
- }
- return false;
- }
-
- public MockAllocatedEvalautor getEvaluator() {
- return this.evaluator;
- }
-
- public Optional<MockActiveContext> getParentContext() {
- return this.parentContext;
- }
-
- @Override
- public void close() {
- this.mockRuntimeDriver.add(new CloseContext(this));
- }
-
- @Override
- public void submitTask(final Configuration taskConf) {
- final String taskID = MockUtils.getValue(taskConf, TaskConfigurationOptions.Identifier.class);
- final MockRunningTask task = new MockRunningTask(this.mockRuntimeDriver, taskID, this);
- this.mockRuntimeDriver.add(new CreateTask(task, this.mockRuntimeDriver.getTaskReturnValueProvider()));
- }
-
- @Override
- public void submitContext(final Configuration contextConfiguration) {
- final String childContextID = MockUtils.getValue(contextConfiguration, ContextIdentifier.class);
- final MockActiveContext context = new MockActiveContext(
- this.mockRuntimeDriver,
- this.evaluator,
- Optional.of(this),
- childContextID);
- this.mockRuntimeDriver.add(new CreateContext(context));
- }
-
- @Override
- public void submitContextAndService(
- final Configuration contextConfiguration,
- final Configuration serviceConfiguration) {
- submitContext(contextConfiguration);
- }
-
- @Override
- public void sendMessage(final byte[] message) {
- this.mockRuntimeDriver.add(new SendMessageDriverToContext(this, message));
- }
-
- @Override
- public String getEvaluatorId() {
- return this.evaluator.getId();
- }
-
- @Override
- public Optional<String> getParentId() {
- return this.parentContext.isPresent() ?
- Optional.of(this.parentContext.get().getId()) :
- Optional.<String>empty();
- }
-
- @Override
- public EvaluatorDescriptor getEvaluatorDescriptor() {
- return this.evaluator.getEvaluatorDescriptor();
- }
-
- @Override
- public String getId() {
- return this.contextID;
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockAllocatedEvalautor.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockAllocatedEvalautor.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockAllocatedEvalautor.java
deleted file mode 100644
index b08d557..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockAllocatedEvalautor.java
+++ /dev/null
@@ -1,149 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.evaluator.AllocatedEvaluator;
-import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
-import org.apache.reef.driver.evaluator.EvaluatorProcess;
-import org.apache.reef.driver.task.TaskConfigurationOptions;
-import org.apache.reef.evaluator.context.parameters.ContextIdentifier;
-import org.apache.reef.mock.request.CloseEvaluator;
-import org.apache.reef.mock.request.CreateContextAndTask;
-import org.apache.reef.tang.Configuration;
-import org.apache.reef.util.Optional;
-
-import java.io.File;
-
-/**
- * mock allocated evaluator.
- */
-@Unstable
-@Private
-public final class MockAllocatedEvalautor implements AllocatedEvaluator {
- public static final String ROOT_CONTEXT_IDENTIFIER_PREFIX = "ROOT.CONTEXT.";
-
- private final MockRuntimeDriver mockRuntimeDriver;
-
- private final String identifier;
-
- private final EvaluatorDescriptor evaluatorDescriptor;
-
- private final MockActiveContext rootContext;
-
- private boolean closed = false;
-
- MockAllocatedEvalautor(
- final MockRuntimeDriver mockRuntimeDriver,
- final String identifier,
- final EvaluatorDescriptor evaluatorDescriptor) {
- this.mockRuntimeDriver = mockRuntimeDriver;
- this.identifier = identifier;
- this.evaluatorDescriptor = evaluatorDescriptor;
- this.rootContext = new MockActiveContext(
- mockRuntimeDriver,
- this,
- Optional.<MockActiveContext>empty(),
- ROOT_CONTEXT_IDENTIFIER_PREFIX + identifier);
- }
-
- public MockActiveContext getRootContext() {
- return this.rootContext;
- }
-
- @Override
- public void addFile(final File file) {
- // ignore
- }
-
- @Override
- public void addLibrary(final File file) {
- // ignore
- }
-
- @Override
- public EvaluatorDescriptor getEvaluatorDescriptor() {
- return this.evaluatorDescriptor;
- }
-
- @Override
- public void setProcess(final EvaluatorProcess process) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void close() {
- if (!this.closed) {
- this.mockRuntimeDriver.add(new CloseEvaluator(this));
- } else {
- throw new IllegalStateException("evaluator already closed");
- }
- }
-
- @Override
- public void submitTask(final Configuration taskConfiguration) {
- this.rootContext.submitTask(taskConfiguration);
- }
-
- @Override
- public void submitContext(final Configuration contextConfiguration) {
- this.rootContext.submitContext(contextConfiguration);
- }
-
- @Override
- public void submitContextAndService(
- final Configuration contextConfiguration,
- final Configuration serviceConfiguration) {
- this.rootContext.submitContextAndService(contextConfiguration, serviceConfiguration);
- }
-
- @Override
- public void submitContextAndTask(
- final Configuration contextConfiguration,
- final Configuration taskConfiguration) {
- final String contextID = MockUtils.getValue(contextConfiguration, ContextIdentifier.class);
- final String taskID = MockUtils.getValue(taskConfiguration, TaskConfigurationOptions.Identifier.class);
- final MockActiveContext mockContext = new MockActiveContext(
- this.mockRuntimeDriver,
- this,
- Optional.of(this.rootContext),
- contextID);
- final MockRunningTask mockTask = new MockRunningTask(this.mockRuntimeDriver, taskID, mockContext);
- this.mockRuntimeDriver.add(
- new CreateContextAndTask(
- mockContext,
- mockTask,
- this.mockRuntimeDriver.getTaskReturnValueProvider()));
- }
-
- @Override
- public void submitContextAndServiceAndTask(
- final Configuration contextConfiguration,
- final Configuration serviceConfiguration,
- final Configuration taskConfiguration) {
- submitContextAndTask(contextConfiguration, taskConfiguration);
- }
-
- @Override
- public String getId() {
- return this.identifier;
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockClock.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockClock.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockClock.java
deleted file mode 100644
index cd32b4f..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockClock.java
+++ /dev/null
@@ -1,120 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.mock.MockRuntime;
-import org.apache.reef.tang.InjectionFuture;
-import org.apache.reef.wake.EventHandler;
-import org.apache.reef.wake.time.Clock;
-import org.apache.reef.wake.time.Time;
-import org.apache.reef.wake.time.event.Alarm;
-import org.apache.reef.wake.time.runtime.event.ClientAlarm;
-
-import javax.inject.Inject;
-import java.util.ArrayList;
-import java.util.Iterator;
-import java.util.List;
-
-/**
- * The MockClock can be used to drive alarms set by the client application.
- */
-@Unstable
-@Private
-public final class MockClock implements Clock {
-
- private final InjectionFuture<MockRuntime> runtime;
-
- private final List<Alarm> alarmList = new ArrayList<>();
-
- private long currentTime = 0;
-
- private boolean closed = false;
-
- @Inject
- MockClock(final InjectionFuture<MockRuntime> runtime) {
- this.runtime = runtime;
- }
-
- /**
- * Advances the clock by the offset amount.
- * @param offset amount to advance clock
- */
- public void advanceClock(final int offset) {
- this.currentTime += offset;
- final Iterator<Alarm> iter = this.alarmList.iterator();
- while (iter.hasNext()) {
- final Alarm alarm = iter.next();
- if (alarm.getTimestamp() <= this.currentTime) {
- alarm.run();
- iter.remove();
- }
- }
- }
-
- /**
- * @return the current mock clock time
- */
- public long getCurrentTime() {
- return this.currentTime;
- }
-
- @Override
- public Time scheduleAlarm(final int offset, final EventHandler<Alarm> handler) {
- final Alarm alarm = new ClientAlarm(this.currentTime + offset, handler);
- alarmList.add(alarm);
- return alarm;
- }
-
- @Override
- public void close() {
- if (!closed) {
- this.runtime.get().stop();
- this.closed = true;
- }
- }
-
- @Override
- public void stop() {
- close();
- }
-
- @Override
- public void stop(final Throwable exception) {
- close();
- }
-
- @Override
- public boolean isIdle() {
- return this.alarmList.size() == 0;
- }
-
- @Override
- public boolean isClosed() {
- return this.closed;
- }
-
- @Override
- public void run() {
- this.runtime.get().start();
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockClosedContext.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockClosedContext.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockClosedContext.java
deleted file mode 100644
index 31d5845..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockClosedContext.java
+++ /dev/null
@@ -1,71 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.context.ActiveContext;
-import org.apache.reef.driver.context.ClosedContext;
-import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
-import org.apache.reef.util.Optional;
-
-/**
- * mock closed context.
- */
-@Unstable
-@Private
-public final class MockClosedContext implements ClosedContext {
-
- private final MockActiveContext mockActiveContext;
-
- public MockClosedContext(final MockActiveContext activeContext) {
- this.mockActiveContext = activeContext;
- }
-
- public MockActiveContext getMockActiveContext() {
- return this.mockActiveContext;
- }
-
- @Override
- public ActiveContext getParentContext() {
- return this.mockActiveContext.getParentContext().isPresent() ?
- this.mockActiveContext.getParentContext().get() : null;
- }
-
- @Override
- public String getId() {
- return this.mockActiveContext.getId();
- }
-
- @Override
- public String getEvaluatorId() {
- return this.mockActiveContext.getEvaluatorId();
- }
-
- @Override
- public Optional<String> getParentId() {
- return this.mockActiveContext.getParentId();
- }
-
- @Override
- public EvaluatorDescriptor getEvaluatorDescriptor() {
- return this.mockActiveContext.getEvaluatorDescriptor();
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockCompletedTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockCompletedTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockCompletedTask.java
deleted file mode 100644
index 6ad6b35..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockCompletedTask.java
+++ /dev/null
@@ -1,57 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.context.ActiveContext;
-import org.apache.reef.driver.task.CompletedTask;
-
-/**
- * mock completed task.
- */
-@Unstable
-@Private
-public final class MockCompletedTask implements CompletedTask {
-
- private final MockRunningTask task;
-
- private final byte[] returnValue;
-
- public MockCompletedTask(final MockRunningTask task, final byte[] returnValue) {
- this.task = task;
- this.returnValue = returnValue;
- }
-
- @Override
- public ActiveContext getActiveContext() {
- return this.task.getActiveContext();
- }
-
- @Override
- public String getId() {
- return this.task.getId();
- }
-
- @Override
- public byte[] get() {
- return this.returnValue;
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockEvaluatorDescriptor.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockEvaluatorDescriptor.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockEvaluatorDescriptor.java
deleted file mode 100644
index 2d98af0..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockEvaluatorDescriptor.java
+++ /dev/null
@@ -1,64 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.catalog.NodeDescriptor;
-import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
-import org.apache.reef.driver.evaluator.EvaluatorProcess;
-
-/**
- * mock evaluator descriptor.
- */
-@Unstable
-@Private
-public final class MockEvaluatorDescriptor implements EvaluatorDescriptor {
- private final NodeDescriptor nodeDescriptor;
-
- MockEvaluatorDescriptor(final NodeDescriptor nodeDescriptor) {
- this.nodeDescriptor = nodeDescriptor;
- }
-
- @Override
- public NodeDescriptor getNodeDescriptor() {
- return this.nodeDescriptor;
- }
-
- @Override
- public EvaluatorProcess getProcess() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public int getMemory() {
- return 0;
- }
-
- @Override
- public int getNumberOfCores() {
- return 1;
- }
-
- @Override
- public String getRuntimeName() {
- return "mock";
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockEvaluatorRequestor.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockEvaluatorRequestor.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockEvaluatorRequestor.java
deleted file mode 100644
index 7f90039..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockEvaluatorRequestor.java
+++ /dev/null
@@ -1,85 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.catalog.NodeDescriptor;
-import org.apache.reef.driver.evaluator.EvaluatorRequest;
-import org.apache.reef.driver.evaluator.EvaluatorRequestor;
-import org.apache.reef.mock.request.AllocateEvaluator;
-import org.apache.reef.tang.InjectionFuture;
-
-import javax.inject.Inject;
-import java.util.UUID;
-
-/**
- * mock evaluator requestor.
- */
-@Unstable
-@Private
-public final class MockEvaluatorRequestor implements EvaluatorRequestor {
-
- private final InjectionFuture<MockRuntimeDriver> mockRuntimeDriver;
-
- private final InjectionFuture<MockClock> clock;
-
- @Inject
- MockEvaluatorRequestor(
- final InjectionFuture<MockClock> clock,
- final InjectionFuture<MockRuntimeDriver> mockRuntimeDriver) {
- this.clock = clock;
- this.mockRuntimeDriver = mockRuntimeDriver;
- }
-
- @Override
- public void submit(final EvaluatorRequest req) {
- if (this.clock.get().isClosed()) {
- throw new IllegalStateException("clock closed");
- }
- final NodeDescriptor nodeDescriptor = new MockNodeDescriptor();
- final MockEvaluatorDescriptor evaluatorDescriptor = new MockEvaluatorDescriptor(nodeDescriptor);
- for (int i = 0; i < req.getNumber(); i++) {
- final MockAllocatedEvalautor mockEvaluator = new MockAllocatedEvalautor(
- this.mockRuntimeDriver.get(), UUID.randomUUID().toString(), evaluatorDescriptor);
- this.mockRuntimeDriver.get().add(new AllocateEvaluator(mockEvaluator));
- }
- }
-
- @Override
- public Builder newRequest() {
- if (this.clock.get().isClosed()) {
- throw new IllegalStateException("clock closed");
- }
- return new Builder();
- }
-
-
- /**
- * {@link EvaluatorRequest.Builder} extended with a new submit method.
- * {@link EvaluatorRequest}s are built using this builder.
- */
- private final class Builder extends EvaluatorRequest.Builder<Builder> {
- @Override
- public void submit() {
- MockEvaluatorRequestor.this.submit(this.build());
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockFailedContext.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockFailedContext.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockFailedContext.java
deleted file mode 100644
index 55eafe1..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockFailedContext.java
+++ /dev/null
@@ -1,93 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.context.ActiveContext;
-import org.apache.reef.driver.context.FailedContext;
-import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
-import org.apache.reef.util.Optional;
-
-/**
- * mock failed context.
- */
-@Unstable
-@Private
-public final class MockFailedContext implements FailedContext {
-
- private final MockActiveContext context;
-
- public MockFailedContext(final MockActiveContext context) {
- this.context = context;
- }
-
- @Override
- public Optional<ActiveContext> getParentContext() {
- return this.context.getParentContext().isPresent() ?
- Optional.of((ActiveContext)this.context.getParentContext().get()) :
- Optional.<ActiveContext>empty();
- }
-
- @Override
- public String getMessage() {
- return "mock";
- }
-
- @Override
- public Optional<String> getDescription() {
- return Optional.empty();
- }
-
- @Override
- public Optional<Throwable> getReason() {
- return Optional.empty();
- }
-
- @Override
- public Optional<byte[]> getData() {
- return Optional.empty();
- }
-
- @Override
- public Throwable asError() {
- return new Exception("mock");
- }
-
- @Override
- public String getEvaluatorId() {
- return this.context.getEvaluatorId();
- }
-
- @Override
- public Optional<String> getParentId() {
- return this.context.getParentId();
- }
-
- @Override
- public EvaluatorDescriptor getEvaluatorDescriptor() {
- return this.context.getEvaluatorDescriptor();
- }
-
- @Override
- public String getId() {
- return this.context.getId();
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockFailedEvaluator.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockFailedEvaluator.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockFailedEvaluator.java
deleted file mode 100644
index d9c0c3c..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockFailedEvaluator.java
+++ /dev/null
@@ -1,79 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.context.FailedContext;
-import org.apache.reef.driver.evaluator.FailedEvaluator;
-import org.apache.reef.driver.task.FailedTask;
-import org.apache.reef.exception.EvaluatorException;
-import org.apache.reef.util.Optional;
-
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * mock failed evaluator.
- */
-@Unstable
-@Private
-public final class MockFailedEvaluator implements FailedEvaluator {
-
- private final String evaluatorID;
-
- private final List<FailedContext> failedContextList;
-
- private final Optional<FailedTask> failedTask;
-
- public MockFailedEvaluator(
- final String evaluatorID,
- final List<FailedContext> failedContextList,
- final Optional<FailedTask> failedTask) {
- this.evaluatorID = evaluatorID;
- this.failedContextList = failedContextList;
- this.failedTask = failedTask;
- }
-
- public MockFailedEvaluator(final String evaluatorID) {
- this.evaluatorID = evaluatorID;
- this.failedContextList = new ArrayList<>();
- this.failedTask = Optional.empty();
- }
-
- @Override
- public EvaluatorException getEvaluatorException() {
- return null;
- }
-
- @Override
- public List<FailedContext> getFailedContextList() {
- return this.failedContextList;
- }
-
- @Override
- public Optional<FailedTask> getFailedTask() {
- return this.failedTask;
- }
-
- @Override
- public String getId() {
- return this.evaluatorID;
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockNodeDescriptor.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockNodeDescriptor.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockNodeDescriptor.java
deleted file mode 100644
index be04994..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockNodeDescriptor.java
+++ /dev/null
@@ -1,68 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.catalog.NodeDescriptor;
-import org.apache.reef.driver.catalog.RackDescriptor;
-
-import java.net.InetSocketAddress;
-import java.util.ArrayList;
-import java.util.List;
-
-/**
- * mock node descriptor.
- */
-@Unstable
-@Private
-public final class MockNodeDescriptor implements NodeDescriptor {
- @Override
- public InetSocketAddress getInetSocketAddress() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public RackDescriptor getRackDescriptor() {
- return new RackDescriptor() {
- @Override
- public List<NodeDescriptor> getNodes() {
- final List<NodeDescriptor> nodes = new ArrayList<>();
- nodes.add(MockNodeDescriptor.this);
- return nodes;
- }
-
- @Override
- public String getName() {
- return "mock";
- }
- };
- }
-
- @Override
- public String getName() {
- return "mock";
- }
-
- @Override
- public String getId() {
- return "mock";
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockRunningTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockRunningTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockRunningTask.java
deleted file mode 100644
index bec26f4..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockRunningTask.java
+++ /dev/null
@@ -1,97 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.context.ActiveContext;
-import org.apache.reef.driver.task.RunningTask;
-import org.apache.reef.mock.request.CloseTask;
-import org.apache.reef.mock.request.SendMessageDriverToTask;
-import org.apache.reef.mock.request.SuspendTask;
-import org.apache.reef.runtime.common.driver.task.TaskRepresenter;
-import org.apache.reef.util.Optional;
-
-/**
- * mock running task.
- */
-@Unstable
-@Private
-public final class MockRunningTask implements RunningTask {
-
- private final MockRuntimeDriver mockRuntimeDriver;
-
- private final String taskID;
-
- private final ActiveContext context;
-
- MockRunningTask(
- final MockRuntimeDriver mockRuntimeDriver,
- final String taskID,
- final ActiveContext context) {
- this.mockRuntimeDriver = mockRuntimeDriver;
- this.taskID = taskID;
- this.context = context;
- }
-
- public String evaluatorID() {
- return this.context.getEvaluatorId();
- }
-
- @Override
- public ActiveContext getActiveContext() {
- return this.context;
- }
-
- @Override
- public void send(final byte[] message) {
- this.mockRuntimeDriver.add(new SendMessageDriverToTask(this, message));
- }
-
- @Override
- public void suspend(final byte[] message) {
- this.mockRuntimeDriver.add(new SuspendTask(this, Optional.of(message)));
- }
-
- @Override
- public void suspend() {
- this.mockRuntimeDriver.add(new SuspendTask(this, Optional.<byte[]>empty()));
- }
-
- @Override
- public void close(final byte[] message) {
- this.mockRuntimeDriver.add(new CloseTask(this, this.mockRuntimeDriver.getTaskReturnValueProvider()));
- }
-
- @Override
- public void close() {
- this.mockRuntimeDriver.add(new CloseTask(this, this.mockRuntimeDriver.getTaskReturnValueProvider()));
- }
-
- @Override
- public TaskRepresenter getTaskRepresenter() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public String getId() {
- return this.taskID;
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockRuntimeDriver.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockRuntimeDriver.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockRuntimeDriver.java
deleted file mode 100644
index 0b89e61..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockRuntimeDriver.java
+++ /dev/null
@@ -1,454 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.context.ActiveContext;
-import org.apache.reef.driver.context.ContextMessage;
-import org.apache.reef.driver.context.FailedContext;
-import org.apache.reef.driver.evaluator.AllocatedEvaluator;
-import org.apache.reef.driver.evaluator.CompletedEvaluator;
-import org.apache.reef.driver.evaluator.FailedEvaluator;
-import org.apache.reef.driver.parameters.*;
-import org.apache.reef.driver.task.*;
-import org.apache.reef.io.Tuple;
-import org.apache.reef.mock.MockRuntime;
-import org.apache.reef.mock.MockTaskReturnValueProvider;
-import org.apache.reef.mock.ProcessRequest;
-import org.apache.reef.mock.request.*;
-import org.apache.reef.tang.InjectionFuture;
-import org.apache.reef.tang.annotations.Parameter;
-import org.apache.reef.util.Optional;
-import org.apache.reef.wake.EventHandler;
-import org.apache.reef.wake.time.Clock;
-import org.apache.reef.wake.time.event.StartTime;
-import org.apache.reef.wake.time.event.StopTime;
-
-import javax.inject.Inject;
-import java.util.*;
-
-/**
- * mock runtime driver.
- */
-@Unstable
-@Private
-public final class MockRuntimeDriver implements MockRuntime {
-
- private final InjectionFuture<MockClock> clock;
-
- private final List<ProcessRequest> processRequestQueue = new ArrayList<>();
-
- private final Set<EventHandler<StartTime>> driverStartHandlers;
-
- private final Set<EventHandler<StopTime>> driverStopHandlers;
-
- private final Set<EventHandler<AllocatedEvaluator>> allocatedEvaluatorHandlers;
-
- private final Set<EventHandler<CompletedEvaluator>> completedEvaluatorHandlers;
-
- private final Set<EventHandler<FailedEvaluator>> failedEvaluatorHandlers;
-
- private final Set<EventHandler<TaskRunningHandlers>> taskRunningHandlers;
-
- private final Set<EventHandler<FailedTask>> taskFailedHandlers;
-
- private final Set<EventHandler<TaskMessage>> taskMessageHandlers;
-
- private final Set<EventHandler<CompletedTask>> taskCompletedHandlers;
-
- private final Set<EventHandler<SuspendedTask>> taskSuspendedHandlers;
-
- private final Set<EventHandler<ActiveContext>> contextActiveHandlers;
-
- private final Set<EventHandler<CloseContext>> contextClosedHandlers;
-
- private final Set<EventHandler<ContextMessage>> contextMessageHandlers;
-
- private final Set<EventHandler<FailedContext>> contextFailedHandlers;
-
- private final Map<String, MockAllocatedEvalautor> allocatedEvaluatorMap = new HashMap<>();
-
- private final Map<String, List<MockActiveContext>> allocatedContextsMap = new HashMap<>();
-
- private final Map<String, MockRunningTask> runningTasks = new HashMap<>();
-
- private final MockTaskReturnValueProvider taskReturnValueProvider;
-
- @Inject
- MockRuntimeDriver(
- final InjectionFuture<MockClock> clock,
- final MockTaskReturnValueProvider taskReturnValueProvider,
- @Parameter(DriverStartHandler.class) final Set<EventHandler<StartTime>> driverStartHandlers,
- @Parameter(Clock.StopHandler.class) final Set<EventHandler<StopTime>> driverStopHandlers,
- @Parameter(EvaluatorAllocatedHandlers.class) final Set<EventHandler<AllocatedEvaluator>>
- allocatedEvaluatorHandlers,
- @Parameter(EvaluatorCompletedHandlers.class) final Set<EventHandler<CompletedEvaluator>>
- completedEvaluatorHandlers,
- @Parameter(EvaluatorFailedHandlers.class) final Set<EventHandler<FailedEvaluator>> failedEvaluatorHandlers,
- @Parameter(TaskRunningHandlers.class) final Set<EventHandler<TaskRunningHandlers>> taskRunningHandlers,
- @Parameter(TaskFailedHandlers.class) final Set<EventHandler<FailedTask>> taskFailedHandlers,
- @Parameter(TaskMessageHandlers.class) final Set<EventHandler<TaskMessage>> taskMessageHandlers,
- @Parameter(TaskCompletedHandlers.class) final Set<EventHandler<CompletedTask>> taskCompletedHandlers,
- @Parameter(TaskSuspendedHandlers.class) final Set<EventHandler<SuspendedTask>> taskSuspendedHandlers,
- @Parameter(ContextActiveHandlers.class) final Set<EventHandler<ActiveContext>> contextActiveHandlers,
- @Parameter(ContextClosedHandlers.class) final Set<EventHandler<CloseContext>> contextClosedHandlers,
- @Parameter(ContextMessageHandlers.class) final Set<EventHandler<ContextMessage>> contextMessageHandlers,
- @Parameter(ContextFailedHandlers.class) final Set<EventHandler<FailedContext>> contextFailedHandlers) {
- this.clock = clock;
- this.taskReturnValueProvider = taskReturnValueProvider;
- this.driverStartHandlers = driverStartHandlers;
- this.driverStopHandlers = driverStopHandlers;
- this.allocatedEvaluatorHandlers = allocatedEvaluatorHandlers;
- this.completedEvaluatorHandlers = completedEvaluatorHandlers;
- this.failedEvaluatorHandlers = failedEvaluatorHandlers;
- this.taskRunningHandlers = taskRunningHandlers;
- this.taskFailedHandlers = taskFailedHandlers;
- this.taskMessageHandlers = taskMessageHandlers;
- this.taskCompletedHandlers = taskCompletedHandlers;
- this.taskSuspendedHandlers = taskSuspendedHandlers;
- this.contextActiveHandlers = contextActiveHandlers;
- this.contextClosedHandlers = contextClosedHandlers;
- this.contextMessageHandlers = contextMessageHandlers;
- this.contextFailedHandlers = contextFailedHandlers;
- }
-
- @Override
- public Collection<AllocatedEvaluator> getCurrentAllocatedEvaluators() {
- if (this.clock.get().isClosed()) {
- throw new IllegalStateException("clock is closed");
- }
- return new ArrayList<AllocatedEvaluator>(this.allocatedEvaluatorMap.values());
- }
-
- @Override
- public void fail(final AllocatedEvaluator evaluator) {
- if (this.clock.get().isClosed()) {
- throw new IllegalStateException("clock is closed");
- }
- if (this.allocatedEvaluatorMap.containsKey(evaluator.getId())) {
- FailedTask failedTask = null;
- if (this.runningTasks.containsKey(evaluator.getId())) {
- final RunningTask task = this.runningTasks.remove(evaluator.getId());
- failedTask = new FailedTask(
- task.getId(),
- "mock",
- Optional.<String>empty(),
- Optional.<Throwable>empty(),
- Optional.<byte[]>empty(),
- Optional.<ActiveContext>of(task.getActiveContext()));
- }
- final List<FailedContext> failedContexts = new ArrayList<>();
- for (final MockActiveContext context : this.allocatedContextsMap.get(evaluator.getId())) {
- failedContexts.add(new MockFailedContext(context));
- }
- this.allocatedContextsMap.remove(evaluator.getId());
-
- post(this.failedEvaluatorHandlers, new MockFailedEvaluator(
- evaluator.getId(), failedContexts,
- failedTask == null ? Optional.<FailedTask>empty() : Optional.of(failedTask)));
- } else {
- throw new IllegalStateException("unknown evaluator " + evaluator);
- }
- }
-
- @Override
- public Collection<ActiveContext> getCurrentActiveContexts() {
- if (this.clock.get().isClosed()) {
- throw new IllegalStateException("clock is closed");
- }
- final List<ActiveContext> currentActiveContexts = new ArrayList<>();
- for (final List<MockActiveContext> contexts : this.allocatedContextsMap.values()) {
- currentActiveContexts.addAll(contexts);
- }
- return currentActiveContexts;
- }
-
- @Override
- public void fail(final ActiveContext context) {
- if (this.clock.get().isClosed()) {
- throw new IllegalStateException("clock is closed");
- }
- final MockAllocatedEvalautor evaluator = ((MockActiveContext) context).getEvaluator();
- post(this.contextFailedHandlers, new MockFailedContext((MockActiveContext) context));
- if (!((MockActiveContext) context).getParentContext().isPresent()) {
- // root context failure shuts evalautor down
- fail(evaluator);
- } else {
- this.allocatedContextsMap.get(evaluator.getId()).remove(context);
- }
- }
-
- @Override
- public Collection<RunningTask> getCurrentRunningTasks() {
- if (this.clock.get().isClosed()) {
- throw new IllegalStateException("clock is closed");
- }
- return new ArrayList<RunningTask>(this.runningTasks.values());
- }
-
- @Override
- public void fail(final RunningTask task) {
- if (this.clock.get().isClosed()) {
- throw new IllegalStateException("clock is closed");
- }
- final String evaluatorID = task.getActiveContext().getEvaluatorId();
- if (this.runningTasks.containsKey(evaluatorID) &&
- this.runningTasks.get(evaluatorID).equals(task)) {
- this.runningTasks.remove(evaluatorID);
- post(taskFailedHandlers, new FailedTask(
- task.getId(),
- "mock",
- Optional.<String>empty(),
- Optional.<Throwable>empty(),
- Optional.<byte[]>empty(),
- Optional.of(task.getActiveContext())));
- } else {
- throw new IllegalStateException("unknown running task " + task);
- }
- }
-
- @Override
- public void start() {
- post(this.driverStartHandlers, new StartTime(this.clock.get().getCurrentTime()));
- }
-
- @Override
- public void stop() {
- post(this.driverStopHandlers, new StopTime(this.clock.get().getCurrentTime()));
- }
-
- @Override
- public boolean hasProcessRequest() {
- return this.processRequestQueue.size() > 0;
- }
-
- @Override
- public ProcessRequest getNextProcessRequest() {
- if (this.processRequestQueue.size() > 0) {
- return this.processRequestQueue.remove(0);
- } else {
- return null;
- }
- }
-
- @Override
- public void succeed(final ProcessRequest pr) {
- if (this.clock.get().isClosed()) {
- throw new IllegalStateException("clock is closed");
- }
- final ProcessRequestInternal request = (ProcessRequestInternal) pr;
- switch (request.getType()) {
- case ALLOCATE_EVALUATOR:
- final MockAllocatedEvalautor allocatedEvalautor = ((AllocateEvaluator)request).getSuccessEvent();
- validateAndCreate(allocatedEvalautor);
- post(this.allocatedEvaluatorHandlers, allocatedEvalautor);
- post(this.contextActiveHandlers, allocatedEvalautor.getRootContext());
- break;
- case CLOSE_EVALUATOR:
- final CompletedEvaluator closedEvaluator = ((CloseEvaluator)request).getSuccessEvent();
- validateAndClose(closedEvaluator);
- post(this.completedEvaluatorHandlers, closedEvaluator);
- break;
- case CREATE_CONTEXT:
- final MockActiveContext createContext = ((CreateContext) request).getSuccessEvent();
- validateAndCreate(createContext);
- post(this.contextActiveHandlers, createContext);
- break;
- case CLOSE_CONTEXT:
- final MockClosedContext closeContext = ((CloseContext) request).getSuccessEvent();
- validateAndClose(closeContext);
- post(this.contextClosedHandlers, closeContext);
- break;
- case CREATE_TASK:
- final MockRunningTask createTask = ((CreateTask)request).getSuccessEvent();
- validateAndCreate(createTask);
- post(this.taskRunningHandlers, request.getSuccessEvent());
- break;
- case SUSPEND_TASK:
- final MockRunningTask suspendedTask = ((SuspendTask)request).getTask();
- validateAndClose(suspendedTask);
- post(this.taskSuspendedHandlers, request.getSuccessEvent());
- break;
- case CLOSE_TASK:
- case COMPLETE_TASK:
- final MockRunningTask completedTask = ((CompleteTask)request).getTask();
- validateAndClose(completedTask);
- post(this.taskCompletedHandlers, request.getSuccessEvent());
- break;
- case CREATE_CONTEXT_AND_TASK:
- final CreateContextAndTask createContextTask = (CreateContextAndTask) request;
- final Tuple<MockActiveContext, MockRunningTask> events = createContextTask.getSuccessEvent();
- validateAndCreate(events.getKey());
- post(this.contextActiveHandlers, events.getKey());
- validateAndCreate(events.getValue());
- post(this.taskRunningHandlers, events.getValue());
- break;
- case SEND_MESSAGE_DRIVER_TO_TASK:
- // ignore
- break;
- case SEND_MESSAGE_DRIVER_TO_CONTEXT:
- // ignore
- break;
- default:
- throw new IllegalStateException("unknown type");
- }
-
- if (request.doAutoComplete()) {
- add(request.getCompletionProcessRequest());
- } else if (!this.clock.get().isClosed() && isIdle()) {
- this.clock.get().close();
- }
- }
-
- @Override
- public void fail(final ProcessRequest pr) {
- if (this.clock.get().isClosed()) {
- throw new IllegalStateException("clock is closed");
- }
- final ProcessRequestInternal request = (ProcessRequestInternal) pr;
- switch (request.getType()) {
- case ALLOCATE_EVALUATOR:
- post(this.failedEvaluatorHandlers, request.getFailureEvent());
- break;
- case CLOSE_EVALUATOR:
- final CompletedEvaluator evaluator = ((CloseEvaluator)request).getSuccessEvent();
- validateAndClose(evaluator);
- post(this.failedEvaluatorHandlers, request.getFailureEvent());
- break;
- case CREATE_CONTEXT:
- post(this.contextFailedHandlers, request.getFailureEvent());
- break;
- case CLOSE_CONTEXT:
- final MockClosedContext context = ((CloseContext)request).getSuccessEvent();
- validateAndClose(context);
- if (context.getParentContext() == null) {
- add(new CloseEvaluator(context.getMockActiveContext().getEvaluator()));
- }
- post(this.contextFailedHandlers, request.getFailureEvent());
- break;
- case CREATE_TASK:
- post(this.taskFailedHandlers, request.getFailureEvent());
- break;
- case SUSPEND_TASK:
- validateAndClose(((SuspendTask)request).getTask());
- post(this.taskFailedHandlers, request.getFailureEvent());
- break;
- case CLOSE_TASK:
- case COMPLETE_TASK:
- validateAndClose(((CloseTask)request).getTask());
- post(this.taskFailedHandlers, request.getFailureEvent());
- break;
- case CREATE_CONTEXT_AND_TASK:
- final CreateContextAndTask createContextTask = (CreateContextAndTask) request;
- final Tuple<MockFailedContext, FailedTask> events = createContextTask.getFailureEvent();
- post(this.taskFailedHandlers, events.getValue());
- post(this.contextFailedHandlers, events.getKey());
- break;
- case SEND_MESSAGE_DRIVER_TO_TASK:
- // ignore
- break;
- case SEND_MESSAGE_DRIVER_TO_CONTEXT:
- // ignore
- break;
- default:
- throw new IllegalStateException("unknown type");
- }
-
- if (!this.clock.get().isClosed() && isIdle()) {
- this.clock.get().close();
- }
- }
-
- MockTaskReturnValueProvider getTaskReturnValueProvider() {
- return this.taskReturnValueProvider;
- }
- /**
- * Used by mock REEF entities (e.g., AllocatedEvaluator, RunningTask) to inject
- * process requests from initiated actions e.g., RunningTask.close().
- * @param request to inject
- */
- void add(final ProcessRequest request) {
- this.processRequestQueue.add(request);
- }
-
- private boolean isIdle() {
- return this.clock.get().isIdle() &&
- this.processRequestQueue.isEmpty() &&
- this.allocatedEvaluatorMap.isEmpty();
- }
-
- private <T> void post(final Set<EventHandler<T>> handlers, final Object event) {
- for (final EventHandler<T> handler : handlers) {
- handler.onNext((T) event);
- }
- }
-
- private void validateAndCreate(final MockActiveContext context) {
- if (!this.allocatedEvaluatorMap.containsKey(context.getEvaluatorId())) {
- throw new IllegalStateException("unknown evaluator id " + context.getEvaluatorId());
- } else if (!this.allocatedContextsMap.containsKey(context.getEvaluatorId())) {
- this.allocatedContextsMap.put(context.getEvaluatorId(), new ArrayList<MockActiveContext>());
- }
- this.allocatedContextsMap.get(context.getEvaluatorId()).add(context);
- }
-
- private void validateAndClose(final MockClosedContext context) {
- if (!this.allocatedContextsMap.containsKey(context.getEvaluatorId())) {
- throw new IllegalStateException("unknown evaluator id " + context.getEvaluatorId());
- }
- final List<MockActiveContext> contexts = this.allocatedContextsMap.get(context.getEvaluatorId());
- if (!contexts.get(contexts.size() - 1).equals(context.getMockActiveContext())) {
- throw new IllegalStateException("closing context that is not on the top of the stack");
- }
- contexts.remove(context.getMockActiveContext());
- }
-
- private void validateAndCreate(final MockRunningTask task) {
- if (this.runningTasks.containsKey(task.evaluatorID())) {
- throw new IllegalStateException("task already running on evaluator " +
- task.evaluatorID());
- }
- this.runningTasks.put(task.evaluatorID(), task);
- }
-
- private void validateAndClose(final MockRunningTask task) {
- if (!this.runningTasks.containsKey(task.getActiveContext().getEvaluatorId())) {
- throw new IllegalStateException("no task running on evaluator");
- }
- this.runningTasks.remove(task.getActiveContext().getEvaluatorId());
- }
-
- private void validateAndCreate(final MockAllocatedEvalautor evalutor) {
- if (this.allocatedEvaluatorMap.containsKey(evalutor.getId())) {
- throw new IllegalStateException("evaluator id " + evalutor.getId() + " already exists");
- }
- this.allocatedEvaluatorMap.put(evalutor.getId(), evalutor);
- this.allocatedContextsMap.put(evalutor.getId(), new ArrayList<MockActiveContext>());
- this.allocatedContextsMap.get(evalutor.getId()).add(evalutor.getRootContext());
- }
-
- private void validateAndClose(final CompletedEvaluator evalautor) {
- if (!this.allocatedEvaluatorMap.containsKey(evalautor.getId())) {
- throw new IllegalStateException("unknown evaluator id " + evalautor.getId());
- }
- this.allocatedEvaluatorMap.remove(evalautor.getId());
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockSuspendedTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockSuspendedTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockSuspendedTask.java
deleted file mode 100644
index 84569ff..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockSuspendedTask.java
+++ /dev/null
@@ -1,54 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.context.ActiveContext;
-import org.apache.reef.driver.task.SuspendedTask;
-
-/**
- * mock suspended task.
- */
-@Unstable
-@Private
-public final class MockSuspendedTask implements SuspendedTask {
-
- private final MockRunningTask task;
-
- public MockSuspendedTask(final MockRunningTask task) {
- this.task = task;
- }
-
- @Override
- public ActiveContext getActiveContext() {
- return this.task.getActiveContext();
- }
-
- @Override
- public byte[] get() {
- return new byte[0];
- }
-
- @Override
- public String getId() {
- return this.task.getId();
- }
-}
[16/16] reef git commit: Merge branch 'master' of
https://git-wip-us.apache.org/repos/asf/reef into REEF-335
Posted by do...@apache.org.
Merge branch 'master' of https://git-wip-us.apache.org/repos/asf/reef into REEF-335
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/386069e4
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/386069e4
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/386069e4
Branch: refs/heads/REEF-335
Commit: 386069e4e509346ea5a9f9bb11e5e95b8c07a016
Parents: 133c064 5ed56eb
Author: Doug Service <do...@apache.org>
Authored: Mon Apr 30 23:19:24 2018 +0000
Committer: Doug Service <do...@apache.org>
Committed: Mon Apr 30 23:19:24 2018 +0000
----------------------------------------------------------------------
dev/ClosedPullRequests.md | 13 +
lang/cs/App.config | 14 +-
.../Org.Apache.REEF.Bridge.CLR.DotNet.csproj | 13 -
.../Org.Apache.REEF.Bridge.JAR.DotNet.csproj | 2 +-
.../EvaluatorRequestorClr2Java.cpp | 5 +-
.../Org.Apache.REEF.Bridge.DotNet.vcxproj | 15 +-
.../AzureBatchCommandBuilderTests.cs | 56 ++
.../Org.Apache.REEF.Client.Tests.DotNet.csproj | 9 -
.../Org.Apache.REEF.Client.Tests.csproj | 1 +
.../AvroAzureBatchJobSubmissionParameters.cs | 107 ++++
.../AzureBatch/AzureBatchDotNetClient.cs | 139 +++++
.../AzureBatchRuntimeClientConfiguration.cs | 78 +++
.../Parameters/AzureBatchAccountKey.cs | 25 +
.../Parameters/AzureBatchAccountName.cs | 26 +
.../Parameters/AzureBatchAccountUri.cs | 26 +
.../AzureBatch/Parameters/AzureBatchPoolId.cs | 26 +
.../Parameters/AzureStorageAccountKey.cs | 26 +
.../Parameters/AzureStorageAccountName.cs | 26 +
.../Parameters/AzureStorageContainerName.cs | 26 +
.../Parameters/DriverStderrFilePath.cs | 29 ++
.../Parameters/DriverStdoutFilePath.cs | 29 ++
.../AzureBatch/Parameters/EnableDebugLogging.cs | 28 +
.../AzureBatch/Parameters/IsWindows.cs | 26 +
.../AzureBatch/Service/AzureBatchService.cs | 138 +++++
.../AzureBatch/Storage/AzureStorageClient.cs | 109 ++++
.../AzureBatch/Util/AbstractCommandBuilder.cs | 72 +++
.../AzureBatchCommandProviderConfiguration.cs | 42 ++
.../AzureBatch/Util/AzureBatchFileNames.cs | 55 ++
.../AzureBatch/Util/ICommandBuilder.cs | 35 ++
.../AzureBatch/Util/JobJarMaker.cs | 108 ++++
.../AzureBatch/Util/LinuxCommandBuilder.cs | 44 ++
.../AzureBatch/Util/WindowsCommandBuilder.cs | 52 ++
.../Common/DriverFolderPreparationHelper.cs | 46 +-
.../Org.Apache.REEF.Client.DotNet.csproj | 34 +-
.../Org.Apache.REEF.Client.csproj | 60 ++-
lang/cs/Org.Apache.REEF.Client/packages.config | 13 +
.../Org.Apache.REEF.Common.Tests.DotNet.csproj | 10 -
.../Org.Apache.REEF.Common.DotNet.csproj | 7 -
.../Runtime/RuntimeName.cs | 5 +
.../Evaluator/EvaluatorRequest.cs | 20 +-
.../Evaluator/EvaluatorRequestBuilder.cs | 16 +-
.../Evaluator/IEvaluatorRequest.cs | 6 +
.../Org.Apache.REEF.Driver.DotNet.csproj | 6 -
...rg.Apache.REEF.Evaluator.Tests.DotNet.csproj | 4 -
.../Org.Apache.REEF.Evaluator.DotNet.csproj | 6 -
...ache.REEF.Examples.AllHandlers.DotNet.csproj | 10 -
...he.REEF.Examples.DriverRestart.DotNet.csproj | 10 -
.../HelloREEF.cs | 25 +-
...Apache.REEF.Examples.HelloREEF.DotNet.csproj | 10 -
.../Org.Apache.REEF.Examples.HelloREEF/Run.cs | 9 +-
.../Org.Apache.REEF.Examples.DotNet.csproj | 6 -
.../Org.Apache.REEF.FatNuGet.DotNet.csproj | 21 +-
.../Org.Apache.REEF.IMRU.Examples.DotNet.csproj | 10 -
.../Org.Apache.REEF.IMRU.Tests.DotNet.csproj | 10 -
.../Org.Apache.REEF.IMRU.DotNet.csproj | 10 -
.../Org.Apache.REEF.IO.TestClient.DotNet.csproj | 10 -
.../Org.Apache.REEF.IO.Tests.DotNet.csproj | 13 -
.../Org.Apache.REEF.IO.Tests.csproj | 24 +-
.../TestAzureBlockBlobFileSystemE2E.cs | 5 +-
.../cs/Org.Apache.REEF.IO.Tests/packages.config | 10 +
.../AzureBlob/AzureCloudBlobContainer.cs | 4 +-
.../FileSystem/AzureBlob/AzureCloudBlockBlob.cs | 6 +-
.../Org.Apache.REEF.IO.DotNet.csproj | 13 -
.../Org.Apache.REEF.IO.csproj | 24 +-
lang/cs/Org.Apache.REEF.IO/packages.config | 16 +-
...e.REEF.Network.Examples.Client.DotNet.csproj | 10 -
...g.Apache.REEF.Network.Examples.DotNet.csproj | 10 -
.../Org.Apache.REEF.Network.Tests.DotNet.csproj | 4 -
.../Org.Apache.REEF.Network.DotNet.csproj | 7 -
.../Org.Apache.REEF.Tang.Examples.DotNet.csproj | 5 -
.../Org.Apache.REEF.Tang.Tests.DotNet.csproj | 5 -
.../Org.Apache.REEF.Tang.DotNet.csproj | 7 -
.../Org.Apache.REEF.Tests.DotNet.csproj | 6 -
.../Org.Apache.REEF.Tests.csproj | 22 +-
lang/cs/Org.Apache.REEF.Tests/packages.config | 16 +-
.../Org.Apache.REEF.Wake.DotNet.csproj | 6 -
lang/cs/Org.Apache.REEF.sln | Bin 37102 -> 37230 bytes
lang/cs/TestRunner.DotNet.proj | 2 +-
lang/cs/build.Common.DotNet.props | 8 +-
lang/cs/build.DotNet.props | 3 +-
lang/cs/build.DotNetApp.props | 7 +-
lang/cs/build.DotNetLibrary.props | 12 +-
lang/cs/build.DotNetTest.props | 10 +-
lang/cs/build.DotNetTest.targets | 2 +-
lang/cs/xunit.DotNet.props | 15 +-
lang/java/reef-bridge-client/pom.xml | 5 +
.../src/main/avro/JobSubmissionParameters.avsc | 17 +
...zureBatchBootstrapDriverConfigGenerator.java | 85 +++
.../client/AzureBatchBootstrapREEFLauncher.java | 139 +++++
lang/java/reef-bridge-java/pom.xml | 5 +
.../javabridge/EvaluatorRequestorBridge.java | 4 +-
.../reef/driver/evaluator/EvaluatorRequest.java | 29 +-
.../common/driver/EvaluatorRequestorImpl.java | 1 +
.../common/driver/api/ResourceRequestEvent.java | 5 +
.../driver/api/ResourceRequestEventImpl.java | 19 +
.../org/apache/reef/mock/AutoCompletable.java | 45 --
.../mock/DefaultTaskReturnValueProvider.java | 41 --
.../org/apache/reef/mock/MockConfiguration.java | 173 ------
.../java/org/apache/reef/mock/MockFailure.java | 67 ---
.../java/org/apache/reef/mock/MockRuntime.java | 76 ---
.../reef/mock/MockTaskReturnValueProvider.java | 44 --
.../org/apache/reef/mock/ProcessRequest.java | 54 --
.../reef/mock/driver/AutoCompletable.java | 45 ++
.../driver/DefaultTaskReturnValueProvider.java | 41 ++
.../reef/mock/driver/MockConfiguration.java | 208 ++++++++
.../mock/driver/MockDriverRestartContext.java | 172 ++++++
.../apache/reef/mock/driver/MockFailure.java | 74 +++
.../apache/reef/mock/driver/MockRuntime.java | 92 ++++
.../driver/MockTaskReturnValueProvider.java | 44 ++
.../apache/reef/mock/driver/ProcessRequest.java | 54 ++
.../apache/reef/mock/driver/package-info.java | 40 ++
.../mock/driver/request/AllocateEvaluator.java | 72 +++
.../reef/mock/driver/request/CloseContext.java | 76 +++
.../mock/driver/request/CloseEvaluator.java | 78 +++
.../reef/mock/driver/request/CloseTask.java | 89 ++++
.../reef/mock/driver/request/CompleteTask.java | 82 +++
.../reef/mock/driver/request/CreateContext.java | 76 +++
.../driver/request/CreateContextAndTask.java | 98 ++++
.../reef/mock/driver/request/CreateTask.java | 89 ++++
.../driver/request/ProcessRequestInternal.java | 44 ++
.../request/SendMessageDriverToContext.java | 81 +++
.../driver/request/SendMessageDriverToTask.java | 81 +++
.../reef/mock/driver/request/SuspendTask.java | 90 ++++
.../reef/mock/driver/request/package-info.java | 23 +
.../mock/driver/runtime/MockActiveContext.java | 139 +++++
.../driver/runtime/MockAllocatedEvaluator.java | 172 ++++++
.../reef/mock/driver/runtime/MockClock.java | 120 +++++
.../mock/driver/runtime/MockClosedContext.java | 71 +++
.../mock/driver/runtime/MockCompletedTask.java | 57 ++
.../driver/runtime/MockEvaluatorDescriptor.java | 64 +++
.../driver/runtime/MockEvaluatorRequestor.java | 85 +++
.../mock/driver/runtime/MockFailedContext.java | 93 ++++
.../driver/runtime/MockFailedEvaluator.java | 79 +++
.../mock/driver/runtime/MockNodeDescriptor.java | 68 +++
.../mock/driver/runtime/MockRunningTask.java | 97 ++++
.../mock/driver/runtime/MockRuntimeDriver.java | 522 +++++++++++++++++++
.../mock/driver/runtime/MockSuspendedTask.java | 54 ++
.../reef/mock/driver/runtime/MockUtils.java | 48 ++
.../reef/mock/driver/runtime/package-info.java | 23 +
.../java/org/apache/reef/mock/package-info.java | 40 --
.../reef/mock/request/AllocateEvaluator.java | 72 ---
.../apache/reef/mock/request/CloseContext.java | 76 ---
.../reef/mock/request/CloseEvaluator.java | 78 ---
.../org/apache/reef/mock/request/CloseTask.java | 89 ----
.../apache/reef/mock/request/CompleteTask.java | 82 ---
.../apache/reef/mock/request/CreateContext.java | 76 ---
.../reef/mock/request/CreateContextAndTask.java | 98 ----
.../apache/reef/mock/request/CreateTask.java | 89 ----
.../mock/request/ProcessRequestInternal.java | 44 --
.../request/SendMessageDriverToContext.java | 81 ---
.../mock/request/SendMessageDriverToTask.java | 81 ---
.../apache/reef/mock/request/SuspendTask.java | 90 ----
.../apache/reef/mock/request/package-info.java | 23 -
.../reef/mock/runtime/MockActiveContext.java | 139 -----
.../mock/runtime/MockAllocatedEvalautor.java | 149 ------
.../org/apache/reef/mock/runtime/MockClock.java | 120 -----
.../reef/mock/runtime/MockClosedContext.java | 71 ---
.../reef/mock/runtime/MockCompletedTask.java | 57 --
.../mock/runtime/MockEvaluatorDescriptor.java | 64 ---
.../mock/runtime/MockEvaluatorRequestor.java | 85 ---
.../reef/mock/runtime/MockFailedContext.java | 93 ----
.../reef/mock/runtime/MockFailedEvaluator.java | 79 ---
.../reef/mock/runtime/MockNodeDescriptor.java | 68 ---
.../reef/mock/runtime/MockRunningTask.java | 97 ----
.../reef/mock/runtime/MockRuntimeDriver.java | 454 ----------------
.../reef/mock/runtime/MockSuspendedTask.java | 54 --
.../org/apache/reef/mock/runtime/MockUtils.java | 48 --
.../apache/reef/mock/runtime/package-info.java | 23 -
.../org/apache/reef/mock/BasicMockTests.java | 205 --------
.../org/apache/reef/mock/MockApplication.java | 275 ----------
.../apache/reef/mock/driver/BasicMockTests.java | 163 ++++++
.../reef/mock/driver/MockApplication.java | 275 ++++++++++
.../apache/reef/mock/driver/package-info.java | 23 +
.../java/org/apache/reef/mock/package-info.java | 23 -
.../yarn/driver/YarnContainerManager.java | 3 +
.../driver/YarnDriverRestartConfiguration.java | 2 +-
.../driver/YarnDriverRuntimeRestartManager.java | 2 +-
.../yarn/driver/YarnResourceRequestHandler.java | 6 +-
178 files changed, 5879 insertions(+), 4004 deletions(-)
----------------------------------------------------------------------
[14/16] reef git commit: [REEF-2012] Add driver restart capabilities
to reef runtime mock
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/SendMessageDriverToContext.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/SendMessageDriverToContext.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/SendMessageDriverToContext.java
new file mode 100644
index 0000000..d90486a
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/SendMessageDriverToContext.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.request;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.mock.driver.ProcessRequest;
+
+/**
+ * send message from driver to context process request.
+ */
+@Unstable
+@Private
+public final class SendMessageDriverToContext implements
+ ProcessRequestInternal<Object, Object> {
+
+ private final ActiveContext context;
+
+ private final byte[] message;
+
+ public SendMessageDriverToContext(final ActiveContext context, final byte[] message) {
+ this.context = context;
+ this.message = message;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.SEND_MESSAGE_DRIVER_TO_CONTEXT;
+ }
+
+ public ActiveContext getContext() {
+ return this.context;
+ }
+
+ public byte[] getMessage() {
+ return this.message;
+ }
+
+ @Override
+ public Object getSuccessEvent() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object getFailureEvent() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean doAutoComplete() {
+ return false;
+ }
+
+ @Override
+ public void setAutoComplete(final boolean value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ProcessRequest getCompletionProcessRequest() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/SendMessageDriverToTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/SendMessageDriverToTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/SendMessageDriverToTask.java
new file mode 100644
index 0000000..bf6c3dd
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/SendMessageDriverToTask.java
@@ -0,0 +1,81 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.request;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.task.RunningTask;
+import org.apache.reef.mock.driver.ProcessRequest;
+
+/**
+ * send message from driver to task process request.
+ */
+@Unstable
+@Private
+public final class SendMessageDriverToTask implements
+ ProcessRequestInternal<Object, Object> {
+
+ private RunningTask task;
+
+ private final byte[] message;
+
+ public SendMessageDriverToTask(final RunningTask task, final byte[] message) {
+ this.task = task;
+ this.message = message;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.SEND_MESSAGE_DRIVER_TO_TASK;
+ }
+
+ public RunningTask getTask() {
+ return task;
+ }
+
+ public byte[] getMessage() {
+ return message;
+ }
+
+ @Override
+ public Object getSuccessEvent() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Object getFailureEvent() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public boolean doAutoComplete() {
+ return false;
+ }
+
+ @Override
+ public void setAutoComplete(final boolean value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ProcessRequest getCompletionProcessRequest() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/SuspendTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/SuspendTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/SuspendTask.java
new file mode 100644
index 0000000..a4dbcee
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/SuspendTask.java
@@ -0,0 +1,90 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.request;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.task.FailedTask;
+import org.apache.reef.driver.task.SuspendedTask;
+import org.apache.reef.mock.driver.ProcessRequest;
+import org.apache.reef.mock.driver.runtime.MockRunningTask;
+import org.apache.reef.mock.driver.runtime.MockSuspendedTask;
+import org.apache.reef.util.Optional;
+
+/**
+ * suspend task process request.
+ */
+@Unstable
+@Private
+public final class SuspendTask implements ProcessRequestInternal<SuspendedTask, FailedTask> {
+
+ private final MockRunningTask task;
+
+ private final Optional<byte[]> message;
+
+ public SuspendTask(final MockRunningTask task, final Optional<byte[]> message) {
+ this.task = task;
+ this.message = message;
+ }
+
+ public MockRunningTask getTask() {
+ return task;
+ }
+
+ @Override
+ public Type getType() {
+ return Type.SUSPEND_TASK;
+ }
+
+ public Optional<byte[]> getMessage() {
+ return message;
+ }
+
+ @Override
+ public MockSuspendedTask getSuccessEvent() {
+ return new MockSuspendedTask(this.task);
+ }
+
+ @Override
+ public FailedTask getFailureEvent() {
+ return new FailedTask(
+ this.task.getId(),
+ "mock",
+ Optional.<String>empty(),
+ Optional.<Throwable>empty(),
+ Optional.<byte[]>empty(),
+ Optional.of(this.task.getActiveContext()));
+ }
+
+ @Override
+ public boolean doAutoComplete() {
+ return false;
+ }
+
+ @Override
+ public void setAutoComplete(final boolean value) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public ProcessRequest getCompletionProcessRequest() {
+ throw new UnsupportedOperationException();
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/package-info.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/package-info.java
new file mode 100644
index 0000000..22dbfbc
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/request/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+/**
+ * process request implementations.
+ */
+package org.apache.reef.mock.driver.request;
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockActiveContext.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockActiveContext.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockActiveContext.java
new file mode 100644
index 0000000..292dd9d
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockActiveContext.java
@@ -0,0 +1,139 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
+import org.apache.reef.driver.task.TaskConfigurationOptions;
+import org.apache.reef.evaluator.context.parameters.ContextIdentifier;
+import org.apache.reef.mock.driver.request.CloseContext;
+import org.apache.reef.mock.driver.request.CreateContext;
+import org.apache.reef.mock.driver.request.CreateTask;
+import org.apache.reef.mock.driver.request.SendMessageDriverToContext;
+import org.apache.reef.tang.Configuration;
+import org.apache.reef.util.Optional;
+
+/**
+ * mock active context.
+ */
+@Unstable
+@Private
+public final class MockActiveContext implements ActiveContext {
+
+ private final MockRuntimeDriver mockRuntimeDriver;
+
+ private final MockAllocatedEvaluator evaluator;
+
+ private final Optional<MockActiveContext> parentContext;
+
+ private final String contextID;
+
+ MockActiveContext(
+ final MockRuntimeDriver mockRuntimeDriver,
+ final MockAllocatedEvaluator evaluator,
+ final Optional<MockActiveContext> parentContext,
+ final String contextID) {
+ this.mockRuntimeDriver = mockRuntimeDriver;
+ this.evaluator = evaluator;
+ this.parentContext = parentContext;
+ this.contextID = contextID;
+ }
+
+ @Override
+ public int hashCode() {
+ final String id = this.getEvaluatorId() + ":" + contextID;
+ return id.hashCode();
+ }
+
+ public boolean equals(final Object that) {
+ if (that instanceof MockActiveContext) {
+ return this.getEvaluatorId().equals(((MockActiveContext)that).getEvaluatorId()) &&
+ this.contextID.equals(((MockActiveContext)that).contextID);
+ }
+ return false;
+ }
+
+ public MockAllocatedEvaluator getEvaluator() {
+ return this.evaluator;
+ }
+
+ public Optional<MockActiveContext> getParentContext() {
+ return this.parentContext;
+ }
+
+ @Override
+ public void close() {
+ this.mockRuntimeDriver.add(new CloseContext(this));
+ }
+
+ @Override
+ public void submitTask(final Configuration taskConf) {
+ final String taskID = MockUtils.getValue(taskConf, TaskConfigurationOptions.Identifier.class);
+ final MockRunningTask task = new MockRunningTask(this.mockRuntimeDriver, taskID, this);
+ this.mockRuntimeDriver.add(new CreateTask(task, this.mockRuntimeDriver.getTaskReturnValueProvider()));
+ }
+
+ @Override
+ public void submitContext(final Configuration contextConfiguration) {
+ final String childContextID = MockUtils.getValue(contextConfiguration, ContextIdentifier.class);
+ final MockActiveContext context = new MockActiveContext(
+ this.mockRuntimeDriver,
+ this.evaluator,
+ Optional.of(this),
+ childContextID);
+ this.mockRuntimeDriver.add(new CreateContext(context));
+ }
+
+ @Override
+ public void submitContextAndService(
+ final Configuration contextConfiguration,
+ final Configuration serviceConfiguration) {
+ submitContext(contextConfiguration);
+ }
+
+ @Override
+ public void sendMessage(final byte[] message) {
+ this.mockRuntimeDriver.add(new SendMessageDriverToContext(this, message));
+ }
+
+ @Override
+ public String getEvaluatorId() {
+ return this.evaluator.getId();
+ }
+
+ @Override
+ public Optional<String> getParentId() {
+ return this.parentContext.isPresent() ?
+ Optional.of(this.parentContext.get().getId()) :
+ Optional.<String>empty();
+ }
+
+ @Override
+ public EvaluatorDescriptor getEvaluatorDescriptor() {
+ return this.evaluator.getEvaluatorDescriptor();
+ }
+
+ @Override
+ public String getId() {
+ return this.contextID;
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockAllocatedEvaluator.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockAllocatedEvaluator.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockAllocatedEvaluator.java
new file mode 100644
index 0000000..d3c0581
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockAllocatedEvaluator.java
@@ -0,0 +1,172 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.evaluator.AllocatedEvaluator;
+import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
+import org.apache.reef.driver.evaluator.EvaluatorProcess;
+import org.apache.reef.driver.task.TaskConfigurationOptions;
+import org.apache.reef.evaluator.context.parameters.ContextIdentifier;
+import org.apache.reef.mock.driver.request.CloseEvaluator;
+import org.apache.reef.mock.driver.request.CreateContext;
+import org.apache.reef.mock.driver.request.CreateContextAndTask;
+import org.apache.reef.tang.Configuration;
+import org.apache.reef.util.Optional;
+
+import java.io.File;
+
+/**
+ * mock allocated evaluator.
+ */
+@Unstable
+@Private
+public final class MockAllocatedEvaluator implements AllocatedEvaluator {
+ public static final String ROOT_CONTEXT_IDENTIFIER_PREFIX = "ROOT.CONTEXT.";
+
+ private final MockRuntimeDriver mockRuntimeDriver;
+
+ private final String identifier;
+
+ private final EvaluatorDescriptor evaluatorDescriptor;
+
+ private MockActiveContext rootContext = null;
+
+ private boolean closed = false;
+
+ MockAllocatedEvaluator(
+ final MockRuntimeDriver mockRuntimeDriver,
+ final String identifier,
+ final EvaluatorDescriptor evaluatorDescriptor) {
+ this.mockRuntimeDriver = mockRuntimeDriver;
+ this.identifier = identifier;
+ this.evaluatorDescriptor = evaluatorDescriptor;
+ }
+
+ public MockActiveContext getRootContext() {
+ return this.rootContext;
+ }
+
+ @Override
+ public void addFile(final File file) {
+ // ignore
+ }
+
+ @Override
+ public void addLibrary(final File file) {
+ // ignore
+ }
+
+ @Override
+ public EvaluatorDescriptor getEvaluatorDescriptor() {
+ return this.evaluatorDescriptor;
+ }
+
+ @Override
+ public void setProcess(final EvaluatorProcess process) {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void close() {
+ if (!this.closed) {
+ this.mockRuntimeDriver.add(new CloseEvaluator(this));
+ } else {
+ throw new IllegalStateException("evaluator already closed");
+ }
+ }
+
+ @Override
+ public void submitTask(final Configuration taskConfiguration) {
+ if (this.rootContext != null) {
+ throw new IllegalStateException("Root context already created");
+ }
+ this.rootContext = new MockActiveContext(
+ mockRuntimeDriver,
+ this,
+ Optional.<MockActiveContext>empty(),
+ ROOT_CONTEXT_IDENTIFIER_PREFIX + identifier);
+ final String taskID = MockUtils.getValue(taskConfiguration, TaskConfigurationOptions.Identifier.class);
+ final MockRunningTask mockTask = new MockRunningTask(this.mockRuntimeDriver, taskID, this.rootContext);
+ this.mockRuntimeDriver.add(
+ new CreateContextAndTask(
+ this.rootContext,
+ mockTask,
+ this.mockRuntimeDriver.getTaskReturnValueProvider()));
+ }
+
+ @Override
+ public void submitContext(final Configuration contextConfiguration) {
+ if (this.rootContext != null) {
+ throw new IllegalStateException("Root context already created");
+ }
+ final String rootContextID = MockUtils.getValue(contextConfiguration, ContextIdentifier.class);
+ this.rootContext = new MockActiveContext(
+ this.mockRuntimeDriver,
+ this,
+ Optional.<MockActiveContext>empty(),
+ rootContextID);
+ this.mockRuntimeDriver.add(new CreateContext(this.rootContext));
+ }
+
+ @Override
+ public void submitContextAndService(
+ final Configuration contextConfiguration,
+ final Configuration serviceConfiguration) {
+ submitContext(contextConfiguration);
+ // ignore services
+ }
+
+ @Override
+ public void submitContextAndTask(
+ final Configuration contextConfiguration,
+ final Configuration taskConfiguration) {
+ if (this.rootContext != null) {
+ throw new IllegalStateException("Root context already created");
+ }
+ final String contextID = MockUtils.getValue(contextConfiguration, ContextIdentifier.class);
+ final String taskID = MockUtils.getValue(taskConfiguration, TaskConfigurationOptions.Identifier.class);
+ this.rootContext = new MockActiveContext(
+ this.mockRuntimeDriver,
+ this,
+ Optional.<MockActiveContext>empty(),
+ contextID);
+ final MockRunningTask mockTask = new MockRunningTask(this.mockRuntimeDriver, taskID, this.rootContext);
+ this.mockRuntimeDriver.add(
+ new CreateContextAndTask(
+ this.rootContext,
+ mockTask,
+ this.mockRuntimeDriver.getTaskReturnValueProvider()));
+ }
+
+ @Override
+ public void submitContextAndServiceAndTask(
+ final Configuration contextConfiguration,
+ final Configuration serviceConfiguration,
+ final Configuration taskConfiguration) {
+ submitContextAndTask(contextConfiguration, taskConfiguration);
+ }
+
+ @Override
+ public String getId() {
+ return this.identifier;
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockClock.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockClock.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockClock.java
new file mode 100644
index 0000000..297647a
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockClock.java
@@ -0,0 +1,120 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.mock.driver.MockRuntime;
+import org.apache.reef.tang.InjectionFuture;
+import org.apache.reef.wake.EventHandler;
+import org.apache.reef.wake.time.Clock;
+import org.apache.reef.wake.time.Time;
+import org.apache.reef.wake.time.event.Alarm;
+import org.apache.reef.wake.time.runtime.event.ClientAlarm;
+
+import javax.inject.Inject;
+import java.util.ArrayList;
+import java.util.Iterator;
+import java.util.List;
+
+/**
+ * The MockClock can be used to drive alarms set by the client application.
+ */
+@Unstable
+@Private
+public final class MockClock implements Clock {
+
+ private final InjectionFuture<MockRuntime> runtime;
+
+ private final List<Alarm> alarmList = new ArrayList<>();
+
+ private long currentTime = 0;
+
+ private boolean closed = false;
+
+ @Inject
+ MockClock(final InjectionFuture<MockRuntime> runtime) {
+ this.runtime = runtime;
+ }
+
+ /**
+ * Advances the clock by the offset amount.
+ * @param offset amount to advance clock
+ */
+ public void advanceClock(final int offset) {
+ this.currentTime += offset;
+ final Iterator<Alarm> iter = this.alarmList.iterator();
+ while (iter.hasNext()) {
+ final Alarm alarm = iter.next();
+ if (alarm.getTimestamp() <= this.currentTime) {
+ alarm.run();
+ iter.remove();
+ }
+ }
+ }
+
+ /**
+ * @return the current mock clock time
+ */
+ public long getCurrentTime() {
+ return this.currentTime;
+ }
+
+ @Override
+ public Time scheduleAlarm(final int offset, final EventHandler<Alarm> handler) {
+ final Alarm alarm = new ClientAlarm(this.currentTime + offset, handler);
+ alarmList.add(alarm);
+ return alarm;
+ }
+
+ @Override
+ public void close() {
+ if (!closed) {
+ this.runtime.get().stop();
+ this.closed = true;
+ }
+ }
+
+ @Override
+ public void stop() {
+ close();
+ }
+
+ @Override
+ public void stop(final Throwable exception) {
+ close();
+ }
+
+ @Override
+ public boolean isIdle() {
+ return this.alarmList.size() == 0;
+ }
+
+ @Override
+ public boolean isClosed() {
+ return this.closed;
+ }
+
+ @Override
+ public void run() {
+ this.runtime.get().start();
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockClosedContext.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockClosedContext.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockClosedContext.java
new file mode 100644
index 0000000..d008373
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockClosedContext.java
@@ -0,0 +1,71 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.context.ClosedContext;
+import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
+import org.apache.reef.util.Optional;
+
+/**
+ * mock closed context.
+ */
+@Unstable
+@Private
+public final class MockClosedContext implements ClosedContext {
+
+ private final MockActiveContext mockActiveContext;
+
+ public MockClosedContext(final MockActiveContext activeContext) {
+ this.mockActiveContext = activeContext;
+ }
+
+ public MockActiveContext getMockActiveContext() {
+ return this.mockActiveContext;
+ }
+
+ @Override
+ public ActiveContext getParentContext() {
+ return this.mockActiveContext.getParentContext().isPresent() ?
+ this.mockActiveContext.getParentContext().get() : null;
+ }
+
+ @Override
+ public String getId() {
+ return this.mockActiveContext.getId();
+ }
+
+ @Override
+ public String getEvaluatorId() {
+ return this.mockActiveContext.getEvaluatorId();
+ }
+
+ @Override
+ public Optional<String> getParentId() {
+ return this.mockActiveContext.getParentId();
+ }
+
+ @Override
+ public EvaluatorDescriptor getEvaluatorDescriptor() {
+ return this.mockActiveContext.getEvaluatorDescriptor();
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockCompletedTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockCompletedTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockCompletedTask.java
new file mode 100644
index 0000000..e141787
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockCompletedTask.java
@@ -0,0 +1,57 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.task.CompletedTask;
+
+/**
+ * mock completed task.
+ */
+@Unstable
+@Private
+public final class MockCompletedTask implements CompletedTask {
+
+ private final MockRunningTask task;
+
+ private final byte[] returnValue;
+
+ public MockCompletedTask(final MockRunningTask task, final byte[] returnValue) {
+ this.task = task;
+ this.returnValue = returnValue;
+ }
+
+ @Override
+ public ActiveContext getActiveContext() {
+ return this.task.getActiveContext();
+ }
+
+ @Override
+ public String getId() {
+ return this.task.getId();
+ }
+
+ @Override
+ public byte[] get() {
+ return this.returnValue;
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockEvaluatorDescriptor.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockEvaluatorDescriptor.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockEvaluatorDescriptor.java
new file mode 100644
index 0000000..4341766
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockEvaluatorDescriptor.java
@@ -0,0 +1,64 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.catalog.NodeDescriptor;
+import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
+import org.apache.reef.driver.evaluator.EvaluatorProcess;
+
+/**
+ * mock evaluator descriptor.
+ */
+@Unstable
+@Private
+public final class MockEvaluatorDescriptor implements EvaluatorDescriptor {
+ private final NodeDescriptor nodeDescriptor;
+
+ MockEvaluatorDescriptor(final NodeDescriptor nodeDescriptor) {
+ this.nodeDescriptor = nodeDescriptor;
+ }
+
+ @Override
+ public NodeDescriptor getNodeDescriptor() {
+ return this.nodeDescriptor;
+ }
+
+ @Override
+ public EvaluatorProcess getProcess() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public int getMemory() {
+ return 0;
+ }
+
+ @Override
+ public int getNumberOfCores() {
+ return 1;
+ }
+
+ @Override
+ public String getRuntimeName() {
+ return "mock";
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockEvaluatorRequestor.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockEvaluatorRequestor.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockEvaluatorRequestor.java
new file mode 100644
index 0000000..ec80422
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockEvaluatorRequestor.java
@@ -0,0 +1,85 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.catalog.NodeDescriptor;
+import org.apache.reef.driver.evaluator.EvaluatorRequest;
+import org.apache.reef.driver.evaluator.EvaluatorRequestor;
+import org.apache.reef.mock.driver.request.AllocateEvaluator;
+import org.apache.reef.tang.InjectionFuture;
+
+import javax.inject.Inject;
+import java.util.UUID;
+
+/**
+ * mock evaluator requestor.
+ */
+@Unstable
+@Private
+public final class MockEvaluatorRequestor implements EvaluatorRequestor {
+
+ private final InjectionFuture<MockRuntimeDriver> mockRuntimeDriver;
+
+ private final InjectionFuture<MockClock> clock;
+
+ @Inject
+ MockEvaluatorRequestor(
+ final InjectionFuture<MockClock> clock,
+ final InjectionFuture<MockRuntimeDriver> mockRuntimeDriver) {
+ this.clock = clock;
+ this.mockRuntimeDriver = mockRuntimeDriver;
+ }
+
+ @Override
+ public void submit(final EvaluatorRequest req) {
+ if (this.clock.get().isClosed()) {
+ throw new IllegalStateException("clock closed");
+ }
+ final NodeDescriptor nodeDescriptor = new MockNodeDescriptor();
+ final MockEvaluatorDescriptor evaluatorDescriptor = new MockEvaluatorDescriptor(nodeDescriptor);
+ for (int i = 0; i < req.getNumber(); i++) {
+ final MockAllocatedEvaluator mockEvaluator = new MockAllocatedEvaluator(
+ this.mockRuntimeDriver.get(), UUID.randomUUID().toString(), evaluatorDescriptor);
+ this.mockRuntimeDriver.get().add(new AllocateEvaluator(mockEvaluator));
+ }
+ }
+
+ @Override
+ public Builder newRequest() {
+ if (this.clock.get().isClosed()) {
+ throw new IllegalStateException("clock closed");
+ }
+ return new Builder();
+ }
+
+
+ /**
+ * {@link EvaluatorRequest.Builder} extended with a new submit method.
+ * {@link EvaluatorRequest}s are built using this builder.
+ */
+ private final class Builder extends EvaluatorRequest.Builder<Builder> {
+ @Override
+ public void submit() {
+ MockEvaluatorRequestor.this.submit(this.build());
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockFailedContext.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockFailedContext.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockFailedContext.java
new file mode 100644
index 0000000..d57cc2c
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockFailedContext.java
@@ -0,0 +1,93 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.context.FailedContext;
+import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
+import org.apache.reef.util.Optional;
+
+/**
+ * mock failed context.
+ */
+@Unstable
+@Private
+public final class MockFailedContext implements FailedContext {
+
+ private final MockActiveContext context;
+
+ public MockFailedContext(final MockActiveContext context) {
+ this.context = context;
+ }
+
+ @Override
+ public Optional<ActiveContext> getParentContext() {
+ return this.context.getParentContext().isPresent() ?
+ Optional.of((ActiveContext)this.context.getParentContext().get()) :
+ Optional.<ActiveContext>empty();
+ }
+
+ @Override
+ public String getMessage() {
+ return "mock";
+ }
+
+ @Override
+ public Optional<String> getDescription() {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<Throwable> getReason() {
+ return Optional.empty();
+ }
+
+ @Override
+ public Optional<byte[]> getData() {
+ return Optional.empty();
+ }
+
+ @Override
+ public Throwable asError() {
+ return new Exception("mock");
+ }
+
+ @Override
+ public String getEvaluatorId() {
+ return this.context.getEvaluatorId();
+ }
+
+ @Override
+ public Optional<String> getParentId() {
+ return this.context.getParentId();
+ }
+
+ @Override
+ public EvaluatorDescriptor getEvaluatorDescriptor() {
+ return this.context.getEvaluatorDescriptor();
+ }
+
+ @Override
+ public String getId() {
+ return this.context.getId();
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockFailedEvaluator.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockFailedEvaluator.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockFailedEvaluator.java
new file mode 100644
index 0000000..aed9c24
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockFailedEvaluator.java
@@ -0,0 +1,79 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.context.FailedContext;
+import org.apache.reef.driver.evaluator.FailedEvaluator;
+import org.apache.reef.driver.task.FailedTask;
+import org.apache.reef.exception.EvaluatorException;
+import org.apache.reef.util.Optional;
+
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * mock failed evaluator.
+ */
+@Unstable
+@Private
+public final class MockFailedEvaluator implements FailedEvaluator {
+
+ private final String evaluatorID;
+
+ private final List<FailedContext> failedContextList;
+
+ private final Optional<FailedTask> failedTask;
+
+ public MockFailedEvaluator(
+ final String evaluatorID,
+ final List<FailedContext> failedContextList,
+ final Optional<FailedTask> failedTask) {
+ this.evaluatorID = evaluatorID;
+ this.failedContextList = failedContextList;
+ this.failedTask = failedTask;
+ }
+
+ public MockFailedEvaluator(final String evaluatorID) {
+ this.evaluatorID = evaluatorID;
+ this.failedContextList = new ArrayList<>();
+ this.failedTask = Optional.empty();
+ }
+
+ @Override
+ public EvaluatorException getEvaluatorException() {
+ return null;
+ }
+
+ @Override
+ public List<FailedContext> getFailedContextList() {
+ return this.failedContextList;
+ }
+
+ @Override
+ public Optional<FailedTask> getFailedTask() {
+ return this.failedTask;
+ }
+
+ @Override
+ public String getId() {
+ return this.evaluatorID;
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockNodeDescriptor.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockNodeDescriptor.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockNodeDescriptor.java
new file mode 100644
index 0000000..8278a29
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockNodeDescriptor.java
@@ -0,0 +1,68 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.catalog.NodeDescriptor;
+import org.apache.reef.driver.catalog.RackDescriptor;
+
+import java.net.InetSocketAddress;
+import java.util.ArrayList;
+import java.util.List;
+
+/**
+ * mock node descriptor.
+ */
+@Unstable
+@Private
+public final class MockNodeDescriptor implements NodeDescriptor {
+ @Override
+ public InetSocketAddress getInetSocketAddress() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public RackDescriptor getRackDescriptor() {
+ return new RackDescriptor() {
+ @Override
+ public List<NodeDescriptor> getNodes() {
+ final List<NodeDescriptor> nodes = new ArrayList<>();
+ nodes.add(MockNodeDescriptor.this);
+ return nodes;
+ }
+
+ @Override
+ public String getName() {
+ return "mock";
+ }
+ };
+ }
+
+ @Override
+ public String getName() {
+ return "mock";
+ }
+
+ @Override
+ public String getId() {
+ return "mock";
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockRunningTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockRunningTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockRunningTask.java
new file mode 100644
index 0000000..711635a
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockRunningTask.java
@@ -0,0 +1,97 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.task.RunningTask;
+import org.apache.reef.mock.driver.request.CloseTask;
+import org.apache.reef.mock.driver.request.SendMessageDriverToTask;
+import org.apache.reef.mock.driver.request.SuspendTask;
+import org.apache.reef.runtime.common.driver.task.TaskRepresenter;
+import org.apache.reef.util.Optional;
+
+/**
+ * mock running task.
+ */
+@Unstable
+@Private
+public final class MockRunningTask implements RunningTask {
+
+ private final MockRuntimeDriver mockRuntimeDriver;
+
+ private final String taskID;
+
+ private final ActiveContext context;
+
+ MockRunningTask(
+ final MockRuntimeDriver mockRuntimeDriver,
+ final String taskID,
+ final ActiveContext context) {
+ this.mockRuntimeDriver = mockRuntimeDriver;
+ this.taskID = taskID;
+ this.context = context;
+ }
+
+ public String evaluatorID() {
+ return this.context.getEvaluatorId();
+ }
+
+ @Override
+ public ActiveContext getActiveContext() {
+ return this.context;
+ }
+
+ @Override
+ public void send(final byte[] message) {
+ this.mockRuntimeDriver.add(new SendMessageDriverToTask(this, message));
+ }
+
+ @Override
+ public void suspend(final byte[] message) {
+ this.mockRuntimeDriver.add(new SuspendTask(this, Optional.of(message)));
+ }
+
+ @Override
+ public void suspend() {
+ this.mockRuntimeDriver.add(new SuspendTask(this, Optional.<byte[]>empty()));
+ }
+
+ @Override
+ public void close(final byte[] message) {
+ this.mockRuntimeDriver.add(new CloseTask(this, this.mockRuntimeDriver.getTaskReturnValueProvider()));
+ }
+
+ @Override
+ public void close() {
+ this.mockRuntimeDriver.add(new CloseTask(this, this.mockRuntimeDriver.getTaskReturnValueProvider()));
+ }
+
+ @Override
+ public TaskRepresenter getTaskRepresenter() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getId() {
+ return this.taskID;
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockRuntimeDriver.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockRuntimeDriver.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockRuntimeDriver.java
new file mode 100644
index 0000000..42ea28d
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockRuntimeDriver.java
@@ -0,0 +1,522 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.context.ContextMessage;
+import org.apache.reef.driver.context.FailedContext;
+import org.apache.reef.driver.evaluator.AllocatedEvaluator;
+import org.apache.reef.driver.evaluator.CompletedEvaluator;
+import org.apache.reef.driver.evaluator.FailedEvaluator;
+import org.apache.reef.driver.parameters.*;
+import org.apache.reef.driver.restart.DriverRestartCompleted;
+import org.apache.reef.driver.restart.DriverRestarted;
+import org.apache.reef.driver.task.*;
+import org.apache.reef.io.Tuple;
+import org.apache.reef.mock.driver.MockDriverRestartContext;
+import org.apache.reef.mock.driver.MockRuntime;
+import org.apache.reef.mock.driver.MockTaskReturnValueProvider;
+import org.apache.reef.mock.driver.ProcessRequest;
+import org.apache.reef.mock.driver.request.*;
+import org.apache.reef.tang.InjectionFuture;
+import org.apache.reef.tang.annotations.Parameter;
+import org.apache.reef.util.Optional;
+import org.apache.reef.wake.EventHandler;
+import org.apache.reef.wake.time.Clock;
+import org.apache.reef.wake.time.event.StartTime;
+import org.apache.reef.wake.time.event.StopTime;
+
+import javax.inject.Inject;
+import java.util.*;
+
+/**
+ * mock runtime driver.
+ */
+@Unstable
+@Private
+public final class MockRuntimeDriver implements MockRuntime {
+
+ private final InjectionFuture<MockClock> clock;
+
+ private final List<ProcessRequest> processRequestQueue = new ArrayList<>();
+
+ private final Set<EventHandler<StartTime>> driverStartHandlers;
+
+ private final Set<EventHandler<StopTime>> driverStopHandlers;
+
+ private final Set<EventHandler<AllocatedEvaluator>> allocatedEvaluatorHandlers;
+
+ private final Set<EventHandler<CompletedEvaluator>> completedEvaluatorHandlers;
+
+ private final Set<EventHandler<FailedEvaluator>> failedEvaluatorHandlers;
+
+ private final Set<EventHandler<TaskRunningHandlers>> taskRunningHandlers;
+
+ private final Set<EventHandler<FailedTask>> taskFailedHandlers;
+
+ private final Set<EventHandler<TaskMessage>> taskMessageHandlers;
+
+ private final Set<EventHandler<CompletedTask>> taskCompletedHandlers;
+
+ private final Set<EventHandler<SuspendedTask>> taskSuspendedHandlers;
+
+ private final Set<EventHandler<ActiveContext>> contextActiveHandlers;
+
+ private final Set<EventHandler<CloseContext>> contextClosedHandlers;
+
+ private final Set<EventHandler<ContextMessage>> contextMessageHandlers;
+
+ private final Set<EventHandler<FailedContext>> contextFailedHandlers;
+
+ private final Set<EventHandler<DriverRestarted>> driverRestartHandlers;
+
+ private final Set<EventHandler<RunningTask>> driverRestartRunningTaskHandlers;
+
+ private final Set<EventHandler<ActiveContext>> driverRestartActiveContextHandlers;
+
+ private final Set<EventHandler<DriverRestartCompleted>> driverRestartCompletedHandlers;
+
+ private final Set<EventHandler<FailedEvaluator>> driverRestartFailedEvaluatorHandlers;
+
+ private final Map<String, MockAllocatedEvaluator> allocatedEvaluatorMap = new HashMap<>();
+
+ private final Map<String, List<MockActiveContext>> allocatedContextsMap = new HashMap<>();
+
+ private final Map<String, MockRunningTask> runningTasks = new HashMap<>();
+
+ private final MockTaskReturnValueProvider taskReturnValueProvider;
+
+ @Inject
+ MockRuntimeDriver(
+ final InjectionFuture<MockClock> clock,
+ final MockTaskReturnValueProvider taskReturnValueProvider,
+ @Parameter(DriverStartHandler.class) final Set<EventHandler<StartTime>> driverStartHandlers,
+ @Parameter(Clock.StopHandler.class) final Set<EventHandler<StopTime>> driverStopHandlers,
+ @Parameter(EvaluatorAllocatedHandlers.class) final Set<EventHandler<AllocatedEvaluator>>
+ allocatedEvaluatorHandlers,
+ @Parameter(EvaluatorCompletedHandlers.class) final Set<EventHandler<CompletedEvaluator>>
+ completedEvaluatorHandlers,
+ @Parameter(EvaluatorFailedHandlers.class) final Set<EventHandler<FailedEvaluator>> failedEvaluatorHandlers,
+ @Parameter(TaskRunningHandlers.class) final Set<EventHandler<TaskRunningHandlers>> taskRunningHandlers,
+ @Parameter(TaskFailedHandlers.class) final Set<EventHandler<FailedTask>> taskFailedHandlers,
+ @Parameter(TaskMessageHandlers.class) final Set<EventHandler<TaskMessage>> taskMessageHandlers,
+ @Parameter(TaskCompletedHandlers.class) final Set<EventHandler<CompletedTask>> taskCompletedHandlers,
+ @Parameter(TaskSuspendedHandlers.class) final Set<EventHandler<SuspendedTask>> taskSuspendedHandlers,
+ @Parameter(ContextActiveHandlers.class) final Set<EventHandler<ActiveContext>> contextActiveHandlers,
+ @Parameter(ContextClosedHandlers.class) final Set<EventHandler<CloseContext>> contextClosedHandlers,
+ @Parameter(ContextMessageHandlers.class) final Set<EventHandler<ContextMessage>> contextMessageHandlers,
+ @Parameter(ContextFailedHandlers.class) final Set<EventHandler<FailedContext>> contextFailedHandlers,
+ @Parameter(DriverRestartHandler.class) final Set<EventHandler<DriverRestarted>>
+ driverRestartHandlers,
+ @Parameter(DriverRestartTaskRunningHandlers.class) final Set<EventHandler<RunningTask>>
+ driverRestartRunningTaskHandlers,
+ @Parameter(DriverRestartContextActiveHandlers.class) final Set<EventHandler<ActiveContext>>
+ driverRestartActiveContextHandlers,
+ @Parameter(DriverRestartCompletedHandlers.class) final Set<EventHandler<DriverRestartCompleted>>
+ driverRestartCompletedHandlers,
+ @Parameter(DriverRestartFailedEvaluatorHandlers.class) final Set<EventHandler<FailedEvaluator>>
+ driverRestartFailedEvaluatorHandlers){
+ this.clock = clock;
+ this.taskReturnValueProvider = taskReturnValueProvider;
+ this.driverStartHandlers = driverStartHandlers;
+ this.driverStopHandlers = driverStopHandlers;
+ this.allocatedEvaluatorHandlers = allocatedEvaluatorHandlers;
+ this.completedEvaluatorHandlers = completedEvaluatorHandlers;
+ this.failedEvaluatorHandlers = failedEvaluatorHandlers;
+ this.taskRunningHandlers = taskRunningHandlers;
+ this.taskFailedHandlers = taskFailedHandlers;
+ this.taskMessageHandlers = taskMessageHandlers;
+ this.taskCompletedHandlers = taskCompletedHandlers;
+ this.taskSuspendedHandlers = taskSuspendedHandlers;
+ this.contextActiveHandlers = contextActiveHandlers;
+ this.contextClosedHandlers = contextClosedHandlers;
+ this.contextMessageHandlers = contextMessageHandlers;
+ this.contextFailedHandlers = contextFailedHandlers;
+ this.driverRestartHandlers = driverRestartHandlers;
+ this.driverRestartRunningTaskHandlers = driverRestartRunningTaskHandlers;
+ this.driverRestartActiveContextHandlers = driverRestartActiveContextHandlers;
+ this.driverRestartCompletedHandlers = driverRestartCompletedHandlers;
+ this.driverRestartFailedEvaluatorHandlers = driverRestartFailedEvaluatorHandlers;
+ }
+
+ @Override
+ public Collection<AllocatedEvaluator> getCurrentAllocatedEvaluators() {
+ if (this.clock.get().isClosed()) {
+ throw new IllegalStateException("clock is closed");
+ }
+ return new ArrayList<AllocatedEvaluator>(this.allocatedEvaluatorMap.values());
+ }
+
+ @Override
+ public void fail(final AllocatedEvaluator evaluator) {
+ if (this.clock.get().isClosed()) {
+ throw new IllegalStateException("clock is closed");
+ }
+ if (this.allocatedEvaluatorMap.remove(evaluator.getId()) == null) {
+ throw new IllegalStateException("unknown evaluator " + evaluator);
+ }
+ FailedTask failedTask = null;
+ if (this.runningTasks.containsKey(evaluator.getId())) {
+ final RunningTask task = this.runningTasks.remove(evaluator.getId());
+ failedTask = new FailedTask(
+ task.getId(),
+ "mock",
+ Optional.<String>empty(),
+ Optional.<Throwable>empty(),
+ Optional.<byte[]>empty(),
+ Optional.<ActiveContext>of(task.getActiveContext()));
+ }
+ final List<FailedContext> failedContexts = new ArrayList<>();
+ for (final MockActiveContext context : this.allocatedContextsMap.get(evaluator.getId())) {
+ failedContexts.add(new MockFailedContext(context));
+ }
+ post(this.failedEvaluatorHandlers, new MockFailedEvaluator(
+ evaluator.getId(), failedContexts, Optional.ofNullable(failedTask)));
+ }
+
+ @Override
+ public Collection<ActiveContext> getCurrentActiveContexts() {
+ if (this.clock.get().isClosed()) {
+ throw new IllegalStateException("clock is closed");
+ }
+ final List<ActiveContext> currentActiveContexts = new ArrayList<>();
+ for (final List<MockActiveContext> contexts : this.allocatedContextsMap.values()) {
+ currentActiveContexts.addAll(contexts);
+ }
+ return currentActiveContexts;
+ }
+
+ @Override
+ public void fail(final ActiveContext context) {
+ if (this.clock.get().isClosed()) {
+ throw new IllegalStateException("clock is closed");
+ }
+
+ final MockAllocatedEvaluator evaluator = ((MockActiveContext) context).getEvaluator();
+
+ // Root context failure shuts evaluator down.
+ if (!((MockActiveContext) context).getParentContext().isPresent()) {
+ allocatedEvaluatorMap.remove(evaluator.getId());
+ post(this.completedEvaluatorHandlers, new CompletedEvaluator() {
+ @Override
+ public String getId() {
+ return evaluator.getId();
+ }
+ });
+ }
+
+ this.allocatedContextsMap.get(evaluator.getId()).remove(context);
+ post(this.contextFailedHandlers, new MockFailedContext((MockActiveContext) context));
+ }
+
+ @Override
+ public Collection<RunningTask> getCurrentRunningTasks() {
+ if (this.clock.get().isClosed()) {
+ throw new IllegalStateException("clock is closed");
+ }
+ return new ArrayList<RunningTask>(this.runningTasks.values());
+ }
+
+ @Override
+ public void fail(final RunningTask task) {
+ if (this.clock.get().isClosed()) {
+ throw new IllegalStateException("clock is closed");
+ }
+ final String evaluatorID = task.getActiveContext().getEvaluatorId();
+ if (this.runningTasks.containsKey(evaluatorID) &&
+ this.runningTasks.get(evaluatorID).equals(task)) {
+ this.runningTasks.remove(evaluatorID);
+ post(taskFailedHandlers, new FailedTask(
+ task.getId(),
+ "mock",
+ Optional.<String>empty(),
+ Optional.<Throwable>empty(),
+ Optional.<byte[]>empty(),
+ Optional.of(task.getActiveContext())));
+ } else {
+ throw new IllegalStateException("unknown running task " + task);
+ }
+ }
+
+ @Override
+ public void start() {
+ post(this.driverStartHandlers, new StartTime(this.clock.get().getCurrentTime()));
+ }
+
+ @Override
+ public void stop() {
+ post(this.driverStopHandlers, new StopTime(this.clock.get().getCurrentTime()));
+ }
+
+ @Override
+ public void restart(final MockDriverRestartContext restartContext, final boolean isTimeout, final long duration) {
+ post(this.driverRestartHandlers, restartContext.getDriverRestarted());
+ for (final RunningTask runningTask : restartContext.getRunningTasks()) {
+ post(this.driverRestartRunningTaskHandlers, runningTask);
+ }
+ for (final ActiveContext activeContext : restartContext.getIdleActiveContexts()) {
+ post(this.driverRestartActiveContextHandlers, activeContext);
+ }
+ post(this.driverRestartCompletedHandlers, restartContext.getDriverRestartCompleted(isTimeout, duration));
+ for (final FailedEvaluator failedEvaluator : restartContext.getFailedEvaluators()) {
+ post(this.driverRestartFailedEvaluatorHandlers, failedEvaluator);
+ }
+ }
+
+ @Override
+ public MockDriverRestartContext failDriver(final int attempt, final StartTime startTime) {
+ final List<MockActiveContext> activeContexts = new ArrayList<>();
+ for (final List<MockActiveContext> contexts : this.allocatedContextsMap.values()) {
+ if (contexts.size() > 0) {
+ activeContexts.add(contexts.get(contexts.size() - 1));
+ }
+ }
+ return new MockDriverRestartContext(
+ attempt,
+ startTime,
+ new ArrayList<>(this.allocatedEvaluatorMap.values()),
+ activeContexts,
+ new ArrayList<>(this.runningTasks.values()));
+ }
+
+ @Override
+ public boolean hasProcessRequest() {
+ return this.processRequestQueue.size() > 0;
+ }
+
+ @Override
+ public ProcessRequest getNextProcessRequest() {
+ if (this.processRequestQueue.size() > 0) {
+ return this.processRequestQueue.remove(0);
+ } else {
+ return null;
+ }
+ }
+
+ @Override
+ public void succeed(final ProcessRequest pr) {
+ if (this.clock.get().isClosed()) {
+ throw new IllegalStateException("clock is closed");
+ }
+ final ProcessRequestInternal request = (ProcessRequestInternal) pr;
+ switch (request.getType()) {
+ case ALLOCATE_EVALUATOR:
+ final MockAllocatedEvaluator allocatedEvaluator = ((AllocateEvaluator)request).getSuccessEvent();
+ validateAndCreate(allocatedEvaluator);
+ post(this.allocatedEvaluatorHandlers, allocatedEvaluator);
+ break;
+ case CLOSE_EVALUATOR:
+ final CompletedEvaluator closedEvaluator = ((CloseEvaluator)request).getSuccessEvent();
+ validateAndClose(closedEvaluator);
+ post(this.completedEvaluatorHandlers, closedEvaluator);
+ break;
+ case CREATE_CONTEXT:
+ final MockActiveContext createContext = ((CreateContext) request).getSuccessEvent();
+ validateAndCreate(createContext);
+ post(this.contextActiveHandlers, createContext);
+ break;
+ case CLOSE_CONTEXT:
+ final MockClosedContext closeContext = ((CloseContext) request).getSuccessEvent();
+ validateAndClose(closeContext);
+ post(this.contextClosedHandlers, closeContext);
+ break;
+ case CREATE_TASK:
+ final MockRunningTask createTask = ((CreateTask)request).getSuccessEvent();
+ validateAndCreate(createTask);
+ post(this.taskRunningHandlers, request.getSuccessEvent());
+ break;
+ case SUSPEND_TASK:
+ final MockRunningTask suspendedTask = ((SuspendTask)request).getTask();
+ validateAndClose(suspendedTask);
+ post(this.taskSuspendedHandlers, request.getSuccessEvent());
+ break;
+ case CLOSE_TASK:
+ case COMPLETE_TASK:
+ final MockRunningTask completedTask = ((CompleteTask)request).getTask();
+ validateAndClose(completedTask);
+ post(this.taskCompletedHandlers, request.getSuccessEvent());
+ break;
+ case CREATE_CONTEXT_AND_TASK:
+ final CreateContextAndTask createContextTask = (CreateContextAndTask) request;
+ final Tuple<MockActiveContext, MockRunningTask> events = createContextTask.getSuccessEvent();
+ validateAndCreate(events.getKey());
+ post(this.contextActiveHandlers, events.getKey());
+ validateAndCreate(events.getValue());
+ post(this.taskRunningHandlers, events.getValue());
+ break;
+ case SEND_MESSAGE_DRIVER_TO_TASK:
+ // ignore
+ break;
+ case SEND_MESSAGE_DRIVER_TO_CONTEXT:
+ // ignore
+ break;
+ default:
+ throw new IllegalStateException("unknown type");
+ }
+
+ if (request.doAutoComplete()) {
+ add(request.getCompletionProcessRequest());
+ } else if (!this.clock.get().isClosed() && isIdle()) {
+ this.clock.get().close();
+ }
+ }
+
+ @Override
+ public void fail(final ProcessRequest pr) {
+ if (this.clock.get().isClosed()) {
+ throw new IllegalStateException("clock is closed");
+ }
+ final ProcessRequestInternal request = (ProcessRequestInternal) pr;
+ switch (request.getType()) {
+ case ALLOCATE_EVALUATOR:
+ post(this.failedEvaluatorHandlers, request.getFailureEvent());
+ break;
+ case CLOSE_EVALUATOR:
+ final CompletedEvaluator evaluator = ((CloseEvaluator)request).getSuccessEvent();
+ validateAndClose(evaluator);
+ post(this.failedEvaluatorHandlers, request.getFailureEvent());
+ break;
+ case CREATE_CONTEXT:
+ post(this.contextFailedHandlers, request.getFailureEvent());
+ break;
+ case CLOSE_CONTEXT:
+ final MockClosedContext context = ((CloseContext)request).getSuccessEvent();
+ validateAndClose(context);
+ if (context.getParentContext() == null) {
+ add(new CloseEvaluator(context.getMockActiveContext().getEvaluator()));
+ }
+ post(this.contextFailedHandlers, request.getFailureEvent());
+ break;
+ case CREATE_TASK:
+ post(this.taskFailedHandlers, request.getFailureEvent());
+ break;
+ case SUSPEND_TASK:
+ validateAndClose(((SuspendTask)request).getTask());
+ post(this.taskFailedHandlers, request.getFailureEvent());
+ break;
+ case CLOSE_TASK:
+ case COMPLETE_TASK:
+ validateAndClose(((CloseTask)request).getTask());
+ post(this.taskFailedHandlers, request.getFailureEvent());
+ break;
+ case CREATE_CONTEXT_AND_TASK:
+ final CreateContextAndTask createContextTask = (CreateContextAndTask) request;
+ final Tuple<MockFailedContext, FailedTask> events = createContextTask.getFailureEvent();
+ post(this.taskFailedHandlers, events.getValue());
+ post(this.contextFailedHandlers, events.getKey());
+ break;
+ case SEND_MESSAGE_DRIVER_TO_TASK:
+ // ignore
+ break;
+ case SEND_MESSAGE_DRIVER_TO_CONTEXT:
+ // ignore
+ break;
+ default:
+ throw new IllegalStateException("unknown type");
+ }
+
+ if (!this.clock.get().isClosed() && isIdle()) {
+ this.clock.get().close();
+ }
+ }
+
+ @Override
+ public void publish(final ContextMessage contextMessage) {
+ for (final EventHandler<ContextMessage> handler : this.contextMessageHandlers) {
+ handler.onNext(contextMessage);
+ }
+ }
+
+ MockTaskReturnValueProvider getTaskReturnValueProvider() {
+ return this.taskReturnValueProvider;
+ }
+ /**
+ * Used by mock REEF entities (e.g., AllocatedEvaluator, RunningTask) to inject
+ * process requests from initiated actions e.g., RunningTask.close().
+ * @param request to inject
+ */
+ void add(final ProcessRequest request) {
+ this.processRequestQueue.add(request);
+ }
+
+ private boolean isIdle() {
+ return this.clock.get().isIdle() &&
+ this.processRequestQueue.isEmpty() &&
+ this.allocatedEvaluatorMap.isEmpty();
+ }
+
+ private <T> void post(final Set<EventHandler<T>> handlers, final Object event) {
+ for (final EventHandler<T> handler : handlers) {
+ handler.onNext((T) event);
+ }
+ }
+
+ private void validateAndCreate(final MockActiveContext context) {
+ if (!this.allocatedEvaluatorMap.containsKey(context.getEvaluatorId())) {
+ throw new IllegalStateException("unknown evaluator id " + context.getEvaluatorId());
+ } else if (!this.allocatedContextsMap.containsKey(context.getEvaluatorId())) {
+ this.allocatedContextsMap.put(context.getEvaluatorId(), new ArrayList<MockActiveContext>());
+ }
+ this.allocatedContextsMap.get(context.getEvaluatorId()).add(context);
+ }
+
+ private void validateAndClose(final MockClosedContext context) {
+ if (!this.allocatedContextsMap.containsKey(context.getEvaluatorId())) {
+ throw new IllegalStateException("unknown evaluator id " + context.getEvaluatorId());
+ }
+ final List<MockActiveContext> contexts = this.allocatedContextsMap.get(context.getEvaluatorId());
+ if (!contexts.get(contexts.size() - 1).equals(context.getMockActiveContext())) {
+ throw new IllegalStateException("closing context that is not on the top of the stack");
+ }
+ contexts.remove(context.getMockActiveContext());
+ }
+
+ private void validateAndCreate(final MockRunningTask task) {
+ if (this.runningTasks.containsKey(task.evaluatorID())) {
+ throw new IllegalStateException("task already running on evaluator " +
+ task.evaluatorID());
+ }
+ this.runningTasks.put(task.evaluatorID(), task);
+ }
+
+ private void validateAndClose(final MockRunningTask task) {
+ if (!this.runningTasks.containsKey(task.getActiveContext().getEvaluatorId())) {
+ throw new IllegalStateException("no task running on evaluator");
+ }
+ this.runningTasks.remove(task.getActiveContext().getEvaluatorId());
+ }
+
+ private void validateAndCreate(final MockAllocatedEvaluator evaluator) {
+ if (this.allocatedEvaluatorMap.containsKey(evaluator.getId())) {
+ throw new IllegalStateException("evaluator id " + evaluator.getId() + " already exists");
+ }
+ this.allocatedEvaluatorMap.put(evaluator.getId(), evaluator);
+ this.allocatedContextsMap.put(evaluator.getId(), new ArrayList<MockActiveContext>());
+ }
+
+ private void validateAndClose(final CompletedEvaluator evaluator) {
+ if (!this.allocatedEvaluatorMap.containsKey(evaluator.getId())) {
+ throw new IllegalStateException("unknown evaluator id " + evaluator.getId());
+ }
+ this.allocatedEvaluatorMap.remove(evaluator.getId());
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockSuspendedTask.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockSuspendedTask.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockSuspendedTask.java
new file mode 100644
index 0000000..9f25ae32
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockSuspendedTask.java
@@ -0,0 +1,54 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.task.SuspendedTask;
+
+/**
+ * mock suspended task.
+ */
+@Unstable
+@Private
+public final class MockSuspendedTask implements SuspendedTask {
+
+ private final MockRunningTask task;
+
+ public MockSuspendedTask(final MockRunningTask task) {
+ this.task = task;
+ }
+
+ @Override
+ public ActiveContext getActiveContext() {
+ return this.task.getActiveContext();
+ }
+
+ @Override
+ public byte[] get() {
+ return new byte[0];
+ }
+
+ @Override
+ public String getId() {
+ return this.task.getId();
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockUtils.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockUtils.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockUtils.java
new file mode 100644
index 0000000..23d3b46
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/MockUtils.java
@@ -0,0 +1,48 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver.runtime;
+
+import org.apache.reef.annotations.Unstable;
+import org.apache.reef.annotations.audience.Private;
+import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.Injector;
+import org.apache.reef.tang.Tang;
+import org.apache.reef.tang.annotations.Name;
+import org.apache.reef.tang.exceptions.InjectionException;
+
+/**
+ * mock utilities.
+ */
+@Unstable
+@Private
+final class MockUtils {
+
+ private MockUtils() {
+ }
+
+ public static <U, T extends Name<U>> U getValue(final Configuration configuration, final Class<T> name) {
+ try {
+ final Injector injector = Tang.Factory.getTang().newInjector(configuration);
+ return injector.getNamedInstance(name);
+ } catch (InjectionException e) {
+ throw new IllegalStateException(e);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/package-info.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/package-info.java
new file mode 100644
index 0000000..50e7eac
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/driver/runtime/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+/**
+ * mock runtime implementation.
+ */
+package org.apache.reef.mock.driver.runtime;
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/package-info.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/package-info.java
deleted file mode 100644
index fdda864..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/package-info.java
+++ /dev/null
@@ -1,40 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-/**
- * Mock runtime API.
- *
- * Mock runtime is meant to mimic the semantics of the REEF runtime and
- * allow:
- * 1. Applications to driver the forward progress of processing REEF events.
- * See {@link org.apache.reef.mock.MockRuntime} API
- * 2. Control the advancement of the Clock and Alarm callbacks.
- * See {@link org.apache.reef.mock.runtime.MockClock}
- * 3. Inject failures into the REEF applications.
- * See {@link org.apache.reef.mock.MockFailure}
- *
- * Use {@link org.apache.reef.mock.MockConfiguration} to bind your REEF
- * driver application event handlers.
- *
- * Use {@link org.apache.reef.mock.MockRuntime#start()} to trigger the
- * driver start event and {@link org.apache.reef.mock.MockRuntime#stop()}}
- * or {@link org.apache.reef.mock.runtime.MockClock#close()} to trigger the driver
- * stop event.
- */
-package org.apache.reef.mock;
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/AllocateEvaluator.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/AllocateEvaluator.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/AllocateEvaluator.java
deleted file mode 100644
index 9d6e400..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/AllocateEvaluator.java
+++ /dev/null
@@ -1,72 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.request;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.evaluator.FailedEvaluator;
-import org.apache.reef.mock.ProcessRequest;
-import org.apache.reef.mock.runtime.MockAllocatedEvalautor;
-import org.apache.reef.mock.runtime.MockFailedEvaluator;
-
-/**
- * Allocate Evaluator process request.
- */
-@Unstable
-@Private
-public final class AllocateEvaluator implements
- ProcessRequestInternal<MockAllocatedEvalautor, FailedEvaluator> {
-
- private final MockAllocatedEvalautor evaluator;
-
- public AllocateEvaluator(final MockAllocatedEvalautor evaluator) {
- this.evaluator = evaluator;
- }
-
- @Override
- public Type getType() {
- return Type.ALLOCATE_EVALUATOR;
- }
-
- @Override
- public MockAllocatedEvalautor getSuccessEvent() {
- return this.evaluator;
- }
-
- @Override
- public FailedEvaluator getFailureEvent() {
- return new MockFailedEvaluator(evaluator.getId());
- }
-
- @Override
- public boolean doAutoComplete() {
- return false;
- }
-
- @Override
- public void setAutoComplete(final boolean value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ProcessRequest getCompletionProcessRequest() {
- throw new UnsupportedOperationException();
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CloseContext.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CloseContext.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CloseContext.java
deleted file mode 100644
index 00bdf3c..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CloseContext.java
+++ /dev/null
@@ -1,76 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.request;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.context.ClosedContext;
-import org.apache.reef.driver.context.FailedContext;
-import org.apache.reef.mock.AutoCompletable;
-import org.apache.reef.mock.ProcessRequest;
-import org.apache.reef.mock.runtime.MockActiveContext;
-import org.apache.reef.mock.runtime.MockClosedContext;
-import org.apache.reef.mock.runtime.MockFailedContext;
-
-/**
- * close context process request.
- */
-@Unstable
-@Private
-public final class CloseContext implements
- ProcessRequestInternal<ClosedContext, FailedContext>,
- AutoCompletable {
-
- private final MockActiveContext context;
-
- public CloseContext(final MockActiveContext context) {
- this.context = context;
- }
-
- @Override
- public Type getType() {
- return Type.CLOSE_CONTEXT;
- }
-
- @Override
- public MockClosedContext getSuccessEvent() {
- return new MockClosedContext(this.context);
- }
-
- @Override
- public FailedContext getFailureEvent() {
- return new MockFailedContext(this.context);
- }
-
- @Override
- public boolean doAutoComplete() {
- return !this.context.getParentContext().isPresent();
- }
-
- @Override
- public void setAutoComplete(final boolean value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ProcessRequest getCompletionProcessRequest() {
- return new CloseEvaluator(this.context.getEvaluator());
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CloseEvaluator.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CloseEvaluator.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CloseEvaluator.java
deleted file mode 100644
index 6ef2b9f..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/request/CloseEvaluator.java
+++ /dev/null
@@ -1,78 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.request;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.driver.evaluator.CompletedEvaluator;
-import org.apache.reef.driver.evaluator.FailedEvaluator;
-import org.apache.reef.mock.ProcessRequest;
-import org.apache.reef.mock.runtime.MockAllocatedEvalautor;
-import org.apache.reef.mock.runtime.MockFailedEvaluator;
-
-/**
- * close evaluator request.
- */
-@Unstable
-@Private
-public final class CloseEvaluator implements ProcessRequestInternal<CompletedEvaluator, FailedEvaluator> {
-
- private final MockAllocatedEvalautor evaluator;
-
- public CloseEvaluator(final MockAllocatedEvalautor evaluator) {
- this.evaluator = evaluator;
- }
-
- @Override
- public Type getType() {
- return Type.CLOSE_EVALUATOR;
- }
-
- @Override
- public CompletedEvaluator getSuccessEvent() {
- return new CompletedEvaluator() {
- @Override
- public String getId() {
- return evaluator.getId();
- }
- };
- }
-
- @Override
- public FailedEvaluator getFailureEvent() {
- // TODO[initialize remaining failed contstructer fields]
- return new MockFailedEvaluator(evaluator.getId());
- }
-
- @Override
- public boolean doAutoComplete() {
- return false;
- }
-
- @Override
- public void setAutoComplete(final boolean value) {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public ProcessRequest getCompletionProcessRequest() {
- throw new UnsupportedOperationException();
- }
-}
[06/16] reef git commit: [REEF-1965] Implement .NET REEF runtime for
Azure Batch
Posted by do...@apache.org.
[REEF-1965] Implement .NET REEF runtime for Azure Batch
This implements
* Azure Batch DotNet client that submits REEF Job to Azure Batch pool.
* HelloReef example runs in Azure Batch.
JIRA: [REEF-1965](https://issues.apache.org/jira/browse/REEF-1965)
Closes #1441
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/8d0bad24
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/8d0bad24
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/8d0bad24
Branch: refs/heads/REEF-335
Commit: 8d0bad248c5421d1d906ce5fc67b0863817c70b0
Parents: aa30e7f
Author: Chenxi Zhao <ch...@microsoft.com>
Authored: Thu Nov 30 17:48:31 2017 -0800
Committer: Sergiy Matusevych <mo...@apache.org>
Committed: Wed Apr 25 15:50:23 2018 -0700
----------------------------------------------------------------------
.../AzureBatchCommandBuilderTests.cs | 56 ++++++++
.../Org.Apache.REEF.Client.Tests.csproj | 1 +
.../AvroAzureBatchJobSubmissionParameters.cs | 107 ++++++++++++++
.../AzureBatch/AzureBatchDotNetClient.cs | 139 +++++++++++++++++++
.../AzureBatchRuntimeClientConfiguration.cs | 78 +++++++++++
.../Parameters/AzureBatchAccountKey.cs | 25 ++++
.../Parameters/AzureBatchAccountName.cs | 26 ++++
.../Parameters/AzureBatchAccountUri.cs | 26 ++++
.../AzureBatch/Parameters/AzureBatchPoolId.cs | 26 ++++
.../Parameters/AzureStorageAccountKey.cs | 26 ++++
.../Parameters/AzureStorageAccountName.cs | 26 ++++
.../Parameters/AzureStorageContainerName.cs | 26 ++++
.../Parameters/DriverStderrFilePath.cs | 29 ++++
.../Parameters/DriverStdoutFilePath.cs | 29 ++++
.../AzureBatch/Parameters/EnableDebugLogging.cs | 28 ++++
.../AzureBatch/Parameters/IsWindows.cs | 26 ++++
.../AzureBatch/Service/AzureBatchService.cs | 138 ++++++++++++++++++
.../AzureBatch/Storage/AzureStorageClient.cs | 109 +++++++++++++++
.../AzureBatch/Util/AbstractCommandBuilder.cs | 72 ++++++++++
.../AzureBatchCommandProviderConfiguration.cs | 42 ++++++
.../AzureBatch/Util/AzureBatchFileNames.cs | 55 ++++++++
.../AzureBatch/Util/ICommandBuilder.cs | 35 +++++
.../AzureBatch/Util/JobJarMaker.cs | 108 ++++++++++++++
.../AzureBatch/Util/LinuxCommandBuilder.cs | 44 ++++++
.../AzureBatch/Util/WindowsCommandBuilder.cs | 52 +++++++
.../Common/DriverFolderPreparationHelper.cs | 46 ++++--
.../Org.Apache.REEF.Client.DotNet.csproj | 1 +
.../Org.Apache.REEF.Client.csproj | 60 +++++++-
lang/cs/Org.Apache.REEF.Client/packages.config | 13 ++
.../Runtime/RuntimeName.cs | 5 +
.../HelloREEF.cs | 25 +++-
.../Org.Apache.REEF.Examples.HelloREEF/Run.cs | 9 +-
lang/cs/Org.Apache.REEF.sln | Bin 37102 -> 37230 bytes
lang/java/reef-bridge-client/pom.xml | 5 +
.../src/main/avro/JobSubmissionParameters.avsc | 17 +++
...zureBatchBootstrapDriverConfigGenerator.java | 85 ++++++++++++
.../client/AzureBatchBootstrapREEFLauncher.java | 135 ++++++++++++++++++
lang/java/reef-bridge-java/pom.xml | 5 +
38 files changed, 1712 insertions(+), 23 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client.Tests/AzureBatchCommandBuilderTests.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/AzureBatchCommandBuilderTests.cs b/lang/cs/Org.Apache.REEF.Client.Tests/AzureBatchCommandBuilderTests.cs
new file mode 100644
index 0000000..489753b
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client.Tests/AzureBatchCommandBuilderTests.cs
@@ -0,0 +1,56 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Client.AzureBatch.Util;
+using Org.Apache.REEF.Tang.Implementations.Tang;
+using Org.Apache.REEF.Tang.Interface;
+using Xunit;
+
+namespace Org.Apache.REEF.Client.Tests
+{
+ public class AzureBatchCommandBuilder
+ {
+ [Fact]
+ public void WindowsCommandBuilderDriverTest()
+ {
+ // Prepare
+ const int driverMemory = 100;
+ AbstractCommandBuilder builder = TestContext.GetWindowsCommandBuilder();
+ string expected = "powershell.exe /c \"Add-Type -AssemblyName System.IO.Compression.FileSystem; " +
+ "[System.IO.Compression.ZipFile]::ExtractToDirectory(\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\local.jar\\\", " +
+ "\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\reef\\\");reef\\Org.Apache.REEF.Bridge.exe " +
+ "java -Xmx100m -XX:PermSize=128m -XX:MaxPermSize=128m -classpath 'reef/local/*;reef/global/*;' " +
+ "-Dproc_reef org.apache.reef.bridge.client.AzureBatchBootstrapREEFLauncher " +
+ "reef\\job-submission-params.json\";";
+
+ // Action
+ string actual = builder.BuildDriverCommand(driverMemory);
+
+ // Assert
+ Assert.Equal(expected, actual);
+ }
+
+ private class TestContext
+ {
+ public static AbstractCommandBuilder GetWindowsCommandBuilder()
+ {
+ IInjector injector = TangFactory.GetTang().NewInjector();
+ return injector.GetInstance<WindowsCommandBuilder>();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj b/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj
index 384e178..0602f70 100644
--- a/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj
+++ b/lang/cs/Org.Apache.REEF.Client.Tests/Org.Apache.REEF.Client.Tests.csproj
@@ -51,6 +51,7 @@ under the License.
<Compile Include="$(SolutionDir)\SharedAssemblyInfo.cs">
<Link>Properties\SharedAssemblyInfo.cs</Link>
</Compile>
+ <Compile Include="AzureBatchCommandBuilderTests.cs" />
<Compile Include="HDInsightYarnClientTests.cs" />
<Compile Include="JobRequestBuilderTests.cs" />
<Compile Include="JobResourceUploaderTests.cs" />
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/Avro/AzureBatch/AvroAzureBatchJobSubmissionParameters.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/Avro/AzureBatch/AvroAzureBatchJobSubmissionParameters.cs b/lang/cs/Org.Apache.REEF.Client/Avro/AzureBatch/AvroAzureBatchJobSubmissionParameters.cs
new file mode 100644
index 0000000..78bfe4a
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/Avro/AzureBatch/AvroAzureBatchJobSubmissionParameters.cs
@@ -0,0 +1,107 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Runtime.Serialization;
+using Org.Apache.REEF.Utilities.Attributes;
+
+namespace Org.Apache.REEF.Client.Avro.AzureBatch
+{
+ /// <summary>
+ /// Used to serialize and deserialize Avro record
+ /// org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters.
+ /// This is a (mostly) auto-generated class.
+ /// For instructions on how to regenerate, please view the README.md in the same folder.
+ /// </summary>
+ [Private]
+ [DataContract(Namespace = "org.apache.reef.reef.bridge.client.avro")]
+ public sealed class AvroAzureBatchJobSubmissionParameters
+ {
+ private const string JsonSchema = @"{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroAzureBatchJobSubmissionParameters"",""doc"":""Job submission parameters used by the Azure Batch runtime"",""fields"":[{""name"":""sharedJobSubmissionParameters"",""type"":{""type"":""record"",""name"":""org.apache.reef.reef.bridge.client.avro.AvroJobSubmissionParameters"",""doc"":""General cross-language job submission parameters shared by all runtimes"",""fields"":[{""name"":""jobId"",""type"":""string""},{""name"":""jobSubmissionFolder"",""type"":""string""}]}},{""name"":""AzureBatchAccountKey"",""type"":""string""},{""name"":""AzureBatchAccountName"",""type"":""string""},{""name"":""AzureBatchAccountUri"",""type"":""string""},{""name"":""AzureBatchPoolId"",""type"":""string""},{""name"":""AzureStorageAccountKey"",""type"":""string""},{""name"":""AzureStorageAccountName"",""type"":""string""},{""name"":""AzureStorageContainerName"",""type"":""string""},{""name"":
""AzureBatchIsWindows"",""type"":""boolean""}]}";
+
+ /// <summary>
+ /// Gets the schema.
+ /// </summary>
+ public static string Schema
+ {
+ get
+ {
+ return JsonSchema;
+ }
+ }
+
+ /// <summary>
+ /// Gets or sets the sharedJobSubmissionParameters field.
+ /// </summary>
+ [DataMember]
+ public AvroJobSubmissionParameters sharedJobSubmissionParameters { get; set; }
+
+ /// <summary>
+ /// Gets or sets the AzureBatchAccountKey field.
+ /// </summary>
+ [DataMember]
+ public string AzureBatchAccountKey { get; set; }
+
+ /// <summary>
+ /// Gets or sets the AzureBatchAccountName field.
+ /// </summary>
+ [DataMember]
+ public string AzureBatchAccountName { get; set; }
+
+ /// <summary>
+ /// Gets or sets the AzureBatchAccountUri field.
+ /// </summary>
+ [DataMember]
+ public string AzureBatchAccountUri { get; set; }
+
+ /// <summary>
+ /// Gets or sets the AzureBatchPoolId field.
+ /// </summary>
+ [DataMember]
+ public string AzureBatchPoolId { get; set; }
+
+ /// <summary>
+ /// Gets or sets the AzureStorageAccountKey field.
+ /// </summary>
+ [DataMember]
+ public string AzureStorageAccountKey { get; set; }
+
+ /// <summary>
+ /// Gets or sets the AzureStorageAccountName field.
+ /// </summary>
+ [DataMember]
+ public string AzureStorageAccountName { get; set; }
+
+ /// <summary>
+ /// Gets or sets the AzureStorageContainerName field.
+ /// </summary>
+ [DataMember]
+ public string AzureStorageContainerName { get; set; }
+
+ /// <summary>
+ /// Gets or sets the AzureBatchIsWindows field.
+ /// </summary>
+ [DataMember]
+ public bool AzureBatchIsWindows { get; set; }
+
+ /// <summary>
+ /// Initializes a new instance of the <see cref="AvroAzureBatchJobSubmissionParameters"/> class.
+ /// </summary>
+ public AvroAzureBatchJobSubmissionParameters()
+ {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/AzureBatchDotNetClient.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/AzureBatchDotNetClient.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/AzureBatchDotNetClient.cs
new file mode 100644
index 0000000..7f9bdb3
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/AzureBatchDotNetClient.cs
@@ -0,0 +1,139 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System;
+using System.Threading.Tasks;
+using Org.Apache.REEF.Client.API;
+using Org.Apache.REEF.Client.AzureBatch.Storage;
+using Org.Apache.REEF.Client.AzureBatch;
+using Org.Apache.REEF.Client.AzureBatch.Util;
+using Org.Apache.REEF.Client.Common;
+using Org.Apache.REEF.Client.YARN.RestClient.DataModel;
+using Org.Apache.REEF.Common.Files;
+using Org.Apache.REEF.Tang.Annotations;
+using Org.Apache.REEF.Tang.Interface;
+using Org.Apache.REEF.Utilities.Logging;
+
+namespace Org.Apache.REEF.Client.DotNet.AzureBatch
+{
+ public sealed class AzureBatchDotNetClient : IREEFClient
+ {
+ private static readonly Logger LOGGER = Logger.GetLogger(typeof(AzureBatchDotNetClient));
+
+ /// Maximum number of characters allowed in Azure Batch job name. This limit is imposed by Azure Batch.
+ private const int AzureBatchMaxCharsJobName = 64;
+
+ private readonly IInjector _injector;
+ private readonly DriverFolderPreparationHelper _driverFolderPreparationHelper;
+ private readonly REEFFileNames _fileNames;
+ private readonly AzureStorageClient _azureStorageClient;
+ private readonly JobRequestBuilderFactory _jobRequestBuilderFactory;
+ private readonly AzureBatchService _batchService;
+ private readonly JobJarMaker _jobJarMaker;
+ private readonly AzureBatchFileNames _azbatchFileNames;
+
+ [Inject]
+ private AzureBatchDotNetClient(
+ IInjector injector,
+ IResourceArchiveFileGenerator resourceArchiveFileGenerator,
+ DriverFolderPreparationHelper driverFolderPreparationHelper,
+ AzureStorageClient azureStorageClient,
+ REEFFileNames fileNames,
+ AzureBatchFileNames azbatchFileNames,
+ JobRequestBuilderFactory jobRequestBuilderFactory,
+ AzureBatchService batchService,
+ JobJarMaker jobJarMaker)
+ {
+ _injector = injector;
+ _fileNames = fileNames;
+ _azbatchFileNames = azbatchFileNames;
+ _driverFolderPreparationHelper = driverFolderPreparationHelper;
+ _azureStorageClient = azureStorageClient;
+ _jobRequestBuilderFactory = jobRequestBuilderFactory;
+ _batchService = batchService;
+ _jobJarMaker = jobJarMaker;
+ }
+
+ public JobRequestBuilder NewJobRequestBuilder()
+ {
+ return _jobRequestBuilderFactory.NewInstance();
+ }
+
+ public Task<FinalState> GetJobFinalStatus(string appId)
+ {
+ // FinalState is DataModel in YARN. For Azure Batch runtime, this is not supported.
+ throw new NotImplementedException();
+ }
+
+ public void Submit(JobRequest jobRequest)
+ {
+ var configModule = AzureBatchRuntimeClientConfiguration.ConfigurationModule;
+ string jobId = jobRequest.JobIdentifier;
+ string azureBatchjobId = CreateAzureJobId(jobId);
+ string commandLine = GetCommand(jobRequest.JobParameters);
+ string jarPath = _jobJarMaker.CreateJobSubmissionJAR(jobRequest, azureBatchjobId);
+ string destination = _azbatchFileNames.GetStorageJobFolder(azureBatchjobId);
+ Uri blobUri = _azureStorageClient.UploadFile(destination, jarPath).Result;
+ string sasToken = _azureStorageClient.CreateContainerSharedAccessSignature();
+ _batchService.CreateJob(azureBatchjobId, blobUri, commandLine, sasToken);
+ }
+
+ private string GetCommand(JobParameters jobParameters)
+ {
+ var commandProviderConfigModule = AzureBatchCommandBuilderConfiguration.ConfigurationModule;
+ if (jobParameters.JavaLogLevel == JavaLoggingSetting.Verbose)
+ {
+ commandProviderConfigModule = commandProviderConfigModule
+ .Set(AzureBatchCommandBuilderConfiguration.JavaDebugLogging, true.ToString().ToLowerInvariant());
+ }
+
+ if (jobParameters.StdoutFilePath.IsPresent())
+ {
+ commandProviderConfigModule = commandProviderConfigModule
+ .Set(AzureBatchCommandBuilderConfiguration.DriverStdoutFilePath, jobParameters.StdoutFilePath.Value);
+ }
+
+ if (jobParameters.StderrFilePath.IsPresent())
+ {
+ commandProviderConfigModule = commandProviderConfigModule
+ .Set(AzureBatchCommandBuilderConfiguration.DriverStderrFilePath, jobParameters.StderrFilePath.Value);
+ }
+
+ var azureBatchJobCommandBuilder = _injector.ForkInjector(commandProviderConfigModule.Build())
+ .GetInstance<ICommandBuilder>();
+
+ var command = azureBatchJobCommandBuilder.BuildDriverCommand(jobParameters.DriverMemoryInMB);
+
+ return command;
+ }
+
+ public IJobSubmissionResult SubmitAndGetJobStatus(JobRequest jobRequest)
+ {
+ Submit(jobRequest);
+ /// Azure Batch is not able to comminicate to client through driver end point. It behaves the same as Submit(JobRequest jobRequest).
+ return null;
+ }
+
+ private string CreateAzureJobId(string jobId)
+ {
+ string guid = Guid.NewGuid().ToString();
+ string jobNameShort = jobId.Length + 1 + guid.Length < AzureBatchMaxCharsJobName ?
+ jobId : jobId.Substring(0, AzureBatchMaxCharsJobName - guid.Length - 1);
+ return jobNameShort + "-" + guid;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/AzureBatchRuntimeClientConfiguration.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/AzureBatchRuntimeClientConfiguration.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/AzureBatchRuntimeClientConfiguration.cs
new file mode 100644
index 0000000..31db860
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/AzureBatchRuntimeClientConfiguration.cs
@@ -0,0 +1,78 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Client.API;
+using Org.Apache.REEF.Client.AzureBatch.Parameters;
+using Org.Apache.REEF.Client.DotNet.AzureBatch;
+using Org.Apache.REEF.Tang.Formats;
+using Org.Apache.REEF.Tang.Interface;
+using Org.Apache.REEF.Tang.Util;
+using System;
+using System.IO;
+
+namespace Org.Apache.REEF.Client.AzureBatch
+{
+ /// <summary>
+ /// The Configuration for the Azure Batch Client
+ /// </summary>
+ public sealed class AzureBatchRuntimeClientConfiguration : ConfigurationModuleBuilder
+ {
+ public static readonly string AzBatchConfigurationFileEnvironmentVariable = "REEF_AZBATCH_CONF";
+
+ public static readonly RequiredParameter<string> AzureBatchAccountUri = new RequiredParameter<string>();
+ public static readonly RequiredParameter<string> AzureBatchAccountName = new RequiredParameter<string>();
+ public static readonly RequiredParameter<string> AzureBatchAccountKey = new RequiredParameter<string>();
+ public static readonly RequiredParameter<string> AzureBatchPoolId = new RequiredParameter<string>();
+
+ public static readonly RequiredParameter<string> AzureStorageAccountName = new RequiredParameter<string>();
+ public static readonly RequiredParameter<string> AzureStorageAccountKey = new RequiredParameter<string>();
+ public static readonly RequiredParameter<string> AzureStorageContainerName = new RequiredParameter<string>();
+
+ public static ConfigurationModule ConfigurationModule = new AzureBatchRuntimeClientConfiguration()
+ .BindImplementation(GenericType<IREEFClient>.Class, GenericType<AzureBatchDotNetClient>.Class)
+ .BindNamedParameter(GenericType<AzureBatchAccountUri>.Class, AzureBatchAccountUri)
+ .BindNamedParameter(GenericType<AzureBatchAccountName>.Class, AzureBatchAccountName)
+ .BindNamedParameter(GenericType<AzureBatchAccountKey>.Class, AzureBatchAccountKey)
+ .BindNamedParameter(GenericType<AzureBatchPoolId>.Class, AzureBatchPoolId)
+ .BindNamedParameter(GenericType<AzureStorageAccountName>.Class, AzureStorageAccountName)
+ .BindNamedParameter(GenericType<AzureStorageAccountKey>.Class, AzureStorageAccountKey)
+ .BindNamedParameter(GenericType<AzureStorageContainerName>.Class, AzureStorageContainerName)
+ .Build();
+
+ public static IConfiguration FromTextFile(string file)
+ {
+ return new AvroConfigurationSerializer().FromFile(file);
+ }
+
+ public static IConfiguration FromEnvironment()
+ {
+ string configurationPath = Environment.GetEnvironmentVariable(AzBatchConfigurationFileEnvironmentVariable);
+
+ if (configurationPath == null)
+ {
+ throw new ArgumentException(@"Environment Variable {AzureBatchClientConfiguration.AzBatchConfigurationFileEnvironmentVariable} not set");
+ }
+
+ if (!File.Exists(configurationPath))
+ {
+ throw new ArgumentException(@"File located by Environment Variable {AzureBatchClientConfiguration.AzBatchConfigurationFileEnvironmentVariable} cannot be read.");
+ }
+
+ return FromTextFile(configurationPath);
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchAccountKey.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchAccountKey.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchAccountKey.cs
new file mode 100644
index 0000000..9d5aa71
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchAccountKey.cs
@@ -0,0 +1,25 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Parameters
+{
+ [NamedParameter(Documentation = "The Azure Batch Account Key")]
+ public sealed class AzureBatchAccountKey : Name<string>
+ {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchAccountName.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchAccountName.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchAccountName.cs
new file mode 100644
index 0000000..a0c1151
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchAccountName.cs
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Parameters
+{
+ [NamedParameter(Documentation = "The Azure Batch Account Name")]
+ public sealed class AzureBatchAccountName : Name<string>
+ {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchAccountUri.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchAccountUri.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchAccountUri.cs
new file mode 100644
index 0000000..2a3466c
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchAccountUri.cs
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Parameters
+{
+ [NamedParameter(Documentation = "The Azure Batch account URI")]
+ public sealed class AzureBatchAccountUri : Name<string>
+ {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchPoolId.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchPoolId.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchPoolId.cs
new file mode 100644
index 0000000..da7766d
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureBatchPoolId.cs
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Parameters
+{
+ [NamedParameter(Documentation = "The Azure Batch Pool Id")]
+ public sealed class AzureBatchPoolId : Name<string>
+ {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureStorageAccountKey.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureStorageAccountKey.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureStorageAccountKey.cs
new file mode 100644
index 0000000..eef3f00
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureStorageAccountKey.cs
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Parameters
+{
+ [NamedParameter(Documentation = "The Azure Storage Account Key")]
+ public sealed class AzureStorageAccountKey : Name<string>
+ {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureStorageAccountName.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureStorageAccountName.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureStorageAccountName.cs
new file mode 100644
index 0000000..0238feb
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureStorageAccountName.cs
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Parameters
+{
+ [NamedParameter(Documentation = "The Azure Storage Account Name")]
+ public sealed class AzureStorageAccountName : Name<string>
+ {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureStorageContainerName.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureStorageContainerName.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureStorageContainerName.cs
new file mode 100644
index 0000000..bf93730
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/AzureStorageContainerName.cs
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Parameters
+{
+ [NamedParameter(Documentation = "The Azure Storage Container Name")]
+ public sealed class AzureStorageContainerName : Name<string>
+ {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/DriverStderrFilePath.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/DriverStderrFilePath.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/DriverStderrFilePath.cs
new file mode 100644
index 0000000..2b96e33
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/DriverStderrFilePath.cs
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Parameters
+{
+ [NamedParameter("Driver stderr file path for Azure Batch.", defaultValue: "<LOG_DIR>/driver.stderr")]
+ internal sealed class DriverStderrFilePath : Name<string>
+ {
+ private DriverStderrFilePath()
+ {
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/DriverStdoutFilePath.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/DriverStdoutFilePath.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/DriverStdoutFilePath.cs
new file mode 100644
index 0000000..1f2e546
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/DriverStdoutFilePath.cs
@@ -0,0 +1,29 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Parameters
+{
+ [NamedParameter("Driver stdout file path for Azure Batch.", defaultValue: "<LOG_DIR>/driver.stdout")]
+ internal sealed class DriverStdoutFilePath : Name<string>
+ {
+ private DriverStdoutFilePath()
+ {
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/EnableDebugLogging.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/EnableDebugLogging.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/EnableDebugLogging.cs
new file mode 100644
index 0000000..ceb7f9e
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/EnableDebugLogging.cs
@@ -0,0 +1,28 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Parameters
+{
+ [NamedParameter(
+ "Boolean indicating whether debug logging should be enabled in Java part of runtime.",
+ defaultValue: "false")]
+ public sealed class EnableDebugLogging : Name<bool>
+ {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/IsWindows.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/IsWindows.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/IsWindows.cs
new file mode 100644
index 0000000..3b16fee
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Parameters/IsWindows.cs
@@ -0,0 +1,26 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Parameters
+{
+ [NamedParameter(Documentation = "Are the Azure Batch VMs linux or Windows based")]
+ public sealed class IsWindows : Name<bool>
+ {
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Service/AzureBatchService.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Service/AzureBatchService.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Service/AzureBatchService.cs
new file mode 100644
index 0000000..b3e40da
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Service/AzureBatchService.cs
@@ -0,0 +1,138 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System;
+using System.Collections.Generic;
+using System.Threading.Tasks;
+using Microsoft.Azure.Batch;
+using Microsoft.Azure.Batch.Common;
+using Org.Apache.REEF.Client.AzureBatch.Parameters;
+using Org.Apache.REEF.Client.AzureBatch.Util;
+using Org.Apache.REEF.Tang.Annotations;
+using Org.Apache.REEF.Utilities.Logging;
+using BatchSharedKeyCredential = Microsoft.Azure.Batch.Auth.BatchSharedKeyCredentials;
+
+namespace Org.Apache.REEF.Client.DotNet.AzureBatch
+{
+ public sealed class AzureBatchService : IDisposable
+ {
+ private static readonly Logger LOGGER = Logger.GetLogger(typeof(AzureBatchService));
+ private static readonly TimeSpan RetryDeltaBackOff = TimeSpan.FromSeconds(5);
+ private const string AzureStorageContainerSasToken = "AZURE_STORAGE_CONTAINER_SAS_TOKEN_ENV";
+ private const int MaxRetries = 3;
+
+ public BatchSharedKeyCredential Credentials { get; private set; }
+ public string PoolId { get; private set; }
+
+ private BatchClient Client { get; set; }
+ private bool disposed;
+
+ [Inject]
+ public AzureBatchService(
+ [Parameter(typeof(AzureBatchAccountUri))] string azureBatchAccountUri,
+ [Parameter(typeof(AzureBatchAccountName))] string azureBatchAccountName,
+ [Parameter(typeof(AzureBatchAccountKey))] string azureBatchAccountKey,
+ [Parameter(typeof(AzureBatchPoolId))] string azureBatchPoolId)
+ {
+ BatchSharedKeyCredential credentials = new BatchSharedKeyCredential(azureBatchAccountUri, azureBatchAccountName, azureBatchAccountKey);
+
+ this.Client = BatchClient.Open(credentials);
+ this.Credentials = credentials;
+ this.PoolId = azureBatchPoolId;
+ this.Client.CustomBehaviors.Add(new RetryPolicyProvider(new ExponentialRetry(RetryDeltaBackOff, MaxRetries)));
+ }
+
+ /// <summary>
+ /// Dispose of this object and all members
+ /// </summary>
+ public void Dispose()
+ {
+ this.Dispose(true);
+ GC.SuppressFinalize(this);
+ }
+
+ ~AzureBatchService()
+ {
+ this.Dispose(false);
+ }
+
+ /// <summary>
+ /// Disposes of this object
+ /// </summary>
+ private void Dispose(bool disposing)
+ {
+ if (this.disposed)
+ {
+ return;
+ }
+
+ if (disposing)
+ {
+ this.Client.Dispose();
+ }
+
+ this.disposed = true;
+ }
+
+ #region Job related operations
+
+ public void CreateJob(string jobId, Uri resourceFile, string commandLine, string storageContainerSAS)
+ {
+ CloudJob unboundJob = this.Client.JobOperations.CreateJob();
+ unboundJob.Id = jobId;
+ unboundJob.PoolInformation = new PoolInformation() { PoolId = this.PoolId };
+ unboundJob.JobManagerTask = new JobManagerTask()
+ {
+ Id = jobId,
+ CommandLine = commandLine,
+ RunExclusive = false,
+
+ ResourceFiles = resourceFile != null
+ ? new List<ResourceFile>() { new ResourceFile(resourceFile.AbsoluteUri, AzureBatchFileNames.GetTaskJarFileName()) }
+ : new List<ResourceFile>(),
+
+ EnvironmentSettings = new List<EnvironmentSetting> { new EnvironmentSetting(AzureStorageContainerSasToken, storageContainerSAS) },
+
+ // This setting will signal Batch to generate an access token and pass it
+ // to the Job Manager Task (aka the Driver) as an environment variable.
+ // For more info, see
+ // https://docs.microsoft.com/en-us/dotnet/api/microsoft.azure.batch.cloudtask.authenticationtokensettings
+ AuthenticationTokenSettings = new AuthenticationTokenSettings() { Access = AccessScope.Job }
+ };
+
+ unboundJob.Commit();
+
+ LOGGER.Log(Level.Info, "Submitted job {0}, commandLine {1} ", jobId, commandLine);
+ }
+
+ public CloudJob GetJob(string jobId, DetailLevel detailLevel)
+ {
+ using (Task<CloudJob> getJobTask = this.GetJobAsync(jobId, detailLevel))
+ {
+ getJobTask.Wait();
+ return getJobTask.Result;
+ }
+ }
+
+ public Task<CloudJob> GetJobAsync(string jobId, DetailLevel detailLevel)
+ {
+ return this.Client.JobOperations.GetJobAsync(jobId, detailLevel);
+ }
+
+ #endregion
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Storage/AzureStorageClient.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Storage/AzureStorageClient.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Storage/AzureStorageClient.cs
new file mode 100644
index 0000000..ce39f1e
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Storage/AzureStorageClient.cs
@@ -0,0 +1,109 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Microsoft.WindowsAzure.Storage;
+using Microsoft.WindowsAzure.Storage.Blob;
+using Org.Apache.REEF.Client.AzureBatch.Parameters;
+using Org.Apache.REEF.Tang.Annotations;
+using Org.Apache.REEF.Utilities.Logging;
+using System;
+using System.IO;
+using System.Threading.Tasks;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Storage
+{
+ internal sealed class AzureStorageClient
+ {
+ private static readonly Logger LOGGER = Logger.GetLogger(typeof(AzureStorageClient));
+ private const string StorageConnectionStringFormat = "DefaultEndpointsProtocol=https;AccountName={0};AccountKey={1}";
+ private const int SASTokenValidityMinutes = 60;
+
+ private readonly string _storageAccountName;
+ private readonly string _storageAccountKey;
+ private readonly string _storageContainerName;
+
+ private readonly string _storageConnectionString;
+
+ [Inject]
+ AzureStorageClient(
+ [Parameter(typeof(AzureStorageAccountName))] string storageAccountName,
+ [Parameter(typeof(AzureStorageAccountKey))] string storageAccountKey,
+ [Parameter(typeof(AzureStorageContainerName))] string storageContainerName)
+ {
+ this._storageAccountName = storageAccountName;
+ this._storageAccountKey = storageAccountKey;
+ this._storageContainerName = storageContainerName;
+
+ this._storageConnectionString = string.Format(StorageConnectionStringFormat,
+ new object[] { storageAccountName, storageAccountKey });
+ }
+
+ /// <summary>
+ /// Uploads a given file to the given destination folder in Azure Storage.
+ /// </summary>
+ /// <param name="destination">Destination in Azure Storage where given file will be uploaded.</param>
+ /// <param name="filePath">Path to the file to be uploaded.</param>
+ /// <returns>Storage SAS URI for uploaded file.</returns>
+ public async Task<Uri> UploadFile(string destination, string filePath)
+ {
+ CloudBlobContainer blobContainer = await this.GetOrCreateCloudBlobContainer();
+ CloudBlobDirectory directory = blobContainer.GetDirectoryReference(destination);
+
+ string fileName = Path.GetFileName(filePath);
+ CloudBlockBlob blob = directory.GetBlockBlobReference(fileName);
+ await blob.UploadFromFileAsync(filePath);
+
+ string sas = blob.GetSharedAccessSignature(CreateSASPolicy());
+ string uri = blob.Uri.AbsoluteUri;
+ Uri uploadedFile = new Uri(uri + sas);
+ LOGGER.Log(Level.Info, "Uploaded {0} jar file to {1}", filePath, uploadedFile);
+ return uploadedFile;
+ }
+
+ public string CreateContainerSharedAccessSignature()
+ {
+ CloudBlobClient cloudBlobClient = CloudStorageAccount.Parse(this._storageConnectionString).CreateCloudBlobClient();
+ CloudBlobContainer cloudBlobContainer = cloudBlobClient.GetContainerReference(this._storageContainerName);
+ cloudBlobContainer.CreateIfNotExists();
+ return cloudBlobContainer.GetSharedAccessSignature(CreateSASPolicy());
+ }
+
+ private CloudBlobClient GetCloudBlobClient()
+ {
+ return CloudStorageAccount.Parse(this._storageConnectionString).CreateCloudBlobClient();
+ }
+
+ private async Task<CloudBlobContainer> GetOrCreateCloudBlobContainer()
+ {
+ CloudBlobClient blobClient = this.GetCloudBlobClient();
+ CloudBlobContainer blobContainer = blobClient.GetContainerReference(this._storageContainerName);
+ await blobContainer.CreateIfNotExistsAsync();
+
+ return blobContainer;
+ }
+
+ private SharedAccessBlobPolicy CreateSASPolicy()
+ {
+ return new SharedAccessBlobPolicy()
+ {
+ SharedAccessStartTime = DateTime.UtcNow,
+ SharedAccessExpiryTime = DateTime.UtcNow.AddMinutes(SASTokenValidityMinutes),
+ Permissions = SharedAccessBlobPermissions.Read | SharedAccessBlobPermissions.Write
+ };
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/AbstractCommandBuilder.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/AbstractCommandBuilder.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/AbstractCommandBuilder.cs
new file mode 100644
index 0000000..4363bd7
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/AbstractCommandBuilder.cs
@@ -0,0 +1,72 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.IO;
+using System.Text;
+using Org.Apache.REEF.Common.Files;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Util
+{
+ internal abstract class AbstractCommandBuilder : ICommandBuilder
+ {
+ private const string JavaExe = @"java";
+ private const string JvmOptionsPermSize = @"-XX:PermSize=128m";
+ private const string JvmOptionsMaxPermSizeFormat = @"-XX:MaxPermSize=128m";
+ private const string JvmOptionsMaxMemoryAllocationPoolSizeFormat = @"-Xmx{0}m";
+ private const string ClassPathToken = @"-classpath";
+ private const string ProcReefProperty = @"-Dproc_reef";
+ private const string LauncherClassName = @"org.apache.reef.bridge.client.AzureBatchBootstrapREEFLauncher";
+ protected readonly REEFFileNames _fileNames;
+ protected readonly string _osCommandFormat;
+ protected readonly string _commandPrefix;
+ protected readonly AzureBatchFileNames _azureBatchFileNames;
+
+ protected AbstractCommandBuilder(
+ REEFFileNames fileNames,
+ AzureBatchFileNames azureBatchFileNames,
+ string commandPrefix,
+ string osCommandFormat)
+ {
+ _fileNames = fileNames;
+ _osCommandFormat = osCommandFormat;
+ _commandPrefix = commandPrefix;
+ _azureBatchFileNames = azureBatchFileNames;
+ }
+
+ public string BuildDriverCommand(int driverMemory)
+ {
+ var sb = new StringBuilder();
+ sb.Append(_fileNames.GetBridgeExePath())
+ .Append(" " + JavaExe)
+ .Append(" " + string.Format(JvmOptionsMaxMemoryAllocationPoolSizeFormat, driverMemory))
+ .Append(" " + JvmOptionsPermSize)
+ .Append(" " + JvmOptionsMaxPermSizeFormat)
+ .Append(" " + ClassPathToken)
+ .Append(" " + GetDriverClasspath())
+ .Append(" " + ProcReefProperty)
+ .Append(" " + LauncherClassName)
+ .Append(" " + Path.Combine(_fileNames.GetReefFolderName(), _fileNames.GetJobSubmissionParametersFile()));
+ return string.Format(_osCommandFormat, _commandPrefix + sb.ToString());
+ }
+
+ /// <summary>
+ /// Returns the driver classpath string which is compatible with the intricacies of the OS.
+ /// </summary>
+ /// <returns>classpath parameter string.</returns>
+ protected abstract string GetDriverClasspath();
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/AzureBatchCommandProviderConfiguration.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/AzureBatchCommandProviderConfiguration.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/AzureBatchCommandProviderConfiguration.cs
new file mode 100644
index 0000000..f17d312
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/AzureBatchCommandProviderConfiguration.cs
@@ -0,0 +1,42 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Client.YARN.Parameters;
+using Org.Apache.REEF.Tang.Formats;
+using Org.Apache.REEF.Tang.Util;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Util
+{
+ internal sealed class AzureBatchCommandBuilderConfiguration : ConfigurationModuleBuilder
+ {
+ public static readonly OptionalParameter<string> DriverStdoutFilePath = new OptionalParameter<string>();
+ public static readonly OptionalParameter<string> DriverStderrFilePath = new OptionalParameter<string>();
+ public static readonly OptionalParameter<bool> JavaDebugLogging = new OptionalParameter<bool>();
+
+ public static ConfigurationModule ConfigurationModule
+ {
+ get
+ {
+ return new AzureBatchCommandBuilderConfiguration()
+ .BindNamedParameter(GenericType<DriverStdoutFilePath>.Class, DriverStdoutFilePath)
+ .BindNamedParameter(GenericType<DriverStderrFilePath>.Class, DriverStderrFilePath)
+ .BindNamedParameter(GenericType<EnableDebugLogging>.Class, JavaDebugLogging)
+ .Build();
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/AzureBatchFileNames.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/AzureBatchFileNames.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/AzureBatchFileNames.cs
new file mode 100644
index 0000000..fc98c41
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/AzureBatchFileNames.cs
@@ -0,0 +1,55 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Common.Files;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Util
+{
+ /// <summary>
+ /// Access to the various places things go according to the REEF Azure Batch runtime.
+ /// </summary>
+ internal sealed class AzureBatchFileNames
+ {
+ private const string StorageJobFolderPath = "apps/reef/jobs/";
+ private const string TaskJarFileName = "local.jar";
+ private readonly REEFFileNames _reefFileNames;
+
+ [Inject]
+ private AzureBatchFileNames(REEFFileNames reefFileNames)
+ {
+ _reefFileNames = reefFileNames;
+ }
+
+ /// <summary>
+ /// </summary>
+ /// <param name="jobId">Job Submission Id</param>
+ /// <returns>The relative path to the folder storing the job assets.</returns>
+ public string GetStorageJobFolder(string jobId)
+ {
+ return StorageJobFolderPath + jobId;
+ }
+
+ /// <summary>
+ /// </summary>
+ /// <returns>The name under which the task jar will be stored.</returns>
+ public static string GetTaskJarFileName()
+ {
+ return TaskJarFileName;
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/ICommandBuilder.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/ICommandBuilder.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/ICommandBuilder.cs
new file mode 100644
index 0000000..23e83d4
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/ICommandBuilder.cs
@@ -0,0 +1,35 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Util
+{
+ /// <summary>
+ /// Build the launch command for Java REEF processes for Azure Batch.
+ /// </summary>
+ [DefaultImplementation(typeof(WindowsCommandBuilder))]
+ internal interface ICommandBuilder
+ {
+ /// <summary>
+ /// Assembles the command to execute the Driver.
+ /// </summary>
+ /// <param name="driverMemory">The memory in megabytes used by driver.</param>
+ /// <returns>The command string.</returns>
+ string BuildDriverCommand(int driverMemory);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/JobJarMaker.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/JobJarMaker.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/JobJarMaker.cs
new file mode 100644
index 0000000..48e62e5
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/JobJarMaker.cs
@@ -0,0 +1,108 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using Org.Apache.REEF.Client.API;
+using Org.Apache.REEF.Client.Avro;
+using Org.Apache.REEF.Client.Avro.AzureBatch;
+using Org.Apache.REEF.Client.AzureBatch.Parameters;
+using Org.Apache.REEF.Client.Common;
+using Org.Apache.REEF.Common.Avro;
+using Org.Apache.REEF.Common.Files;
+using Org.Apache.REEF.Tang.Annotations;
+using System;
+using System.IO;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Util
+{
+ internal sealed class JobJarMaker
+ {
+ private readonly IResourceArchiveFileGenerator _resourceArchiveFileGenerator;
+ private readonly DriverFolderPreparationHelper _driverFolderPreparationHelper;
+ private readonly AvroAzureBatchJobSubmissionParameters _avroAzureBatchJobSubmissionParameters;
+ private readonly REEFFileNames _fileNames;
+
+ [Inject]
+ JobJarMaker(
+ IResourceArchiveFileGenerator resourceArchiveFileGenerator,
+ DriverFolderPreparationHelper driverFolderPreparationHelper,
+ REEFFileNames fileNames,
+ [Parameter(typeof(AzureBatchAccountKey))] string azureBatchAccountKey,
+ [Parameter(typeof(AzureBatchAccountName))] string azureBatchAccountName,
+ [Parameter(typeof(AzureBatchAccountUri))] string azureBatchAccountUri,
+ [Parameter(typeof(AzureBatchPoolId))] string azureBatchPoolId,
+ [Parameter(typeof(AzureStorageAccountKey))] string azureStorageAccountKey,
+ [Parameter(typeof(AzureStorageAccountName))] string azureStorageAccountName,
+ [Parameter(typeof(AzureStorageContainerName))] string azureStorageContainerName)
+ {
+ _resourceArchiveFileGenerator = resourceArchiveFileGenerator;
+ _driverFolderPreparationHelper = driverFolderPreparationHelper;
+ _fileNames = fileNames;
+ _avroAzureBatchJobSubmissionParameters = new AvroAzureBatchJobSubmissionParameters
+ {
+ AzureBatchAccountKey = azureBatchAccountKey,
+ AzureBatchAccountName = azureBatchAccountName,
+ AzureBatchAccountUri = azureBatchAccountUri,
+ AzureBatchPoolId = azureBatchPoolId,
+ AzureStorageAccountKey = azureStorageAccountKey,
+ AzureStorageAccountName = azureStorageAccountName,
+ AzureStorageContainerName = azureStorageContainerName,
+ AzureBatchIsWindows = true
+ };
+ }
+
+ /// <summary>
+ /// Creates a JAR file for the job submission.
+ /// </summary>
+ /// <param name="jobRequest">Job request received from the client code.</param>
+ /// <param name="azureBatchjobId">Azure Batch job Id going to be launched.</param>
+ /// <returns>A string path to file.</returns>
+ public string CreateJobSubmissionJAR(JobRequest jobRequest, string azureBatchjobId)
+ {
+ _avroAzureBatchJobSubmissionParameters.sharedJobSubmissionParameters = new AvroJobSubmissionParameters
+ {
+ jobId = jobRequest.JobIdentifier,
+ //// This is dummy in Azure Batch, as it does not use jobSubmissionFolder in Azure Batch.
+ jobSubmissionFolder = Path.PathSeparator.ToString()
+ };
+
+ string localDriverFolderPath = CreateDriverFolder(azureBatchjobId);
+
+ _driverFolderPreparationHelper.PrepareDriverFolderWithGlobalBridgeJar(jobRequest.AppParameters, localDriverFolderPath);
+ SerializeJobFile(localDriverFolderPath, _avroAzureBatchJobSubmissionParameters);
+
+ return _resourceArchiveFileGenerator.CreateArchiveToUpload(localDriverFolderPath);
+ }
+
+ private string CreateDriverFolder(string azureBatchjobId)
+ {
+ return Path.GetFullPath(Path.Combine(Path.GetTempPath(), azureBatchjobId) + Path.DirectorySeparatorChar);
+ }
+
+ private void SerializeJobFile(string localDriverFolderPath, AvroAzureBatchJobSubmissionParameters jobParameters)
+ {
+ var serializedArgs = AvroJsonSerializer<AvroAzureBatchJobSubmissionParameters>.ToBytes(jobParameters);
+
+ var submissionJobArgsFilePath = Path.Combine(localDriverFolderPath,
+ _fileNames.GetReefFolderName(), _fileNames.GetJobSubmissionParametersFile());
+
+ using (var jobArgsFileStream = new FileStream(submissionJobArgsFilePath, FileMode.CreateNew))
+ {
+ jobArgsFileStream.Write(serializedArgs, 0, serializedArgs.Length);
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/LinuxCommandBuilder.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/LinuxCommandBuilder.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/LinuxCommandBuilder.cs
new file mode 100644
index 0000000..eed2163
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/LinuxCommandBuilder.cs
@@ -0,0 +1,44 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System;
+using Org.Apache.REEF.Common.Files;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Util
+{
+ internal sealed class LinuxCommandBuilder : AbstractCommandBuilder
+ {
+ private static readonly string CommandPrefix =
+ "unzip " + AzureBatchFileNames.GetTaskJarFileName() + " -d 'reef/'" + ";";
+ private const string ClassPathSeparator = ":";
+ private const string OsCommandFormat = "/bin/sh c \"{0}\"";
+
+ [Inject]
+ private LinuxCommandBuilder(
+ REEFFileNames fileNames,
+ AzureBatchFileNames azureBatchFileNames) : base(fileNames, azureBatchFileNames,
+ CommandPrefix, OsCommandFormat)
+ {
+ }
+
+ protected override string GetDriverClasspath()
+ {
+ throw new NotImplementedException();
+ }
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/WindowsCommandBuilder.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/WindowsCommandBuilder.cs b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/WindowsCommandBuilder.cs
new file mode 100644
index 0000000..b7d6d2f
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Client/AzureBatch/Util/WindowsCommandBuilder.cs
@@ -0,0 +1,52 @@
+// Licensed to the Apache Software Foundation (ASF) under one
+// or more contributor license agreements. See the NOTICE file
+// distributed with this work for additional information
+// regarding copyright ownership. The ASF licenses this file
+// to you under the Apache License, Version 2.0 (the
+// "License"); you may not use this file except in compliance
+// with the License. You may obtain a copy of the License at
+//
+// http://www.apache.org/licenses/LICENSE-2.0
+//
+// Unless required by applicable law or agreed to in writing,
+// software distributed under the License is distributed on an
+// "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+// KIND, either express or implied. See the License for the
+// specific language governing permissions and limitations
+// under the License.
+
+using System.Collections.Generic;
+using Org.Apache.REEF.Common.Files;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Client.AzureBatch.Util
+{
+ internal sealed class WindowsCommandBuilder : AbstractCommandBuilder
+ {
+ private static readonly string CommandPrefix = @"Add-Type -AssemblyName System.IO.Compression.FileSystem; " +
+ "[System.IO.Compression.ZipFile]::ExtractToDirectory(\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\" +
+ AzureBatchFileNames.GetTaskJarFileName() + "\\\", " +
+ "\\\"$env:AZ_BATCH_TASK_WORKING_DIR\\reef\\\");";
+ private const string ClassPathSeparator = ";";
+ private const string OsCommandFormat = "powershell.exe /c \"{0}\";";
+
+ [Inject]
+ private WindowsCommandBuilder(
+ REEFFileNames fileNames,
+ AzureBatchFileNames azureBatchFileNames) : base(fileNames, azureBatchFileNames,
+ CommandPrefix, OsCommandFormat)
+ {
+ }
+
+ protected override string GetDriverClasspath()
+ {
+ List<string> classpathList = new List<string>()
+ {
+ string.Format("{0}/{1}/*", _fileNames.GetReefFolderName(), _fileNames.GetLocalFolderName()),
+ string.Format("{0}/{1}/*", _fileNames.GetReefFolderName(), _fileNames.GetGlobalFolderName())
+ };
+
+ return string.Format("'{0};'", string.Join(ClassPathSeparator, classpathList));
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs b/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs
index f51a3c4..90e865d 100644
--- a/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs
+++ b/lang/cs/Org.Apache.REEF.Client/Common/DriverFolderPreparationHelper.cs
@@ -82,26 +82,32 @@ namespace Org.Apache.REEF.Client.Common
/// <param name="driverFolderPath"></param>
internal void PrepareDriverFolder(AppParameters appParameters, string driverFolderPath)
{
- Logger.Log(Level.Verbose, "Preparing Driver filesystem layout in {0}", driverFolderPath);
-
- // Setup the folder structure
- CreateDefaultFolderStructure(appParameters, driverFolderPath);
-
// Add the appParameters into that folder structure
_fileSets.AddJobFiles(appParameters);
- // Add the reef-bridge-client jar to the global files in the manner of JavaClientLauncher.cs.
+ // Add the reef-bridge-client jar to the local files in the manner of JavaClientLauncher.cs.
_fileSets.AddToLocalFiles(Directory.GetFiles(JarFolder)
.Where(file => !string.IsNullOrWhiteSpace(file))
.Where(jarFile => Path.GetFileName(jarFile).ToLower().StartsWith(ClientConstants.ClientJarFilePrefix)));
- // Create the driver configuration
- CreateDriverConfiguration(appParameters, driverFolderPath);
+ InternalPrepareDriverFolder(appParameters, driverFolderPath);
+ }
- // Initiate the final copy
- _fileSets.CopyToDriverFolder(driverFolderPath);
+ /// <summary>
+ /// Prepares the working directory for a Driver in driverFolderPath.
+ /// </summary>
+ /// <param name="appParameters"></param>
+ /// <param name="driverFolderPath"></param>
+ internal void PrepareDriverFolderWithGlobalBridgeJar(AppParameters appParameters, string driverFolderPath)
+ {
+ // Add the appParameters into that folder structure
+ _fileSets.AddJobFiles(appParameters);
- Logger.Log(Level.Info, "Done preparing Driver filesystem layout in {0}", driverFolderPath);
+ // Add the reef-bridge-client jar to the global files in the manner of JavaClientLauncher.cs.
+ _fileSets.AddToGlobalFiles(Directory.GetFiles(JarFolder)
+ .Where(jarFile => Path.GetFileName(jarFile).ToLower().StartsWith(ClientConstants.ClientJarFilePrefix)));
+
+ InternalPrepareDriverFolder(appParameters, driverFolderPath);
}
/// <summary>
@@ -144,7 +150,7 @@ namespace Org.Apache.REEF.Client.Common
File.WriteAllBytes(fileName, resourceHelper.GetBytes(fileResources.Value));
}
}
-
+
// generate .config file for bridge executable
var config = DefaultDriverConfigurationFileContents;
if (!string.IsNullOrEmpty(appParameters.DriverConfigurationFileContents))
@@ -165,5 +171,21 @@ namespace Org.Apache.REEF.Client.Common
Logger.Log(Level.Verbose, "Create EvaluatorConfigFile {0} with config {1}.", evaluatorConfigFilName, evaluatorAppConfigString);
File.WriteAllText(evaluatorConfigFilName, evaluatorAppConfigString);
}
+
+ private void InternalPrepareDriverFolder(AppParameters appParameters, string driverFolderPath)
+ {
+ Logger.Log(Level.Info, "Preparing Driver filesystem layout in {0}", driverFolderPath);
+
+ // Setup the folder structure
+ CreateDefaultFolderStructure(appParameters, driverFolderPath);
+
+ // Create the driver configuration
+ CreateDriverConfiguration(appParameters, driverFolderPath);
+
+ // Initiate the final copy
+ _fileSets.CopyToDriverFolder(driverFolderPath);
+
+ Logger.Log(Level.Info, "Done preparing Driver filesystem layout in {0}", driverFolderPath);
+ }
}
}
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj
index a7cc2e8..835faf9 100644
--- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj
+++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.DotNet.csproj
@@ -34,6 +34,7 @@ under the License.
</EmbeddedResource>
</ItemGroup>
<ItemGroup>
+ <PackageReference Include="Azure.Batch" Version="8.0.0" />
<PackageReference Include="Microsoft.Avro.Core" Version="$(AvroVersion)" />
<PackageReference Include="System.Reactive.Interfaces" Version="$(SystemReactiveVersion)" />
<PackageReference Include="EnterpriseLibrary.TransientFaultHandling.Core" Version="$(TransientFaultHandlingVersion)" />
http://git-wip-us.apache.org/repos/asf/reef/blob/8d0bad24/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
index 6245647..90c365e 100644
--- a/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
+++ b/lang/cs/Org.Apache.REEF.Client/Org.Apache.REEF.Client.csproj
@@ -37,11 +37,40 @@ under the License.
<UseVSHostingProcess>false</UseVSHostingProcess>
</PropertyGroup>
<ItemGroup>
+ <Reference Include="Microsoft.Azure.Batch, Version=8.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>..\packages\Azure.Batch.8.0.1\lib\net452\Microsoft.Azure.Batch.dll</HintPath>
+ </Reference>
+ <Reference Include="Microsoft.Azure.KeyVault.Core, Version=1.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>..\packages\Microsoft.Azure.KeyVault.Core.1.0.0\lib\net40\Microsoft.Azure.KeyVault.Core.dll</HintPath>
+ <Private>True</Private>
+ </Reference>
<Reference Include="Microsoft.CSharp" />
+ <Reference Include="Microsoft.Data.Edm, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>..\packages\Microsoft.Data.Edm.5.8.2\lib\net40\Microsoft.Data.Edm.dll</HintPath>
+ <Private>True</Private>
+ </Reference>
+ <Reference Include="Microsoft.Data.OData, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>..\packages\Microsoft.Data.OData.5.8.2\lib\net40\Microsoft.Data.OData.dll</HintPath>
+ <Private>True</Private>
+ </Reference>
+ <Reference Include="Microsoft.Data.Services.Client, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>..\packages\Microsoft.Data.Services.Client.5.8.2\lib\net40\Microsoft.Data.Services.Client.dll</HintPath>
+ <Private>True</Private>
+ </Reference>
<Reference Include="Microsoft.Practices.TransientFaultHandling.Core, Version=5.1.1209.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
<HintPath>$(PackagesDir)\TransientFaultHandling.Core.5.1.1209.1\lib\NET4\Microsoft.Practices.TransientFaultHandling.Core.dll</HintPath>
<Private>True</Private>
</Reference>
+ <Reference Include="Microsoft.Rest.ClientRuntime, Version=2.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>..\packages\Microsoft.Rest.ClientRuntime.2.3.10\lib\net452\Microsoft.Rest.ClientRuntime.dll</HintPath>
+ <Private>True</Private>
+ </Reference>
+ <Reference Include="Microsoft.Rest.ClientRuntime.Azure, Version=3.0.0.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>..\packages\Microsoft.Rest.ClientRuntime.Azure.3.3.10\lib\net452\Microsoft.Rest.ClientRuntime.Azure.dll</HintPath>
+ </Reference>
+ <Reference Include="Microsoft.WindowsAzure.Storage, Version=8.1.3.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>..\packages\WindowsAzure.Storage.8.1.3\lib\net45\Microsoft.WindowsAzure.Storage.dll</HintPath>
+ </Reference>
<Reference Include="Newtonsoft.Json">
<HintPath>$(PackagesDir)\Newtonsoft.Json.$(NewtonsoftJsonVersion)\lib\net45\Newtonsoft.Json.dll</HintPath>
</Reference>
@@ -50,10 +79,15 @@ under the License.
</Reference>
<Reference Include="System" />
<Reference Include="System.Core" />
+ <Reference Include="System.Data" />
<Reference Include="System.IO.Compression" />
<Reference Include="System.IO.Compression.FileSystem" />
<Reference Include="System.Net.Http" />
<Reference Include="System.Net.Http.WebRequest" />
+ <Reference Include="System.Spatial, Version=5.8.1.0, Culture=neutral, PublicKeyToken=31bf3856ad364e35, processorArchitecture=MSIL">
+ <HintPath>..\packages\System.Spatial.5.8.2\lib\net40\System.Spatial.dll</HintPath>
+ <Private>True</Private>
+ </Reference>
<Reference Include="System.Xml" />
<Reference Include="System.Xml.Linq" />
<Reference Include="System.Runtime.Serialization" />
@@ -83,11 +117,34 @@ under the License.
<Compile Include="API\TcpPortConfigurationModule.cs" />
<Compile Include="Avro\AvroAppSubmissionParameters.cs" />
<Compile Include="Avro\AvroJobSubmissionParameters.cs" />
+ <Compile Include="Avro\AzureBatch\AvroAzureBatchJobSubmissionParameters.cs" />
<Compile Include="Avro\Local\AvroLocalAppSubmissionParameters.cs" />
<Compile Include="Avro\Local\AvroLocalJobSubmissionParameters.cs" />
<Compile Include="Avro\YARN\AvroYarnAppSubmissionParameters.cs" />
<Compile Include="Avro\YARN\AvroYarnJobSubmissionParameters.cs" />
<Compile Include="Avro\YARN\AvroYarnClusterJobSubmissionParameters.cs" />
+ <Compile Include="AzureBatch\AzureBatchRuntimeClientConfiguration.cs" />
+ <Compile Include="AzureBatch\Util\AzureBatchCommandProviderConfiguration.cs" />
+ <Compile Include="AzureBatch\AzureBatchDotNetClient.cs" />
+ <Compile Include="AzureBatch\Storage\AzureStorageClient.cs" />
+ <Compile Include="AzureBatch\Service\AzureBatchService.cs" />
+ <Compile Include="AzureBatch\Util\AzureBatchFileNames.cs" />
+ <Compile Include="AzureBatch\Util\ICommandBuilder.cs" />
+ <Compile Include="AzureBatch\Parameters\AzureBatchAccountKey.cs" />
+ <Compile Include="AzureBatch\Parameters\AzureBatchAccountName.cs" />
+ <Compile Include="AzureBatch\Parameters\AzureBatchAccountUri.cs" />
+ <Compile Include="AzureBatch\Parameters\AzureBatchPoolId.cs" />
+ <Compile Include="AzureBatch\Parameters\AzureStorageAccountKey.cs" />
+ <Compile Include="AzureBatch\Parameters\AzureStorageAccountName.cs" />
+ <Compile Include="AzureBatch\Parameters\AzureStorageContainerName.cs" />
+ <Compile Include="AzureBatch\Parameters\DriverStderrFilePath.cs" />
+ <Compile Include="AzureBatch\Parameters\DriverStdoutFilePath.cs" />
+ <Compile Include="AzureBatch\Parameters\EnableDebugLogging.cs" />
+ <Compile Include="AzureBatch\Parameters\IsWindows.cs" />
+ <Compile Include="AzureBatch\Util\AbstractCommandBuilder.cs" />
+ <Compile Include="AzureBatch\Util\JobJarMaker.cs" />
+ <Compile Include="AzureBatch\Util\LinuxCommandBuilder.cs" />
+ <Compile Include="AzureBatch\Util\WindowsCommandBuilder.cs" />
<Compile Include="Common\DotNetFile.cs" />
<Compile Include="Common\ClientConstants.cs" />
<Compile Include="Common\DriverFolderPreparationHelper.cs" />
@@ -260,6 +317,7 @@ under the License.
<Visible>false</Visible>
</EmbeddedResource>
</ItemGroup>
+ <ItemGroup />
<Import Project="$(MSBuildToolsPath)\Microsoft.CSharp.targets" />
<Import Project="$(SolutionDir)\.nuget\NuGet.targets" Condition="Exists('$(SolutionDir)\.nuget\NuGet.targets')" />
<Import Project="$(PackagesDir)\StyleCop.MSBuild.$(StyleCopVersion)\build\StyleCop.MSBuild.Targets" Condition="Exists('$(PackagesDir)\StyleCop.MSBuild.$(StyleCopVersion)\build\StyleCop.MSBuild.Targets')" />
@@ -371,4 +429,4 @@ under the License.
</Target>
<Target Name="BeforeBuild" DependsOnTargets="$(BeforeBuildDependsOn);RewriteClientResources">
</Target>
-</Project>
+</Project>
\ No newline at end of file
[12/16] reef git commit: [REEF-2012] Add driver restart capabilities
to reef runtime mock
Posted by do...@apache.org.
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockUtils.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockUtils.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockUtils.java
deleted file mode 100644
index 0b073c8..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/MockUtils.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock.runtime;
-
-import org.apache.reef.annotations.Unstable;
-import org.apache.reef.annotations.audience.Private;
-import org.apache.reef.tang.Configuration;
-import org.apache.reef.tang.Injector;
-import org.apache.reef.tang.Tang;
-import org.apache.reef.tang.annotations.Name;
-import org.apache.reef.tang.exceptions.InjectionException;
-
-/**
- * mock utilities.
- */
-@Unstable
-@Private
-final class MockUtils {
-
- private MockUtils() {
- }
-
- public static <U, T extends Name<U>> U getValue(final Configuration configuration, final Class<T> name) {
- try {
- final Injector injector = Tang.Factory.getTang().newInjector(configuration);
- return injector.getNamedInstance(name);
- } catch (InjectionException e) {
- throw new IllegalStateException(e);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/package-info.java b/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/package-info.java
deleted file mode 100644
index b5cf639..0000000
--- a/lang/java/reef-runtime-mock/src/main/java/org/apache/reef/mock/runtime/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-/**
- * mock runtime implementation.
- */
-package org.apache.reef.mock.runtime;
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/BasicMockTests.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/BasicMockTests.java b/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/BasicMockTests.java
deleted file mode 100644
index 984a9f4..0000000
--- a/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/BasicMockTests.java
+++ /dev/null
@@ -1,205 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock;
-
-import org.apache.reef.driver.context.ActiveContext;
-import org.apache.reef.driver.evaluator.AllocatedEvaluator;
-import org.apache.reef.driver.task.RunningTask;
-import org.apache.reef.mock.request.ProcessRequestInternal;
-import org.apache.reef.mock.runtime.MockAllocatedEvalautor;
-import org.apache.reef.mock.runtime.MockClock;
-import org.apache.reef.tang.Configuration;
-import org.apache.reef.tang.Injector;
-import org.apache.reef.tang.Tang;
-import org.junit.Before;
-import org.junit.Test;
-
-import static org.junit.Assert.assertEquals;
-import static org.junit.Assert.assertTrue;
-
-/**
- * basic mock tests.
- */
-final class BasicMockTests {
-
- private MockApplication mockApplication;
-
- private MockRuntime mockRuntime;
-
- private MockClock mockClock;
-
- @Before
- public void initialize() throws Exception {
- final Configuration conf = MockConfiguration.CONF
- .set(MockConfiguration.ON_DRIVER_STARTED, MockApplication.StartHandler.class)
- .set(MockConfiguration.ON_DRIVER_STOP, MockApplication.StopHandler.class)
- .set(MockConfiguration.ON_CONTEXT_ACTIVE, MockApplication.ActiveContextHandler.class)
- .set(MockConfiguration.ON_CONTEXT_CLOSED, MockApplication.ContextClosedHandler.class)
- .set(MockConfiguration.ON_CONTEXT_FAILED, MockApplication.FailedContextHandler.class)
- .set(MockConfiguration.ON_EVALUATOR_ALLOCATED, MockApplication.AllocatedEvaluatorHandler.class)
- .set(MockConfiguration.ON_EVALUATOR_COMPLETED, MockApplication.CompletedEvaluatorHandler.class)
- .set(MockConfiguration.ON_EVALUATOR_FAILED, MockApplication.FailedEvaluatorHandler.class)
- .set(MockConfiguration.ON_TASK_COMPLETED, MockApplication.CompletedTaskHandler.class)
- .set(MockConfiguration.ON_TASK_FAILED, MockApplication.FailedTaskHandler.class)
- .set(MockConfiguration.ON_TASK_RUNNING, MockApplication.RunningTaskHandler.class)
- .set(MockConfiguration.ON_TASK_SUSPENDED, MockApplication.SuspendedTaskHandler.class)
- .build();
-
- final Injector injector = Tang.Factory.getTang().newInjector(conf);
- this.mockApplication = injector.getInstance(MockApplication.class);
- this.mockRuntime = injector.getInstance(MockRuntime.class);
- this.mockClock = injector.getInstance(MockClock.class);
-
- this.mockClock.run();
- }
-
- @Test
- public void testSuccessRequests() throws Exception {
- assertTrue("mock application received start event", this.mockApplication.isRunning());
-
- this.mockApplication.requestEvaluators(1);
- assertTrue("check for process event", this.mockRuntime.hasProcessRequest());
- final ProcessRequest allocateEvaluatorRequest = this.mockRuntime.getNextProcessRequest();
- assertEquals("allocate evalautor request", ProcessRequest.Type.ALLOCATE_EVALUATOR,
- allocateEvaluatorRequest.getType());
- final AllocatedEvaluator evaluator =
- ((ProcessRequestInternal<AllocatedEvaluator, Object>)allocateEvaluatorRequest)
- .getSuccessEvent();
- this.mockRuntime.succeed(allocateEvaluatorRequest);
- assertTrue("evaluator allocation succeeded",
- this.mockApplication.getAllocatedEvaluators().contains(evaluator));
- final ActiveContext rootContext = this.mockApplication.getContext(evaluator,
- MockAllocatedEvalautor.ROOT_CONTEXT_IDENTIFIER_PREFIX + evaluator.getId());
- assertTrue("root context", rootContext != null);
-
-
- // submit a task
- this.mockApplication.submitTask(rootContext, "test-task");
- assertTrue("create task queued", this.mockRuntime.hasProcessRequest());
- final ProcessRequest createTaskRequest = this.mockRuntime.getNextProcessRequest();
- assertEquals("create task request", ProcessRequest.Type.CREATE_TASK,
- createTaskRequest.getType());
- final RunningTask task = (RunningTask) ((ProcessRequestInternal)createTaskRequest).getSuccessEvent();
- this.mockRuntime.succeed(createTaskRequest);
- assertTrue("task running", this.mockApplication.getRunningTasks().contains(task));
-
- // check task auto complete
- assertTrue("check for request", this.mockRuntime.hasProcessRequest());
- final ProcessRequestInternal completedTask =
- (ProcessRequestInternal) this.mockRuntime.getNextProcessRequest();
- assertEquals("complete task request", ProcessRequest.Type.COMPLETE_TASK,
- completedTask.getType());
- this.mockRuntime.succeed(completedTask);
- assertEquals("no running tasks", 0, this.mockApplication.getRunningTasks().size());
-
- // create a sub-context
- this.mockApplication.submitContext(rootContext, "child");
- assertTrue("check for request", this.mockRuntime.hasProcessRequest());
- final ProcessRequestInternal createContextRequest =
- (ProcessRequestInternal) this.mockRuntime.getNextProcessRequest();
- assertEquals("create context request", ProcessRequest.Type.CREATE_CONTEXT,
- createContextRequest.getType());
- this.mockRuntime.succeed(createContextRequest);
- final ActiveContext context = this.mockApplication.getContext(evaluator, "child");
- assertTrue("child context", context.getParentId().get().equals(rootContext.getId()));
- }
-
- @Test
- public void testFailureRequests() throws Exception {
- assertTrue("mock application received start event", this.mockApplication.isRunning());
-
- this.mockApplication.requestEvaluators(1);
- assertTrue("check for process event", this.mockRuntime.hasProcessRequest());
- ProcessRequest allocateEvaluatorRequest = this.mockRuntime.getNextProcessRequest();
- this.mockRuntime.fail(allocateEvaluatorRequest);
- assertEquals("evaluator allocation failed", 1,
- this.mockApplication.getFailedEvaluators().size());
-
- this.mockApplication.requestEvaluators(1);
- allocateEvaluatorRequest = this.mockRuntime.getNextProcessRequest();
- final AllocatedEvaluator evaluator =
- (AllocatedEvaluator)((ProcessRequestInternal)allocateEvaluatorRequest).getSuccessEvent();
- this.mockRuntime.succeed(allocateEvaluatorRequest);
- final ActiveContext rootContext = this.mockApplication
- .getContext(evaluator, MockAllocatedEvalautor.ROOT_CONTEXT_IDENTIFIER_PREFIX + evaluator.getId());
-
-
- // submit a task
- this.mockApplication.submitTask(rootContext, "test-task");
- assertTrue("create task queued", this.mockRuntime.hasProcessRequest());
- final ProcessRequest createTaskRequest = this.mockRuntime.getNextProcessRequest();
- assertEquals("create task request", ProcessRequest.Type.CREATE_TASK,
- createTaskRequest.getType());
- this.mockRuntime.fail(createTaskRequest);
- assertEquals("task running", 1, this.mockApplication.getFailedTasks().size());
-
- // create a sub-context
- this.mockApplication.submitContext(rootContext, "child");
- assertTrue("check for request", this.mockRuntime.hasProcessRequest());
- final ProcessRequestInternal createContextRequest =
- (ProcessRequestInternal) this.mockRuntime.getNextProcessRequest();
- this.mockRuntime.fail(createContextRequest);
- assertEquals("child context", 1, this.mockApplication.getFailedContext().size());
- }
-
- @Test
- public void testMockFailures() {
- // make sure we're running
- assertTrue("mock application received start event", this.mockApplication.isRunning());
-
- // allocate an evaluator and get root context
- this.mockApplication.requestEvaluators(1);
- this.mockRuntime.succeed(this.mockRuntime.getNextProcessRequest());
- final AllocatedEvaluator evaluator = this.mockRuntime.getCurrentAllocatedEvaluators().iterator().next();
- final ActiveContext rootContext = this.mockApplication.getContext(evaluator,
- MockAllocatedEvalautor.ROOT_CONTEXT_IDENTIFIER_PREFIX + evaluator.getId());
-
- // create a child context off of root context
- this.mockApplication.submitContext(rootContext, "child");
- this.mockRuntime.succeed(this.mockRuntime.getNextProcessRequest());
- final ActiveContext childContext = this.mockApplication.getContext(evaluator, "child");
-
- // submit a task from child context
- this.mockApplication.submitTask(childContext, "test-task");
- final ProcessRequest createTaskRequest = this.mockRuntime.getNextProcessRequest();
- createTaskRequest.setAutoComplete(false); // keep it running
- this.mockRuntime.succeed(createTaskRequest);
- final RunningTask task = this.mockRuntime.getCurrentRunningTasks().iterator().next();
-
- // fail task
- this.mockRuntime.fail(task);
- assertEquals("task failed", 1, this.mockApplication.getFailedTasks().size());
-
- // fail child context
- this.mockRuntime.fail(childContext);
- assertTrue("child context failed",
- this.mockApplication.getFailedContext().iterator().next().getId().equals(childContext.getId()));
- // evaluator should still be up
- assertEquals("check evaluator", 0, this.mockApplication.getFailedEvaluators().size());
-
- // fail evaluator
- this.mockRuntime.fail(evaluator);
- assertEquals("evaluator failed", 1, this.mockApplication.getFailedEvaluators().size());
-
- // both contexts should be failed
- assertEquals("root and child contexts failed", 2,
- this.mockApplication.getFailedContext().size());
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/MockApplication.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/MockApplication.java b/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/MockApplication.java
deleted file mode 100644
index 86a105e..0000000
--- a/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/MockApplication.java
+++ /dev/null
@@ -1,275 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- */
-
-package org.apache.reef.mock;
-
-import org.apache.reef.driver.context.ActiveContext;
-import org.apache.reef.driver.context.ClosedContext;
-import org.apache.reef.driver.context.ContextConfiguration;
-import org.apache.reef.driver.context.FailedContext;
-import org.apache.reef.driver.evaluator.AllocatedEvaluator;
-import org.apache.reef.driver.evaluator.CompletedEvaluator;
-import org.apache.reef.driver.evaluator.EvaluatorRequestor;
-import org.apache.reef.driver.evaluator.FailedEvaluator;
-import org.apache.reef.driver.task.*;
-import org.apache.reef.tang.annotations.Unit;
-import org.apache.reef.task.Task;
-import org.apache.reef.wake.EventHandler;
-import org.apache.reef.wake.time.Clock;
-import org.apache.reef.wake.time.event.Alarm;
-import org.apache.reef.wake.time.event.StartTime;
-import org.apache.reef.wake.time.event.StopTime;
-
-import javax.inject.Inject;
-import java.util.*;
-import java.util.logging.Level;
-import java.util.logging.Logger;
-
-/**
- * mock application.
- */
-@Unit
-final class MockApplication {
-
- private static final Logger LOG = Logger.getLogger(MockApplication.class.getName());
-
- private final Clock clock;
-
- private final EvaluatorRequestor evaluatorRequestor;
-
- private final Map<String, Map<String, ActiveContext>> evaluatorId2ContextId2ContextMap = new HashMap<>();
-
- private final Map<String, AllocatedEvaluator> evaluatorMap = new HashMap<>();
-
- private final Map<String, FailedEvaluator> failedEvaluatorMap = new HashMap<>();
-
- private final Map<String, RunningTask> evaluatorIdRunningTaskMap = new HashMap<>();
-
- private final Set<FailedContext> failedContextSet = new HashSet<>();
-
- private final Set<FailedTask> failedTaskSet = new HashSet<>();
-
- private final Set<SuspendedTask> suspendedTaskSet = new HashSet<>();
-
- private boolean running = false;
-
- @Inject
- MockApplication(final Clock clock, final EvaluatorRequestor evaluatorRequestor) {
- this.clock = clock;
- this.evaluatorRequestor = evaluatorRequestor;
- }
-
- ActiveContext getContext(final AllocatedEvaluator evaluator, final String identifier) {
- return this.evaluatorId2ContextId2ContextMap.get(evaluator.getId()).get(identifier);
- }
-
- Collection<RunningTask> getRunningTasks() {
- return Collections.unmodifiableCollection(this.evaluatorIdRunningTaskMap.values());
- }
-
- Collection<AllocatedEvaluator> getAllocatedEvaluators() {
- return Collections.unmodifiableCollection(this.evaluatorMap.values());
- }
-
- Collection<FailedEvaluator> getFailedEvaluators() {
- return Collections.unmodifiableCollection(this.failedEvaluatorMap.values());
- }
-
- Collection<FailedTask> getFailedTasks() {
- return Collections.unmodifiableCollection(this.failedTaskSet);
- }
-
- Collection<FailedContext> getFailedContext() {
- return Collections.unmodifiableCollection(this.failedContextSet);
- }
-
- void requestEvaluators(final int numEvaluators) {
- LOG.log(Level.INFO, "request {0} Evaluators", numEvaluators);
- evaluatorRequestor.newRequest()
- .setMemory(128)
- .setNumberOfCores(1)
- .setNumber(numEvaluators)
- .submit();
- }
-
- void submitTask(final ActiveContext context, final String identifier) {
- context.submitTask(TaskConfiguration.CONF
- .set(TaskConfiguration.IDENTIFIER, identifier)
- .set(TaskConfiguration.TASK, DummyTestTask.class)
- .build());
- }
-
- void submitContext(final ActiveContext context, final String identifier) {
- context.submitContext(ContextConfiguration.CONF
- .set(ContextConfiguration.IDENTIFIER, identifier)
- .build());
- }
-
- boolean isRunning() {
- return this.running;
- }
-
- boolean exists(final AllocatedEvaluator evaluator) {
- return this.evaluatorMap.containsKey(evaluator.getId());
- }
-
- /**
- * Job Driver is ready and the clock is set up: request the evaluatorMap.
- */
- final class StartHandler implements EventHandler<StartTime> {
- @Override
- public void onNext(final StartTime startTime) {
- clock.scheduleAlarm(Integer.MAX_VALUE, new EventHandler<Alarm>() {
- @Override
- public void onNext(final Alarm value) {
- throw new RuntimeException("should not happen");
- }
- });
- running = true;
- }
- }
-
- /**
- * Job Driver is is shutting down: write to the log.
- */
- final class StopHandler implements EventHandler<StopTime> {
- @Override
- public void onNext(final StopTime stopTime) {
- running = false;
- }
- }
-
- /**
- * Receive notification that an Evaluator had been allocated,
- * and submitTask a new Task in that Evaluator.
- */
- final class AllocatedEvaluatorHandler implements EventHandler<AllocatedEvaluator> {
- @Override
- public void onNext(final AllocatedEvaluator eval) {
- evaluatorMap.put(eval.getId(), eval);
- }
- }
-
- /**
- * Receive notification that the Evaluator has been shut down.
- */
- final class CompletedEvaluatorHandler implements EventHandler<CompletedEvaluator> {
- @Override
- public void onNext(final CompletedEvaluator eval) {
- evaluatorMap.remove(eval.getId());
- evaluatorId2ContextId2ContextMap.remove(eval.getId());
- evaluatorIdRunningTaskMap.remove(eval.getId());
- }
- }
-
- final class FailedEvaluatorHandler implements EventHandler<FailedEvaluator> {
-
- @Override
- public void onNext(final FailedEvaluator eval) {
- evaluatorMap.remove(eval.getId());
- evaluatorId2ContextId2ContextMap.remove(eval.getId());
- evaluatorIdRunningTaskMap.remove(eval.getId());
- failedEvaluatorMap.put(eval.getId(), eval);
- failedContextSet.addAll(eval.getFailedContextList());
- }
- }
-
- /**
- * Receive notification that the Context is active.
- */
- final class ActiveContextHandler implements EventHandler<ActiveContext> {
- @Override
- public void onNext(final ActiveContext context) {
- if (!evaluatorId2ContextId2ContextMap.containsKey(context.getEvaluatorId())) {
- evaluatorId2ContextId2ContextMap.put(context.getEvaluatorId(), new HashMap<String, ActiveContext>());
- }
- if (evaluatorId2ContextId2ContextMap.get(context.getEvaluatorId()).containsKey(context.getId())) {
- throw new IllegalStateException(
- String.format("Context %s on evaluator %s already exists on evaluator with " +
- "same identifier", context.getId(), context.getEvaluatorId()));
- }
- evaluatorId2ContextId2ContextMap.get(context.getEvaluatorId()).put(context.getId(), context);
- }
- }
-
- final class ContextClosedHandler implements EventHandler<ClosedContext> {
- @Override
- public void onNext(final ClosedContext value) {
- assert evaluatorId2ContextId2ContextMap.containsKey(value.getEvaluatorId());
- assert evaluatorId2ContextId2ContextMap.get(value.getEvaluatorId()).containsKey(value.getId());
- evaluatorId2ContextId2ContextMap.get(value.getEvaluatorId()).remove(value.getId());
- }
- }
-
- final class FailedContextHandler implements EventHandler<FailedContext> {
- @Override
- public void onNext(final FailedContext value) {
- if (evaluatorId2ContextId2ContextMap.containsKey(value.getEvaluatorId()) &&
- evaluatorId2ContextId2ContextMap.get(value.getEvaluatorId()).containsKey(value.getId())) {
- evaluatorId2ContextId2ContextMap.get(value.getEvaluatorId()).remove(value.getEvaluatorId());
- } else {
- // must have failed before it succeeded
- }
- failedContextSet.add(value);
- }
- }
-
- /**
- * Receive notification that the Task is running.
- */
- final class RunningTaskHandler implements EventHandler<RunningTask> {
- @Override
- public void onNext(final RunningTask task) {
- evaluatorIdRunningTaskMap.put(task.getActiveContext().getEvaluatorId(), task);
- }
- }
-
- /**
- * Receive notification that the Task has completed successfully.
- */
- final class CompletedTaskHandler implements EventHandler<CompletedTask> {
- @Override
- public void onNext(final CompletedTask task) {
- evaluatorIdRunningTaskMap.remove(task.getActiveContext().getEvaluatorId());
- }
- }
-
- final class FailedTaskHandler implements EventHandler<FailedTask> {
- @Override
- public void onNext(final FailedTask value) {
- evaluatorIdRunningTaskMap.remove(value.getActiveContext().get().getEvaluatorId());
- failedTaskSet.add(value);
- }
- }
-
- final class SuspendedTaskHandler implements EventHandler<SuspendedTask> {
- @Override
- public void onNext(final SuspendedTask value) {
- evaluatorIdRunningTaskMap.remove(value.getActiveContext().getEvaluatorId());
- suspendedTaskSet.add(value);
- }
- }
-
- private static final class DummyTestTask implements Task {
- @Override
- public byte[] call(final byte[] memento) throws Exception {
- return new byte[0];
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/driver/BasicMockTests.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/driver/BasicMockTests.java b/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/driver/BasicMockTests.java
new file mode 100644
index 0000000..8b52295
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/driver/BasicMockTests.java
@@ -0,0 +1,163 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver;
+
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.evaluator.AllocatedEvaluator;
+import org.apache.reef.driver.task.RunningTask;
+import org.apache.reef.mock.driver.request.ProcessRequestInternal;
+import org.apache.reef.mock.driver.runtime.MockAllocatedEvaluator;
+import org.apache.reef.mock.driver.runtime.MockClock;
+import org.apache.reef.tang.Configuration;
+import org.apache.reef.tang.Injector;
+import org.apache.reef.tang.Tang;
+import org.junit.Before;
+import org.junit.Test;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+/**
+ * basic mock tests.
+ */
+final class BasicMockTests {
+
+ private MockApplication mockApplication;
+
+ private MockRuntime mockRuntime;
+
+ private MockClock mockClock;
+
+ @Before
+ public void initialize() throws Exception {
+ final Configuration conf = MockConfiguration.CONF
+ .set(MockConfiguration.ON_DRIVER_STARTED, MockApplication.StartHandler.class)
+ .set(MockConfiguration.ON_DRIVER_STOP, MockApplication.StopHandler.class)
+ .set(MockConfiguration.ON_CONTEXT_ACTIVE, MockApplication.ActiveContextHandler.class)
+ .set(MockConfiguration.ON_CONTEXT_CLOSED, MockApplication.ContextClosedHandler.class)
+ .set(MockConfiguration.ON_CONTEXT_FAILED, MockApplication.FailedContextHandler.class)
+ .set(MockConfiguration.ON_EVALUATOR_ALLOCATED, MockApplication.AllocatedEvaluatorHandler.class)
+ .set(MockConfiguration.ON_EVALUATOR_COMPLETED, MockApplication.CompletedEvaluatorHandler.class)
+ .set(MockConfiguration.ON_EVALUATOR_FAILED, MockApplication.FailedEvaluatorHandler.class)
+ .set(MockConfiguration.ON_TASK_COMPLETED, MockApplication.CompletedTaskHandler.class)
+ .set(MockConfiguration.ON_TASK_FAILED, MockApplication.FailedTaskHandler.class)
+ .set(MockConfiguration.ON_TASK_RUNNING, MockApplication.RunningTaskHandler.class)
+ .set(MockConfiguration.ON_TASK_SUSPENDED, MockApplication.SuspendedTaskHandler.class)
+ .build();
+
+ final Injector injector = Tang.Factory.getTang().newInjector(conf);
+ this.mockApplication = injector.getInstance(MockApplication.class);
+ this.mockRuntime = injector.getInstance(MockRuntime.class);
+ this.mockClock = injector.getInstance(MockClock.class);
+
+ this.mockClock.run();
+ }
+
+ @Test
+ public void testSuccessRequests() throws Exception {
+ assertTrue("mock application received start event", this.mockApplication.isRunning());
+
+ this.mockApplication.requestEvaluators(1);
+ assertTrue("check for process event", this.mockRuntime.hasProcessRequest());
+ final ProcessRequest allocateEvaluatorRequest = this.mockRuntime.getNextProcessRequest();
+ assertEquals("allocate evaluator request", ProcessRequest.Type.ALLOCATE_EVALUATOR,
+ allocateEvaluatorRequest.getType());
+ final AllocatedEvaluator evaluator =
+ ((ProcessRequestInternal<AllocatedEvaluator, Object>)allocateEvaluatorRequest)
+ .getSuccessEvent();
+ this.mockRuntime.succeed(allocateEvaluatorRequest);
+ assertTrue("evaluator allocation succeeded",
+ this.mockApplication.getAllocatedEvaluators().contains(evaluator));
+ final String contextId = "foo";
+ this.mockApplication.submitContext(evaluator, contextId);
+ final ActiveContext rootContext = ((MockAllocatedEvaluator) evaluator).getRootContext();
+ assertTrue("root context", rootContext != null);
+
+
+ // submit a task
+ this.mockApplication.submitTask(rootContext, "test-task");
+ assertTrue("create task queued", this.mockRuntime.hasProcessRequest());
+ final ProcessRequest createTaskRequest = this.mockRuntime.getNextProcessRequest();
+ assertEquals("create task request", ProcessRequest.Type.CREATE_TASK,
+ createTaskRequest.getType());
+ final RunningTask task = (RunningTask) ((ProcessRequestInternal)createTaskRequest).getSuccessEvent();
+ this.mockRuntime.succeed(createTaskRequest);
+ assertTrue("task running", this.mockApplication.getRunningTasks().contains(task));
+
+ // check task auto complete
+ assertTrue("check for request", this.mockRuntime.hasProcessRequest());
+ final ProcessRequestInternal completedTask =
+ (ProcessRequestInternal) this.mockRuntime.getNextProcessRequest();
+ assertEquals("complete task request", ProcessRequest.Type.COMPLETE_TASK,
+ completedTask.getType());
+ this.mockRuntime.succeed(completedTask);
+ assertEquals("no running tasks", 0, this.mockApplication.getRunningTasks().size());
+ }
+
+ @Test
+ public void testFailureRequests() throws Exception {
+ assertTrue("mock application received start event", this.mockApplication.isRunning());
+
+ this.mockApplication.requestEvaluators(1);
+ assertTrue("check for process event", this.mockRuntime.hasProcessRequest());
+ ProcessRequest allocateEvaluatorRequest = this.mockRuntime.getNextProcessRequest();
+ this.mockRuntime.fail(allocateEvaluatorRequest);
+ assertEquals("evaluator allocation failed", 1,
+ this.mockApplication.getFailedEvaluators().size());
+
+ this.mockApplication.requestEvaluators(1);
+ allocateEvaluatorRequest = this.mockRuntime.getNextProcessRequest();
+ final AllocatedEvaluator evaluator =
+ (AllocatedEvaluator)((ProcessRequestInternal)allocateEvaluatorRequest).getSuccessEvent();
+ this.mockRuntime.succeed(allocateEvaluatorRequest);
+ this.mockApplication.submitContext(evaluator, "FOO");
+ final ActiveContext rootContext = this.mockApplication
+ .getContext(evaluator, "FOO");
+
+
+ // submit a task
+ this.mockApplication.submitTask(rootContext, "test-task");
+ assertTrue("create task queued", this.mockRuntime.hasProcessRequest());
+ final ProcessRequest createTaskRequest = this.mockRuntime.getNextProcessRequest();
+ assertEquals("create task request", ProcessRequest.Type.CREATE_TASK,
+ createTaskRequest.getType());
+ this.mockRuntime.fail(createTaskRequest);
+ assertEquals("task running", 1, this.mockApplication.getFailedTasks().size());
+ }
+
+ @Test
+ public void testMockFailures() {
+ // make sure we're running
+ assertTrue("mock application received start event", this.mockApplication.isRunning());
+
+ // allocate an evaluator and get root context
+ this.mockApplication.requestEvaluators(1);
+ this.mockRuntime.succeed(this.mockRuntime.getNextProcessRequest());
+ final AllocatedEvaluator evaluator = this.mockRuntime.getCurrentAllocatedEvaluators().iterator().next();
+ this.mockApplication.submitContext(evaluator, "FOO");
+ // fail evaluator
+ this.mockRuntime.fail(evaluator);
+ assertEquals("evaluator failed", 1, this.mockApplication.getFailedEvaluators().size());
+
+ // both contexts should be failed
+ assertEquals("root and child contexts failed", 2,
+ this.mockApplication.getFailedContext().size());
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/driver/MockApplication.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/driver/MockApplication.java b/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/driver/MockApplication.java
new file mode 100644
index 0000000..3ce66b2
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/driver/MockApplication.java
@@ -0,0 +1,275 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+package org.apache.reef.mock.driver;
+
+import org.apache.reef.driver.context.ActiveContext;
+import org.apache.reef.driver.context.ClosedContext;
+import org.apache.reef.driver.context.ContextConfiguration;
+import org.apache.reef.driver.context.FailedContext;
+import org.apache.reef.driver.evaluator.AllocatedEvaluator;
+import org.apache.reef.driver.evaluator.CompletedEvaluator;
+import org.apache.reef.driver.evaluator.EvaluatorRequestor;
+import org.apache.reef.driver.evaluator.FailedEvaluator;
+import org.apache.reef.driver.task.*;
+import org.apache.reef.tang.annotations.Unit;
+import org.apache.reef.task.Task;
+import org.apache.reef.wake.EventHandler;
+import org.apache.reef.wake.time.Clock;
+import org.apache.reef.wake.time.event.Alarm;
+import org.apache.reef.wake.time.event.StartTime;
+import org.apache.reef.wake.time.event.StopTime;
+
+import javax.inject.Inject;
+import java.util.*;
+import java.util.logging.Level;
+import java.util.logging.Logger;
+
+/**
+ * mock application.
+ */
+@Unit
+final class MockApplication {
+
+ private static final Logger LOG = Logger.getLogger(MockApplication.class.getName());
+
+ private final Clock clock;
+
+ private final EvaluatorRequestor evaluatorRequestor;
+
+ private final Map<String, Map<String, ActiveContext>> evaluatorId2ContextId2ContextMap = new HashMap<>();
+
+ private final Map<String, AllocatedEvaluator> evaluatorMap = new HashMap<>();
+
+ private final Map<String, FailedEvaluator> failedEvaluatorMap = new HashMap<>();
+
+ private final Map<String, RunningTask> evaluatorIdRunningTaskMap = new HashMap<>();
+
+ private final Set<FailedContext> failedContextSet = new HashSet<>();
+
+ private final Set<FailedTask> failedTaskSet = new HashSet<>();
+
+ private final Set<SuspendedTask> suspendedTaskSet = new HashSet<>();
+
+ private boolean running = false;
+
+ @Inject
+ MockApplication(final Clock clock, final EvaluatorRequestor evaluatorRequestor) {
+ this.clock = clock;
+ this.evaluatorRequestor = evaluatorRequestor;
+ }
+
+ ActiveContext getContext(final AllocatedEvaluator evaluator, final String identifier) {
+ return this.evaluatorId2ContextId2ContextMap.get(evaluator.getId()).get(identifier);
+ }
+
+ Collection<RunningTask> getRunningTasks() {
+ return Collections.unmodifiableCollection(this.evaluatorIdRunningTaskMap.values());
+ }
+
+ Collection<AllocatedEvaluator> getAllocatedEvaluators() {
+ return Collections.unmodifiableCollection(this.evaluatorMap.values());
+ }
+
+ Collection<FailedEvaluator> getFailedEvaluators() {
+ return Collections.unmodifiableCollection(this.failedEvaluatorMap.values());
+ }
+
+ Collection<FailedTask> getFailedTasks() {
+ return Collections.unmodifiableCollection(this.failedTaskSet);
+ }
+
+ Collection<FailedContext> getFailedContext() {
+ return Collections.unmodifiableCollection(this.failedContextSet);
+ }
+
+ void requestEvaluators(final int numEvaluators) {
+ LOG.log(Level.INFO, "request {0} Evaluators", numEvaluators);
+ evaluatorRequestor.newRequest()
+ .setMemory(128)
+ .setNumberOfCores(1)
+ .setNumber(numEvaluators)
+ .submit();
+ }
+
+ void submitTask(final ActiveContext context, final String identifier) {
+ context.submitTask(TaskConfiguration.CONF
+ .set(TaskConfiguration.IDENTIFIER, identifier)
+ .set(TaskConfiguration.TASK, DummyTestTask.class)
+ .build());
+ }
+
+ void submitContext(final AllocatedEvaluator evaluator, final String identifier) {
+ evaluator.submitContext(ContextConfiguration.CONF
+ .set(ContextConfiguration.IDENTIFIER, identifier)
+ .build());
+ }
+
+ boolean isRunning() {
+ return this.running;
+ }
+
+ boolean exists(final AllocatedEvaluator evaluator) {
+ return this.evaluatorMap.containsKey(evaluator.getId());
+ }
+
+ /**
+ * Job Driver is ready and the clock is set up: request the evaluatorMap.
+ */
+ final class StartHandler implements EventHandler<StartTime> {
+ @Override
+ public void onNext(final StartTime startTime) {
+ clock.scheduleAlarm(Integer.MAX_VALUE, new EventHandler<Alarm>() {
+ @Override
+ public void onNext(final Alarm value) {
+ throw new RuntimeException("should not happen");
+ }
+ });
+ running = true;
+ }
+ }
+
+ /**
+ * Job Driver is is shutting down: write to the log.
+ */
+ final class StopHandler implements EventHandler<StopTime> {
+ @Override
+ public void onNext(final StopTime stopTime) {
+ running = false;
+ }
+ }
+
+ /**
+ * Receive notification that an Evaluator had been allocated,
+ * and submitTask a new Task in that Evaluator.
+ */
+ final class AllocatedEvaluatorHandler implements EventHandler<AllocatedEvaluator> {
+ @Override
+ public void onNext(final AllocatedEvaluator eval) {
+ evaluatorMap.put(eval.getId(), eval);
+ }
+ }
+
+ /**
+ * Receive notification that the Evaluator has been shut down.
+ */
+ final class CompletedEvaluatorHandler implements EventHandler<CompletedEvaluator> {
+ @Override
+ public void onNext(final CompletedEvaluator eval) {
+ evaluatorMap.remove(eval.getId());
+ evaluatorId2ContextId2ContextMap.remove(eval.getId());
+ evaluatorIdRunningTaskMap.remove(eval.getId());
+ }
+ }
+
+ final class FailedEvaluatorHandler implements EventHandler<FailedEvaluator> {
+
+ @Override
+ public void onNext(final FailedEvaluator eval) {
+ evaluatorMap.remove(eval.getId());
+ evaluatorId2ContextId2ContextMap.remove(eval.getId());
+ evaluatorIdRunningTaskMap.remove(eval.getId());
+ failedEvaluatorMap.put(eval.getId(), eval);
+ failedContextSet.addAll(eval.getFailedContextList());
+ }
+ }
+
+ /**
+ * Receive notification that the Context is active.
+ */
+ final class ActiveContextHandler implements EventHandler<ActiveContext> {
+ @Override
+ public void onNext(final ActiveContext context) {
+ if (!evaluatorId2ContextId2ContextMap.containsKey(context.getEvaluatorId())) {
+ evaluatorId2ContextId2ContextMap.put(context.getEvaluatorId(), new HashMap<String, ActiveContext>());
+ }
+ if (evaluatorId2ContextId2ContextMap.get(context.getEvaluatorId()).containsKey(context.getId())) {
+ throw new IllegalStateException(
+ String.format("Context %s on evaluator %s already exists on evaluator with " +
+ "same identifier", context.getId(), context.getEvaluatorId()));
+ }
+ evaluatorId2ContextId2ContextMap.get(context.getEvaluatorId()).put(context.getId(), context);
+ }
+ }
+
+ final class ContextClosedHandler implements EventHandler<ClosedContext> {
+ @Override
+ public void onNext(final ClosedContext value) {
+ assert evaluatorId2ContextId2ContextMap.containsKey(value.getEvaluatorId());
+ assert evaluatorId2ContextId2ContextMap.get(value.getEvaluatorId()).containsKey(value.getId());
+ evaluatorId2ContextId2ContextMap.get(value.getEvaluatorId()).remove(value.getId());
+ }
+ }
+
+ final class FailedContextHandler implements EventHandler<FailedContext> {
+ @Override
+ public void onNext(final FailedContext value) {
+ if (evaluatorId2ContextId2ContextMap.containsKey(value.getEvaluatorId()) &&
+ evaluatorId2ContextId2ContextMap.get(value.getEvaluatorId()).containsKey(value.getId())) {
+ evaluatorId2ContextId2ContextMap.get(value.getEvaluatorId()).remove(value.getEvaluatorId());
+ } else {
+ // must have failed before it succeeded
+ }
+ failedContextSet.add(value);
+ }
+ }
+
+ /**
+ * Receive notification that the Task is running.
+ */
+ final class RunningTaskHandler implements EventHandler<RunningTask> {
+ @Override
+ public void onNext(final RunningTask task) {
+ evaluatorIdRunningTaskMap.put(task.getActiveContext().getEvaluatorId(), task);
+ }
+ }
+
+ /**
+ * Receive notification that the Task has completed successfully.
+ */
+ final class CompletedTaskHandler implements EventHandler<CompletedTask> {
+ @Override
+ public void onNext(final CompletedTask task) {
+ evaluatorIdRunningTaskMap.remove(task.getActiveContext().getEvaluatorId());
+ }
+ }
+
+ final class FailedTaskHandler implements EventHandler<FailedTask> {
+ @Override
+ public void onNext(final FailedTask value) {
+ evaluatorIdRunningTaskMap.remove(value.getActiveContext().get().getEvaluatorId());
+ failedTaskSet.add(value);
+ }
+ }
+
+ final class SuspendedTaskHandler implements EventHandler<SuspendedTask> {
+ @Override
+ public void onNext(final SuspendedTask value) {
+ evaluatorIdRunningTaskMap.remove(value.getActiveContext().getEvaluatorId());
+ suspendedTaskSet.add(value);
+ }
+ }
+
+ private static final class DummyTestTask implements Task {
+ @Override
+ public byte[] call(final byte[] memento) throws Exception {
+ return new byte[0];
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/driver/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/driver/package-info.java b/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/driver/package-info.java
new file mode 100644
index 0000000..75f80b6
--- /dev/null
+++ b/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/driver/package-info.java
@@ -0,0 +1,23 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ *
+ */
+/**
+ * mock runtime tests.
+ */
+package org.apache.reef.mock.driver;
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/package-info.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/package-info.java b/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/package-info.java
deleted file mode 100644
index e93688c..0000000
--- a/lang/java/reef-runtime-mock/src/test/java/org/apache/reef/mock/package-info.java
+++ /dev/null
@@ -1,23 +0,0 @@
-/*
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing,
- * software distributed under the License is distributed on an
- * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
- * KIND, either express or implied. See the License for the
- * specific language governing permissions and limitations
- * under the License.
- *
- */
-/**
- * mock runtime tests.
- */
-package org.apache.reef.mock;
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRestartConfiguration.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRestartConfiguration.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRestartConfiguration.java
index dc175e2..0f88c46 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRestartConfiguration.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRestartConfiguration.java
@@ -40,7 +40,7 @@ import org.apache.reef.tang.formats.OptionalImpl;
@Unstable
public final class YarnDriverRestartConfiguration extends ConfigurationModuleBuilder {
/**
- * The Evaluator Preserver implementation used for YARN. Defaults to DFSEvalutorPreserver.
+ * The Evaluator Preserver implementation used for YARN. Defaults to DFSEvaluatorPreserver.
*/
public static final OptionalImpl<EvaluatorPreserver> EVALUATOR_PRESERVER = new OptionalImpl<>();
http://git-wip-us.apache.org/repos/asf/reef/blob/5ed56eba/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRuntimeRestartManager.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRuntimeRestartManager.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRuntimeRestartManager.java
index bdf1779..9ba8d28 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRuntimeRestartManager.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnDriverRuntimeRestartManager.java
@@ -156,7 +156,7 @@ public final class YarnDriverRuntimeRestartManager implements DriverRuntimeResta
/**
* Used by {@link org.apache.reef.driver.restart.DriverRestartManager}.
* Gets the list of previous containers from the resource manager,
- * compares that list to the YarnDriverRuntimeRestartManager's own list based on the evalutor preserver,
+ * compares that list to the YarnDriverRuntimeRestartManager's own list based on the evaluator preserver,
* and determine which evaluators are alive and which have failed during restart.
* @return a map of Evaluator ID to {@link EvaluatorRestartInfo} for evaluators that have either failed or survived
* driver restart.
[04/16] reef git commit: [TRIVIAL] Close a bunch of stale Pull
Requests
Posted by do...@apache.org.
[TRIVIAL] Close a bunch of stale Pull Requests
Pull Requests:
This closes #1434
This closes #1404
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/aa30e7fe
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/aa30e7fe
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/aa30e7fe
Branch: refs/heads/REEF-335
Commit: aa30e7fec2f4d771dc48bd87048333fa2f4b0b16
Parents: 8dc3331
Author: Markus Weimer <we...@apache.org>
Authored: Wed Apr 25 13:26:47 2018 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Wed Apr 25 13:26:47 2018 -0700
----------------------------------------------------------------------
dev/ClosedPullRequests.md | 12 ++++++++++++
1 file changed, 12 insertions(+)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/aa30e7fe/dev/ClosedPullRequests.md
----------------------------------------------------------------------
diff --git a/dev/ClosedPullRequests.md b/dev/ClosedPullRequests.md
new file mode 100644
index 0000000..335b7f1
--- /dev/null
+++ b/dev/ClosedPullRequests.md
@@ -0,0 +1,12 @@
+Pull Requests closed via commits
+================================
+
+In this file, we collect the number of pull requests we closed via commits that
+only closed that PR, but did not accept the code. This can become neccesary when
+the original submitter of the pull request becomes unresponsive in the review
+process.
+
+| PR Number | Closed When | Closed by |
+|-----------|-------------|-------------------|
+| 1434 | 2018-04-25 | weimer@apache.org |
+| 1404 | 2018-04-25 | weimer@apache.org |
[07/16] reef git commit: [REEF-2011] Enable node label expressions in
evaluator requestor.
Posted by do...@apache.org.
[REEF-2011] Enable node label expressions in evaluator requestor.
This add the notion of node label expressions to resource manager
requests in both Java and C#.
JIRA:
[REEF-2011](https://issues.apache.org/jira/browse/REEF-2011)
Pull Request:
Closes #1451
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/24bfea49
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/24bfea49
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/24bfea49
Branch: refs/heads/REEF-335
Commit: 24bfea49dbf5c956033c214bddcb6c8ff5cd2a57
Parents: 8d0bad2
Author: Tyson Condie <tc...@apache.org>
Authored: Wed Apr 25 16:33:07 2018 -0700
Committer: Sergiy Matusevych <mo...@apache.org>
Committed: Thu Apr 26 15:48:10 2018 -0700
----------------------------------------------------------------------
.../EvaluatorRequestorClr2Java.cpp | 5 ++--
.../Evaluator/EvaluatorRequest.cs | 20 ++++++++------
.../Evaluator/EvaluatorRequestBuilder.cs | 16 ++++++++++-
.../Evaluator/IEvaluatorRequest.cs | 6 ++++
.../javabridge/EvaluatorRequestorBridge.java | 4 ++-
.../reef/driver/evaluator/EvaluatorRequest.java | 29 ++++++++++++++++++--
.../common/driver/EvaluatorRequestorImpl.java | 1 +
.../common/driver/api/ResourceRequestEvent.java | 5 ++++
.../driver/api/ResourceRequestEventImpl.java | 19 +++++++++++++
.../yarn/driver/YarnContainerManager.java | 3 ++
.../yarn/driver/YarnResourceRequestHandler.java | 6 +++-
11 files changed, 98 insertions(+), 16 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/24bfea49/lang/cs/Org.Apache.REEF.Bridge/EvaluatorRequestorClr2Java.cpp
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Bridge/EvaluatorRequestorClr2Java.cpp b/lang/cs/Org.Apache.REEF.Bridge/EvaluatorRequestorClr2Java.cpp
index 90fe5d4..4dc9bd1 100644
--- a/lang/cs/Org.Apache.REEF.Bridge/EvaluatorRequestorClr2Java.cpp
+++ b/lang/cs/Org.Apache.REEF.Bridge/EvaluatorRequestorClr2Java.cpp
@@ -53,7 +53,7 @@ namespace Org {
ManagedLog::LOGGER->LogStart("EvaluatorRequestorClr2Java::Submit");
JNIEnv *env = RetrieveEnv(_jvm);
jclass jclassEvaluatorRequestor = env->GetObjectClass(_jobjectEvaluatorRequestor);
- jmethodID jmidSubmit = env->GetMethodID(jclassEvaluatorRequestor, "submit", "(IIIZLjava/lang/String;Ljava/lang/String;Ljava/util/ArrayList;)V");
+ jmethodID jmidSubmit = env->GetMethodID(jclassEvaluatorRequestor, "submit", "(IIIZLjava/lang/String;Ljava/lang/String;Ljava/util/ArrayList;Ljava/lang/String;)V");
if (jmidSubmit == NULL) {
fprintf(stdout, " jmidSubmit is NULL\n");
@@ -69,7 +69,8 @@ namespace Org {
request->RelaxLocality,
JavaStringFromManagedString(env, request->Rack),
JavaStringFromManagedString(env, request->RuntimeName),
- JavaArrayListFromManagedList(env, request->NodeNames));
+ JavaArrayListFromManagedList(env, request->NodeNames),
+ JavaStringFromManagedString(env, request->NodeLabelExpression));
ManagedLog::LOGGER->LogStop("EvaluatorRequestorClr2Java::Submit");
}
http://git-wip-us.apache.org/repos/asf/reef/blob/24bfea49/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs b/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs
index 2c8365d..f4e827f 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequest.cs
@@ -29,43 +29,43 @@ namespace Org.Apache.REEF.Driver.Evaluator
internal class EvaluatorRequest : IEvaluatorRequest
{
internal EvaluatorRequest()
- : this(0, 0, 1, string.Empty, Guid.NewGuid().ToString("N"), string.Empty, Enumerable.Empty<string>().ToList(), true)
+ : this(0, 0, 1, string.Empty, Guid.NewGuid().ToString("N"), string.Empty, Enumerable.Empty<string>().ToList(), true, String.Empty)
{
}
internal EvaluatorRequest(int number, int megaBytes)
- : this(number, megaBytes, 1, string.Empty, Guid.NewGuid().ToString("N"), string.Empty, Enumerable.Empty<string>().ToList(), true)
+ : this(number, megaBytes, 1, string.Empty, Guid.NewGuid().ToString("N"), string.Empty, Enumerable.Empty<string>().ToList(), true, string.Empty)
{
}
internal EvaluatorRequest(int number, int megaBytes, int core)
- : this(number, megaBytes, core, string.Empty, Guid.NewGuid().ToString("N"), string.Empty, Enumerable.Empty<string>().ToList(), true)
+ : this(number, megaBytes, core, string.Empty, Guid.NewGuid().ToString("N"), string.Empty, Enumerable.Empty<string>().ToList(), true, string.Empty)
{
}
internal EvaluatorRequest(int number, int megaBytes, string rack)
- : this(number, megaBytes, 1, rack, Guid.NewGuid().ToString("N"), string.Empty, Enumerable.Empty<string>().ToList(), true)
+ : this(number, megaBytes, 1, rack, Guid.NewGuid().ToString("N"), string.Empty, Enumerable.Empty<string>().ToList(), true, String.Empty)
{
}
internal EvaluatorRequest(int number, int megaBytes, int core, string rack)
- : this(number, megaBytes, core, rack, Guid.NewGuid().ToString("N"), string.Empty, Enumerable.Empty<string>().ToList(), true)
+ : this(number, megaBytes, core, rack, Guid.NewGuid().ToString("N"), string.Empty, Enumerable.Empty<string>().ToList(), true, string.Empty)
{
}
internal EvaluatorRequest(int number, int megaBytes, int core, string rack, string evaluatorBatchId, ICollection<string> nodeNames)
- : this(number, megaBytes, core, rack, evaluatorBatchId, string.Empty, nodeNames, true)
+ : this(number, megaBytes, core, rack, evaluatorBatchId, string.Empty, nodeNames, true, string.Empty)
{
}
internal EvaluatorRequest(int number, int megaBytes, int core, string rack, string evaluatorBatchId, ICollection<string> nodeNames, bool relaxLocality)
- : this(number, megaBytes, core, rack, evaluatorBatchId, string.Empty, nodeNames, relaxLocality)
+ : this(number, megaBytes, core, rack, evaluatorBatchId, string.Empty, nodeNames, relaxLocality, string.Empty)
{
}
- internal EvaluatorRequest(int number, int megaBytes, int core, string rack, string evaluatorBatchId, string runtimeName, ICollection<string> nodeNames, bool relaxLocality)
+ internal EvaluatorRequest(int number, int megaBytes, int core, string rack, string evaluatorBatchId, string runtimeName, ICollection<string> nodeNames, bool relaxLocality, string nodeLabelExpression)
{
Number = number;
MemoryMegaBytes = megaBytes;
@@ -75,6 +75,7 @@ namespace Org.Apache.REEF.Driver.Evaluator
RuntimeName = runtimeName;
NodeNames = nodeNames;
RelaxLocality = relaxLocality;
+ NodeLabelExpression = NodeLabelExpression;
}
[DataMember]
@@ -101,6 +102,9 @@ namespace Org.Apache.REEF.Driver.Evaluator
[DataMember]
public bool RelaxLocality { get; private set; }
+ [DataMember]
+ public string NodeLabelExpression { get; private set; }
+
internal static EvaluatorRequestBuilder NewBuilder()
{
return new EvaluatorRequestBuilder();
http://git-wip-us.apache.org/repos/asf/reef/blob/24bfea49/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequestBuilder.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequestBuilder.cs b/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequestBuilder.cs
index 6ccd1fb..3554bb6 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequestBuilder.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Evaluator/EvaluatorRequestBuilder.cs
@@ -29,6 +29,7 @@ namespace Org.Apache.REEF.Driver.Evaluator
private string _runtimeName;
private ICollection<string> _nodeNames;
private bool _relaxLocality;
+ private string _nodeLabelExpression;
internal EvaluatorRequestBuilder(IEvaluatorRequest request)
{
@@ -40,6 +41,7 @@ namespace Org.Apache.REEF.Driver.Evaluator
_runtimeName = request.RuntimeName;
_nodeNames = request.NodeNames;
_relaxLocality = request.RelaxLocality;
+ _nodeLabelExpression = request.NodeLabelExpression;
}
internal EvaluatorRequestBuilder()
@@ -52,6 +54,7 @@ namespace Org.Apache.REEF.Driver.Evaluator
_runtimeName = string.Empty;
_nodeNames = Enumerable.Empty<string>().ToList();
_relaxLocality = true;
+ _nodeLabelExpression = string.Empty;
}
public int Number { get; private set; }
@@ -158,12 +161,23 @@ namespace Org.Apache.REEF.Driver.Evaluator
}
/// <summary>
+ /// Set the node label expression.
+ /// </summary>
+ /// <param name="nodeLabelExpression">describing a desired node type.</param>
+ /// <returns></returns>
+ public EvaluatorRequestBuilder SetNodeLabelExpression(string nodeLabelExpression)
+ {
+ _nodeLabelExpression = nodeLabelExpression;
+ return this;
+ }
+
+ /// <summary>
/// Build the EvaluatorRequest.
/// </summary>
/// <returns></returns>
public IEvaluatorRequest Build()
{
- return new EvaluatorRequest(Number, MegaBytes, VirtualCore, rack: _rackName, evaluatorBatchId: _evaluatorBatchId, runtimeName: _runtimeName, nodeNames: _nodeNames, relaxLocality: _relaxLocality);
+ return new EvaluatorRequest(Number, MegaBytes, VirtualCore, rack: _rackName, evaluatorBatchId: _evaluatorBatchId, runtimeName: _runtimeName, nodeNames: _nodeNames, relaxLocality: _relaxLocality, nodeLabelExpression: _nodeLabelExpression);
}
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/24bfea49/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequest.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequest.cs b/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequest.cs
index 357ffe3..0ed0545 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequest.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Evaluator/IEvaluatorRequest.cs
@@ -73,5 +73,11 @@ namespace Org.Apache.REEF.Driver.Evaluator
/// the corresponding any-level request should have locality relaxation set to false.
/// </summary>
bool RelaxLocality { get; }
+
+ /// <summary>
+ /// For specifying a node label expression that can be used by the resource manager
+ /// to aquire certain container types.
+ /// </summary>
+ string NodeLabelExpression { get; }
}
}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/reef/blob/24bfea49/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/EvaluatorRequestorBridge.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/EvaluatorRequestorBridge.java b/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/EvaluatorRequestorBridge.java
index 518537d..2327846 100644
--- a/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/EvaluatorRequestorBridge.java
+++ b/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/EvaluatorRequestorBridge.java
@@ -69,7 +69,8 @@ public final class EvaluatorRequestorBridge extends NativeBridge {
final boolean relaxLocality,
final String rack,
final String runtimeName,
- final ArrayList<String> nodeNames) {
+ final ArrayList<String> nodeNames,
+ final String nodeLabelExpression) {
if (this.isBlocked) {
throw new RuntimeException("Cannot request additional Evaluator, this is probably because " +
"the Driver has crashed and restarted, and cannot ask for new container due to YARN-2433.");
@@ -89,6 +90,7 @@ public final class EvaluatorRequestorBridge extends NativeBridge {
.setRuntimeName(runtimeName)
.setRelaxLocality(relaxLocality)
.addNodeNames(nodeNames)
+ .setNodeLabelExpression(nodeLabelExpression)
.build();
LOG.log(Level.FINE, "submitting evaluator request {0}", request);
http://git-wip-us.apache.org/repos/asf/reef/blob/24bfea49/lang/java/reef-common/src/main/java/org/apache/reef/driver/evaluator/EvaluatorRequest.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/driver/evaluator/EvaluatorRequest.java b/lang/java/reef-common/src/main/java/org/apache/reef/driver/evaluator/EvaluatorRequest.java
index 4c48a24..b539a9b 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/driver/evaluator/EvaluatorRequest.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/driver/evaluator/EvaluatorRequest.java
@@ -41,6 +41,7 @@ public final class EvaluatorRequest {
private final List<String> rackNames;
private final String runtimeName;
private final boolean relaxLocality;
+ private final String nodeLabelExpression;
EvaluatorRequest(final int number,
final int megaBytes,
@@ -56,7 +57,7 @@ public final class EvaluatorRequest {
final List<String> nodeNames,
final List<String> rackNames,
final String runtimeName) {
- this(number, megaBytes, cores, nodeNames, rackNames, runtimeName, true);
+ this(number, megaBytes, cores, nodeNames, rackNames, runtimeName, true, null);
}
@@ -66,7 +67,8 @@ public final class EvaluatorRequest {
final List<String> nodeNames,
final List<String> rackNames,
final String runtimeName,
- final boolean relaxLocality) {
+ final boolean relaxLocality,
+ final String nodeLabelExpression) {
this.number = number;
this.megaBytes = megaBytes;
this.cores = cores;
@@ -74,6 +76,7 @@ public final class EvaluatorRequest {
this.rackNames = rackNames;
this.runtimeName = runtimeName;
this.relaxLocality = relaxLocality;
+ this.nodeLabelExpression = nodeLabelExpression;
}
/**
@@ -158,6 +161,13 @@ public final class EvaluatorRequest {
return relaxLocality;
}
+ /**
+ * Node label expression.
+ * @return string expression
+ */
+ public String getNodeLabelExpression() {
+ return nodeLabelExpression;
+ }
/**
* {@link EvaluatorRequest}s are build using this Builder.
@@ -171,6 +181,7 @@ public final class EvaluatorRequest {
private final List<String> rackNames = new ArrayList<>();
private String runtimeName = "";
private boolean relaxLocality = true; //if not set, default to true
+ private String nodeLabelExpression = null;
@Private
public Builder() {
@@ -194,6 +205,7 @@ public final class EvaluatorRequest {
for (final String rackName : request.getRackNames()) {
addRackName(rackName);
}
+ setNodeLabelExpression(request.getNodeLabelExpression());
}
/**
@@ -300,12 +312,23 @@ public final class EvaluatorRequest {
}
/**
+ * A string expression that describes the node type being requested.
+ * @param nodeLabelExpr describing node type
+ * @return this Builder.
+ */
+ public T setNodeLabelExpression(final String nodeLabelExpr) {
+ this.nodeLabelExpression = nodeLabelExpr;
+ return (T) this;
+ }
+
+ /**
* Builds the {@link EvaluatorRequest}.
*/
@Override
public EvaluatorRequest build() {
return new EvaluatorRequest(this.n, this.megaBytes, this.cores, this.nodeNames,
- this.rackNames, this.runtimeName, this.relaxLocality);
+ this.rackNames, this.runtimeName, this.relaxLocality,
+ this.nodeLabelExpression);
}
/**
http://git-wip-us.apache.org/repos/asf/reef/blob/24bfea49/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/EvaluatorRequestorImpl.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/EvaluatorRequestorImpl.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/EvaluatorRequestorImpl.java
index 08b385d..a007805 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/EvaluatorRequestorImpl.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/EvaluatorRequestorImpl.java
@@ -109,6 +109,7 @@ public final class EvaluatorRequestorImpl implements EvaluatorRequestor {
.addRackNames(req.getRackNames())
.setRelaxLocality(relaxLocality)
.setRuntimeName(req.getRuntimeName())
+ .setNodeLabelExpression(req.getNodeLabelExpression())
.build();
this.resourceRequestHandler.onNext(request);
}
http://git-wip-us.apache.org/repos/asf/reef/blob/24bfea49/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/api/ResourceRequestEvent.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/api/ResourceRequestEvent.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/api/ResourceRequestEvent.java
index b8ce952..735662f 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/api/ResourceRequestEvent.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/api/ResourceRequestEvent.java
@@ -73,6 +73,11 @@ public interface ResourceRequestEvent {
Optional<Boolean> getRelaxLocality();
/**
+ * @return label expression that can further describe the desired resources.
+ */
+ Optional<String> getNodeLabelExpression();
+
+ /**
* @return The runtime name
*/
String getRuntimeName();
http://git-wip-us.apache.org/repos/asf/reef/blob/24bfea49/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/api/ResourceRequestEventImpl.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/api/ResourceRequestEventImpl.java b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/api/ResourceRequestEventImpl.java
index f385087..4b43265 100644
--- a/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/api/ResourceRequestEventImpl.java
+++ b/lang/java/reef-common/src/main/java/org/apache/reef/runtime/common/driver/api/ResourceRequestEventImpl.java
@@ -36,6 +36,7 @@ public final class ResourceRequestEventImpl implements ResourceRequestEvent {
private final Optional<Integer> priority;
private final Optional<Integer> virtualCores;
private final Optional<Boolean> relaxLocality;
+ private final Optional<String> nodeLabelExpression;
private final String runtimeName;
private ResourceRequestEventImpl(final Builder builder) {
@@ -46,6 +47,7 @@ public final class ResourceRequestEventImpl implements ResourceRequestEvent {
this.priority = Optional.ofNullable(builder.priority);
this.virtualCores = Optional.ofNullable(builder.virtualCores);
this.relaxLocality = Optional.ofNullable(builder.relaxLocality);
+ this.nodeLabelExpression = Optional.ofNullable(builder.nodeLabelExpression);
this.runtimeName = builder.runtimeName == null ? "" : builder.runtimeName;
}
@@ -89,6 +91,11 @@ public final class ResourceRequestEventImpl implements ResourceRequestEvent {
return runtimeName;
}
+ @Override
+ public Optional<String> getNodeLabelExpression() {
+ return nodeLabelExpression;
+ }
+
public static Builder newBuilder() {
return new Builder();
}
@@ -105,6 +112,7 @@ public final class ResourceRequestEventImpl implements ResourceRequestEvent {
private Integer virtualCores;
private Boolean relaxLocality;
private String runtimeName;
+ private String nodeLabelExpression;
/**
* Create a builder from an existing ResourceRequestEvent.
@@ -118,6 +126,9 @@ public final class ResourceRequestEventImpl implements ResourceRequestEvent {
this.virtualCores = resourceRequestEvent.getVirtualCores().orElse(null);
this.relaxLocality = resourceRequestEvent.getRelaxLocality().orElse(null);
this.runtimeName = resourceRequestEvent.getRuntimeName();
+ this.nodeLabelExpression = resourceRequestEvent
+ .getNodeLabelExpression()
+ .orElse(null);
return this;
}
@@ -209,6 +220,14 @@ public final class ResourceRequestEventImpl implements ResourceRequestEvent {
return this;
}
+ /**
+ * @see ResourceRequestEvent#getNodeLabelExpression()
+ */
+ public Builder setNodeLabelExpression(final String nodeLabelExpression) {
+ this.nodeLabelExpression = nodeLabelExpression;
+ return this;
+ }
+
@Override
public ResourceRequestEvent build() {
return new ResourceRequestEventImpl(this);
http://git-wip-us.apache.org/repos/asf/reef/blob/24bfea49/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
index de8d0b1..299af5b 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
@@ -562,8 +562,11 @@ final class YarnContainerManager implements AMRMClientAsync.CallbackHandler, NMC
}
private boolean isSameKindOfRequest(final AMRMClient.ContainerRequest r1, final AMRMClient.ContainerRequest r2) {
+ final boolean nodeLabelExpressionIsEqual = r1.getNodeLabelExpression() == r2.getNodeLabelExpression() ||
+ (r1.getNodeLabelExpression() != null && r1.getNodeLabelExpression().equals(r2.getNodeLabelExpression()));
return r1.getPriority().compareTo(r2.getPriority()) == 0
&& r1.getCapability().compareTo(r2.getCapability()) == 0
+ && nodeLabelExpressionIsEqual
&& r1.getRelaxLocality() == r2.getRelaxLocality()
&& ListUtils.isEqualList(r1.getNodes(), r2.getNodes())
&& ListUtils.isEqualList(r1.getRacks(), r2.getRacks());
http://git-wip-us.apache.org/repos/asf/reef/blob/24bfea49/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandler.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandler.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandler.java
index baab946..8c6f367 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandler.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnResourceRequestHandler.java
@@ -18,6 +18,7 @@
*/
package org.apache.reef.runtime.yarn.driver;
+import org.apache.commons.lang.StringUtils;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.client.api.AMRMClient;
@@ -64,12 +65,15 @@ public final class YarnResourceRequestHandler implements ResourceRequestHandler
final Priority pri = getPriority(resourceRequestEvent);
final Resource resource = getResource(resourceRequestEvent);
final boolean relaxLocality = resourceRequestEvent.getRelaxLocality().orElse(true);
+ final String nodeLabelExpression = resourceRequestEvent.getNodeLabelExpression().orElse("");
final AMRMClient.ContainerRequest[] containerRequests =
new AMRMClient.ContainerRequest[resourceRequestEvent.getResourceCount()];
for (int i = 0; i < resourceRequestEvent.getResourceCount(); i++) {
- containerRequests[i] = new AMRMClient.ContainerRequest(resource, nodes, racks, pri, relaxLocality);
+ containerRequests[i] =
+ new AMRMClient.ContainerRequest(resource, nodes, racks, pri, relaxLocality,
+ StringUtils.isEmpty(nodeLabelExpression) ? null : nodeLabelExpression);
}
this.yarnContainerRequestHandler.onContainerRequest(containerRequests);
}