You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by do...@apache.org on 2015/11/03 17:25:59 UTC
incubator-reef git commit: [REEF-881] Create a ProgressProvider on
the C# side
Repository: incubator-reef
Updated Branches:
refs/heads/master 7d1ccf1ef -> 5b65c73e9
[REEF-881] Create a ProgressProvider on the C# side
This addressed the issue by
* Creating the ProgressProvider class in C# and piping it to Java.
JIRA:
[REEF-881](https://issues.apache.org/jira/browse/REEF-881)
Pull Request:
This closes #607
Project: http://git-wip-us.apache.org/repos/asf/incubator-reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-reef/commit/5b65c73e
Tree: http://git-wip-us.apache.org/repos/asf/incubator-reef/tree/5b65c73e
Diff: http://git-wip-us.apache.org/repos/asf/incubator-reef/diff/5b65c73e
Branch: refs/heads/master
Commit: 5b65c73e9787867d8d1df4c6192956a965f5d8a3
Parents: 7d1ccf1
Author: Andrew Chung <af...@gmail.com>
Authored: Fri Oct 30 17:48:44 2015 -0700
Committer: Dongjoon Hyun <do...@apache.org>
Committed: Wed Nov 4 01:24:43 2015 +0900
----------------------------------------------------------------------
.../cs/Org.Apache.REEF.Bridge/JavaClrBridge.cpp | 39 +++++++++++++----
.../Bridge/BridgeHandlerManager.cs | 2 +
.../Bridge/ClrSystemHandlerWrapper.cs | 10 +++++
.../Bridge/DriverBridge.cs | 15 +++++--
.../Bridge/DriverBridgeConfigurationOptions.cs | 1 -
.../Defaults/DefaultProgressProvider.cs | 45 ++++++++++++++++++++
.../DriverConfiguration.cs | 6 +++
.../Org.Apache.REEF.Driver/IProgressProvider.cs | 38 +++++++++++++++++
.../Org.Apache.REEF.Driver.csproj | 2 +
.../apache/reef/bridge/client/Constants.java | 1 +
.../reef/javabridge/BridgeHandlerManager.java | 9 ++++
.../apache/reef/javabridge/NativeInterop.java | 2 +
.../reef/javabridge/generic/JobDriver.java | 14 ++++++
.../yarn/driver/YarnContainerManager.java | 14 ++++--
.../webserver/HttpServerReefEventHandler.java | 7 +--
15 files changed, 187 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/cs/Org.Apache.REEF.Bridge/JavaClrBridge.cpp
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Bridge/JavaClrBridge.cpp b/lang/cs/Org.Apache.REEF.Bridge/JavaClrBridge.cpp
index 06bf5a3..a740e70 100644
--- a/lang/cs/Org.Apache.REEF.Bridge/JavaClrBridge.cpp
+++ b/lang/cs/Org.Apache.REEF.Bridge/JavaClrBridge.cpp
@@ -93,6 +93,8 @@ void populateJavaBridgeHandlerManager(JNIEnv * env, jobject jbridgeHandlerManage
env->CallVoidMethod(jbridgeHandlerManager, jsetDriverRestartCompletedHandlerMid, bridgeHandlerManager->DriverRestartCompletedHandler);
jmethodID jsetDriverRestartFailedEvaluatorHandlerMid = env->GetMethodID(cls, "setDriverRestartFailedEvaluatorHandler", "(J)V");
env->CallVoidMethod(jbridgeHandlerManager, jsetDriverRestartFailedEvaluatorHandlerMid, bridgeHandlerManager->DriverRestartFailedEvaluatorHandler);
+ jmethodID jsetProgressProviderMid = env->GetMethodID(cls, "setProgressProvider", "(J)V");
+ env->CallVoidMethod(jbridgeHandlerManager, jsetProgressProviderMid, bridgeHandlerManager->ProgressProvider);
}
// Loading Clr Assembly. Note that we do not use ManagerLogger in this method since the
@@ -528,10 +530,10 @@ JNIEXPORT void JNICALL Java_org_apache_reef_javabridge_NativeInterop_clrSystemDr
}
/*
-* Class: org_apache_reef_javabridge_NativeInterop
-* Method: clrSystemDriverRestartCompletedHandlerOnNext
-* Signature: (JLorg/apache/reef/javabridge/generic/DriverRestartCompletedBridge;)V
-*/
+ * Class: org_apache_reef_javabridge_NativeInterop
+ * Method: clrSystemDriverRestartCompletedHandlerOnNext
+ * Signature: (JLorg/apache/reef/javabridge/generic/DriverRestartCompletedBridge;)V
+ */
JNIEXPORT void JNICALL Java_org_apache_reef_javabridge_NativeInterop_clrSystemDriverRestartCompletedHandlerOnNext
(JNIEnv * env, jclass cls , jlong handler, jobject jdriverRestartCompleted) {
ManagedLog::LOGGER->Log("+Java_org_apache_reef_javabridge_NativeInterop_clrSystemDriverRestartCompletedHandlerOnNext");
@@ -546,10 +548,10 @@ JNIEXPORT void JNICALL Java_org_apache_reef_javabridge_NativeInterop_clrSystemDr
}
/*
-* Class: org_apache_reef_javabridge_NativeInterop
-* Method: clrSystemDriverRestartFailedEvaluatorHandlerOnNext
-* Signature: (JLorg/apache/reef/javabridge/FailedEvaluatorBridge;Lorg/apache/reef/javabridge/InteropLogger;)V
-*/
+ * Class: org_apache_reef_javabridge_NativeInterop
+ * Method: clrSystemDriverRestartFailedEvaluatorHandlerOnNext
+ * Signature: (JLorg/apache/reef/javabridge/FailedEvaluatorBridge;Lorg/apache/reef/javabridge/InteropLogger;)V
+ */
JNIEXPORT void JNICALL Java_org_apache_reef_javabridge_NativeInterop_clrSystemDriverRestartFailedEvaluatorHandlerOnNext
(JNIEnv * env, jclass cls, jlong handler, jobject jfailedEvaluator, jobject jlogger) {
ManagedLog::LOGGER->Log("+Java_org_apache_reef_javabridge_NativeInterop_clrSystemDriverRestartFailedEvaluatorHandlerOnNext");
@@ -563,6 +565,24 @@ JNIEXPORT void JNICALL Java_org_apache_reef_javabridge_NativeInterop_clrSystemDr
}
}
+/*
+ * Class: org_apache_reef_javabridge_NativeInterop
+ * Method: clrSystemProgressProviderGetProgress
+ * Signature: (J)F
+ */
+JNIEXPORT jfloat JNICALL Java_org_apache_reef_javabridge_NativeInterop_clrSystemProgressProviderGetProgress
+(JNIEnv * env, jclass cls, jlong handler) {
+ ManagedLog::LOGGER->Log("+Java_org_apache_reef_javabridge_NativeInterop_clrSystemProgressProviderGetProgress");
+ try {
+ return (jfloat)ClrSystemHandlerWrapper::Call_ProgressProvider_GetProgress(handler);
+ }
+ catch (System::Exception^ ex) {
+ String^ errorMessage = "Exception in Call_ProgressProvider_GetProgress";
+ ManagedLog::LOGGER->LogError(errorMessage, ex);
+ return 0;
+ }
+}
+
static JNINativeMethod methods[] = {
{ "loadClrAssembly", "(Ljava/lang/String;)V", (void*)&Java_org_apache_reef_javabridge_NativeInterop_loadClrAssembly },
@@ -621,6 +641,9 @@ static JNINativeMethod methods[] = {
{ "clrSystemDriverRestartCompletedHandlerOnNext", "(JLorg/apache/reef/javabridge/generic/DriverRestartCompletedBridge;)V",
(void*)&Java_org_apache_reef_javabridge_NativeInterop_clrSystemDriverRestartCompletedHandlerOnNext },
+
+ { "clrSystemProgressProviderGetProgress", "(J)F",
+ (void*)&Java_org_apache_reef_javabridge_NativeInterop_clrSystemProgressProviderGetProgress },
};
JNIEXPORT void JNICALL
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/cs/Org.Apache.REEF.Driver/Bridge/BridgeHandlerManager.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Bridge/BridgeHandlerManager.cs b/lang/cs/Org.Apache.REEF.Driver/Bridge/BridgeHandlerManager.cs
index bf0b04c..377ad94 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Bridge/BridgeHandlerManager.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Bridge/BridgeHandlerManager.cs
@@ -57,5 +57,7 @@ namespace Org.Apache.REEF.Driver.Bridge
public ulong DriverRestartFailedEvaluatorHandler { get; internal set; }
public ulong HttpServerHandler { get; internal set; }
+
+ public ulong ProgressProvider { get; internal set; }
}
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/cs/Org.Apache.REEF.Driver/Bridge/ClrSystemHandlerWrapper.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Bridge/ClrSystemHandlerWrapper.cs b/lang/cs/Org.Apache.REEF.Driver/Bridge/ClrSystemHandlerWrapper.cs
index 381aeba..d67d8f4 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Bridge/ClrSystemHandlerWrapper.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Bridge/ClrSystemHandlerWrapper.cs
@@ -230,6 +230,16 @@ namespace Org.Apache.REEF.Driver.Bridge
}
}
+ public static float Call_ProgressProvider_GetProgress(ulong handle)
+ {
+ using (LOGGER.LogFunction("ClrSystemHandlerWrapper::Call_ProgressProvider_GetProgress"))
+ {
+ GCHandle gc = GCHandle.FromIntPtr((IntPtr)handle);
+ IProgressProvider obj = (IProgressProvider)gc.Target;
+ return obj.Progress;
+ }
+ }
+
public static BridgeHandlerManager Call_ClrSystemStartHandler_OnStart(
DateTime startTime,
string httpServerPort,
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/cs/Org.Apache.REEF.Driver/Bridge/DriverBridge.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Bridge/DriverBridge.cs b/lang/cs/Org.Apache.REEF.Driver/Bridge/DriverBridge.cs
index d770d3a..35db431 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Bridge/DriverBridge.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Bridge/DriverBridge.cs
@@ -21,7 +21,6 @@ using System;
using System.Collections.Generic;
using System.Diagnostics;
using System.Globalization;
-using System.Linq;
using Org.Apache.REEF.Common.Context;
using Org.Apache.REEF.Driver.Context;
using Org.Apache.REEF.Driver.Evaluator;
@@ -29,10 +28,10 @@ using Org.Apache.REEF.Driver.Task;
using Org.Apache.REEF.Tang.Annotations;
using Org.Apache.REEF.Tang.Interface;
using Org.Apache.REEF.Utilities.Logging;
-using Org.Apache.REEF.Wake.Time.Event;
using Org.Apache.REEF.Common.Evaluator.Parameters;
using Org.Apache.REEF.Driver.Bridge.Clr2java;
using Org.Apache.REEF.Driver.Bridge.Events;
+using Org.Apache.REEF.Driver.Defaults;
namespace Org.Apache.REEF.Driver.Bridge
{
@@ -118,6 +117,8 @@ namespace Org.Apache.REEF.Driver.Bridge
private readonly ISet<IConfigurationProvider> _configurationProviders;
+ private readonly IProgressProvider _progressProvider;
+
[Inject]
public DriverBridge(
[Parameter(Value = typeof(DriverBridgeConfigurationOptions.DriverStartedHandlers))] ISet<IObserver<IDriverStarted>> driverStartHandlers,
@@ -142,7 +143,8 @@ namespace Org.Apache.REEF.Driver.Bridge
[Parameter(Value = typeof(DriverBridgeConfigurationOptions.TraceListenersSet))] ISet<TraceListener> traceListeners,
[Parameter(Value = typeof(EvaluatorConfigurationProviders))] ISet<IConfigurationProvider> configurationProviders,
[Parameter(Value = typeof(DriverBridgeConfigurationOptions.TraceLevel))] string traceLevel,
- HttpServerHandler httpServerHandler)
+ HttpServerHandler httpServerHandler,
+ IProgressProvider progressProvider)
{
foreach (TraceListener listener in traceListeners)
{
@@ -182,6 +184,7 @@ namespace Org.Apache.REEF.Driver.Bridge
_driverRestartFailedEvaluatorHandlers = driverRestartFailedEvaluatorHandlers;
_httpServerHandler = httpServerHandler;
_configurationProviders = configurationProviders;
+ _progressProvider = progressProvider;
_allocatedEvaluatorSubscriber = new ClrSystemHandler<IAllocatedEvaluator>();
_completedEvaluatorSubscriber = new ClrSystemHandler<ICompletedEvaluator>();
@@ -340,6 +343,12 @@ namespace Org.Apache.REEF.Driver.Bridge
_logger.Log(Level.Verbose, "subscribed to IHttpMessage handler :" + _httpServerHandler);
bridgeHandlerManager.HttpServerHandler = ClrHandlerHelper.CreateHandler(_httpServerEventSubscriber);
+ // bind progress provider to provide application progress
+ // Bind null handler if user does not specify their own implementation of IProgressProvider. This is
+ // used to get around the overhead of Interop calls since the Java side checks for null handler here.
+ bridgeHandlerManager.ProgressProvider = _progressProvider is DefaultProgressProvider ?
+ ClrHandlerHelper.CreateNullHandler() : ClrHandlerHelper.CreateHandler(_progressProvider);
+
return bridgeHandlerManager;
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/cs/Org.Apache.REEF.Driver/Bridge/DriverBridgeConfigurationOptions.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Bridge/DriverBridgeConfigurationOptions.cs b/lang/cs/Org.Apache.REEF.Driver/Bridge/DriverBridgeConfigurationOptions.cs
index d48a6ce..16940d2 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Bridge/DriverBridgeConfigurationOptions.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/Bridge/DriverBridgeConfigurationOptions.cs
@@ -27,7 +27,6 @@ using Org.Apache.REEF.Driver.Defaults;
using Org.Apache.REEF.Driver.Evaluator;
using Org.Apache.REEF.Driver.Task;
using Org.Apache.REEF.Tang.Annotations;
-using Org.Apache.REEF.Wake.Time.Event;
[module: SuppressMessage("StyleCop.CSharp.MaintainabilityRules", "SA1402:FileMayOnlyContainASingleClass", Justification = "allow name parameter class to be embedded")]
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/cs/Org.Apache.REEF.Driver/Defaults/DefaultProgressProvider.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Defaults/DefaultProgressProvider.cs b/lang/cs/Org.Apache.REEF.Driver/Defaults/DefaultProgressProvider.cs
new file mode 100644
index 0000000..b2ab63c
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Driver/Defaults/DefaultProgressProvider.cs
@@ -0,0 +1,45 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Driver.Defaults
+{
+ /// <summary>
+ /// The default implementation of <see cref="IProgressProvider"/>.
+ /// </summary>
+ internal sealed class DefaultProgressProvider : IProgressProvider
+ {
+ [Inject]
+ private DefaultProgressProvider()
+ {
+ }
+
+ /// <summary>
+ /// Always returns a progress of 0 to the Resource Manager.
+ /// </summary>
+ public float Progress
+ {
+ get
+ {
+ return 0f;
+ }
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/cs/Org.Apache.REEF.Driver/DriverConfiguration.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/DriverConfiguration.cs b/lang/cs/Org.Apache.REEF.Driver/DriverConfiguration.cs
index 821517e..a055dc1 100644
--- a/lang/cs/Org.Apache.REEF.Driver/DriverConfiguration.cs
+++ b/lang/cs/Org.Apache.REEF.Driver/DriverConfiguration.cs
@@ -184,6 +184,11 @@ namespace Org.Apache.REEF.Driver
/// </summary>
public static readonly OptionalParameter<int> DriverRestartEvaluatorRecoverySeconds = new OptionalParameter<int>();
+ /// <summary>
+ /// The progress provider that will be injected at the Driver.
+ /// </summary>
+ public static readonly OptionalImpl<IProgressProvider> ProgressProvider = new OptionalImpl<IProgressProvider>();
+
public static ConfigurationModule ConfigurationModule
{
get
@@ -233,6 +238,7 @@ namespace Org.Apache.REEF.Driver
MaxApplicationSubmissions)
.BindNamedParameter(GenericType<DriverBridgeConfigurationOptions.DriverRestartEvaluatorRecoverySeconds>.Class,
DriverRestartEvaluatorRecoverySeconds)
+ .BindImplementation(GenericType<IProgressProvider>.Class, ProgressProvider)
.Build();
}
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/cs/Org.Apache.REEF.Driver/IProgressProvider.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/IProgressProvider.cs b/lang/cs/Org.Apache.REEF.Driver/IProgressProvider.cs
new file mode 100644
index 0000000..d5b5fb2
--- /dev/null
+++ b/lang/cs/Org.Apache.REEF.Driver/IProgressProvider.cs
@@ -0,0 +1,38 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing,
+ * software distributed under the License is distributed on an
+ * "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY
+ * KIND, either express or implied. See the License for the
+ * specific language governing permissions and limitations
+ * under the License.
+ */
+
+using Org.Apache.REEF.Driver.Defaults;
+using Org.Apache.REEF.Tang.Annotations;
+
+namespace Org.Apache.REEF.Driver
+{
+ /// <summary>
+ /// Provides the progress of the job to the Resource Manager
+ /// </summary>
+ [DefaultImplementation(typeof(DefaultProgressProvider))]
+ public interface IProgressProvider
+ {
+ /// <summary>
+ /// Provides the progress of the job to the Resource Manager.
+ /// Value should be a float between 0 and 1. If value greater than 1,
+ /// 1 will be reported. If value less than 0, 0 will be reported.
+ /// </summary>
+ float Progress { get; }
+ }
+}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.csproj
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.csproj b/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.csproj
index 1316cb6..0c0a47e 100644
--- a/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.csproj
+++ b/lang/cs/Org.Apache.REEF.Driver/Org.Apache.REEF.Driver.csproj
@@ -103,6 +103,7 @@ under the License.
<Compile Include="Context\IClosedContext.cs" />
<Compile Include="Context\IContext.cs" />
<Compile Include="Context\IFailedContext.cs" />
+ <Compile Include="Defaults\DefaultProgressProvider.cs" />
<Compile Include="Defaults\DefaultClientCloseHandler.cs" />
<Compile Include="Defaults\DefaultClientCloseWithMessageHandler.cs" />
<Compile Include="Defaults\DefaultClientMessageHandler.cs" />
@@ -144,6 +145,7 @@ under the License.
<Compile Include="IDriverRestarted.cs" />
<Compile Include="IDriverStarted.cs" />
<Compile Include="IDriverRestartCompleted.cs" />
+ <Compile Include="IProgressProvider.cs" />
<Compile Include="Properties\AssemblyInfo.cs" />
<Compile Include="Task\ICompletedTask.cs" />
<Compile Include="Task\IFailedTask.cs" />
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/Constants.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/Constants.java b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/Constants.java
index e5b397c..ac41f9f 100644
--- a/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/Constants.java
+++ b/lang/java/reef-bridge-client/src/main/java/org/apache/reef/bridge/client/Constants.java
@@ -51,6 +51,7 @@ public final class Constants {
.set(DriverConfiguration.ON_DRIVER_STARTED, JobDriver.StartHandler.class)
.set(DriverConfiguration.ON_TASK_SUSPENDED, JobDriver.SuspendedTaskHandler.class)
.set(DriverConfiguration.ON_EVALUATOR_COMPLETED, JobDriver.CompletedEvaluatorHandler.class)
+ .set(DriverConfiguration.PROGRESS_PROVIDER, JobDriver.ProgressProvider.class)
.build();
/**
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/BridgeHandlerManager.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/BridgeHandlerManager.java b/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/BridgeHandlerManager.java
index ce04454..1684e26 100644
--- a/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/BridgeHandlerManager.java
+++ b/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/BridgeHandlerManager.java
@@ -44,6 +44,7 @@ public final class BridgeHandlerManager {
private long driverRestartRunningTaskHandler = 0;
private long driverRestartCompletedHandler = 0;
private long driverRestartFailedEvaluatorHandler = 0;
+ private long progressProvider = 0;
public BridgeHandlerManager() {
}
@@ -183,4 +184,12 @@ public final class BridgeHandlerManager {
public void setDriverRestartFailedEvaluatorHandler(final long driverRestartFailedEvaluatorHandler) {
this.driverRestartFailedEvaluatorHandler = driverRestartFailedEvaluatorHandler;
}
+
+ public long getProgressProvider() {
+ return progressProvider;
+ }
+
+ public void setProgressProvider(final long progressProvider) {
+ this.progressProvider = progressProvider;
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/NativeInterop.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/NativeInterop.java b/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/NativeInterop.java
index 28a7faf..32b10d0 100644
--- a/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/NativeInterop.java
+++ b/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/NativeInterop.java
@@ -139,6 +139,8 @@ public final class NativeInterop {
final InteropLogger interopLogger
);
+ public static native float clrSystemProgressProviderGetProgress(final long handle);
+
/**
* Empty private constructor to prohibit instantiation of utility class.
*/
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/generic/JobDriver.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/generic/JobDriver.java b/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/generic/JobDriver.java
index 422cb62..03ce202 100644
--- a/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/generic/JobDriver.java
+++ b/lang/java/reef-bridge-java/src/main/java/org/apache/reef/javabridge/generic/JobDriver.java
@@ -776,4 +776,18 @@ public final class JobDriver {
}
}
}
+
+ /**
+ * Gets the progress of the application from .NET side.
+ */
+ public final class ProgressProvider implements org.apache.reef.driver.ProgressProvider {
+ @Override
+ public float getProgress() {
+ if (JobDriver.this.handlerManager != null && JobDriver.this.handlerManager.getProgressProvider() != 0) {
+ return NativeInterop.clrSystemProgressProviderGetProgress(JobDriver.this.handlerManager.getProgressProvider());
+ }
+
+ return 0f;
+ }
+ }
}
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
index 51a8022..4b7ce49 100644
--- a/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
+++ b/lang/java/reef-runtime-yarn/src/main/java/org/apache/reef/runtime/yarn/driver/YarnContainerManager.java
@@ -41,6 +41,7 @@ import org.apache.reef.runtime.common.driver.resourcemanager.ResourceStatusEvent
import org.apache.reef.runtime.common.driver.resourcemanager.RuntimeStatusEventImpl;
import org.apache.reef.runtime.common.files.REEFFileNames;
import org.apache.reef.runtime.yarn.driver.parameters.YarnHeartbeatPeriod;
+import org.apache.reef.tang.InjectionFuture;
import org.apache.reef.tang.annotations.Parameter;
import org.apache.reef.util.Optional;
import org.apache.reef.wake.remote.Encoder;
@@ -82,7 +83,7 @@ final class YarnContainerManager
private final String jobSubmissionDirectory;
private final REEFFileNames reefFileNames;
private final RackNameFormatter rackNameFormatter;
- private final ProgressProvider progressProvider;
+ private final InjectionFuture<ProgressProvider> progressProvider;
@Inject
YarnContainerManager(
@@ -97,7 +98,7 @@ final class YarnContainerManager
@Parameter(JobSubmissionDirectory.class) final String jobSubmissionDirectory,
final TrackingURLProvider trackingURLProvider,
final RackNameFormatter rackNameFormatter,
- final ProgressProvider progressProvider) throws IOException {
+ final InjectionFuture<ProgressProvider> progressProvider) throws IOException {
this.reefEventHandlers = reefEventHandlers;
this.driverStatusManager = driverStatusManager;
@@ -162,7 +163,14 @@ final class YarnContainerManager
@Override
public float getProgress() {
- return progressProvider.getProgress();
+ try {
+ return Math.max(Math.min(1, progressProvider.get().getProgress()), 0);
+ } catch (final Exception e) {
+ // An Exception must be caught and logged here because YARN swallows the Exception and fails the job.
+ LOG.log(Level.WARNING, "An exception occurred in ProgressProvider.getProgress(), with message : " +
+ e.getMessage() + ". Returning 0 as progress.");
+ return 0f;
+ }
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-reef/blob/5b65c73e/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java
----------------------------------------------------------------------
diff --git a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java
index aa7dd42..7f68277 100644
--- a/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java
+++ b/lang/java/reef-webserver/src/main/java/org/apache/reef/webserver/HttpServerReefEventHandler.java
@@ -22,6 +22,7 @@ import org.apache.reef.driver.ProgressProvider;
import org.apache.reef.driver.evaluator.EvaluatorDescriptor;
import org.apache.reef.driver.parameters.ClientCloseHandlers;
import org.apache.reef.runtime.common.files.REEFFileNames;
+import org.apache.reef.tang.InjectionFuture;
import org.apache.reef.tang.Tang;
import org.apache.reef.tang.annotations.Parameter;
import org.apache.reef.tang.exceptions.InjectionException;
@@ -62,7 +63,7 @@ public final class HttpServerReefEventHandler implements HttpHandler {
private final ReefEventStateManager reefStateManager;
private final Set<EventHandler<Void>> clientCloseHandlers;
private final LoggingScopeFactory loggingScopeFactory;
- private final ProgressProvider progressProvider;
+ private final InjectionFuture<ProgressProvider> progressProvider;
/**
* Log level string prefix in the log lines.
@@ -81,7 +82,7 @@ public final class HttpServerReefEventHandler implements HttpHandler {
@Parameter(LogLevelName.class) final String logLevel,
final LoggingScopeFactory loggingScopeFactory,
final REEFFileNames reefFileNames,
- final ProgressProvider progressProvider) {
+ final InjectionFuture<ProgressProvider> progressProvider) {
this.reefStateManager = reefStateManager;
this.clientCloseHandlers = clientCloseHandlers;
this.loggingScopeFactory = loggingScopeFactory;
@@ -194,7 +195,7 @@ public final class HttpServerReefEventHandler implements HttpHandler {
}
break;
case "progress":
- response.getWriter().println(progressProvider.getProgress());
+ response.getWriter().println(progressProvider.get().getProgress());
break;
default:
response.getWriter().println(String.format("Unsupported query for entity: [%s].", target));