You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by we...@apache.org on 2016/04/20 00:10:42 UTC
reef git commit: [REEF-1325] Fix
PoisonTest.TestPoisonedEvaluatorStartHanlder failure in AppVeyor
Repository: reef
Updated Branches:
refs/heads/master 6b058ba1f -> 7d62431bf
[REEF-1325] Fix PoisonTest.TestPoisonedEvaluatorStartHanlder failure in AppVeyor
This change:
* adds ICompletedTask handler to PoisonTest (which fixes task failues)
* expands IFailedEvaluator handler to show information about associated FailedTask
* refactors reading from log in ReefFunctionalTest to allow reading log file content
in a test, not just in Validate functions.
A follow-up JIRA will investigate why task completes after evaluator failure.
JIRA:
[REEF-1325](https://issues.apache.org/jira/browse/REEF-1325)
Pull request:
This closes #955
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/7d62431b
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/7d62431b
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/7d62431b
Branch: refs/heads/master
Commit: 7d62431bf94f87ed395dfa6cb4ff32abd52d9c67
Parents: 6b058ba
Author: Mariia Mykhailova <ma...@apache.org>
Authored: Wed Apr 13 14:21:19 2016 -0700
Committer: Markus Weimer <we...@apache.org>
Committed: Tue Apr 19 15:09:14 2016 -0700
----------------------------------------------------------------------
.../Poison/PoisonedEventHandler.cs | 1 -
.../Functional/FaultTolerant/PoisonTest.cs | 23 +++-
.../Functional/ReefFunctionalTest.cs | 123 +++++++++----------
3 files changed, 77 insertions(+), 70 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/7d62431b/lang/cs/Org.Apache.REEF.Common/Poison/PoisonedEventHandler.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Common/Poison/PoisonedEventHandler.cs b/lang/cs/Org.Apache.REEF.Common/Poison/PoisonedEventHandler.cs
index 4ab6722..a5f07da 100644
--- a/lang/cs/Org.Apache.REEF.Common/Poison/PoisonedEventHandler.cs
+++ b/lang/cs/Org.Apache.REEF.Common/Poison/PoisonedEventHandler.cs
@@ -22,7 +22,6 @@ using Org.Apache.REEF.Wake.Time.Event;
using Org.Apache.REEF.Wake.Time.Runtime;
using System;
using System.Reactive;
-using System.Threading.Tasks;
namespace Org.Apache.REEF.Common.Poison
{
http://git-wip-us.apache.org/repos/asf/reef/blob/7d62431b/lang/cs/Org.Apache.REEF.Tests/Functional/FaultTolerant/PoisonTest.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/FaultTolerant/PoisonTest.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/FaultTolerant/PoisonTest.cs
index 72df68c..7aab1ac 100644
--- a/lang/cs/Org.Apache.REEF.Tests/Functional/FaultTolerant/PoisonTest.cs
+++ b/lang/cs/Org.Apache.REEF.Tests/Functional/FaultTolerant/PoisonTest.cs
@@ -29,6 +29,7 @@ using System.Threading;
using Org.Apache.REEF.Common.Context;
using Org.Apache.REEF.Common.Events;
using Org.Apache.REEF.Common.Poison;
+using Org.Apache.REEF.Driver.Task;
using Org.Apache.REEF.Tang.Implementations.Tang;
using Org.Apache.REEF.Tang.Implementations.Configuration;
@@ -39,6 +40,7 @@ namespace Org.Apache.REEF.Tests.Functional.FaultTolerant
{
private static readonly Logger Logger = Logger.GetLogger(typeof(PoisonTest));
+ private const string Prefix = "Poison: ";
private const string FailedEvaluatorMessage = "I have succeeded in seeing a failed evaluator.";
private const string TaskId = "1234567";
@@ -59,6 +61,7 @@ namespace Org.Apache.REEF.Tests.Functional.FaultTolerant
.Set(DriverConfiguration.OnEvaluatorAllocated, GenericType<PoisonedEvaluatorDriver>.Class)
.Set(DriverConfiguration.OnEvaluatorFailed, GenericType<PoisonedEvaluatorDriver>.Class)
.Set(DriverConfiguration.OnContextActive, GenericType<PoisonedEvaluatorDriver>.Class)
+ .Set(DriverConfiguration.OnTaskCompleted, GenericType<PoisonedEvaluatorDriver>.Class)
.Build();
}
@@ -66,7 +69,8 @@ namespace Org.Apache.REEF.Tests.Functional.FaultTolerant
IObserver<IDriverStarted>,
IObserver<IAllocatedEvaluator>,
IObserver<IActiveContext>,
- IObserver<IFailedEvaluator>
+ IObserver<IFailedEvaluator>,
+ IObserver<ICompletedTask>
{
private readonly IEvaluatorRequestor _requestor;
@@ -108,6 +112,20 @@ namespace Org.Apache.REEF.Tests.Functional.FaultTolerant
public void OnNext(IFailedEvaluator value)
{
Logger.Log(Level.Error, FailedEvaluatorMessage);
+ if (value.FailedTask.Value == null)
+ {
+ // TODO[JIRA REEF-1343]: fail the test if there's no failed task
+ Logger.Log(Level.Error, "No failed task associated with failed evaluator");
+ }
+ else
+ {
+ Logger.Log(Level.Error, "Failed task id '" + value.FailedTask.Value.Id + "'");
+ }
+ }
+ public void OnNext(ICompletedTask value)
+ {
+ // TODO[JIRA REEF-1343]: fail the test if receive ICompletedTask after failed evaluator
+ Logger.Log(Level.Info, "ICompletedTask");
}
public void OnError(Exception error)
@@ -134,8 +152,9 @@ namespace Org.Apache.REEF.Tests.Functional.FaultTolerant
public byte[] Call(byte[] memento)
{
- Logger.Log(Level.Verbose, "Will sleep for 2 seconds (expecting to be poisoned faster).");
+ Logger.Log(Level.Info, Prefix + "Will sleep for 2 seconds (expecting to be poisoned faster).");
Thread.Sleep(2000);
+ Logger.Log(Level.Info, Prefix + "Task sleep finished successfully.");
return null;
}
}
http://git-wip-us.apache.org/repos/asf/reef/blob/7d62431b/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs
index ca3451b..ad38652 100644
--- a/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs
+++ b/lang/cs/Org.Apache.REEF.Tests/Functional/ReefFunctionalTest.cs
@@ -156,37 +156,18 @@ namespace Org.Apache.REEF.Tests.Functional
const string successIndication = "EXIT: ActiveContextClr2Java::Close";
const string failedTaskIndication = "Java_org_apache_reef_javabridge_NativeInterop_clrSystemFailedTaskHandlerOnNext";
const string failedEvaluatorIndication = "Java_org_apache_reef_javabridge_NativeInterop_clrSystemFailedEvaluatorHandlerOnNext";
- string[] lines = null;
- for (int i = 0; i < 60; i++)
- {
- try
- {
- lines = File.ReadAllLines(GetLogFile(DriverStdout, "driver", testFolder));
- break;
- }
- catch (Exception)
- {
- Thread.Sleep(SleepTime);
- }
- }
-
- if (lines != null)
- {
- Logger.Log(Level.Verbose, "Lines read from log file : " + lines.Count());
- string[] successIndicators = lines.Where(s => s.Contains(successIndication)).ToArray();
- string[] failedTaskIndicators = lines.Where(s => s.Contains(failedTaskIndication)).ToArray();
- string[] failedEvaluatorIndicators = lines.Where(s => s.Contains(failedEvaluatorIndication)).ToArray();
- Assert.True(numberOfContextsToClose == successIndicators.Length,
- "Expected number of contexts to close (" + numberOfContextsToClose + ") differs from actual number of success indicators (" + successIndicators.Length + ")");
- Assert.True(numberOfTasksToFail == failedTaskIndicators.Length,
- "Expected number of tasks to fail (" + numberOfTasksToFail + ") differs from actual number of failed task indicators (" + failedTaskIndicators.Length + ")");
- Assert.True(numberOfEvaluatorsToFail == failedEvaluatorIndicators.Length,
- "Expected number of evaluators to fail (" + numberOfEvaluatorsToFail + ") differs from actual number of failed evaluator indicators (" + failedEvaluatorIndicators.Length + ")");
- }
- else
- {
- Assert.True(false, "Cannot read from log file " + DriverStdout);
- }
+ string[] lines = ReadLogFile(DriverStdout, "driver", testFolder);
+
+ Logger.Log(Level.Verbose, "Lines read from log file : " + lines.Count());
+ string[] successIndicators = lines.Where(s => s.Contains(successIndication)).ToArray();
+ string[] failedTaskIndicators = lines.Where(s => s.Contains(failedTaskIndication)).ToArray();
+ string[] failedEvaluatorIndicators = lines.Where(s => s.Contains(failedEvaluatorIndication)).ToArray();
+ Assert.True(numberOfContextsToClose == successIndicators.Length,
+ "Expected number of contexts to close (" + numberOfContextsToClose + ") differs from actual number of success indicators (" + successIndicators.Length + ")");
+ Assert.True(numberOfTasksToFail == failedTaskIndicators.Length,
+ "Expected number of tasks to fail (" + numberOfTasksToFail + ") differs from actual number of failed task indicators (" + failedTaskIndicators.Length + ")");
+ Assert.True(numberOfEvaluatorsToFail == failedEvaluatorIndicators.Length,
+ "Expected number of evaluators to fail (" + numberOfEvaluatorsToFail + ") differs from actual number of failed evaluator indicators (" + failedEvaluatorIndicators.Length + ")");
}
/// <summary>
@@ -209,46 +190,25 @@ namespace Org.Apache.REEF.Tests.Functional
/// </summary>
protected void ValidateMessageSuccessfullyLogged(IList<string> messages, string subfolder, string fileName, string testFolder, int numberOfOccurrences = 1)
{
- string[] lines = null;
- for (int i = 0; i < 60; i++)
+ string[] lines = ReadLogFile(fileName, subfolder, testFolder);
+ foreach (string message in messages)
{
- try
+ string[] successIndicators = lines.Where(s => s.Contains(message)).ToArray();
+ if (numberOfOccurrences > 0)
{
- lines = File.ReadAllLines(GetLogFile(fileName, subfolder, testFolder));
- break;
+ Assert.True(numberOfOccurrences == successIndicators.Count(),
+ "Expected number of message occurrences " + numberOfOccurrences + " differs from actual " + successIndicators.Count());
}
- catch (Exception e)
+ else if (numberOfOccurrences == 0)
{
- Logger.Log(Level.Verbose, e.ToString());
- Thread.Sleep(SleepTime);
+ Assert.True(0 == successIndicators.Count(),
+ "Message not expected to occur but occurs " + successIndicators.Count() + " times");
}
- }
-
- if (lines != null)
- {
- foreach (string message in messages)
+ else
{
- string[] successIndicators = lines.Where(s => s.Contains(message)).ToArray();
- if (numberOfOccurrences > 0)
- {
- Assert.True(numberOfOccurrences == successIndicators.Count(),
- "Expected number of message occurrences " + numberOfOccurrences + " differs from actual " + successIndicators.Count());
- }
- else if (numberOfOccurrences == 0)
- {
- Assert.True(0 == successIndicators.Count(),
- "Message not expected to occur but occurs " + successIndicators.Count() + " times");
- }
- else
- {
- Assert.True(successIndicators.Count() > 0, "Message expected to occur, but did not.");
- }
+ Assert.True(successIndicators.Count() > 0, "Message expected to occur, but did not.");
}
}
- else
- {
- Assert.True(false, "Cannot read from log file " + fileName);
- }
}
protected void PeriodicUploadLog(object source, ElapsedEventArgs e)
@@ -263,10 +223,39 @@ namespace Org.Apache.REEF.Tests.Functional
}
}
- protected string GetLogFile(string logFileName, string subfolder = "driver", string testFolder = DefaultRuntimeFolder)
+ internal string[] ReadLogFile(string logFileName, string subfolder = "driver", string testFolder = DefaultRuntimeFolder)
+ {
+ string fileName = string.Empty;
+ string[] lines = null;
+ for (int i = 0; i < 60; i++)
+ {
+ try
+ {
+ fileName = GetLogFileName(logFileName, subfolder, testFolder);
+ lines = File.ReadAllLines(fileName);
+ break;
+ }
+ catch (Exception e)
+ {
+ if (i == 59)
+ {
+ // log only last exception before failure
+ Logger.Log(Level.Verbose, e.ToString());
+ }
+ if (i < 59)
+ {
+ Thread.Sleep(SleepTime);
+ }
+ }
+ }
+ Assert.True(lines != null, "Cannot read from log file " + fileName);
+ return lines;
+ }
+
+ protected string GetLogFileName(string logFileName, string subfolder = "driver", string testFolder = DefaultRuntimeFolder)
{
string driverContainerDirectory = Directory.GetDirectories(Path.Combine(Directory.GetCurrentDirectory(), testFolder), subfolder, SearchOption.AllDirectories).SingleOrDefault();
- Logger.Log(Level.Verbose, "GetLogFile, driverContainerDirectory:" + driverContainerDirectory);
+ Logger.Log(Level.Verbose, "GetLogFileName, driverContainerDirectory:" + driverContainerDirectory);
if (string.IsNullOrWhiteSpace(driverContainerDirectory))
{
@@ -282,8 +271,8 @@ namespace Org.Apache.REEF.Tests.Functional
private void UploadDriverLog()
{
- string driverStdout = GetLogFile(DriverStdout);
- string driverStderr = GetLogFile(DriverStderr);
+ string driverStdout = GetLogFileName(DriverStdout);
+ string driverStderr = GetLogFileName(DriverStderr);
CloudStorageAccount storageAccount = CloudStorageAccount.Parse(GetStorageConnectionString());
CloudBlobClient blobClient = storageAccount.CreateCloudBlobClient();
CloudBlobContainer container = blobClient.GetContainerReference(DateTime.Now.ToString("yyyy-MM-dd", CultureInfo.InvariantCulture));