You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@reef.apache.org by ju...@apache.org on 2016/12/20 22:29:07 UTC
reef git commit: [REEF-1628] Clean up temporary files after REEF test
runs
Repository: reef
Updated Branches:
refs/heads/master 09d56bcfb -> 415bb9eb3
[REEF-1628] Clean up temporary files after REEF test runs
This change:
* cleans up KMeansInput-* files generated by TestKMeans.
* cleans up temp folders and files created by REEF.IO.Tests.
* cleans up IMRUBroadcastReduceWithFilePartitionDataSetTest.
JIRA:
[REEF-1628](https://issues.apache.org/jira/browse/REEF-1628)
Pull request:
This closes #1206
Project: http://git-wip-us.apache.org/repos/asf/reef/repo
Commit: http://git-wip-us.apache.org/repos/asf/reef/commit/415bb9eb
Tree: http://git-wip-us.apache.org/repos/asf/reef/tree/415bb9eb
Diff: http://git-wip-us.apache.org/repos/asf/reef/diff/415bb9eb
Branch: refs/heads/master
Commit: 415bb9eb3dabf93153a20c462c80eb71b679e342
Parents: 09d56bc
Author: Mariia Mykhailova <ma...@apache.org>
Authored: Thu Dec 15 11:51:27 2016 -0800
Committer: Julia Wang <jw...@yahoo.com>
Committed: Tue Dec 20 14:26:36 2016 -0800
----------------------------------------------------------------------
.../TestFilePartitionInputDataSet.cs | 18 +++++++++---
.../TestTempFolderCreator.cs | 15 ++++++++--
...oadcastReduceWithFilePartitionDataSetTest.cs | 29 ++++++++++++--------
.../Functional/ML/KMeans/TestKMeans.cs | 16 +++++++++++
4 files changed, 59 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/reef/blob/415bb9eb/lang/cs/Org.Apache.REEF.IO.Tests/TestFilePartitionInputDataSet.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestFilePartitionInputDataSet.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestFilePartitionInputDataSet.cs
index 001c322..0695f9f 100644
--- a/lang/cs/Org.Apache.REEF.IO.Tests/TestFilePartitionInputDataSet.cs
+++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestFilePartitionInputDataSet.cs
@@ -35,7 +35,7 @@ namespace Org.Apache.REEF.IO.Tests
/// <summary>
/// Tests for Org.Apache.REEF.IO.PartitionedData.FileSystem.
/// </summary>
- public class TestFilePartitionInputDataSet
+ public class TestFilePartitionInputDataSet : IDisposable
{
private static readonly Logger Logger = Logger.GetLogger(typeof(TestFilePartitionInputDataSet));
@@ -266,15 +266,25 @@ namespace Org.Apache.REEF.IO.Tests
}
}
Assert.Equal(count, 5);
+
+ // clean up created folders
+ Directory.Delete("./test2/", true);
}
- private void MakeLocalTestFile(string filePath, byte[] bytes)
+ public void Dispose()
{
- if (File.Exists(filePath))
+ if (File.Exists(sourceFilePath1))
+ {
+ File.Delete(sourceFilePath1);
+ }
+ if (File.Exists(sourceFilePath2))
{
- File.Delete(filePath);
+ File.Delete(sourceFilePath2);
}
+ }
+ private void MakeLocalTestFile(string filePath, byte[] bytes)
+ {
using (var s = File.Create(filePath))
{
foreach (var b in bytes)
http://git-wip-us.apache.org/repos/asf/reef/blob/415bb9eb/lang/cs/Org.Apache.REEF.IO.Tests/TestTempFolderCreator.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.IO.Tests/TestTempFolderCreator.cs b/lang/cs/Org.Apache.REEF.IO.Tests/TestTempFolderCreator.cs
index d3be4fd..fdd0e25 100644
--- a/lang/cs/Org.Apache.REEF.IO.Tests/TestTempFolderCreator.cs
+++ b/lang/cs/Org.Apache.REEF.IO.Tests/TestTempFolderCreator.cs
@@ -37,6 +37,9 @@ namespace Org.Apache.REEF.IO.Tests
var f2 = (string)i.GetNamedInstance(typeof(TempFileFolder));
var f = Path.GetFullPath(f2);
Assert.True(f1.StartsWith(f));
+
+ // clean up created folders
+ Directory.Delete(f, true);
}
/// <summary>
@@ -46,7 +49,7 @@ namespace Org.Apache.REEF.IO.Tests
public void TestTempFileFolerParameter()
{
var b = TempFileConfigurationModule.ConfigurationModule
- .Set(TempFileConfigurationModule.TempFileFolerParameter, @".\test1\abc\")
+ .Set(TempFileConfigurationModule.TempFileFolerParameter, "./test1/abc/")
.Build();
var i = TangFactory.GetTang().NewInjector(b);
var tempFileCreator = i.GetInstance<ITempFileCreator>();
@@ -54,6 +57,9 @@ namespace Org.Apache.REEF.IO.Tests
var f2 = (string)i.GetNamedInstance(typeof(TempFileFolder));
var f = Path.GetFullPath(f2);
Assert.True(f1.StartsWith(f));
+
+ // clean up created folders
+ Directory.Delete("./test1/", true);
}
/// <summary>
@@ -63,15 +69,18 @@ namespace Org.Apache.REEF.IO.Tests
public void TestCreateTempFileFoler()
{
var b = TempFileConfigurationModule.ConfigurationModule
- .Set(TempFileConfigurationModule.TempFileFolerParameter, @"./test1/abc/")
+ .Set(TempFileConfigurationModule.TempFileFolerParameter, @"./test3/abc/")
.Build();
var i = TangFactory.GetTang().NewInjector(b);
var tempFileCreator = i.GetInstance<ITempFileCreator>();
var f1 = tempFileCreator.CreateTempDirectory("ddd\\fff");
var f3 = tempFileCreator.CreateTempDirectory("ddd\\fff", "bbb");
- var f2 = Path.GetFullPath(@"./test1/abc/" + "ddd\\fff");
+ var f2 = Path.GetFullPath(@"./test3/abc/" + "ddd\\fff");
Assert.True(f1.StartsWith(f2));
Assert.True(f3.EndsWith("bbb"));
+
+ // clean up created folders
+ Directory.Delete("./test3/", true);
}
}
}
http://git-wip-us.apache.org/repos/asf/reef/blob/415bb9eb/lang/cs/Org.Apache.REEF.Tests/Functional/IMRU/IMRUBroadcastReduceWithFilePartitionDataSetTest.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/IMRU/IMRUBroadcastReduceWithFilePartitionDataSetTest.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/IMRU/IMRUBroadcastReduceWithFilePartitionDataSetTest.cs
index c68517c..4f5d9f5 100644
--- a/lang/cs/Org.Apache.REEF.Tests/Functional/IMRU/IMRUBroadcastReduceWithFilePartitionDataSetTest.cs
+++ b/lang/cs/Org.Apache.REEF.Tests/Functional/IMRU/IMRUBroadcastReduceWithFilePartitionDataSetTest.cs
@@ -49,7 +49,9 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU
/// <summary>
/// Defines the number of data in input data bytes in the test
/// </summary>
- protected const int DataCount = 3;
+ private const int DataCount = 3;
+
+ private const string tempFileName = "REEF.TestLocalFileSystem.tmp";
/// <summary>
/// This test tests DataLoadingContext with FilePartitionDataSet
@@ -79,6 +81,15 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU
testFolder);
ValidateSuccessForLocalRuntime(numTasks, 0, 0, testFolder, 100);
CleanUp(testFolder);
+
+ for (var i = 0; i < numTasks; i++)
+ {
+ var filePath = Path.Combine(Path.GetTempPath(), tempFileName + i);
+ if (File.Exists(filePath))
+ {
+ File.Delete(filePath);
+ }
+ }
}
/// <summary>
@@ -140,7 +151,7 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU
PartitionDatasetConfig();
}
- protected void PartitionDatasetConfig()
+ private void PartitionDatasetConfig()
{
int count = 0;
int numberOfTasks = 4;
@@ -190,7 +201,7 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU
/// It then iterates the data and verifies the data count.
/// </summary>
/// <typeparam name="T"></typeparam>
- internal sealed class TestSenderMapFunction<T> : IMapFunction<int[], int[]>
+ private sealed class TestSenderMapFunction<T> : IMapFunction<int[], int[]>
{
private int _iterations;
@@ -277,7 +288,6 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU
protected static ConfigurationModule CreateDatasetConfigurationModule(int numberofMappers)
{
- const string tempFileName = "REEF.TestLocalFileSystem.tmp";
string sourceFilePath = Path.Combine(Path.GetTempPath(), tempFileName);
var cm = FileSystemInputPartitionConfiguration<IEnumerable<Row>>.ConfigurationModule;
@@ -295,7 +305,7 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU
/// </summary>
/// <param name="dataNumber"></param>
/// <returns></returns>
- protected static byte[] CreateTestData(int dataNumber)
+ private static byte[] CreateTestData(int dataNumber)
{
var bytes = new byte[dataNumber];
for (int i = 0; i < dataNumber; i++)
@@ -310,13 +320,8 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU
/// </summary>
/// <param name="filePath"></param>
/// <param name="bytes"></param>
- protected static void MakeLocalTestFile(string filePath, byte[] bytes)
+ private static void MakeLocalTestFile(string filePath, byte[] bytes)
{
- if (File.Exists(filePath))
- {
- File.Delete(filePath);
- }
-
using (var s = File.Create(filePath))
{
foreach (var b in bytes)
@@ -329,7 +334,7 @@ namespace Org.Apache.REEF.Tests.Functional.IMRU
/// <summary>
/// Test DeSerializer
/// </summary>
- protected class RowSerializer : IFileDeSerializer<IEnumerable<Row>>
+ private class RowSerializer : IFileDeSerializer<IEnumerable<Row>>
{
[Inject]
private RowSerializer()
http://git-wip-us.apache.org/repos/asf/reef/blob/415bb9eb/lang/cs/Org.Apache.REEF.Tests/Functional/ML/KMeans/TestKMeans.cs
----------------------------------------------------------------------
diff --git a/lang/cs/Org.Apache.REEF.Tests/Functional/ML/KMeans/TestKMeans.cs b/lang/cs/Org.Apache.REEF.Tests/Functional/ML/KMeans/TestKMeans.cs
index 4db2b3b..a1cc91c 100644
--- a/lang/cs/Org.Apache.REEF.Tests/Functional/ML/KMeans/TestKMeans.cs
+++ b/lang/cs/Org.Apache.REEF.Tests/Functional/ML/KMeans/TestKMeans.cs
@@ -101,6 +101,14 @@ namespace Org.Apache.REEF.Tests.Functional.ML.KMeans
{
// do not fail if clean up is unsuccessful
}
+ try
+ {
+ File.Delete(dataFilePath);
+ }
+ catch (Exception)
+ {
+ // do not fail if clean up is unsuccessful
+ }
}
[Fact]
@@ -117,6 +125,14 @@ namespace Org.Apache.REEF.Tests.Functional.ML.KMeans
TestRun(DriverConfiguration(dataFilePath), typeof(KMeansDriverHandlers), Partitions + 1, "KMeansDriverHandlers", "local", testFolder);
ValidateSuccessForLocalRuntime(Partitions + 1, testFolder: testFolder);
CleanUp(testFolder);
+ try
+ {
+ File.Delete(dataFilePath);
+ }
+ catch (Exception)
+ {
+ // do not fail if clean up is unsuccessful
+ }
}
[Fact(Skip = "Requires Yarn Single Node")]