You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by zj...@apache.org on 2014/04/10 20:59:44 UTC
svn commit: r1586420 - in
/hadoop/common/branches/branch-2/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/
hadoop-yarn/hadoop-yar...
Author: zjshen
Date: Thu Apr 10 18:59:44 2014
New Revision: 1586420
URL: http://svn.apache.org/r1586420
Log:
YARN-1920. Fixed TestFileSystemApplicationHistoryStore failure on windows. Contributed by Vinod Kumar Vavilapalli.
svn merge --ignore-ancestry -c 1586414 ../../trunk/
Modified:
hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt?rev=1586420&r1=1586419&r2=1586420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/CHANGES.txt Thu Apr 10 18:59:44 2014
@@ -82,6 +82,9 @@ Release 2.4.1 - UNRELEASED
YARN-1910. Fixed a race condition in TestAMRMTokens that causes the test to
fail more often on Windows. (Xuan Gong via vinodkv)
+ YARN-1920. Fixed TestFileSystemApplicationHistoryStore failure on windows.
+ (Vinod Kumar Vavilapalli via zjshen)
+
Release 2.4.0 - 2014-04-07
INCOMPATIBLE CHANGES
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java?rev=1586420&r1=1586419&r2=1586420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/main/java/org/apache/hadoop/yarn/server/applicationhistoryservice/FileSystemApplicationHistoryStore.java Thu Apr 10 18:59:44 2014
@@ -179,7 +179,7 @@ public class FileSystemApplicationHistor
LOG.info("Completed reading history information of application " + appId);
return historyData;
} catch (IOException e) {
- LOG.error("Error when reading history file of application " + appId);
+ LOG.error("Error when reading history file of application " + appId, e);
throw e;
} finally {
hfReader.close();
@@ -296,7 +296,7 @@ public class FileSystemApplicationHistor
return historyData;
} catch (IOException e) {
LOG.error("Error when reading history file of application attempt"
- + appAttemptId);
+ + appAttemptId, e);
throw e;
} finally {
hfReader.close();
@@ -344,7 +344,7 @@ public class FileSystemApplicationHistor
+ containerId);
return historyData;
} catch (IOException e) {
- LOG.error("Error when reading history file of container " + containerId);
+ LOG.error("Error when reading history file of container " + containerId, e);
throw e;
} finally {
hfReader.close();
@@ -420,7 +420,7 @@ public class FileSystemApplicationHistor
+ appStart.getApplicationId());
} catch (IOException e) {
LOG.error("Error when openning history file of application "
- + appStart.getApplicationId());
+ + appStart.getApplicationId(), e);
throw e;
}
outstandingWriters.put(appStart.getApplicationId(), hfWriter);
@@ -437,7 +437,7 @@ public class FileSystemApplicationHistor
+ appStart.getApplicationId() + " is written");
} catch (IOException e) {
LOG.error("Error when writing start information of application "
- + appStart.getApplicationId());
+ + appStart.getApplicationId(), e);
throw e;
}
}
@@ -456,7 +456,7 @@ public class FileSystemApplicationHistor
+ appFinish.getApplicationId() + " is written");
} catch (IOException e) {
LOG.error("Error when writing finish information of application "
- + appFinish.getApplicationId());
+ + appFinish.getApplicationId(), e);
throw e;
} finally {
hfWriter.close();
@@ -480,7 +480,7 @@ public class FileSystemApplicationHistor
+ appAttemptStart.getApplicationAttemptId() + " is written");
} catch (IOException e) {
LOG.error("Error when writing start information of application attempt "
- + appAttemptStart.getApplicationAttemptId());
+ + appAttemptStart.getApplicationAttemptId(), e);
throw e;
}
}
@@ -501,7 +501,7 @@ public class FileSystemApplicationHistor
+ appAttemptFinish.getApplicationAttemptId() + " is written");
} catch (IOException e) {
LOG.error("Error when writing finish information of application attempt "
- + appAttemptFinish.getApplicationAttemptId());
+ + appAttemptFinish.getApplicationAttemptId(), e);
throw e;
}
}
@@ -521,7 +521,7 @@ public class FileSystemApplicationHistor
+ containerStart.getContainerId() + " is written");
} catch (IOException e) {
LOG.error("Error when writing start information of container "
- + containerStart.getContainerId());
+ + containerStart.getContainerId(), e);
throw e;
}
}
@@ -541,7 +541,7 @@ public class FileSystemApplicationHistor
+ containerFinish.getContainerId() + " is written");
} catch (IOException e) {
LOG.error("Error when writing finish information of container "
- + containerFinish.getContainerId());
+ + containerFinish.getContainerId(), e);
}
}
@@ -676,9 +676,10 @@ public class FileSystemApplicationHistor
private TFile.Reader reader;
private TFile.Reader.Scanner scanner;
+ FSDataInputStream fsdis;
public HistoryFileReader(Path historyFile) throws IOException {
- FSDataInputStream fsdis = fs.open(historyFile);
+ fsdis = fs.open(historyFile);
reader =
new TFile.Reader(fsdis, fs.getFileStatus(historyFile).getLen(),
getConfig());
@@ -707,7 +708,7 @@ public class FileSystemApplicationHistor
}
public void close() {
- IOUtils.cleanup(LOG, scanner, reader);
+ IOUtils.cleanup(LOG, scanner, reader, fsdis);
}
}
Modified: hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java?rev=1586420&r1=1586419&r2=1586420&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java (original)
+++ hadoop/common/branches/branch-2/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-applicationhistoryservice/src/test/java/org/apache/hadoop/yarn/server/applicationhistoryservice/TestFileSystemApplicationHistoryStore.java Thu Apr 10 18:59:44 2014
@@ -23,6 +23,8 @@ import java.net.URI;
import org.junit.Assert;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
@@ -42,6 +44,9 @@ import org.junit.Test;
public class TestFileSystemApplicationHistoryStore extends
ApplicationHistoryStoreTestUtils {
+ private static Log LOG = LogFactory
+ .getLog(TestFileSystemApplicationHistoryStore.class.getName());
+
private FileSystem fs;
private Path fsWorkingPath;
@@ -50,9 +55,12 @@ public class TestFileSystemApplicationHi
fs = new RawLocalFileSystem();
Configuration conf = new Configuration();
fs.initialize(new URI("/"), conf);
- fsWorkingPath = new Path("Test");
+ fsWorkingPath =
+ new Path("target",
+ TestFileSystemApplicationHistoryStore.class.getSimpleName());
fs.delete(fsWorkingPath, true);
- conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI, fsWorkingPath.toString());
+ conf.set(YarnConfiguration.FS_APPLICATION_HISTORY_STORE_URI,
+ fsWorkingPath.toString());
store = new FileSystemApplicationHistoryStore();
store.init(conf);
store.start();
@@ -67,6 +75,7 @@ public class TestFileSystemApplicationHi
@Test
public void testReadWriteHistoryData() throws IOException {
+ LOG.info("Starting testReadWriteHistoryData");
testWriteHistoryData(5);
testReadHistoryData(5);
}
@@ -167,6 +176,7 @@ public class TestFileSystemApplicationHi
@Test
public void testWriteAfterApplicationFinish() throws IOException {
+ LOG.info("Starting testWriteAfterApplicationFinish");
ApplicationId appId = ApplicationId.newInstance(0, 1);
writeApplicationStartData(appId);
writeApplicationFinishData(appId);
@@ -203,6 +213,7 @@ public class TestFileSystemApplicationHi
@Test
public void testMassiveWriteContainerHistoryData() throws IOException {
+ LOG.info("Starting testMassiveWriteContainerHistoryData");
long mb = 1024 * 1024;
long usedDiskBefore = fs.getContentSummary(fsWorkingPath).getLength() / mb;
ApplicationId appId = ApplicationId.newInstance(0, 1);
@@ -221,12 +232,14 @@ public class TestFileSystemApplicationHi
@Test
public void testMissingContainerHistoryData() throws IOException {
+ LOG.info("Starting testMissingContainerHistoryData");
testWriteHistoryData(3, true, false);
testReadHistoryData(3, true, false);
}
@Test
public void testMissingApplicationAttemptHistoryData() throws IOException {
+ LOG.info("Starting testMissingApplicationAttemptHistoryData");
testWriteHistoryData(3, false, true);
testReadHistoryData(3, false, true);
}