You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by ji...@apache.org on 2015/05/01 09:01:33 UTC
[16/50] [abbrv] hadoop git commit: MAPREDUCE-6252. JobHistoryServer
should not fail when encountering a missing directory. Contributed by Craig
Welch.
MAPREDUCE-6252. JobHistoryServer should not fail when encountering a
missing directory. Contributed by Craig Welch.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5e67c4d3
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5e67c4d3
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5e67c4d3
Branch: refs/heads/HDFS-7240
Commit: 5e67c4d384193b38a85655c8f93193596821faa5
Parents: 618ba70
Author: Devaraj K <de...@apache.org>
Authored: Mon Apr 27 15:01:42 2015 +0530
Committer: Devaraj K <de...@apache.org>
Committed: Mon Apr 27 15:01:42 2015 +0530
----------------------------------------------------------------------
hadoop-mapreduce-project/CHANGES.txt | 3 +++
.../mapreduce/v2/hs/HistoryFileManager.java | 19 ++++++++++-------
.../mapreduce/v2/hs/TestHistoryFileManager.java | 22 ++++++++++++++++++++
3 files changed, 37 insertions(+), 7 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e67c4d3/hadoop-mapreduce-project/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/CHANGES.txt b/hadoop-mapreduce-project/CHANGES.txt
index 4166e6a..f9488fb 100644
--- a/hadoop-mapreduce-project/CHANGES.txt
+++ b/hadoop-mapreduce-project/CHANGES.txt
@@ -349,6 +349,9 @@ Release 2.8.0 - UNRELEASED
MAPREDUCE-6333. TestEvents,TestAMWebServicesTasks,TestAppController are
broken due to MAPREDUCE-6297. (Siqi Li via gera)
+ MAPREDUCE-6252. JobHistoryServer should not fail when encountering a
+ missing directory. (Craig Welch via devaraj)
+
Release 2.7.1 - UNRELEASED
INCOMPATIBLE CHANGES
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e67c4d3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
index 65f8a4f..69f814d 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/main/java/org/apache/hadoop/mapreduce/v2/hs/HistoryFileManager.java
@@ -740,17 +740,22 @@ public class HistoryFileManager extends AbstractService {
}
}
- private static List<FileStatus> scanDirectory(Path path, FileContext fc,
+ @VisibleForTesting
+ protected static List<FileStatus> scanDirectory(Path path, FileContext fc,
PathFilter pathFilter) throws IOException {
path = fc.makeQualified(path);
List<FileStatus> jhStatusList = new ArrayList<FileStatus>();
- RemoteIterator<FileStatus> fileStatusIter = fc.listStatus(path);
- while (fileStatusIter.hasNext()) {
- FileStatus fileStatus = fileStatusIter.next();
- Path filePath = fileStatus.getPath();
- if (fileStatus.isFile() && pathFilter.accept(filePath)) {
- jhStatusList.add(fileStatus);
+ try {
+ RemoteIterator<FileStatus> fileStatusIter = fc.listStatus(path);
+ while (fileStatusIter.hasNext()) {
+ FileStatus fileStatus = fileStatusIter.next();
+ Path filePath = fileStatus.getPath();
+ if (fileStatus.isFile() && pathFilter.accept(filePath)) {
+ jhStatusList.add(fileStatus);
+ }
}
+ } catch (FileNotFoundException fe) {
+ LOG.error("Error while scanning directory " + path, fe);
}
return jhStatusList;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5e67c4d3/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
----------------------------------------------------------------------
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
index e2e943a..1c5cc5c 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-hs/src/test/java/org/apache/hadoop/mapreduce/v2/hs/TestHistoryFileManager.java
@@ -21,13 +21,17 @@ package org.apache.hadoop.mapreduce.v2.hs;
import java.io.File;
import java.io.FileOutputStream;
+import java.io.FileNotFoundException;
import java.util.UUID;
+import java.util.List;
import org.junit.Assert;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileContext;
import org.apache.hadoop.hdfs.HdfsConfiguration;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
@@ -45,6 +49,8 @@ import org.junit.Rule;
import org.junit.Test;
import org.junit.rules.TestName;
+import static org.mockito.Mockito.*;
+
public class TestHistoryFileManager {
private static MiniDFSCluster dfsCluster = null;
private static MiniDFSCluster dfsCluster2 = null;
@@ -199,4 +205,20 @@ public class TestHistoryFileManager {
testCreateHistoryDirs(dfsCluster.getConfiguration(0), clock);
}
+ @Test
+ public void testScanDirectory() throws Exception {
+
+ Path p = new Path("any");
+ FileContext fc = mock(FileContext.class);
+ when(fc.makeQualified(p)).thenReturn(p);
+ when(fc.listStatus(p)).thenThrow(new FileNotFoundException());
+
+ List<FileStatus> lfs = HistoryFileManager.scanDirectory(p, fc, null);
+
+ //primarily, succcess is that an exception was not thrown. Also nice to
+ //check this
+ Assert.assertNotNull(lfs);
+
+ }
+
}