You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2011/06/09 23:13:40 UTC
svn commit: r1134079 - in /hadoop/common/branches/yahoo-merge: ./
src/contrib/ec2/ src/docs/ src/java/ src/java/org/apache/hadoop/fs/
src/test/core/ src/test/core/org/apache/hadoop/fs/
Author: suresh
Date: Thu Jun 9 21:13:39 2011
New Revision: 1134079
URL: http://svn.apache.org/viewvc?rev=1134079&view=rev
Log:
Merging change r1127697 for HADOOP-7322 from trunk
Modified:
hadoop/common/branches/yahoo-merge/ (props changed)
hadoop/common/branches/yahoo-merge/CHANGES.txt (contents, props changed)
hadoop/common/branches/yahoo-merge/src/contrib/ec2/ (props changed)
hadoop/common/branches/yahoo-merge/src/docs/ (props changed)
hadoop/common/branches/yahoo-merge/src/java/ (props changed)
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileUtil.java
hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
hadoop/common/branches/yahoo-merge/src/test/core/ (props changed)
hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/TestFileUtil.java
Propchange: hadoop/common/branches/yahoo-merge/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 21:13:39 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk:1038480,1038493,1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199,1101653,1102012,1102068,1102093,1102861,1102940,1103968,1104426,1127591,1129989,1130833,1131330
+/hadoop/common/trunk:1038480,1038493,1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199,1101653,1102012,1102068,1102093,1102861,1102940,1103968,1104426,1127591,1127697,1129989,1130833,1131330
/hadoop/core/branches/branch-0.19/core:713112
Modified: hadoop/common/branches/yahoo-merge/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/CHANGES.txt?rev=1134079&r1=1134078&r2=1134079&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/CHANGES.txt (original)
+++ hadoop/common/branches/yahoo-merge/CHANGES.txt Thu Jun 9 21:13:39 2011
@@ -9,6 +9,9 @@ Trunk (unreleased changes)
HADOOP-7342. Add an utility API in FileUtil for JDK File.list
avoid NPEs on File.list() (Bharath Mundlapudi via mattf)
+ HADOOP-7322. Adding a util method in FileUtil for directory listing,
+ avoid NPEs on File.listFiles() (Bharath Mundlapudi via mattf)
+
HADOOP-6994. Api to get delegation token in AbstractFileSystem. (jitendra)
HADOOP-7171. Support UGI in FileContext API. (jitendra)
Propchange: hadoop/common/branches/yahoo-merge/CHANGES.txt
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 21:13:39 2011
@@ -1,4 +1,4 @@
-/hadoop/common/trunk/CHANGES.txt:1038480,1038493,1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1092832,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199,1101653,1102012,1102068,1102093,1102861,1102940,1103968,1104426,1127591,1129905,1129989,1130833,1131330
+/hadoop/common/trunk/CHANGES.txt:1038480,1038493,1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1092832,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199,1101653,1102012,1102068,1102093,1102861,1102940,1103968,1104426,1127591,1127697,1129905,1129989,1130833,1131330
/hadoop/core/branches/branch-0.18/CHANGES.txt:727226
/hadoop/core/branches/branch-0.19/CHANGES.txt:713112
/hadoop/core/trunk/CHANGES.txt:776175-785643,785929-786278
Propchange: hadoop/common/branches/yahoo-merge/src/contrib/ec2/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 21:13:39 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/contrib/ec2:1038480,1038493,1043117,1076296,1078148,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199,1101653,1102012,1102068,1102093,1102861,1102940,1103968,1104426,1127591,1129989,1130833,1131330
+/hadoop/common/trunk/src/contrib/ec2:1038480,1038493,1043117,1076296,1078148,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199,1101653,1102012,1102068,1102093,1102861,1102940,1103968,1104426,1127591,1127697,1129989,1130833,1131330
/hadoop/core/branches/branch-0.19/core/src/contrib/ec2:713112
/hadoop/core/trunk/src/contrib/ec2:776175-784663
Propchange: hadoop/common/branches/yahoo-merge/src/docs/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 21:13:39 2011
@@ -1,2 +1,2 @@
-/hadoop/common/trunk/src/docs:1038480,1038493,1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199,1101653,1102012,1102068,1102093,1102861,1102940,1103968,1104426,1127591,1129989,1130833,1131330
+/hadoop/common/trunk/src/docs:1038480,1038493,1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199,1101653,1102012,1102068,1102093,1102861,1102940,1103968,1104426,1127591,1127697,1129989,1130833,1131330
/hadoop/core/branches/branch-0.19/src/docs:713112
Propchange: hadoop/common/branches/yahoo-merge/src/java/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 21:13:39 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/java:1038480,1038493,1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199,1101653,1102012,1102068,1102093,1102861,1102940,1103968,1104426,1127591,1129989,1130833,1131330
+/hadoop/common/trunk/src/java:1038480,1038493,1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199,1101653,1102012,1102068,1102093,1102861,1102940,1103968,1104426,1127591,1127697,1129989,1130833,1131330
/hadoop/core/branches/branch-0.19/core/src/java:713112
/hadoop/core/trunk/src/core:776175-785643,785929-786278
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileUtil.java?rev=1134079&r1=1134078&r2=1134079&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/FileUtil.java Thu Jun 9 21:13:39 2011
@@ -324,7 +324,7 @@ public class FileUtil {
if (!dstFS.mkdirs(dst)) {
return false;
}
- File contents[] = src.listFiles();
+ File contents[] = listFiles(src);
for (int i = 0; i < contents.length; i++) {
copy(contents[i], dstFS, new Path(dst, contents[i].getName()),
deleteSource, conf);
@@ -486,8 +486,10 @@ public class FileUtil {
} else {
size = dir.length();
File[] allFiles = dir.listFiles();
- for (int i = 0; i < allFiles.length; i++) {
- size = size + getDU(allFiles[i]);
+ if(allFiles != null) {
+ for (int i = 0; i < allFiles.length; i++) {
+ size = size + getDU(allFiles[i]);
+ }
}
return size;
}
@@ -709,6 +711,25 @@ public class FileUtil {
}
/**
+ * A wrapper for {@link File#listFiles()}. This java.io API returns null
+ * when a dir is not a directory or for any I/O error. Instead of having
+ * null check everywhere File#listFiles() is used, we will add utility API
+ * to get around this problem. For the majority of cases where we prefer
+ * an IOException to be thrown.
+ * @param dir directory for which listing should be performed
+ * @return list of files or empty list
+ * @exception IOException for invalid directory or for a bad disk.
+ */
+ public static File[] listFiles(File dir) throws IOException {
+ File[] files = dir.listFiles();
+ if(files == null) {
+ throw new IOException("Invalid directory or I/O error occurred for dir: "
+ + dir.toString());
+ }
+ return files;
+ }
+
+ /**
* A wrapper for {@link File#list()}. This java.io API returns null
* when a dir is not a directory or for any I/O error. Instead of having
* null check everywhere File#list() is used, we will add utility API
Modified: hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java?rev=1134079&r1=1134078&r2=1134079&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java (original)
+++ hadoop/common/branches/yahoo-merge/src/java/org/apache/hadoop/fs/RawLocalFileSystem.java Thu Jun 9 21:13:39 2011
@@ -274,7 +274,7 @@ public class RawLocalFileSystem extends
if (f.isFile()) {
return f.delete();
} else if ((!recursive) && f.isDirectory() &&
- (f.listFiles().length != 0)) {
+ (FileUtil.listFiles(f).length != 0)) {
throw new IOException("Directory " + f.toString() + " is not empty");
}
return FileUtil.fullyDelete(f);
Propchange: hadoop/common/branches/yahoo-merge/src/test/core/
------------------------------------------------------------------------------
--- svn:mergeinfo (original)
+++ svn:mergeinfo Thu Jun 9 21:13:39 2011
@@ -1,3 +1,3 @@
-/hadoop/common/trunk/src/test/core:1038480,1038493,1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199,1101653,1102012,1102068,1102093,1102861,1102940,1103968,1104426,1127591,1129989,1130833,1131330
+/hadoop/common/trunk/src/test/core:1038480,1038493,1043117,1076296,1078148,1080396,1081598,1082329,1082787-1082788,1084415,1084769,1085043,1085122,1086309,1087159,1087844,1090039,1090485,1091618,1091902,1091970,1092519,1092565,1094750,1095121,1095761,1096522,1096988,1099612,1099633,1100026,1100356,1100369,1100400,1101132,1101199,1101653,1102012,1102068,1102093,1102861,1102940,1103968,1104426,1127591,1127697,1129989,1130833,1131330
/hadoop/core/branches/branch-0.19/core/src/test/core:713112
/hadoop/core/trunk/src/test/core:776175-785643,785929-786278
Modified: hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/TestFileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/TestFileUtil.java?rev=1134079&r1=1134078&r2=1134079&view=diff
==============================================================================
--- hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/TestFileUtil.java (original)
+++ hadoop/common/branches/yahoo-merge/src/test/core/org/apache/hadoop/fs/TestFileUtil.java Thu Jun 9 21:13:39 2011
@@ -117,6 +117,32 @@ public class TestFileUtil {
}
}
+ @Test
+ public void testListFiles() throws IOException {
+ setupDirs();
+ //Test existing files case
+ File[] files = FileUtil.listFiles(partitioned);
+ Assert.assertEquals(2, files.length);
+
+ //Test existing directory with no files case
+ File newDir = new File(tmp.getPath(),"test");
+ newDir.mkdir();
+ Assert.assertTrue("Failed to create test dir", newDir.exists());
+ files = FileUtil.listFiles(newDir);
+ Assert.assertEquals(0, files.length);
+ newDir.delete();
+ Assert.assertFalse("Failed to delete test dir", newDir.exists());
+
+ //Test non-existing directory case, this throws
+ //IOException
+ try {
+ files = FileUtil.listFiles(newDir);
+ Assert.fail("IOException expected on listFiles() for non-existent dir "
+ + newDir.toString());
+ } catch(IOException ioe) {
+ //Expected an IOException
+ }
+ }
@Test
public void testListAPI() throws IOException {