You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by sz...@apache.org on 2012/04/05 20:28:34 UTC
svn commit: r1309994 - in
/hadoop/common/trunk/hadoop-common-project/hadoop-common: CHANGES.txt
pom.xml src/main/java/org/apache/hadoop/fs/FileUtil.java
src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Author: szetszwo
Date: Thu Apr 5 18:28:34 2012
New Revision: 1309994
URL: http://svn.apache.org/viewvc?rev=1309994&view=rev
Log:
HADOOP-6963. In FileUtil.getDU(..), neither include the size of directories nor follow symbolic links. Contributed by Ravi Prakash
Modified:
hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt?rev=1309994&r1=1309993&r2=1309994&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/CHANGES.txt Thu Apr 5 18:28:34 2012
@@ -324,6 +324,9 @@ Release 2.0.0 - UNRELEASED
HADOOP-8243. Security support broken in CLI (manual) failover controller
(todd)
+ HADOOP-6963. In FileUtil.getDU(..), neither include the size of directories
+ nor follow symbolic links. (Ravi Prakash via szetszwo)
+
BREAKDOWN OF HADOOP-7454 SUBTASKS
HADOOP-7455. HA: Introduce HA Service Protocol Interface. (suresh)
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml?rev=1309994&r1=1309993&r2=1309994&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/pom.xml Thu Apr 5 18:28:34 2012
@@ -79,6 +79,11 @@
<scope>compile</scope>
</dependency>
<dependency>
+ <groupId>commons-io</groupId>
+ <artifactId>commons-io</artifactId>
+ <scope>compile</scope>
+ </dependency>
+ <dependency>
<groupId>javax.servlet</groupId>
<artifactId>servlet-api</artifactId>
<scope>compile</scope>
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java?rev=1309994&r1=1309993&r2=1309994&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/main/java/org/apache/hadoop/fs/FileUtil.java Thu Apr 5 18:28:34 2012
@@ -483,11 +483,18 @@ public class FileUtil {
if (!dir.isDirectory()) {
return dir.length();
} else {
- size = dir.length();
File[] allFiles = dir.listFiles();
if(allFiles != null) {
for (int i = 0; i < allFiles.length; i++) {
- size = size + getDU(allFiles[i]);
+ boolean isSymLink;
+ try {
+ isSymLink = org.apache.commons.io.FileUtils.isSymlink(allFiles[i]);
+ } catch(IOException ioe) {
+ isSymLink = true;
+ }
+ if(!isSymLink) {
+ size += getDU(allFiles[i]);
+ }
}
}
return size;
Modified: hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java
URL: http://svn.apache.org/viewvc/hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java?rev=1309994&r1=1309993&r2=1309994&view=diff
==============================================================================
--- hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java (original)
+++ hadoop/common/trunk/hadoop-common-project/hadoop-common/src/test/java/org/apache/hadoop/fs/TestFileUtil.java Thu Apr 5 18:28:34 2012
@@ -93,6 +93,9 @@ public class TestFileUtil {
// create files in partitioned directories
createFile(partitioned, "part-r-00000", "foo");
createFile(partitioned, "part-r-00001", "bar");
+
+ // create a cycle using symlinks. Cycles should be handled
+ FileUtil.symLink(del.toString(), dir1.toString() + "/cycle");
}
/**
@@ -458,4 +461,18 @@ public class TestFileUtil {
return result;
}
+
+ /**
+ * Test that getDU is able to handle cycles caused due to symbolic links
+ * and that directory sizes are not added to the final calculated size
+ * @throws IOException
+ */
+ @Test
+ public void testGetDU() throws IOException {
+ setupDirs();
+
+ long du = FileUtil.getDU(TEST_DIR);
+ //Only two files (in partitioned) have 4 bytes each
+ Assert.assertEquals(du, 8);
+ }
}