You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by om...@apache.org on 2011/03/04 05:14:59 UTC
svn commit: r1077438 - in
/hadoop/common/branches/branch-0.20-security-patches/src:
hdfs/org/apache/hadoop/hdfs/server/namenode/
test/org/apache/hadoop/hdfs/server/namenode/
Author: omalley
Date: Fri Mar 4 04:14:59 2011
New Revision: 1077438
URL: http://svn.apache.org/viewvc?rev=1077438&view=rev
Log:
commit fd0321d4e431ddb12116c6c213bce742033e18dc
Author: Hairong Kuang <ha...@ucdev21.inktomisearch.com>
Date: Tue May 4 19:14:10 2010 +0000
HDFS:1104 from https://issues.apache.org/jira/secure/attachment/12443523/fsckATime_Yahoo0.20.patch
+++ b/YAHOO-CHANGES.txt
+ HDFS-1104. Fsck triggers full GC on NameNode. (hairong)
+
Modified:
hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1077438&r1=1077437&r2=1077438&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Fri Mar 4 04:14:59 2011
@@ -807,10 +807,6 @@ public class FSNamesystem implements FSC
*/
LocatedBlocks getBlockLocations(String clientMachine, String src,
long offset, long length) throws IOException {
- if (isPermissionEnabled) {
- checkPathAccess(src, FsAction.READ);
- }
-
LocatedBlocks blocks = getBlockLocations(src, offset, length, true);
if (blocks != null) {
//sort the blocks
@@ -838,6 +834,10 @@ public class FSNamesystem implements FSC
*/
public LocatedBlocks getBlockLocations(String src, long offset, long length,
boolean doAccessTime) throws IOException {
+ if (isPermissionEnabled) {
+ checkPathAccess(src, FsAction.READ);
+ }
+
if (offset < 0) {
throw new IOException("Negative offset is not supported. File: " + src );
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java?rev=1077438&r1=1077437&r2=1077438&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NameNode.java Fri Mar 4 04:14:59 2011
@@ -445,6 +445,19 @@ public class NameNode implements ClientP
src, offset, length);
}
+ /**
+ * The specification of this method matches that of
+ * {@link getBlockLocations(Path)}
+ * except that it does not update the file's access time.
+ */
+ LocatedBlocks getBlockLocationsNoATime(String src,
+ long offset,
+ long length)
+ throws IOException {
+ myMetrics.numGetBlockLocations.inc();
+ return namesystem.getBlockLocations(src, offset, length, false);
+ }
+
private static String getClientMachine() {
String clientMachine = Server.getRemoteAddress();
if (clientMachine == null) {
Modified: hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=1077438&r1=1077437&r2=1077438&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/hdfs/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java Fri Mar 4 04:14:59 2011
@@ -34,7 +34,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hdfs.DFSClient;
import org.apache.hadoop.hdfs.protocol.Block;
-import org.apache.hadoop.hdfs.protocol.ClientProtocol;
import org.apache.hadoop.hdfs.protocol.DatanodeInfo;
import org.apache.hadoop.hdfs.protocol.HdfsFileStatus;
import org.apache.hadoop.hdfs.protocol.LocatedBlock;
@@ -83,7 +82,7 @@ public class NamenodeFsck {
/** Delete corrupted files. */
public static final int FIXING_DELETE = 2;
- private final ClientProtocol namenode;
+ private final NameNode namenode;
private final NetworkTopology networktopology;
private final int totalDatanodes;
private final short minReplication;
@@ -110,7 +109,7 @@ public class NamenodeFsck {
* @param response the object into which this servelet writes the url contents
* @throws IOException
*/
- NamenodeFsck(Configuration conf, ClientProtocol namenode,
+ NamenodeFsck(Configuration conf, NameNode namenode,
NetworkTopology networktopology,
Map<String,String[]> pmap, PrintWriter out,
int totalDatanodes, short minReplication) {
@@ -195,7 +194,7 @@ public class NamenodeFsck {
return;
}
long fileLen = file.getLen();
- LocatedBlocks blocks = namenode.getBlockLocations(path, 0, fileLen);
+ LocatedBlocks blocks = namenode.getBlockLocationsNoATime(path, 0, fileLen);
if (blocks == null) { // the file is deleted
return;
}
Modified: hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestFsck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestFsck.java?rev=1077438&r1=1077437&r2=1077438&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestFsck.java (original)
+++ hadoop/common/branches/branch-0.20-security-patches/src/test/org/apache/hadoop/hdfs/server/namenode/TestFsck.java Fri Mar 4 04:14:59 2011
@@ -33,10 +33,12 @@ import junit.framework.TestCase;
import org.apache.commons.logging.impl.Log4JLogger;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.hdfs.DFSClient;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.MiniDFSCluster;
import org.apache.hadoop.hdfs.protocol.LocatedBlocks;
@@ -73,12 +75,21 @@ public class TestFsck extends TestCase {
FileSystem fs = null;
try {
Configuration conf = new Configuration();
+ final long precision = 1L;
+ conf.setLong("dfs.access.time.precision", precision);
conf.setLong("dfs.blockreport.intervalMsec", 10000L);
cluster = new MiniDFSCluster(conf, 4, true, null);
fs = cluster.getFileSystem();
- util.createFiles(fs, "/srcdat");
- util.waitReplication(fs, "/srcdat", (short)3);
+ final String fileName = "/srcdat";
+ util.createFiles(fs, fileName);
+ util.waitReplication(fs, fileName, (short)3);
+ FileStatus[] stats = fs.listStatus(new Path(fileName));
+ assertFalse(0==stats.length);
+ final Path file = stats[0].getPath();
+ long aTime = fs.getFileStatus(file).getAccessTime();
+ Thread.sleep(2*precision);
String outStr = runFsck(conf, 0, true, "/");
+ assertEquals(aTime, fs.getFileStatus(file).getAccessTime());
assertTrue(outStr.contains(NamenodeFsck.HEALTHY_STATUS));
System.out.println(outStr);
if (fs != null) {try{fs.close();} catch(Exception e){}}