You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2012/06/12 03:59:25 UTC
svn commit: r1349090 - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/java/ src/main/java/org/apache/hadoop/hdfs/server/namenode/
src/test/java/org/apache/hadoop/hdfs/
src/test/java/org/apache/hadoop/hdfs/server/namenode/
Author: szetszwo
Date: Tue Jun 12 01:59:24 2012
New Revision: 1349090
URL: http://svn.apache.org/viewvc?rev=1349090&view=rev
Log:
svn merge -c 1349088 from trunk for HDFS-3522. If a namenode is in safemode, it should throw SafeModeException when getBlockLocations has zero locations.
Modified:
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/ (props changed)
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs:r1349088
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt?rev=1349090&r1=1349089&r2=1349090&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt Tue Jun 12 01:59:24 2012
@@ -191,6 +191,9 @@ Release 2.0.1-alpha - UNRELEASED
HDFS-3517. TestStartup should bind ephemeral ports. (eli)
+ HDFS-3522. If a namenode is in safemode, it should throw SafeModeException
+ when getBlockLocations has zero locations. (Brandon Li via szetszwo)
+
BREAKDOWN OF HDFS-3042 SUBTASKS
HDFS-2185. HDFS portion of ZK-based FailoverController (todd)
Propchange: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/
------------------------------------------------------------------------------
Merged /hadoop/common/trunk/hadoop-hdfs-project/hadoop-hdfs/src/main/java:r1349088
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java?rev=1349090&r1=1349089&r2=1349090&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java Tue Jun 12 01:59:24 2012
@@ -1074,7 +1074,8 @@ public class FSNamesystem implements Nam
LocatedBlocks getBlockLocations(String clientMachine, String src,
long offset, long length) throws AccessControlException,
FileNotFoundException, UnresolvedLinkException, IOException {
- LocatedBlocks blocks = getBlockLocations(src, offset, length, true, true);
+ LocatedBlocks blocks = getBlockLocations(src, offset, length, true, true,
+ true);
if (blocks != null) {
blockManager.getDatanodeManager().sortLocatedBlocks(
clientMachine, blocks.getLocatedBlocks());
@@ -1088,8 +1089,8 @@ public class FSNamesystem implements Nam
* @throws FileNotFoundException, UnresolvedLinkException, IOException
*/
LocatedBlocks getBlockLocations(String src, long offset, long length,
- boolean doAccessTime, boolean needBlockToken) throws FileNotFoundException,
- UnresolvedLinkException, IOException {
+ boolean doAccessTime, boolean needBlockToken, boolean checkSafeMode)
+ throws FileNotFoundException, UnresolvedLinkException, IOException {
if (isPermissionEnabled) {
checkPathAccess(src, FsAction.READ);
}
@@ -1109,6 +1110,15 @@ public class FSNamesystem implements Nam
Server.getRemoteIp(),
"open", src, null, null);
}
+ if (checkSafeMode && isInSafeMode()) {
+ for (LocatedBlock b : ret.getLocatedBlocks()) {
+ // if safemode & no block locations yet then throw safemodeException
+ if ((b.getLocations() == null) || (b.getLocations().length == 0)) {
+ throw new SafeModeException("Zero blocklocations for " + src,
+ safeMode);
+ }
+ }
+ }
return ret;
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java?rev=1349090&r1=1349089&r2=1349090&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/NamenodeFsck.java Tue Jun 12 01:59:24 2012
@@ -277,7 +277,7 @@ public class NamenodeFsck {
// Get block locations without updating the file access time
// and without block access tokens
LocatedBlocks blocks = namenode.getNamesystem().getBlockLocations(path, 0,
- fileLen, false, false);
+ fileLen, false, false, false);
if (blocks == null) { // the file is deleted
return;
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java?rev=1349090&r1=1349089&r2=1349090&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestSafeMode.java Tue Jun 12 01:59:24 2012
@@ -23,6 +23,7 @@ import java.util.List;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.FileSystem;
import org.apache.hadoop.fs.permission.FsPermission;
@@ -31,9 +32,11 @@ import org.apache.hadoop.hdfs.protocol.H
import org.apache.hadoop.hdfs.server.blockmanagement.BlockManagerTestUtil;
import org.apache.hadoop.hdfs.server.namenode.FSNamesystem;
import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.ipc.RemoteException;
import org.apache.hadoop.hdfs.server.common.HdfsServerConstants.StartupOption;
import org.apache.hadoop.hdfs.server.namenode.NameNode;
import org.apache.hadoop.hdfs.server.namenode.NameNodeAdapter;
+import org.apache.hadoop.hdfs.server.namenode.SafeModeException;
import org.apache.hadoop.test.GenericTestUtils;
import static org.junit.Assert.*;
@@ -372,4 +375,76 @@ public class TestSafeMode {
dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
assertFalse("State was expected to be out of safemode.", dfs.isInSafeMode());
}
+
+ @Test
+ public void testSafeModeWhenZeroBlockLocations() throws IOException {
+
+ try {
+ Path file1 = new Path("/tmp/testManualSafeMode/file1");
+ Path file2 = new Path("/tmp/testManualSafeMode/file2");
+
+ System.out.println("Created file1 and file2.");
+
+ // create two files with one block each.
+ DFSTestUtil.createFile(fs, file1, 1000, (short)1, 0);
+ DFSTestUtil.createFile(fs, file2, 2000, (short)1, 0);
+ checkGetBlockLocationsWorks(fs, file1);
+
+ NameNode namenode = cluster.getNameNode();
+
+ // manually set safemode.
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ assertTrue("should still be in SafeMode", namenode.isInSafeMode());
+ // getBlock locations should still work since block locations exists
+ checkGetBlockLocationsWorks(fs, file1);
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+ assertFalse("should not be in SafeMode", namenode.isInSafeMode());
+
+
+ // Now 2nd part of the tests where there aren't block locations
+ cluster.shutdownDataNodes();
+ cluster.shutdownNameNode(0);
+
+ // now bring up just the NameNode.
+ cluster.restartNameNode();
+ cluster.waitActive();
+
+ System.out.println("Restarted cluster with just the NameNode");
+
+ namenode = cluster.getNameNode();
+
+ assertTrue("No datanode is started. Should be in SafeMode",
+ namenode.isInSafeMode());
+ FileStatus stat = fs.getFileStatus(file1);
+ try {
+ fs.getFileBlockLocations(stat, 0, 1000);
+ assertTrue("Should have got safemode exception", false);
+ } catch (SafeModeException e) {
+ // as expected
+ } catch (RemoteException re) {
+ if (!re.getClassName().equals(SafeModeException.class.getName()))
+ assertTrue("Should have got safemode exception", false);
+ }
+
+
+ dfs.setSafeMode(SafeModeAction.SAFEMODE_LEAVE);
+ assertFalse("Should not be in safemode", namenode.isInSafeMode());
+ checkGetBlockLocationsWorks(fs, file1);
+
+ } finally {
+ if(fs != null) fs.close();
+ if(cluster!= null) cluster.shutdown();
+ }
+ }
+
+ void checkGetBlockLocationsWorks(FileSystem fs, Path fileName) throws IOException {
+ FileStatus stat = fs.getFileStatus(fileName);
+ try {
+ fs.getFileBlockLocations(stat, 0, 1000);
+ } catch (SafeModeException e) {
+ assertTrue("Should have not got safemode exception", false);
+ } catch (RemoteException re) {
+ assertTrue("Should have not got safemode exception", false);
+ }
+ }
}
Modified: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java?rev=1349090&r1=1349089&r2=1349090&view=diff
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java (original)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/NameNodeAdapter.java Tue Jun 12 01:59:24 2012
@@ -56,7 +56,7 @@ public class NameNodeAdapter {
public static LocatedBlocks getBlockLocations(NameNode namenode,
String src, long offset, long length) throws IOException {
return namenode.getNamesystem().getBlockLocations(
- src, offset, length, false, true);
+ src, offset, length, false, true, true);
}
public static HdfsFileStatus getFileInfo(NameNode namenode, String src,