You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by zh...@apache.org on 2015/05/11 21:23:32 UTC
[32/50] hadoop git commit: HDFS-8308. Erasure Coding: NameNode may
get blocked in waitForLoadingFSImage() when loading editlog. Contributed by
Jing Zhao.
HDFS-8308. Erasure Coding: NameNode may get blocked in waitForLoadingFSImage() when loading editlog. Contributed by Jing Zhao.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/2bd2dd7e
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/2bd2dd7e
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/2bd2dd7e
Branch: refs/heads/HDFS-7285
Commit: 2bd2dd7e0e346c5e63c22f20692b7ee3318fd159
Parents: b962860
Author: Jing Zhao <ji...@apache.org>
Authored: Thu Apr 30 19:42:29 2015 -0700
Committer: Zhe Zhang <zh...@apache.org>
Committed: Mon May 11 11:36:22 2015 -0700
----------------------------------------------------------------------
.../hadoop-hdfs/CHANGES-HDFS-EC-7285.txt | 3 +
.../namenode/ErasureCodingZoneManager.java | 3 +-
.../hdfs/server/namenode/FSNamesystem.java | 4 +-
.../org/apache/hadoop/hdfs/DFSTestUtil.java | 12 ++++
.../hadoop/hdfs/TestErasureCodingZones.java | 6 +-
.../server/namenode/TestAddStripedBlocks.java | 61 ++++++++++----------
6 files changed, 52 insertions(+), 37 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bd2dd7e/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
index 3c75152..596bbcf 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES-HDFS-EC-7285.txt
@@ -152,3 +152,6 @@
HDFS-8183. Erasure Coding: Improve DFSStripedOutputStream closing of
datastreamer threads. (Rakesh R via Zhe Zhang)
+
+ HDFS-8308. Erasure Coding: NameNode may get blocked in waitForLoadingFSImage()
+ when loading editlog. (jing9)
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bd2dd7e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java
index 8cda289..14d4e29 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/ErasureCodingZoneManager.java
@@ -79,7 +79,8 @@ public class ErasureCodingZoneManager {
for (XAttr xAttr : xAttrs) {
if (XATTR_ERASURECODING_ZONE.equals(XAttrHelper.getPrefixName(xAttr))) {
String schemaName = new String(xAttr.getValue());
- ECSchema schema = dir.getFSNamesystem().getECSchema(schemaName);
+ ECSchema schema = dir.getFSNamesystem().getSchemaManager()
+ .getSchema(schemaName);
return new ECZoneInfo(inode.getFullPathName(), schema);
}
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bd2dd7e/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
index 1290bb2..86745ec 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSNamesystem.java
@@ -8120,9 +8120,9 @@ public class FSNamesystem implements Namesystem, FSNamesystemMBean,
/**
* Create an erasure coding zone on directory src.
- * @param schema ECSchema for the erasure coding zone
- * @param src the path of a directory which will be the root of the
+ * @param srcArg the path of a directory which will be the root of the
* erasure coding zone. The directory must be empty.
+ * @param schema ECSchema for the erasure coding zone
*
* @throws AccessControlException if the caller is not the superuser.
* @throws UnresolvedLinkException if the path can't be resolved.
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bd2dd7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
index 0c88842..0165189 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/DFSTestUtil.java
@@ -1948,4 +1948,16 @@ public class DFSTestUtil {
lastBlock.setNumBytes(numStripes * BLOCK_STRIPED_CELL_SIZE * NUM_DATA_BLOCKS);
return lastBlock;
}
+
+ /**
+ * Because currently DFSStripedOutputStream does not support hflush/hsync,
+ * tests can use this method to flush all the buffered data to DataNodes.
+ */
+ public static void writeAndFlushStripedOutputStream(
+ DFSStripedOutputStream out, int chunkSize) throws IOException {
+ // FSOutputSummer.BUFFER_NUM_CHUNKS == 9
+ byte[] toWrite = new byte[chunkSize * 9 + 1];
+ out.write(toWrite);
+ out.flushInternal();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bd2dd7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java
index ae2bdd8..f1aec82 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/TestErasureCodingZones.java
@@ -35,8 +35,6 @@ import static org.apache.hadoop.test.GenericTestUtils.assertExceptionContains;
import static org.junit.Assert.*;
public class TestErasureCodingZones {
- private final int NUM_OF_DATANODES = 3;
- private Configuration conf;
private MiniDFSCluster cluster;
private DistributedFileSystem fs;
private static final int BLOCK_SIZE = 1024;
@@ -44,10 +42,10 @@ public class TestErasureCodingZones {
@Before
public void setupCluster() throws IOException {
- conf = new HdfsConfiguration();
+ Configuration conf = new HdfsConfiguration();
conf.setInt(DFSConfigKeys.DFS_BLOCK_SIZE_KEY, BLOCK_SIZE);
cluster = new MiniDFSCluster.Builder(conf).
- numDataNodes(NUM_OF_DATANODES).build();
+ numDataNodes(1).build();
cluster.waitActive();
fs = cluster.getFileSystem();
namesystem = cluster.getNamesystem();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/2bd2dd7e/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
index d03e938..297db14 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestAddStripedBlocks.java
@@ -19,6 +19,7 @@ package org.apache.hadoop.hdfs.server.namenode;
import org.apache.hadoop.fs.FSDataOutputStream;
import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSStripedOutputStream;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.HdfsConfiguration;
@@ -55,6 +56,7 @@ import java.util.ArrayList;
import java.util.List;
import java.util.UUID;
+import static org.apache.hadoop.hdfs.DFSConfigKeys.DFS_BYTES_PER_CHECKSUM_DEFAULT;
import static org.apache.hadoop.hdfs.protocol.HdfsConstants.BLOCK_STRIPED_CELL_SIZE;
import static org.apache.hadoop.hdfs.protocol.HdfsConstants.NUM_DATA_BLOCKS;
import static org.junit.Assert.assertEquals;
@@ -103,52 +105,50 @@ public class TestAddStripedBlocks {
Assert.assertEquals(firstId + HdfsConstants.MAX_BLOCKS_IN_GROUP, secondId);
}
- @Test
+ @Test (timeout=60000)
public void testAddStripedBlock() throws Exception {
final Path file = new Path("/file1");
// create an empty file
FSDataOutputStream out = null;
try {
out = dfs.create(file, (short) 1);
+ DFSTestUtil.writeAndFlushStripedOutputStream(
+ (DFSStripedOutputStream) out.getWrappedStream(),
+ DFS_BYTES_PER_CHECKSUM_DEFAULT);
FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
- LocatedBlock newBlock = cluster.getNamesystem().getAdditionalBlock(
- file.toString(), fileNode.getId(), dfs.getClient().getClientName(),
- null, null, null);
- assertEquals(GROUP_SIZE, newBlock.getLocations().length);
- assertEquals(GROUP_SIZE, newBlock.getStorageIDs().length);
BlockInfo[] blocks = fileNode.getBlocks();
assertEquals(1, blocks.length);
Assert.assertTrue(blocks[0].isStriped());
checkStripedBlockUC((BlockInfoStriped) fileNode.getLastBlock(), true);
+
+ // restart NameNode to check editlog
+ cluster.restartNameNode(true);
+ fsdir = cluster.getNamesystem().getFSDirectory();
+ fileNode = fsdir.getINode4Write(file.toString()).asFile();
+ blocks = fileNode.getBlocks();
+ assertEquals(1, blocks.length);
+ Assert.assertTrue(blocks[0].isStriped());
+ checkStripedBlockUC((BlockInfoStriped) fileNode.getLastBlock(), false);
+
+ // save namespace, restart namenode, and check
+ dfs = cluster.getFileSystem();
+ dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
+ dfs.saveNamespace();
+ dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
+ cluster.restartNameNode(true);
+ fsdir = cluster.getNamesystem().getFSDirectory();
+ fileNode = fsdir.getINode4Write(file.toString()).asFile();
+ blocks = fileNode.getBlocks();
+ assertEquals(1, blocks.length);
+ Assert.assertTrue(blocks[0].isStriped());
+ checkStripedBlockUC((BlockInfoStriped) fileNode.getLastBlock(), false);
} finally {
IOUtils.cleanup(null, out);
}
-
- // restart NameNode to check editlog
- cluster.restartNameNode(true);
- FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
- INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
- BlockInfo[] blocks = fileNode.getBlocks();
- assertEquals(1, blocks.length);
- Assert.assertTrue(blocks[0].isStriped());
- checkStripedBlockUC((BlockInfoStriped) fileNode.getLastBlock(), false);
-
- // save namespace, restart namenode, and check
- dfs = cluster.getFileSystem();
- dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_ENTER);
- dfs.saveNamespace();
- dfs.setSafeMode(HdfsConstants.SafeModeAction.SAFEMODE_LEAVE);
- cluster.restartNameNode(true);
- fsdir = cluster.getNamesystem().getFSDirectory();
- fileNode = fsdir.getINode4Write(file.toString()).asFile();
- blocks = fileNode.getBlocks();
- assertEquals(1, blocks.length);
- Assert.assertTrue(blocks[0].isStriped());
- checkStripedBlockUC((BlockInfoStriped) fileNode.getLastBlock(), false);
}
private void checkStripedBlockUC(BlockInfoStriped block,
@@ -190,11 +190,12 @@ public class TestAddStripedBlocks {
FSDataOutputStream out = null;
try {
out = dfs.create(file, (short) 1);
+ DFSTestUtil.writeAndFlushStripedOutputStream(
+ (DFSStripedOutputStream) out.getWrappedStream(),
+ DFS_BYTES_PER_CHECKSUM_DEFAULT);
FSDirectory fsdir = cluster.getNamesystem().getFSDirectory();
INodeFile fileNode = fsdir.getINode4Write(file.toString()).asFile();
- cluster.getNamesystem().getAdditionalBlock(file.toString(),
- fileNode.getId(), dfs.getClient().getClientName(), null, null, null);
BlockInfoStripedUnderConstruction lastBlk =
(BlockInfoStripedUnderConstruction) fileNode.getLastBlock();
DatanodeInfo[] expectedDNs = DatanodeStorageInfo