You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by jh...@apache.org on 2017/03/10 00:42:53 UTC
[25/33] hadoop git commit: HDFS-10983. OIV tool should make an EC
file explicit. Contributed by Manoj Govindassamy.
HDFS-10983. OIV tool should make an EC file explicit. Contributed by Manoj Govindassamy.
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/5ca6ef0c
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/5ca6ef0c
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/5ca6ef0c
Branch: refs/heads/YARN-5734
Commit: 5ca6ef0c268b1acb3abf12505b9ead6fe7e38a23
Parents: d7762a5
Author: Andrew Wang <wa...@apache.org>
Authored: Wed Mar 8 15:36:19 2017 -0800
Committer: Andrew Wang <wa...@apache.org>
Committed: Wed Mar 8 15:36:19 2017 -0800
----------------------------------------------------------------------
.../server/namenode/FSImageFormatPBINode.java | 1 +
.../OfflineImageReconstructor.java | 4 +
.../offlineImageViewer/PBImageXmlWriter.java | 15 ++-
.../hdfs/server/namenode/TestFSImage.java | 1 +
.../TestOfflineImageViewer.java | 99 +++++++++++++++++++-
5 files changed, 112 insertions(+), 8 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
index 17b1da7..ef334f7 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/server/namenode/FSImageFormatPBINode.java
@@ -332,6 +332,7 @@ public final class FSImageFormatPBINode {
BlockType blockType = PBHelperClient.convert(f.getBlockType());
LoaderContext state = parent.getLoaderContext();
boolean isStriped = f.hasErasureCodingPolicyID();
+ assert ((!isStriped) || (isStriped && !f.hasReplication()));
Short replication = (!isStriped ? (short) f.getReplication() : null);
ErasureCodingPolicy ecPolicy = isStriped ?
ErasureCodingPolicyManager.getPolicyByPolicyID(
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
index ed348d3..e80f4d4 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageReconstructor.java
@@ -647,6 +647,10 @@ class OfflineImageReconstructor {
break;
case "STRIPED":
bld.setBlockType(HdfsProtos.BlockTypeProto.STRIPED);
+ ival = node.removeChildInt(INODE_SECTION_EC_POLICY_ID);
+ if (ival != null) {
+ bld.setErasureCodingPolicyID(ival);
+ }
break;
default:
throw new IOException("INode XML found with unknown <blocktype> " +
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
index f8734cb..5a42a6b 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageXmlWriter.java
@@ -40,7 +40,6 @@ import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheD
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CacheDirectiveInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.ClientNamenodeProtocolProtos.CachePoolInfoProto;
import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockProto;
-import org.apache.hadoop.hdfs.protocol.proto.HdfsProtos.BlockTypeProto;
import org.apache.hadoop.hdfs.protocol.proto.XAttrProtos;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatPBINode;
import org.apache.hadoop.hdfs.server.namenode.FSImageFormatProtobuf.SectionName;
@@ -59,6 +58,7 @@ import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SecretManagerSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotDiffSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.SnapshotSection;
import org.apache.hadoop.hdfs.server.namenode.FsImageProto.StringTableSection;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.util.XMLUtils;
import org.apache.hadoop.util.LimitInputStream;
import com.google.common.collect.ImmutableList;
@@ -132,6 +132,8 @@ public final class PBImageXmlWriter {
public static final String INODE_SECTION_STORAGE_POLICY_ID =
"storagePolicyId";
public static final String INODE_SECTION_BLOCK_TYPE = "blockType";
+ public static final String INODE_SECTION_EC_POLICY_ID =
+ "erasureCodingPolicyId";
public static final String INODE_SECTION_NS_QUOTA = "nsquota";
public static final String INODE_SECTION_DS_QUOTA = "dsquota";
public static final String INODE_SECTION_TYPE_QUOTA = "typeQuota";
@@ -472,8 +474,12 @@ public final class PBImageXmlWriter {
}
private void dumpINodeFile(INodeSection.INodeFile f) {
- o(SECTION_REPLICATION, f.getReplication())
- .o(INODE_SECTION_MTIME, f.getModificationTime())
+ if (f.hasErasureCodingPolicyID()) {
+ o(SECTION_REPLICATION, INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS);
+ } else {
+ o(SECTION_REPLICATION, f.getReplication());
+ }
+ o(INODE_SECTION_MTIME, f.getModificationTime())
.o(INODE_SECTION_ATIME, f.getAccessTime())
.o(INODE_SECTION_PREFERRED_BLOCK_SIZE, f.getPreferredBlockSize())
.o(INODE_SECTION_PERMISSION, dumpPermission(f.getPermission()));
@@ -495,8 +501,9 @@ public final class PBImageXmlWriter {
if (f.hasStoragePolicyID()) {
o(INODE_SECTION_STORAGE_POLICY_ID, f.getStoragePolicyID());
}
- if (f.getBlockType() != BlockTypeProto.CONTIGUOUS) {
+ if (f.hasErasureCodingPolicyID()) {
o(INODE_SECTION_BLOCK_TYPE, f.getBlockType().name());
+ o(INODE_SECTION_EC_POLICY_ID, f.getErasureCodingPolicyID());
}
if (f.hasFileUC()) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
index ae15491..57416ed 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/server/namenode/TestFSImage.java
@@ -223,6 +223,7 @@ public class TestFSImage {
// blocks to/from legacy fsimage
assertEquals(3, fileByLoaded.getBlocks().length);
assertEquals(preferredBlockSize, fileByLoaded.getPreferredBlockSize());
+ assertEquals(file.getFileReplication(), fileByLoaded.getFileReplication());
if (isUC) {
assertEquals(client,
http://git-wip-us.apache.org/repos/asf/hadoop/blob/5ca6ef0c/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index b587017..e52d536 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -76,11 +76,13 @@ import org.apache.hadoop.hdfs.DFSConfigKeys;
import org.apache.hadoop.hdfs.DFSTestUtil;
import org.apache.hadoop.hdfs.DistributedFileSystem;
import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.BlockType;
import org.apache.hadoop.hdfs.protocol.ErasureCodingPolicy;
import org.apache.hadoop.hdfs.protocol.HdfsConstants;
import org.apache.hadoop.hdfs.protocol.HdfsConstants.SafeModeAction;
import org.apache.hadoop.hdfs.server.namenode.ErasureCodingPolicyManager;
import org.apache.hadoop.hdfs.server.namenode.FSImageTestUtil;
+import org.apache.hadoop.hdfs.server.namenode.INodeFile;
import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
import org.apache.hadoop.hdfs.web.WebHdfsFileSystem;
import org.apache.hadoop.io.IOUtils;
@@ -91,9 +93,8 @@ import org.apache.log4j.Level;
import org.junit.AfterClass;
import org.junit.Assert;
import org.junit.BeforeClass;
-import org.junit.Rule;
import org.junit.Test;
-import org.junit.rules.TemporaryFolder;
+import org.xml.sax.Attributes;
import org.xml.sax.InputSource;
import org.xml.sax.SAXException;
import org.xml.sax.helpers.DefaultHandler;
@@ -112,7 +113,6 @@ public class TestOfflineImageViewer {
// namespace as written to dfs, to be compared with viewer's output
final static HashMap<String, FileStatus> writtenFiles = Maps.newHashMap();
static int dirCount = 0;
-
private static File tempDir;
// Create a populated namespace for later testing. Save its contents to a
@@ -358,6 +358,96 @@ public class TestOfflineImageViewer {
assertEquals(0, status);
}
+ /**
+ * SAX handler to verify EC Files and their policies.
+ */
+ class ECXMLHandler extends DefaultHandler {
+
+ private boolean isInode = false;
+ private boolean isAttrRepl = false;
+ private boolean isAttrName = false;
+ private boolean isXAttrs = false;
+ private boolean isAttrECPolicy = false;
+ private boolean isAttrBlockType = false;
+ private String currentInodeName;
+ private String currentECPolicy;
+ private String currentBlockType;
+ private String currentRepl;
+
+ @Override
+ public void startElement(String uri, String localName, String qName,
+ Attributes attributes) throws SAXException {
+ super.startElement(uri, localName, qName, attributes);
+ if (qName.equalsIgnoreCase(PBImageXmlWriter.INODE_SECTION_INODE)) {
+ isInode = true;
+ } else if (isInode && !isXAttrs && qName.equalsIgnoreCase(
+ PBImageXmlWriter.SECTION_NAME)) {
+ isAttrName = true;
+ } else if (isInode && qName.equalsIgnoreCase(
+ PBImageXmlWriter.SECTION_REPLICATION)) {
+ isAttrRepl = true;
+ } else if (isInode &&
+ qName.equalsIgnoreCase(PBImageXmlWriter.INODE_SECTION_EC_POLICY_ID)) {
+ isAttrECPolicy = true;
+ } else if (isInode && qName.equalsIgnoreCase(
+ PBImageXmlWriter.INODE_SECTION_BLOCK_TYPE)) {
+ isAttrBlockType = true;
+ } else if (isInode && qName.equalsIgnoreCase(
+ PBImageXmlWriter.INODE_SECTION_XATTRS)) {
+ isXAttrs = true;
+ }
+ }
+
+ @Override
+ public void endElement(String uri, String localName, String qName)
+ throws SAXException {
+ super.endElement(uri, localName, qName);
+ if (qName.equalsIgnoreCase(PBImageXmlWriter.INODE_SECTION_INODE)) {
+ if (currentInodeName != null && currentInodeName.length() > 0) {
+ if (currentBlockType != null && currentBlockType.equalsIgnoreCase(
+ BlockType.STRIPED.name())) {
+ Assert.assertEquals("INode '"
+ + currentInodeName + "' has unexpected EC Policy!",
+ Byte.parseByte(currentECPolicy),
+ ErasureCodingPolicyManager.getPolicyByPolicyID(
+ HdfsConstants.XOR_2_1_POLICY_ID).getId());
+ Assert.assertEquals("INode '"
+ + currentInodeName + "' has unexpected replication!",
+ currentRepl,
+ Short.toString(INodeFile.DEFAULT_REPL_FOR_STRIPED_BLOCKS));
+ }
+ }
+ isInode = false;
+ currentInodeName = "";
+ currentECPolicy = "";
+ currentRepl = "";
+ } else if (qName.equalsIgnoreCase(
+ PBImageXmlWriter.INODE_SECTION_XATTRS)) {
+ isXAttrs = false;
+ }
+ }
+
+ @Override
+ public void characters(char[] ch, int start, int length)
+ throws SAXException {
+ super.characters(ch, start, length);
+ String value = new String(ch, start, length);
+ if (isAttrName) {
+ currentInodeName = value;
+ isAttrName = false;
+ } else if (isAttrRepl) {
+ currentRepl = value;
+ isAttrRepl = false;
+ } else if (isAttrECPolicy) {
+ currentECPolicy = value;
+ isAttrECPolicy = false;
+ } else if (isAttrBlockType) {
+ currentBlockType = value;
+ isAttrBlockType = false;
+ }
+ }
+ }
+
@Test
public void testPBImageXmlWriter() throws IOException, SAXException,
ParserConfigurationException {
@@ -368,7 +458,8 @@ public class TestOfflineImageViewer {
SAXParserFactory spf = SAXParserFactory.newInstance();
SAXParser parser = spf.newSAXParser();
final String xml = output.toString();
- parser.parse(new InputSource(new StringReader(xml)), new DefaultHandler());
+ ECXMLHandler ecxmlHandler = new ECXMLHandler();
+ parser.parse(new InputSource(new StringReader(xml)), ecxmlHandler);
}
@Test
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org