You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by le...@apache.org on 2016/02/02 19:43:13 UTC
hadoop git commit: HDFS-9721. Allow Delimited PB OIV tool to run upon
fsimage that contains INodeReference. (Xiao Chen via lei)
Repository: hadoop
Updated Branches:
refs/heads/branch-2.8 94950c15c -> dfbd0d4df
HDFS-9721. Allow Delimited PB OIV tool to run upon fsimage that contains INodeReference. (Xiao Chen via lei)
(cherry picked from commit 9d494f0c0eaa05417f3a3e88487d878d1731da36)
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/dfbd0d4d
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/dfbd0d4d
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/dfbd0d4d
Branch: refs/heads/branch-2.8
Commit: dfbd0d4df4b3e1974034f6202330481fb86006ef
Parents: 94950c1
Author: Lei Xu <le...@apache.org>
Authored: Tue Feb 2 10:41:17 2016 -0800
Committer: Lei Xu <le...@apache.org>
Committed: Tue Feb 2 10:43:06 2016 -0800
----------------------------------------------------------------------
hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt | 3 +
.../tools/offlineImageViewer/FSImageLoader.java | 2 +-
.../IgnoreSnapshotException.java | 28 ++++++
.../PBImageDelimitedTextWriter.java | 23 ++++-
.../offlineImageViewer/PBImageTextWriter.java | 90 ++++++++++++++++----
.../TestOfflineImageViewer.java | 36 +++++---
6 files changed, 153 insertions(+), 29 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfbd0d4d/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
index 53a74e0..04405d6 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
+++ b/hadoop-hdfs-project/hadoop-hdfs/CHANGES.txt
@@ -894,6 +894,9 @@ Release 2.8.0 - UNRELEASED
HDFS-9638. Improve DistCp Help and documentation.
(Wei-Chiu Chuang via Yongjun Zhang)
+ HDFS-9721. Allow Delimited PB OIV tool to run upon fsimage that contains
+ INodeReference. (Xiao Chen via lei)
+
OPTIMIZATIONS
HDFS-8026. Trace FSOutputSummer#writeChecksumChunks rather than
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfbd0d4d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
index 4d8eb35..172f599 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FSImageLoader.java
@@ -204,7 +204,7 @@ class FSImageLoader {
return dirs;
}
- private static ImmutableList<Long> loadINodeReferenceSection(InputStream in)
+ static ImmutableList<Long> loadINodeReferenceSection(InputStream in)
throws IOException {
LOG.info("Loading inode references");
ImmutableList.Builder<Long> builder = ImmutableList.builder();
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfbd0d4d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IgnoreSnapshotException.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IgnoreSnapshotException.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IgnoreSnapshotException.java
new file mode 100644
index 0000000..60a6534
--- /dev/null
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IgnoreSnapshotException.java
@@ -0,0 +1,28 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.IOException;
+
+/**
+ * Signals that a snapshot is ignored.
+ */
+public class IgnoreSnapshotException extends IOException {
+ public IgnoreSnapshotException() {
+ }
+}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfbd0d4d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java
index fbe7f3a..3ddc135 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageDelimitedTextWriter.java
@@ -84,6 +84,7 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter {
inodeName.isEmpty() ? "/" : inodeName);
buffer.append(path.toString());
PermissionStatus p = null;
+ boolean isDir = false;
switch (inode.getType()) {
case FILE:
@@ -109,6 +110,7 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter {
append(buffer, 0); // Num bytes.
append(buffer, dir.getNsQuota());
append(buffer, dir.getDsQuota());
+ isDir = true;
break;
case SYMLINK:
INodeSymlink s = inode.getSymlink();
@@ -126,9 +128,28 @@ public class PBImageDelimitedTextWriter extends PBImageTextWriter {
break;
}
assert p != null;
- append(buffer, p.getPermission().toString());
+ String dirString = isDir ? "d" : "-";
+ append(buffer, dirString + p.getPermission().toString());
append(buffer, p.getUserName());
append(buffer, p.getGroupName());
return buffer.toString();
}
+
+ @Override
+ public String getHeader() {
+ StringBuffer buffer = new StringBuffer();
+ buffer.append("Path");
+ append(buffer, "Replication");
+ append(buffer, "ModificationTime");
+ append(buffer, "AccessTime");
+ append(buffer, "PreferredBlockSize");
+ append(buffer, "BlocksCount");
+ append(buffer, "FileSize");
+ append(buffer, "NSQUOTA");
+ append(buffer, "DSQUOTA");
+ append(buffer, "Permission");
+ append(buffer, "UserName");
+ append(buffer, "GroupName");
+ return buffer.toString();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfbd0d4d/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
index d2ccc5c..ae1cd79 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/PBImageTextWriter.java
@@ -18,6 +18,7 @@
package org.apache.hadoop.hdfs.tools.offlineImageViewer;
import com.google.common.base.Preconditions;
+import com.google.common.collect.ImmutableList;
import com.google.common.collect.Lists;
import org.apache.commons.io.FileUtils;
import org.apache.hadoop.conf.Configuration;
@@ -195,13 +196,17 @@ abstract class PBImageTextWriter implements Closeable {
dirMap.put(p.getId(), dir);
}
+ @Override
public String getParentPath(long inode) throws IOException {
if (inode == INodeId.ROOT_INODE_ID) {
return "";
}
Dir parent = dirChildMap.get(inode);
- Preconditions.checkState(parent != null,
- "Can not find parent directory for INode: %s", inode);
+ if (parent == null) {
+ // The inode is an INodeReference, which is generated from snapshot.
+ // For delimited oiv tool, no need to print out metadata in snapshots.
+ PBImageTextWriter.ignoreSnapshotName(inode);
+ }
return parent.getPath();
}
@@ -353,16 +358,22 @@ abstract class PBImageTextWriter implements Closeable {
return "/";
}
byte[] bytes = dirChildMap.get(toBytes(inode));
- Preconditions.checkState(bytes != null && bytes.length == 8,
- "Can not find parent directory for inode %s, "
- + "fsimage might be corrupted", inode);
+ if (bytes == null) {
+ // The inode is an INodeReference, which is generated from snapshot.
+ // For delimited oiv tool, no need to print out metadata in snapshots.
+ PBImageTextWriter.ignoreSnapshotName(inode);
+ }
+ if (bytes.length != 8) {
+ throw new IOException(
+ "bytes array length error. Actual length is " + bytes.length);
+ }
long parent = toLong(bytes);
if (!dirPathCache.containsKey(parent)) {
bytes = dirMap.get(toBytes(parent));
- if (parent != INodeId.ROOT_INODE_ID) {
- Preconditions.checkState(bytes != null,
- "Can not find parent directory for inode %s, "
- + ", the fsimage might be corrupted.", parent);
+ if (parent != INodeId.ROOT_INODE_ID && bytes == null) {
+ // The parent is an INodeReference, which is generated from snapshot.
+ // For delimited oiv tool, no need to print out metadata in snapshots.
+ PBImageTextWriter.ignoreSnapshotName(parent);
}
String parentName = toString(bytes);
String parentPath =
@@ -401,6 +412,7 @@ abstract class PBImageTextWriter implements Closeable {
@Override
public void close() throws IOException {
+ out.flush();
IOUtils.cleanup(null, metadataMap);
}
@@ -411,6 +423,11 @@ abstract class PBImageTextWriter implements Closeable {
*/
abstract protected String getEntry(String parent, INode inode);
+ /**
+ * Get text output for the header line.
+ */
+ abstract protected String getHeader();
+
public void visit(RandomAccessFile file) throws IOException {
Configuration conf = new Configuration();
if (!FSImageUtil.checkFileFormat(file)) {
@@ -442,6 +459,7 @@ abstract class PBImageTextWriter implements Closeable {
}
});
+ ImmutableList<Long> refIdList = null;
for (FileSummary.Section section : sections) {
fin.getChannel().position(section.getOffset());
is = FSImageUtil.wrapInputStreamForCompression(conf,
@@ -449,15 +467,22 @@ abstract class PBImageTextWriter implements Closeable {
fin, section.getLength())));
switch (SectionName.fromString(section.getName())) {
case STRING_TABLE:
+ LOG.info("Loading string table");
stringTable = FSImageLoader.loadStringTable(is);
break;
+ case INODE_REFERENCE:
+ // Load INodeReference so that all INodes can be processed.
+ // Snapshots are not handled and will just be ignored for now.
+ LOG.info("Loading inode references");
+ refIdList = FSImageLoader.loadINodeReferenceSection(is);
+ break;
default:
break;
}
}
loadDirectories(fin, sections, summary, conf);
- loadINodeDirSection(fin, sections, summary, conf);
+ loadINodeDirSection(fin, sections, summary, conf, refIdList);
metadataMap.sync();
output(conf, summary, fin, sections);
}
@@ -468,6 +493,7 @@ abstract class PBImageTextWriter implements Closeable {
throws IOException {
InputStream is;
long startTime = Time.monotonicNow();
+ out.println(getHeader());
for (FileSummary.Section section : sections) {
if (SectionName.fromString(section.getName()) == SectionName.INODE) {
fin.getChannel().position(section.getOffset());
@@ -508,7 +534,7 @@ abstract class PBImageTextWriter implements Closeable {
private void loadINodeDirSection(
FileInputStream fin, List<FileSummary.Section> sections,
- FileSummary summary, Configuration conf)
+ FileSummary summary, Configuration conf, List<Long> refIdList)
throws IOException {
LOG.info("Loading INode directory section.");
long startTime = Time.monotonicNow();
@@ -519,7 +545,7 @@ abstract class PBImageTextWriter implements Closeable {
InputStream is = FSImageUtil.wrapInputStreamForCompression(conf,
summary.getCodec(), new BufferedInputStream(
new LimitInputStream(fin, section.getLength())));
- buildNamespace(is);
+ buildNamespace(is, refIdList);
}
}
long timeTaken = Time.monotonicNow() - startTime;
@@ -549,7 +575,8 @@ abstract class PBImageTextWriter implements Closeable {
/**
* Scan the INodeDirectory section to construct the namespace.
*/
- private void buildNamespace(InputStream in) throws IOException {
+ private void buildNamespace(InputStream in, List<Long> refIdList)
+ throws IOException {
int count = 0;
while (true) {
FsImageProto.INodeDirectorySection.DirEntry e =
@@ -562,12 +589,15 @@ abstract class PBImageTextWriter implements Closeable {
LOG.debug("Scanned {} directories.", count);
}
long parentId = e.getParent();
- // Referred INode is not support for now.
for (int i = 0; i < e.getChildrenCount(); i++) {
long childId = e.getChildren(i);
metadataMap.putDirChild(parentId, childId);
}
- Preconditions.checkState(e.getRefChildrenCount() == 0);
+ for (int i = e.getChildrenCount();
+ i < e.getChildrenCount() + e.getRefChildrenCount(); i++) {
+ int refId = e.getRefChildren(i - e.getChildrenCount());
+ metadataMap.putDirChild(parentId, refIdList.get(refId));
+ }
}
LOG.info("Scanned {} INode directories to build namespace.", count);
}
@@ -575,15 +605,41 @@ abstract class PBImageTextWriter implements Closeable {
private void outputINodes(InputStream in) throws IOException {
INodeSection s = INodeSection.parseDelimitedFrom(in);
LOG.info("Found {} INodes in the INode section", s.getNumInodes());
+ long ignored = 0;
+ long ignoredSnapshots = 0;
for (int i = 0; i < s.getNumInodes(); ++i) {
INode p = INode.parseDelimitedFrom(in);
- String parentPath = metadataMap.getParentPath(p.getId());
- out.println(getEntry(parentPath, p));
+ try {
+ String parentPath = metadataMap.getParentPath(p.getId());
+ out.println(getEntry(parentPath, p));
+ } catch (IOException ioe) {
+ ignored++;
+ if (!(ioe instanceof IgnoreSnapshotException)) {
+ LOG.warn("Exception caught, ignoring node:{}", p.getId(), ioe);
+ } else {
+ ignoredSnapshots++;
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Exception caught, ignoring node:{}.", p.getId(), ioe);
+ }
+ }
+ }
if (LOG.isDebugEnabled() && i % 100000 == 0) {
LOG.debug("Outputted {} INodes.", i);
}
}
+ if (ignored > 0) {
+ LOG.warn("Ignored {} nodes, including {} in snapshots. Please turn on"
+ + " debug log for details", ignored, ignoredSnapshots);
+ }
LOG.info("Outputted {} INodes.", s.getNumInodes());
}
+
+ static void ignoreSnapshotName(long inode) throws IOException {
+ // Ignore snapshots - we want the output similar to -ls -R.
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("No snapshot name found for inode {}", inode);
+ }
+ throw new IgnoreSnapshotException();
+ }
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/dfbd0d4d/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
----------------------------------------------------------------------
diff --git a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
index 467534a..91b79e2 100644
--- a/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
+++ b/hadoop-hdfs-project/hadoop-hdfs/src/test/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
@@ -87,6 +87,7 @@ public class TestOfflineImageViewer {
// namespace as written to dfs, to be compared with viewer's output
final static HashMap<String, FileStatus> writtenFiles = Maps.newHashMap();
+ static int dirCount = 0;
@Rule
public TemporaryFolder folder = new TemporaryFolder();
@@ -113,7 +114,7 @@ public class TestOfflineImageViewer {
DistributedFileSystem hdfs = cluster.getFileSystem();
// Create a reasonable namespace
- for (int i = 0; i < NUM_DIRS; i++) {
+ for (int i = 0; i < NUM_DIRS; i++, dirCount++) {
Path dir = new Path("/dir" + i);
hdfs.mkdirs(dir);
writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString()));
@@ -131,11 +132,13 @@ public class TestOfflineImageViewer {
// Create an empty directory
Path emptydir = new Path("/emptydir");
hdfs.mkdirs(emptydir);
+ dirCount++;
writtenFiles.put(emptydir.toString(), hdfs.getFileStatus(emptydir));
//Create a directory whose name should be escaped in XML
Path invalidXMLDir = new Path("/dirContainingInvalidXMLChar\u0000here");
hdfs.mkdirs(invalidXMLDir);
+ dirCount++;
// Get delegation tokens so we log the delegation token op
Token<?>[] delegationTokens = hdfs
@@ -144,15 +147,24 @@ public class TestOfflineImageViewer {
LOG.debug("got token " + t);
}
- final Path snapshot = new Path("/snapshot");
- hdfs.mkdirs(snapshot);
- hdfs.allowSnapshot(snapshot);
- hdfs.mkdirs(new Path("/snapshot/1"));
- hdfs.delete(snapshot, true);
+ // Create INodeReference
+ final Path src = new Path("/src");
+ hdfs.mkdirs(src);
+ dirCount++;
+ writtenFiles.put(src.toString(), hdfs.getFileStatus(src));
+ final Path orig = new Path("/src/orig");
+ hdfs.mkdirs(orig);
+ hdfs.allowSnapshot(src);
+ hdfs.createSnapshot(src, "snapshot");
+ final Path dst = new Path("/dst");
+ hdfs.rename(orig, dst);
+ dirCount++;
+ writtenFiles.put(dst.toString(), hdfs.getFileStatus(dst));
// Set XAttrs so the fsimage contains XAttr ops
final Path xattr = new Path("/xattr");
hdfs.mkdirs(xattr);
+ dirCount++;
hdfs.setXAttr(xattr, "user.a1", new byte[]{ 0x31, 0x32, 0x33 });
hdfs.setXAttr(xattr, "user.a2", new byte[]{ 0x37, 0x38, 0x39 });
// OIV should be able to handle empty value XAttrs
@@ -232,8 +244,8 @@ public class TestOfflineImageViewer {
matcher = p.matcher(outputString);
assertTrue(matcher.find() && matcher.groupCount() == 1);
int totalDirs = Integer.parseInt(matcher.group(1));
- // totalDirs includes root directory, empty directory, and xattr directory
- assertEquals(NUM_DIRS + 4, totalDirs);
+ // totalDirs includes root directory
+ assertEquals(dirCount + 1, totalDirs);
FileStatus maxFile = Collections.max(writtenFiles.values(),
new Comparator<FileStatus>() {
@@ -285,7 +297,7 @@ public class TestOfflineImageViewer {
// verify the number of directories
FileStatus[] statuses = webhdfs.listStatus(new Path("/"));
- assertEquals(NUM_DIRS + 3, statuses.length); // contains empty and xattr directory
+ assertEquals(dirCount, statuses.length);
// verify the number of files in the directory
statuses = webhdfs.listStatus(new Path("/dir0"));
@@ -393,11 +405,15 @@ public class TestOfflineImageViewer {
BufferedReader reader =
new BufferedReader(new InputStreamReader(input))) {
String line;
+ boolean header = true;
while ((line = reader.readLine()) != null) {
System.out.println(line);
String[] fields = line.split(DELIMITER);
assertEquals(12, fields.length);
- fileNames.add(fields[0]);
+ if (!header) {
+ fileNames.add(fields[0]);
+ }
+ header = false;
}
}