You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by ki...@apache.org on 2014/05/14 03:15:49 UTC
svn commit: r1594440 [2/3] - in
/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs: ./
src/main/bin/ src/main/java/org/apache/hadoop/hdfs/
src/main/java/org/apache/hadoop/hdfs/security/token/delegation/
src/main/java/org/apache/hadoop/hdf...
Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/DepthCounter.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/DepthCounter.java?rev=1594440&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/DepthCounter.java (added)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/DepthCounter.java Wed May 14 01:15:48 2014
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Utility class for tracking descent into the structure of the
+ * Visitor class (ImageVisitor, EditsVisitor etc.)
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class DepthCounter {
+ private int depth = 0;
+
+ public void incLevel() { depth++; }
+ public void decLevel() { if(depth >= 1) depth--; }
+ public int getLevel() { return depth; }
+}
+
Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java?rev=1594440&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java (added)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/FileDistributionVisitor.java Wed May 14 01:15:48 2014
@@ -0,0 +1,193 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.IOException;
+import java.util.LinkedList;
+
+/**
+ * File size distribution visitor.
+ *
+ * <h3>Description.</h3>
+ * This is the tool for analyzing file sizes in the namespace image.
+ * In order to run the tool one should define a range of integers
+ * <tt>[0, maxSize]</tt> by specifying <tt>maxSize</tt> and a <tt>step</tt>.
+ * The range of integers is divided into segments of size <tt>step</tt>:
+ * <tt>[0, s<sub>1</sub>, ..., s<sub>n-1</sub>, maxSize]</tt>,
+ * and the visitor calculates how many files in the system fall into
+ * each segment <tt>[s<sub>i-1</sub>, s<sub>i</sub>)</tt>.
+ * Note that files larger than <tt>maxSize</tt> always fall into
+ * the very last segment.
+ *
+ * <h3>Input.</h3>
+ * <ul>
+ * <li><tt>filename</tt> specifies the location of the image file;</li>
+ * <li><tt>maxSize</tt> determines the range <tt>[0, maxSize]</tt> of files
+ * sizes considered by the visitor;</li>
+ * <li><tt>step</tt> the range is divided into segments of size step.</li>
+ * </ul>
+ *
+ * <h3>Output.</h3>
+ * The output file is formatted as a tab separated two column table:
+ * Size and NumFiles. Where Size represents the start of the segment,
+ * and numFiles is the number of files form the image which size falls in
+ * this segment.
+ */
+class FileDistributionVisitor extends TextWriterImageVisitor {
+ final private LinkedList<ImageElement> elemS = new LinkedList<ImageElement>();
+
+ private final static long MAX_SIZE_DEFAULT = 0x2000000000L; // 1/8 TB = 2^37
+ private final static int INTERVAL_DEFAULT = 0x200000; // 2 MB = 2^21
+
+ private int[] distribution;
+ private long maxSize;
+ private int step;
+
+ private int totalFiles;
+ private int totalDirectories;
+ private int totalBlocks;
+ private long totalSpace;
+ private long maxFileSize;
+
+ private FileContext current;
+
+ private boolean inInode = false;
+
+ /**
+ * File or directory information.
+ */
+ private static class FileContext {
+ String path;
+ long fileSize;
+ int numBlocks;
+ int replication;
+ }
+
+ public FileDistributionVisitor(String filename,
+ long maxSize,
+ int step) throws IOException {
+ super(filename, false);
+ this.maxSize = (maxSize == 0 ? MAX_SIZE_DEFAULT : maxSize);
+ this.step = (step == 0 ? INTERVAL_DEFAULT : step);
+ long numIntervals = this.maxSize / this.step;
+ if(numIntervals >= Integer.MAX_VALUE)
+ throw new IOException("Too many distribution intervals " + numIntervals);
+ this.distribution = new int[1 + (int)(numIntervals)];
+ this.totalFiles = 0;
+ this.totalDirectories = 0;
+ this.totalBlocks = 0;
+ this.totalSpace = 0;
+ this.maxFileSize = 0;
+ }
+
+ @Override
+ void start() throws IOException {}
+
+ @Override
+ void finish() throws IOException {
+ output();
+ super.finish();
+ }
+
+ @Override
+ void finishAbnormally() throws IOException {
+ System.out.println("*** Image processing finished abnormally. Ending ***");
+ output();
+ super.finishAbnormally();
+ }
+
+ private void output() throws IOException {
+ // write the distribution into the output file
+ write("Size\tNumFiles\n");
+ for(int i = 0; i < distribution.length; i++)
+ write(((long)i * step) + "\t" + distribution[i] + "\n");
+ System.out.println("totalFiles = " + totalFiles);
+ System.out.println("totalDirectories = " + totalDirectories);
+ System.out.println("totalBlocks = " + totalBlocks);
+ System.out.println("totalSpace = " + totalSpace);
+ System.out.println("maxFileSize = " + maxFileSize);
+ }
+
+ @Override
+ void leaveEnclosingElement() throws IOException {
+ ImageElement elem = elemS.pop();
+
+ if(elem != ImageElement.INODE &&
+ elem != ImageElement.INODE_UNDER_CONSTRUCTION)
+ return;
+ inInode = false;
+ if(current.numBlocks < 0) {
+ totalDirectories ++;
+ return;
+ }
+ totalFiles++;
+ totalBlocks += current.numBlocks;
+ totalSpace += current.fileSize * current.replication;
+ if(maxFileSize < current.fileSize)
+ maxFileSize = current.fileSize;
+ int high;
+ if(current.fileSize > maxSize)
+ high = distribution.length-1;
+ else
+ high = (int)Math.ceil((double)current.fileSize / step);
+ distribution[high]++;
+ if(totalFiles % 1000000 == 1)
+ System.out.println("Files processed: " + totalFiles
+ + " Current: " + current.path);
+ }
+
+ @Override
+ void visit(ImageElement element, String value) throws IOException {
+ if(inInode) {
+ switch(element) {
+ case INODE_PATH:
+ current.path = (value.equals("") ? "/" : value);
+ break;
+ case REPLICATION:
+ current.replication = Integer.valueOf(value);
+ break;
+ case NUM_BYTES:
+ current.fileSize += Long.valueOf(value);
+ break;
+ default:
+ break;
+ }
+ }
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element) throws IOException {
+ elemS.push(element);
+ if(element == ImageElement.INODE ||
+ element == ImageElement.INODE_UNDER_CONSTRUCTION) {
+ current = new FileContext();
+ inInode = true;
+ }
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element,
+ ImageElement key, String value) throws IOException {
+ elemS.push(element);
+ if(element == ImageElement.INODE ||
+ element == ImageElement.INODE_UNDER_CONSTRUCTION)
+ inInode = true;
+ else if(element == ImageElement.BLOCKS)
+ current.numBlocks = Integer.parseInt(value);
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoader.java?rev=1594440&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoader.java (added)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoader.java Wed May 14 01:15:48 2014
@@ -0,0 +1,83 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * An ImageLoader can accept a DataInputStream to an Hadoop FSImage file
+ * and walk over its structure using the supplied ImageVisitor.
+ *
+ * Each implementation of ImageLoader is designed to rapidly process an
+ * image file. As long as minor changes are made from one layout version
+ * to another, it is acceptable to tweak one implementation to read the next.
+ * However, if the layout version changes enough that it would make a
+ * processor slow or difficult to read, another processor should be created.
+ * This allows each processor to quickly read an image without getting
+ * bogged down in dealing with significant differences between layout versions.
+ */
+interface ImageLoader {
+
+ /**
+ * @param in DataInputStream pointing to an Hadoop FSImage file
+ * @param v Visit to apply to the FSImage file
+ * @param enumerateBlocks Should visitor visit each of the file blocks?
+ */
+ public void loadImage(DataInputStream in, ImageVisitor v,
+ boolean enumerateBlocks) throws IOException;
+
+ /**
+ * Can this processor handle the specified version of FSImage file?
+ *
+ * @param version FSImage version file
+ * @return True if this instance can process the file
+ */
+ public boolean canLoadVersion(int version);
+
+ /**
+ * Factory for obtaining version of image loader that can read
+ * a particular image format.
+ */
+ @InterfaceAudience.Private
+ public class LoaderFactory {
+ // Java doesn't support static methods on interfaces, which necessitates
+ // this factory class
+
+ /**
+ * Find an image loader capable of interpreting the specified
+ * layout version number. If none, return null;
+ *
+ * @param version fsimage layout version number to be processed
+ * @return ImageLoader that can interpret specified version, or null
+ */
+ static public ImageLoader getLoader(int version) {
+ // Easy to add more image processors as they are written
+ ImageLoader[] loaders = { new ImageLoaderCurrent() };
+
+ for (ImageLoader l : loaders) {
+ if (l.canLoadVersion(version))
+ return l;
+ }
+
+ return null;
+ }
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java?rev=1594440&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java (added)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageLoaderCurrent.java Wed May 14 01:15:48 2014
@@ -0,0 +1,821 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.text.DateFormat;
+import java.text.SimpleDateFormat;
+import java.util.Date;
+import java.util.HashMap;
+import java.util.Map;
+
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.permission.FsPermission;
+import org.apache.hadoop.hdfs.protocol.DatanodeInfo.AdminStates;
+import org.apache.hadoop.hdfs.protocol.LayoutFlags;
+import org.apache.hadoop.hdfs.protocol.LayoutVersion.Feature;
+import org.apache.hadoop.hdfs.security.token.delegation.DelegationTokenIdentifier;
+import org.apache.hadoop.hdfs.server.namenode.FSImageSerialization;
+import org.apache.hadoop.hdfs.server.namenode.INodeId;
+import org.apache.hadoop.hdfs.server.namenode.NameNodeLayoutVersion;
+import org.apache.hadoop.hdfs.tools.offlineImageViewer.ImageVisitor.ImageElement;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.io.WritableUtils;
+import org.apache.hadoop.io.compress.CompressionCodec;
+import org.apache.hadoop.io.compress.CompressionCodecFactory;
+import org.apache.hadoop.security.token.delegation.DelegationKey;
+
+/**
+ * ImageLoaderCurrent processes Hadoop FSImage files and walks over
+ * them using a provided ImageVisitor, calling the visitor at each element
+ * enumerated below.
+ *
+ * The only difference between v18 and v19 was the utilization of the
+ * stickybit. Therefore, the same viewer can reader either format.
+ *
+ * Versions -19 fsimage layout (with changes from -16 up):
+ * Image version (int)
+ * Namepsace ID (int)
+ * NumFiles (long)
+ * Generation stamp (long)
+ * INodes (count = NumFiles)
+ * INode
+ * Path (String)
+ * Replication (short)
+ * Modification Time (long as date)
+ * Access Time (long) // added in -16
+ * Block size (long)
+ * Num blocks (int)
+ * Blocks (count = Num blocks)
+ * Block
+ * Block ID (long)
+ * Num bytes (long)
+ * Generation stamp (long)
+ * Namespace Quota (long)
+ * Diskspace Quota (long) // added in -18
+ * Permissions
+ * Username (String)
+ * Groupname (String)
+ * OctalPerms (short -> String) // Modified in -19
+ * Symlink (String) // added in -23
+ * NumINodesUnderConstruction (int)
+ * INodesUnderConstruction (count = NumINodesUnderConstruction)
+ * INodeUnderConstruction
+ * Path (bytes as string)
+ * Replication (short)
+ * Modification time (long as date)
+ * Preferred block size (long)
+ * Num blocks (int)
+ * Blocks
+ * Block
+ * Block ID (long)
+ * Num bytes (long)
+ * Generation stamp (long)
+ * Permissions
+ * Username (String)
+ * Groupname (String)
+ * OctalPerms (short -> String)
+ * Client Name (String)
+ * Client Machine (String)
+ * NumLocations (int)
+ * DatanodeDescriptors (count = numLocations) // not loaded into memory
+ * short // but still in file
+ * long
+ * string
+ * long
+ * int
+ * string
+ * string
+ * enum
+ * CurrentDelegationKeyId (int)
+ * NumDelegationKeys (int)
+ * DelegationKeys (count = NumDelegationKeys)
+ * DelegationKeyLength (vint)
+ * DelegationKey (bytes)
+ * DelegationTokenSequenceNumber (int)
+ * NumDelegationTokens (int)
+ * DelegationTokens (count = NumDelegationTokens)
+ * DelegationTokenIdentifier
+ * owner (String)
+ * renewer (String)
+ * realUser (String)
+ * issueDate (vlong)
+ * maxDate (vlong)
+ * sequenceNumber (vint)
+ * masterKeyId (vint)
+ * expiryTime (long)
+ *
+ */
+class ImageLoaderCurrent implements ImageLoader {
+ protected final DateFormat dateFormat =
+ new SimpleDateFormat("yyyy-MM-dd HH:mm");
+ private static int[] versions = { -16, -17, -18, -19, -20, -21, -22, -23,
+ -24, -25, -26, -27, -28, -30, -31, -32, -33, -34, -35, -36, -37, -38, -39,
+ -40, -41, -42, -43, -44, -45, -46, -47, -48, -49, -50, -51 };
+ private int imageVersion = 0;
+
+ private final Map<Long, Boolean> subtreeMap = new HashMap<Long, Boolean>();
+ private final Map<Long, String> dirNodeMap = new HashMap<Long, String>();
+
+ /* (non-Javadoc)
+ * @see ImageLoader#canProcessVersion(int)
+ */
+ @Override
+ public boolean canLoadVersion(int version) {
+ for(int v : versions)
+ if(v == version) return true;
+
+ return false;
+ }
+
+ /* (non-Javadoc)
+ * @see ImageLoader#processImage(java.io.DataInputStream, ImageVisitor, boolean)
+ */
+ @Override
+ public void loadImage(DataInputStream in, ImageVisitor v,
+ boolean skipBlocks) throws IOException {
+ boolean done = false;
+ try {
+ v.start();
+ v.visitEnclosingElement(ImageElement.FS_IMAGE);
+
+ imageVersion = in.readInt();
+ if( !canLoadVersion(imageVersion))
+ throw new IOException("Cannot process fslayout version " + imageVersion);
+ if (NameNodeLayoutVersion.supports(Feature.ADD_LAYOUT_FLAGS, imageVersion)) {
+ LayoutFlags.read(in);
+ }
+
+ v.visit(ImageElement.IMAGE_VERSION, imageVersion);
+ v.visit(ImageElement.NAMESPACE_ID, in.readInt());
+
+ long numInodes = in.readLong();
+
+ v.visit(ImageElement.GENERATION_STAMP, in.readLong());
+
+ if (NameNodeLayoutVersion.supports(Feature.SEQUENTIAL_BLOCK_ID, imageVersion)) {
+ v.visit(ImageElement.GENERATION_STAMP_V2, in.readLong());
+ v.visit(ImageElement.GENERATION_STAMP_V1_LIMIT, in.readLong());
+ v.visit(ImageElement.LAST_ALLOCATED_BLOCK_ID, in.readLong());
+ }
+
+ if (NameNodeLayoutVersion.supports(Feature.STORED_TXIDS, imageVersion)) {
+ v.visit(ImageElement.TRANSACTION_ID, in.readLong());
+ }
+
+ if (NameNodeLayoutVersion.supports(Feature.ADD_INODE_ID, imageVersion)) {
+ v.visit(ImageElement.LAST_INODE_ID, in.readLong());
+ }
+
+ boolean supportSnapshot = NameNodeLayoutVersion.supports(Feature.SNAPSHOT,
+ imageVersion);
+ if (supportSnapshot) {
+ v.visit(ImageElement.SNAPSHOT_COUNTER, in.readInt());
+ int numSnapshots = in.readInt();
+ v.visit(ImageElement.NUM_SNAPSHOTS_TOTAL, numSnapshots);
+ for (int i = 0; i < numSnapshots; i++) {
+ processSnapshot(in, v);
+ }
+ }
+
+ if (NameNodeLayoutVersion.supports(Feature.FSIMAGE_COMPRESSION, imageVersion)) {
+ boolean isCompressed = in.readBoolean();
+ v.visit(ImageElement.IS_COMPRESSED, String.valueOf(isCompressed));
+ if (isCompressed) {
+ String codecClassName = Text.readString(in);
+ v.visit(ImageElement.COMPRESS_CODEC, codecClassName);
+ CompressionCodecFactory codecFac = new CompressionCodecFactory(
+ new Configuration());
+ CompressionCodec codec = codecFac.getCodecByClassName(codecClassName);
+ if (codec == null) {
+ throw new IOException("Image compression codec not supported: "
+ + codecClassName);
+ }
+ in = new DataInputStream(codec.createInputStream(in));
+ }
+ }
+ processINodes(in, v, numInodes, skipBlocks, supportSnapshot);
+ subtreeMap.clear();
+ dirNodeMap.clear();
+
+ processINodesUC(in, v, skipBlocks);
+
+ if (NameNodeLayoutVersion.supports(Feature.DELEGATION_TOKEN, imageVersion)) {
+ processDelegationTokens(in, v);
+ }
+
+ if (NameNodeLayoutVersion.supports(Feature.CACHING, imageVersion)) {
+ processCacheManagerState(in, v);
+ }
+ v.leaveEnclosingElement(); // FSImage
+ done = true;
+ } finally {
+ if (done) {
+ v.finish();
+ } else {
+ v.finishAbnormally();
+ }
+ }
+ }
+
+ /**
+ * Process CacheManager state from the fsimage.
+ */
+ private void processCacheManagerState(DataInputStream in, ImageVisitor v)
+ throws IOException {
+ v.visit(ImageElement.CACHE_NEXT_ENTRY_ID, in.readLong());
+ final int numPools = in.readInt();
+ for (int i=0; i<numPools; i++) {
+ v.visit(ImageElement.CACHE_POOL_NAME, Text.readString(in));
+ processCachePoolPermission(in, v);
+ v.visit(ImageElement.CACHE_POOL_WEIGHT, in.readInt());
+ }
+ final int numEntries = in.readInt();
+ for (int i=0; i<numEntries; i++) {
+ v.visit(ImageElement.CACHE_ENTRY_PATH, Text.readString(in));
+ v.visit(ImageElement.CACHE_ENTRY_REPLICATION, in.readShort());
+ v.visit(ImageElement.CACHE_ENTRY_POOL_NAME, Text.readString(in));
+ }
+ }
+ /**
+ * Process the Delegation Token related section in fsimage.
+ *
+ * @param in DataInputStream to process
+ * @param v Visitor to walk over records
+ */
+ private void processDelegationTokens(DataInputStream in, ImageVisitor v)
+ throws IOException {
+ v.visit(ImageElement.CURRENT_DELEGATION_KEY_ID, in.readInt());
+ int numDKeys = in.readInt();
+ v.visitEnclosingElement(ImageElement.DELEGATION_KEYS,
+ ImageElement.NUM_DELEGATION_KEYS, numDKeys);
+ for(int i =0; i < numDKeys; i++) {
+ DelegationKey key = new DelegationKey();
+ key.readFields(in);
+ v.visit(ImageElement.DELEGATION_KEY, key.toString());
+ }
+ v.leaveEnclosingElement();
+ v.visit(ImageElement.DELEGATION_TOKEN_SEQUENCE_NUMBER, in.readInt());
+ int numDTokens = in.readInt();
+ v.visitEnclosingElement(ImageElement.DELEGATION_TOKENS,
+ ImageElement.NUM_DELEGATION_TOKENS, numDTokens);
+ for(int i=0; i<numDTokens; i++){
+ DelegationTokenIdentifier id = new DelegationTokenIdentifier();
+ id.readFields(in);
+ long expiryTime = in.readLong();
+ v.visitEnclosingElement(ImageElement.DELEGATION_TOKEN_IDENTIFIER);
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_KIND,
+ id.getKind().toString());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_SEQNO,
+ id.getSequenceNumber());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_OWNER,
+ id.getOwner().toString());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_RENEWER,
+ id.getRenewer().toString());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_REALUSER,
+ id.getRealUser().toString());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_ISSUE_DATE,
+ id.getIssueDate());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_MAX_DATE,
+ id.getMaxDate());
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_EXPIRY_TIME,
+ expiryTime);
+ v.visit(ImageElement.DELEGATION_TOKEN_IDENTIFIER_MASTER_KEY_ID,
+ id.getMasterKeyId());
+ v.leaveEnclosingElement(); // DELEGATION_TOKEN_IDENTIFIER
+ }
+ v.leaveEnclosingElement(); // DELEGATION_TOKENS
+ }
+
+ /**
+ * Process the INodes under construction section of the fsimage.
+ *
+ * @param in DataInputStream to process
+ * @param v Visitor to walk over inodes
+ * @param skipBlocks Walk over each block?
+ */
+ private void processINodesUC(DataInputStream in, ImageVisitor v,
+ boolean skipBlocks) throws IOException {
+ int numINUC = in.readInt();
+
+ v.visitEnclosingElement(ImageElement.INODES_UNDER_CONSTRUCTION,
+ ImageElement.NUM_INODES_UNDER_CONSTRUCTION, numINUC);
+
+ for(int i = 0; i < numINUC; i++) {
+ v.visitEnclosingElement(ImageElement.INODE_UNDER_CONSTRUCTION);
+ byte [] name = FSImageSerialization.readBytes(in);
+ String n = new String(name, "UTF8");
+ v.visit(ImageElement.INODE_PATH, n);
+
+ if (NameNodeLayoutVersion.supports(Feature.ADD_INODE_ID, imageVersion)) {
+ long inodeId = in.readLong();
+ v.visit(ImageElement.INODE_ID, inodeId);
+ }
+
+ v.visit(ImageElement.REPLICATION, in.readShort());
+ v.visit(ImageElement.MODIFICATION_TIME, formatDate(in.readLong()));
+
+ v.visit(ImageElement.PREFERRED_BLOCK_SIZE, in.readLong());
+ int numBlocks = in.readInt();
+ processBlocks(in, v, numBlocks, skipBlocks);
+
+ processPermission(in, v);
+ v.visit(ImageElement.CLIENT_NAME, FSImageSerialization.readString(in));
+ v.visit(ImageElement.CLIENT_MACHINE, FSImageSerialization.readString(in));
+
+ // Skip over the datanode descriptors, which are still stored in the
+ // file but are not used by the datanode or loaded into memory
+ int numLocs = in.readInt();
+ for(int j = 0; j < numLocs; j++) {
+ in.readShort();
+ in.readLong();
+ in.readLong();
+ in.readLong();
+ in.readInt();
+ FSImageSerialization.readString(in);
+ FSImageSerialization.readString(in);
+ WritableUtils.readEnum(in, AdminStates.class);
+ }
+
+ v.leaveEnclosingElement(); // INodeUnderConstruction
+ }
+
+ v.leaveEnclosingElement(); // INodesUnderConstruction
+ }
+
+ /**
+ * Process the blocks section of the fsimage.
+ *
+ * @param in Datastream to process
+ * @param v Visitor to walk over inodes
+ * @param skipBlocks Walk over each block?
+ */
+ private void processBlocks(DataInputStream in, ImageVisitor v,
+ int numBlocks, boolean skipBlocks) throws IOException {
+ v.visitEnclosingElement(ImageElement.BLOCKS,
+ ImageElement.NUM_BLOCKS, numBlocks);
+
+ // directory or symlink or reference node, no blocks to process
+ if(numBlocks < 0) {
+ v.leaveEnclosingElement(); // Blocks
+ return;
+ }
+
+ if(skipBlocks) {
+ int bytesToSkip = ((Long.SIZE * 3 /* fields */) / 8 /*bits*/) * numBlocks;
+ if(in.skipBytes(bytesToSkip) != bytesToSkip)
+ throw new IOException("Error skipping over blocks");
+
+ } else {
+ for(int j = 0; j < numBlocks; j++) {
+ v.visitEnclosingElement(ImageElement.BLOCK);
+ v.visit(ImageElement.BLOCK_ID, in.readLong());
+ v.visit(ImageElement.NUM_BYTES, in.readLong());
+ v.visit(ImageElement.GENERATION_STAMP, in.readLong());
+ v.leaveEnclosingElement(); // Block
+ }
+ }
+ v.leaveEnclosingElement(); // Blocks
+ }
+
+ /**
+ * Extract the INode permissions stored in the fsimage file.
+ *
+ * @param in Datastream to process
+ * @param v Visitor to walk over inodes
+ */
+ private void processPermission(DataInputStream in, ImageVisitor v)
+ throws IOException {
+ v.visitEnclosingElement(ImageElement.PERMISSIONS);
+ v.visit(ImageElement.USER_NAME, Text.readString(in));
+ v.visit(ImageElement.GROUP_NAME, Text.readString(in));
+ FsPermission fsp = new FsPermission(in.readShort());
+ v.visit(ImageElement.PERMISSION_STRING, fsp.toString());
+ v.leaveEnclosingElement(); // Permissions
+ }
+
+ /**
+ * Extract CachePool permissions stored in the fsimage file.
+ *
+ * @param in Datastream to process
+ * @param v Visitor to walk over inodes
+ */
+ private void processCachePoolPermission(DataInputStream in, ImageVisitor v)
+ throws IOException {
+ v.visitEnclosingElement(ImageElement.PERMISSIONS);
+ v.visit(ImageElement.CACHE_POOL_OWNER_NAME, Text.readString(in));
+ v.visit(ImageElement.CACHE_POOL_GROUP_NAME, Text.readString(in));
+ FsPermission fsp = new FsPermission(in.readShort());
+ v.visit(ImageElement.CACHE_POOL_PERMISSION_STRING, fsp.toString());
+ v.leaveEnclosingElement(); // Permissions
+ }
+
+ /**
+ * Process the INode records stored in the fsimage.
+ *
+ * @param in Datastream to process
+ * @param v Visitor to walk over INodes
+ * @param numInodes Number of INodes stored in file
+ * @param skipBlocks Process all the blocks within the INode?
+ * @param supportSnapshot Whether or not the imageVersion supports snapshot
+ * @throws VisitException
+ * @throws IOException
+ */
+ private void processINodes(DataInputStream in, ImageVisitor v,
+ long numInodes, boolean skipBlocks, boolean supportSnapshot)
+ throws IOException {
+ v.visitEnclosingElement(ImageElement.INODES,
+ ImageElement.NUM_INODES, numInodes);
+
+ if (NameNodeLayoutVersion.supports(Feature.FSIMAGE_NAME_OPTIMIZATION, imageVersion)) {
+ if (!supportSnapshot) {
+ processLocalNameINodes(in, v, numInodes, skipBlocks);
+ } else {
+ processLocalNameINodesWithSnapshot(in, v, skipBlocks);
+ }
+ } else { // full path name
+ processFullNameINodes(in, v, numInodes, skipBlocks);
+ }
+
+
+ v.leaveEnclosingElement(); // INodes
+ }
+
+ /**
+ * Process image with full path name
+ *
+ * @param in image stream
+ * @param v visitor
+ * @param numInodes number of indoes to read
+ * @param skipBlocks skip blocks or not
+ * @throws IOException if there is any error occurs
+ */
+ private void processLocalNameINodes(DataInputStream in, ImageVisitor v,
+ long numInodes, boolean skipBlocks) throws IOException {
+ // process root
+ processINode(in, v, skipBlocks, "", false);
+ numInodes--;
+ while (numInodes > 0) {
+ numInodes -= processDirectory(in, v, skipBlocks);
+ }
+ }
+
+ private int processDirectory(DataInputStream in, ImageVisitor v,
+ boolean skipBlocks) throws IOException {
+ String parentName = FSImageSerialization.readString(in);
+ return processChildren(in, v, skipBlocks, parentName);
+ }
+
+ /**
+ * Process image with local path name and snapshot support
+ *
+ * @param in image stream
+ * @param v visitor
+ * @param skipBlocks skip blocks or not
+ */
+ private void processLocalNameINodesWithSnapshot(DataInputStream in,
+ ImageVisitor v, boolean skipBlocks) throws IOException {
+ // process root
+ processINode(in, v, skipBlocks, "", false);
+ processDirectoryWithSnapshot(in, v, skipBlocks);
+ }
+
+ /**
+ * Process directories when snapshot is supported.
+ */
+ private void processDirectoryWithSnapshot(DataInputStream in, ImageVisitor v,
+ boolean skipBlocks) throws IOException {
+ // 1. load dir node id
+ long inodeId = in.readLong();
+
+ String dirName = dirNodeMap.remove(inodeId);
+ Boolean visitedRef = subtreeMap.get(inodeId);
+ if (visitedRef != null) {
+ if (visitedRef.booleanValue()) { // the subtree has been visited
+ return;
+ } else { // first time to visit
+ subtreeMap.put(inodeId, true);
+ }
+ } // else the dir is not linked by a RefNode, thus cannot be revisited
+
+ // 2. load possible snapshots
+ processSnapshots(in, v, dirName);
+ // 3. load children nodes
+ processChildren(in, v, skipBlocks, dirName);
+ // 4. load possible directory diff list
+ processDirectoryDiffList(in, v, dirName);
+ // recursively process sub-directories
+ final int numSubTree = in.readInt();
+ for (int i = 0; i < numSubTree; i++) {
+ processDirectoryWithSnapshot(in, v, skipBlocks);
+ }
+ }
+
+ /**
+ * Process snapshots of a snapshottable directory
+ */
+ private void processSnapshots(DataInputStream in, ImageVisitor v,
+ String rootName) throws IOException {
+ final int numSnapshots = in.readInt();
+ if (numSnapshots >= 0) {
+ v.visitEnclosingElement(ImageElement.SNAPSHOTS,
+ ImageElement.NUM_SNAPSHOTS, numSnapshots);
+ for (int i = 0; i < numSnapshots; i++) {
+ // process snapshot
+ v.visitEnclosingElement(ImageElement.SNAPSHOT);
+ v.visit(ImageElement.SNAPSHOT_ID, in.readInt());
+ v.leaveEnclosingElement();
+ }
+ v.visit(ImageElement.SNAPSHOT_QUOTA, in.readInt());
+ v.leaveEnclosingElement();
+ }
+ }
+
+ private void processSnapshot(DataInputStream in, ImageVisitor v)
+ throws IOException {
+ v.visitEnclosingElement(ImageElement.SNAPSHOT);
+ v.visit(ImageElement.SNAPSHOT_ID, in.readInt());
+ // process root of snapshot
+ v.visitEnclosingElement(ImageElement.SNAPSHOT_ROOT);
+ processINode(in, v, true, "", false);
+ v.leaveEnclosingElement();
+ v.leaveEnclosingElement();
+ }
+
+ private void processDirectoryDiffList(DataInputStream in, ImageVisitor v,
+ String currentINodeName) throws IOException {
+ final int numDirDiff = in.readInt();
+ if (numDirDiff >= 0) {
+ v.visitEnclosingElement(ImageElement.SNAPSHOT_DIR_DIFFS,
+ ImageElement.NUM_SNAPSHOT_DIR_DIFF, numDirDiff);
+ for (int i = 0; i < numDirDiff; i++) {
+ // process directory diffs in reverse chronological oder
+ processDirectoryDiff(in, v, currentINodeName);
+ }
+ v.leaveEnclosingElement();
+ }
+ }
+
+ private void processDirectoryDiff(DataInputStream in, ImageVisitor v,
+ String currentINodeName) throws IOException {
+ v.visitEnclosingElement(ImageElement.SNAPSHOT_DIR_DIFF);
+ int snapshotId = in.readInt();
+ v.visit(ImageElement.SNAPSHOT_DIFF_SNAPSHOTID, snapshotId);
+ v.visit(ImageElement.SNAPSHOT_DIR_DIFF_CHILDREN_SIZE, in.readInt());
+
+ // process snapshotINode
+ boolean useRoot = in.readBoolean();
+ if (!useRoot) {
+ if (in.readBoolean()) {
+ v.visitEnclosingElement(ImageElement.SNAPSHOT_INODE_DIRECTORY_ATTRIBUTES);
+ if (NameNodeLayoutVersion.supports(Feature.OPTIMIZE_SNAPSHOT_INODES, imageVersion)) {
+ processINodeDirectoryAttributes(in, v, currentINodeName);
+ } else {
+ processINode(in, v, true, currentINodeName, true);
+ }
+ v.leaveEnclosingElement();
+ }
+ }
+
+ // process createdList
+ int createdSize = in.readInt();
+ v.visitEnclosingElement(ImageElement.SNAPSHOT_DIR_DIFF_CREATEDLIST,
+ ImageElement.SNAPSHOT_DIR_DIFF_CREATEDLIST_SIZE, createdSize);
+ for (int i = 0; i < createdSize; i++) {
+ String createdNode = FSImageSerialization.readString(in);
+ v.visit(ImageElement.SNAPSHOT_DIR_DIFF_CREATED_INODE, createdNode);
+ }
+ v.leaveEnclosingElement();
+
+ // process deletedList
+ int deletedSize = in.readInt();
+ v.visitEnclosingElement(ImageElement.SNAPSHOT_DIR_DIFF_DELETEDLIST,
+ ImageElement.SNAPSHOT_DIR_DIFF_DELETEDLIST_SIZE, deletedSize);
+ for (int i = 0; i < deletedSize; i++) {
+ v.visitEnclosingElement(ImageElement.SNAPSHOT_DIR_DIFF_DELETED_INODE);
+ processINode(in, v, false, currentINodeName, true);
+ v.leaveEnclosingElement();
+ }
+ v.leaveEnclosingElement();
+ v.leaveEnclosingElement();
+ }
+
+ private void processINodeDirectoryAttributes(DataInputStream in, ImageVisitor v,
+ String parentName) throws IOException {
+ final String pathName = readINodePath(in, parentName);
+ v.visit(ImageElement.INODE_PATH, pathName);
+ processPermission(in, v);
+ v.visit(ImageElement.MODIFICATION_TIME, formatDate(in.readLong()));
+
+ v.visit(ImageElement.NS_QUOTA, in.readLong());
+ v.visit(ImageElement.DS_QUOTA, in.readLong());
+ }
+
+ /** Process children under a directory */
+ private int processChildren(DataInputStream in, ImageVisitor v,
+ boolean skipBlocks, String parentName) throws IOException {
+ int numChildren = in.readInt();
+ for (int i = 0; i < numChildren; i++) {
+ processINode(in, v, skipBlocks, parentName, false);
+ }
+ return numChildren;
+ }
+
+ /**
+ * Process image with full path name
+ *
+ * @param in image stream
+ * @param v visitor
+ * @param numInodes number of indoes to read
+ * @param skipBlocks skip blocks or not
+ * @throws IOException if there is any error occurs
+ */
+ private void processFullNameINodes(DataInputStream in, ImageVisitor v,
+ long numInodes, boolean skipBlocks) throws IOException {
+ for(long i = 0; i < numInodes; i++) {
+ processINode(in, v, skipBlocks, null, false);
+ }
+ }
+
+ private String readINodePath(DataInputStream in, String parentName)
+ throws IOException {
+ String pathName = FSImageSerialization.readString(in);
+ if (parentName != null) { // local name
+ pathName = "/" + pathName;
+ if (!"/".equals(parentName)) { // children of non-root directory
+ pathName = parentName + pathName;
+ }
+ }
+ return pathName;
+ }
+
+ /**
+ * Process an INode
+ *
+ * @param in image stream
+ * @param v visitor
+ * @param skipBlocks skip blocks or not
+ * @param parentName the name of its parent node
+ * @param isSnapshotCopy whether or not the inode is a snapshot copy
+ * @throws IOException
+ */
+ private void processINode(DataInputStream in, ImageVisitor v,
+ boolean skipBlocks, String parentName, boolean isSnapshotCopy)
+ throws IOException {
+ boolean supportSnapshot =
+ NameNodeLayoutVersion.supports(Feature.SNAPSHOT, imageVersion);
+ boolean supportInodeId =
+ NameNodeLayoutVersion.supports(Feature.ADD_INODE_ID, imageVersion);
+
+ v.visitEnclosingElement(ImageElement.INODE);
+ final String pathName = readINodePath(in, parentName);
+ v.visit(ImageElement.INODE_PATH, pathName);
+
+ long inodeId = INodeId.GRANDFATHER_INODE_ID;
+ if (supportInodeId) {
+ inodeId = in.readLong();
+ v.visit(ImageElement.INODE_ID, inodeId);
+ }
+ v.visit(ImageElement.REPLICATION, in.readShort());
+ v.visit(ImageElement.MODIFICATION_TIME, formatDate(in.readLong()));
+ if(NameNodeLayoutVersion.supports(Feature.FILE_ACCESS_TIME, imageVersion))
+ v.visit(ImageElement.ACCESS_TIME, formatDate(in.readLong()));
+ v.visit(ImageElement.BLOCK_SIZE, in.readLong());
+ int numBlocks = in.readInt();
+
+ processBlocks(in, v, numBlocks, skipBlocks);
+
+ if (numBlocks >= 0) { // File
+ if (supportSnapshot) {
+ // make sure subtreeMap only contains entry for directory
+ subtreeMap.remove(inodeId);
+ // process file diffs
+ processFileDiffList(in, v, parentName);
+ if (isSnapshotCopy) {
+ boolean underConstruction = in.readBoolean();
+ if (underConstruction) {
+ v.visit(ImageElement.CLIENT_NAME,
+ FSImageSerialization.readString(in));
+ v.visit(ImageElement.CLIENT_MACHINE,
+ FSImageSerialization.readString(in));
+ }
+ }
+ }
+ processPermission(in, v);
+ } else if (numBlocks == -1) { // Directory
+ if (supportSnapshot && supportInodeId) {
+ dirNodeMap.put(inodeId, pathName);
+ }
+ v.visit(ImageElement.NS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
+ if (NameNodeLayoutVersion.supports(Feature.DISKSPACE_QUOTA, imageVersion))
+ v.visit(ImageElement.DS_QUOTA, numBlocks == -1 ? in.readLong() : -1);
+ if (supportSnapshot) {
+ boolean snapshottable = in.readBoolean();
+ if (!snapshottable) {
+ boolean withSnapshot = in.readBoolean();
+ v.visit(ImageElement.IS_WITHSNAPSHOT_DIR, Boolean.toString(withSnapshot));
+ } else {
+ v.visit(ImageElement.IS_SNAPSHOTTABLE_DIR, Boolean.toString(snapshottable));
+ }
+ }
+ processPermission(in, v);
+ } else if (numBlocks == -2) {
+ v.visit(ImageElement.SYMLINK, Text.readString(in));
+ processPermission(in, v);
+ } else if (numBlocks == -3) { // reference node
+ final boolean isWithName = in.readBoolean();
+ int snapshotId = in.readInt();
+ if (isWithName) {
+ v.visit(ImageElement.SNAPSHOT_LAST_SNAPSHOT_ID, snapshotId);
+ } else {
+ v.visit(ImageElement.SNAPSHOT_DST_SNAPSHOT_ID, snapshotId);
+ }
+
+ final boolean firstReferred = in.readBoolean();
+ if (firstReferred) {
+ // if a subtree is linked by multiple "parents", the corresponding dir
+ // must be referred by a reference node. we put the reference node into
+ // the subtreeMap here and let its value be false. when we later visit
+ // the subtree for the first time, we change the value to true.
+ subtreeMap.put(inodeId, false);
+ v.visitEnclosingElement(ImageElement.SNAPSHOT_REF_INODE);
+ processINode(in, v, skipBlocks, parentName, isSnapshotCopy);
+ v.leaveEnclosingElement(); // referred inode
+ } else {
+ v.visit(ImageElement.SNAPSHOT_REF_INODE_ID, in.readLong());
+ }
+ }
+
+ v.leaveEnclosingElement(); // INode
+ }
+
+ private void processINodeFileAttributes(DataInputStream in, ImageVisitor v,
+ String parentName) throws IOException {
+ final String pathName = readINodePath(in, parentName);
+ v.visit(ImageElement.INODE_PATH, pathName);
+ processPermission(in, v);
+ v.visit(ImageElement.MODIFICATION_TIME, formatDate(in.readLong()));
+ if(NameNodeLayoutVersion.supports(Feature.FILE_ACCESS_TIME, imageVersion)) {
+ v.visit(ImageElement.ACCESS_TIME, formatDate(in.readLong()));
+ }
+
+ v.visit(ImageElement.REPLICATION, in.readShort());
+ v.visit(ImageElement.BLOCK_SIZE, in.readLong());
+ }
+
+ private void processFileDiffList(DataInputStream in, ImageVisitor v,
+ String currentINodeName) throws IOException {
+ final int size = in.readInt();
+ if (size >= 0) {
+ v.visitEnclosingElement(ImageElement.SNAPSHOT_FILE_DIFFS,
+ ImageElement.NUM_SNAPSHOT_FILE_DIFF, size);
+ for (int i = 0; i < size; i++) {
+ processFileDiff(in, v, currentINodeName);
+ }
+ v.leaveEnclosingElement();
+ }
+ }
+
+ private void processFileDiff(DataInputStream in, ImageVisitor v,
+ String currentINodeName) throws IOException {
+ int snapshotId = in.readInt();
+ v.visitEnclosingElement(ImageElement.SNAPSHOT_FILE_DIFF,
+ ImageElement.SNAPSHOT_DIFF_SNAPSHOTID, snapshotId);
+ v.visit(ImageElement.SNAPSHOT_FILE_SIZE, in.readLong());
+ if (in.readBoolean()) {
+ v.visitEnclosingElement(ImageElement.SNAPSHOT_INODE_FILE_ATTRIBUTES);
+ if (NameNodeLayoutVersion.supports(Feature.OPTIMIZE_SNAPSHOT_INODES, imageVersion)) {
+ processINodeFileAttributes(in, v, currentINodeName);
+ } else {
+ processINode(in, v, true, currentINodeName, true);
+ }
+ v.leaveEnclosingElement();
+ }
+ v.leaveEnclosingElement();
+ }
+
+ /**
+ * Helper method to format dates during processing.
+ * @param date Date as read from image file
+ * @return String version of date format
+ */
+ private String formatDate(long date) {
+ return dateFormat.format(new Date(date));
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java?rev=1594440&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java (added)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java Wed May 14 01:15:48 2014
@@ -0,0 +1,212 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.IOException;
+
+/**
+ * An implementation of ImageVisitor can traverse the structure of an
+ * Hadoop fsimage and respond to each of the structures within the file.
+ */
+abstract class ImageVisitor {
+
+ /**
+ * Structural elements of an FSImage that may be encountered within the
+ * file. ImageVisitors are able to handle processing any of these elements.
+ */
+ public enum ImageElement {
+ FS_IMAGE,
+ IMAGE_VERSION,
+ NAMESPACE_ID,
+ IS_COMPRESSED,
+ COMPRESS_CODEC,
+ LAYOUT_VERSION,
+ NUM_INODES,
+ GENERATION_STAMP,
+ GENERATION_STAMP_V2,
+ GENERATION_STAMP_V1_LIMIT,
+ LAST_ALLOCATED_BLOCK_ID,
+ INODES,
+ INODE,
+ INODE_PATH,
+ REPLICATION,
+ MODIFICATION_TIME,
+ ACCESS_TIME,
+ BLOCK_SIZE,
+ NUM_BLOCKS,
+ BLOCKS,
+ BLOCK,
+ BLOCK_ID,
+ NUM_BYTES,
+ NS_QUOTA,
+ DS_QUOTA,
+ PERMISSIONS,
+ SYMLINK,
+ NUM_INODES_UNDER_CONSTRUCTION,
+ INODES_UNDER_CONSTRUCTION,
+ INODE_UNDER_CONSTRUCTION,
+ PREFERRED_BLOCK_SIZE,
+ CLIENT_NAME,
+ CLIENT_MACHINE,
+ USER_NAME,
+ GROUP_NAME,
+ PERMISSION_STRING,
+ CURRENT_DELEGATION_KEY_ID,
+ NUM_DELEGATION_KEYS,
+ DELEGATION_KEYS,
+ DELEGATION_KEY,
+ DELEGATION_TOKEN_SEQUENCE_NUMBER,
+ NUM_DELEGATION_TOKENS,
+ DELEGATION_TOKENS,
+ DELEGATION_TOKEN_IDENTIFIER,
+ DELEGATION_TOKEN_IDENTIFIER_KIND,
+ DELEGATION_TOKEN_IDENTIFIER_SEQNO,
+ DELEGATION_TOKEN_IDENTIFIER_OWNER,
+ DELEGATION_TOKEN_IDENTIFIER_RENEWER,
+ DELEGATION_TOKEN_IDENTIFIER_REALUSER,
+ DELEGATION_TOKEN_IDENTIFIER_ISSUE_DATE,
+ DELEGATION_TOKEN_IDENTIFIER_MAX_DATE,
+ DELEGATION_TOKEN_IDENTIFIER_EXPIRY_TIME,
+ DELEGATION_TOKEN_IDENTIFIER_MASTER_KEY_ID,
+ TRANSACTION_ID,
+ LAST_INODE_ID,
+ INODE_ID,
+
+ SNAPSHOT_COUNTER,
+ NUM_SNAPSHOTS_TOTAL,
+ NUM_SNAPSHOTS,
+ SNAPSHOTS,
+ SNAPSHOT,
+ SNAPSHOT_ID,
+ SNAPSHOT_ROOT,
+ SNAPSHOT_QUOTA,
+ NUM_SNAPSHOT_DIR_DIFF,
+ SNAPSHOT_DIR_DIFFS,
+ SNAPSHOT_DIR_DIFF,
+ SNAPSHOT_DIFF_SNAPSHOTID,
+ SNAPSHOT_DIR_DIFF_CHILDREN_SIZE,
+ SNAPSHOT_INODE_FILE_ATTRIBUTES,
+ SNAPSHOT_INODE_DIRECTORY_ATTRIBUTES,
+ SNAPSHOT_DIR_DIFF_CREATEDLIST,
+ SNAPSHOT_DIR_DIFF_CREATEDLIST_SIZE,
+ SNAPSHOT_DIR_DIFF_CREATED_INODE,
+ SNAPSHOT_DIR_DIFF_DELETEDLIST,
+ SNAPSHOT_DIR_DIFF_DELETEDLIST_SIZE,
+ SNAPSHOT_DIR_DIFF_DELETED_INODE,
+ IS_SNAPSHOTTABLE_DIR,
+ IS_WITHSNAPSHOT_DIR,
+ SNAPSHOT_FILE_DIFFS,
+ SNAPSHOT_FILE_DIFF,
+ NUM_SNAPSHOT_FILE_DIFF,
+ SNAPSHOT_FILE_SIZE,
+ SNAPSHOT_DST_SNAPSHOT_ID,
+ SNAPSHOT_LAST_SNAPSHOT_ID,
+ SNAPSHOT_REF_INODE_ID,
+ SNAPSHOT_REF_INODE,
+
+ CACHE_NEXT_ENTRY_ID,
+ CACHE_NUM_POOLS,
+ CACHE_POOL_NAME,
+ CACHE_POOL_OWNER_NAME,
+ CACHE_POOL_GROUP_NAME,
+ CACHE_POOL_PERMISSION_STRING,
+ CACHE_POOL_WEIGHT,
+ CACHE_NUM_ENTRIES,
+ CACHE_ENTRY_PATH,
+ CACHE_ENTRY_REPLICATION,
+ CACHE_ENTRY_POOL_NAME
+ }
+
+ /**
+ * Begin visiting the fsimage structure. Opportunity to perform
+ * any initialization necessary for the implementing visitor.
+ */
+ abstract void start() throws IOException;
+
+ /**
+ * Finish visiting the fsimage structure. Opportunity to perform any
+ * clean up necessary for the implementing visitor.
+ */
+ abstract void finish() throws IOException;
+
+ /**
+ * Finish visiting the fsimage structure after an error has occurred
+ * during the processing. Opportunity to perform any clean up necessary
+ * for the implementing visitor.
+ */
+ abstract void finishAbnormally() throws IOException;
+
+ /**
+ * Visit non enclosing element of fsimage with specified value.
+ *
+ * @param element FSImage element
+ * @param value Element's value
+ */
+ abstract void visit(ImageElement element, String value) throws IOException;
+
+ // Convenience methods to automatically convert numeric value types to strings
+ void visit(ImageElement element, int value) throws IOException {
+ visit(element, Integer.toString(value));
+ }
+
+ void visit(ImageElement element, long value) throws IOException {
+ visit(element, Long.toString(value));
+ }
+
+ /**
+ * Begin visiting an element that encloses another element, such as
+ * the beginning of the list of blocks that comprise a file.
+ *
+ * @param element Element being visited
+ */
+ abstract void visitEnclosingElement(ImageElement element)
+ throws IOException;
+
+ /**
+ * Begin visiting an element that encloses another element, such as
+ * the beginning of the list of blocks that comprise a file.
+ *
+ * Also provide an additional key and value for the element, such as the
+ * number items within the element.
+ *
+ * @param element Element being visited
+ * @param key Key describing the element being visited
+ * @param value Value associated with element being visited
+ */
+ abstract void visitEnclosingElement(ImageElement element,
+ ImageElement key, String value) throws IOException;
+
+ // Convenience methods to automatically convert value types to strings
+ void visitEnclosingElement(ImageElement element,
+ ImageElement key, int value)
+ throws IOException {
+ visitEnclosingElement(element, key, Integer.toString(value));
+ }
+
+ void visitEnclosingElement(ImageElement element,
+ ImageElement key, long value)
+ throws IOException {
+ visitEnclosingElement(element, key, Long.toString(value));
+ }
+
+ /**
+ * Leave current enclosing element. Called, for instance, at the end of
+ * processing the blocks that compromise a file.
+ */
+ abstract void leaveEnclosingElement() throws IOException;
+}
Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java?rev=1594440&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java (added)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java Wed May 14 01:15:48 2014
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.IOException;
+import java.util.Date;
+
+/**
+ * IndentedImageVisitor walks over an FSImage and displays its structure
+ * using indenting to organize sections within the image file.
+ */
+class IndentedImageVisitor extends TextWriterImageVisitor {
+
+ public IndentedImageVisitor(String filename) throws IOException {
+ super(filename);
+ }
+
+ public IndentedImageVisitor(String filename, boolean printToScreen) throws IOException {
+ super(filename, printToScreen);
+ }
+
+ final private DepthCounter dc = new DepthCounter();// to track leading spacing
+
+ @Override
+ void start() throws IOException {}
+
+ @Override
+ void finish() throws IOException { super.finish(); }
+
+ @Override
+ void finishAbnormally() throws IOException {
+ System.out.println("*** Image processing finished abnormally. Ending ***");
+ super.finishAbnormally();
+ }
+
+ @Override
+ void leaveEnclosingElement() throws IOException {
+ dc.decLevel();
+ }
+
+ @Override
+ void visit(ImageElement element, String value) throws IOException {
+ printIndents();
+ write(element + " = " + value + "\n");
+ }
+
+ @Override
+ void visit(ImageElement element, long value) throws IOException {
+ if ((element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_EXPIRY_TIME) ||
+ (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_ISSUE_DATE) ||
+ (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_MAX_DATE)) {
+ visit(element, new Date(value).toString());
+ } else {
+ visit(element, Long.toString(value));
+ }
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element) throws IOException {
+ printIndents();
+ write(element + "\n");
+ dc.incLevel();
+ }
+
+ // Print element, along with associated key/value pair, in brackets
+ @Override
+ void visitEnclosingElement(ImageElement element,
+ ImageElement key, String value)
+ throws IOException {
+ printIndents();
+ write(element + " [" + key + " = " + value + "]\n");
+ dc.incLevel();
+ }
+
+ /**
+ * Print an appropriate number of spaces for the current level.
+ * FsImages can potentially be millions of lines long, so caching can
+ * significantly speed up output.
+ */
+ final private static String [] indents = { "",
+ " ",
+ " ",
+ " ",
+ " ",
+ " ",
+ " "};
+ private void printIndents() throws IOException {
+ try {
+ write(indents[dc.getLevel()]);
+ } catch (IndexOutOfBoundsException e) {
+ // There's no reason in an fsimage would need a deeper indent
+ for(int i = 0; i < dc.getLevel(); i++)
+ write(" ");
+ }
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java?rev=1594440&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java (added)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java Wed May 14 01:15:48 2014
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.IOException;
+import java.util.Formatter;
+import java.util.LinkedList;
+
+/**
+ * LsImageVisitor displays the blocks of the namespace in a format very similar
+ * to the output of ls/lsr. Entries are marked as directories or not,
+ * permissions listed, replication, username and groupname, along with size,
+ * modification date and full path.
+ *
+ * Note: A significant difference between the output of the lsr command
+ * and this image visitor is that this class cannot sort the file entries;
+ * they are listed in the order they are stored within the fsimage file.
+ * Therefore, the output of this class cannot be directly compared to the
+ * output of the lsr command.
+ */
+class LsImageVisitor extends TextWriterImageVisitor {
+ final private LinkedList<ImageElement> elemQ = new LinkedList<ImageElement>();
+
+ private int numBlocks;
+ private String perms;
+ private int replication;
+ private String username;
+ private String group;
+ private long filesize;
+ private String modTime;
+ private String path;
+ private String linkTarget;
+
+ private boolean inInode = false;
+ final private StringBuilder sb = new StringBuilder();
+ final private Formatter formatter = new Formatter(sb);
+
+ public LsImageVisitor(String filename) throws IOException {
+ super(filename);
+ }
+
+ public LsImageVisitor(String filename, boolean printToScreen) throws IOException {
+ super(filename, printToScreen);
+ }
+
+ /**
+ * Start a new line of output, reset values.
+ */
+ private void newLine() {
+ numBlocks = 0;
+ perms = username = group = path = linkTarget = "";
+ filesize = 0l;
+ replication = 0;
+
+ inInode = true;
+ }
+
+ /**
+ * All the values have been gathered. Print them to the console in an
+ * ls-style format.
+ */
+ private final static int widthRepl = 2;
+ private final static int widthUser = 8;
+ private final static int widthGroup = 10;
+ private final static int widthSize = 10;
+ private final static int widthMod = 10;
+ private final static String lsStr = " %" + widthRepl + "s %" + widthUser +
+ "s %" + widthGroup + "s %" + widthSize +
+ "d %" + widthMod + "s %s";
+ private void printLine() throws IOException {
+ sb.append(numBlocks < 0 ? "d" : "-");
+ sb.append(perms);
+
+ if (0 != linkTarget.length()) {
+ path = path + " -> " + linkTarget;
+ }
+ formatter.format(lsStr, replication > 0 ? replication : "-",
+ username, group, filesize, modTime, path);
+ sb.append("\n");
+
+ write(sb.toString());
+ sb.setLength(0); // clear string builder
+
+ inInode = false;
+ }
+
+ @Override
+ void start() throws IOException {}
+
+ @Override
+ void finish() throws IOException {
+ super.finish();
+ }
+
+ @Override
+ void finishAbnormally() throws IOException {
+ System.out.println("Input ended unexpectedly.");
+ super.finishAbnormally();
+ }
+
+ @Override
+ void leaveEnclosingElement() throws IOException {
+ ImageElement elem = elemQ.pop();
+
+ if(elem == ImageElement.INODE)
+ printLine();
+ }
+
+ // Maintain state of location within the image tree and record
+ // values needed to display the inode in ls-style format.
+ @Override
+ void visit(ImageElement element, String value) throws IOException {
+ if(inInode) {
+ switch(element) {
+ case INODE_PATH:
+ if(value.equals("")) path = "/";
+ else path = value;
+ break;
+ case PERMISSION_STRING:
+ perms = value;
+ break;
+ case REPLICATION:
+ replication = Integer.valueOf(value);
+ break;
+ case USER_NAME:
+ username = value;
+ break;
+ case GROUP_NAME:
+ group = value;
+ break;
+ case NUM_BYTES:
+ filesize += Long.valueOf(value);
+ break;
+ case MODIFICATION_TIME:
+ modTime = value;
+ break;
+ case SYMLINK:
+ linkTarget = value;
+ break;
+ default:
+ // This is OK. We're not looking for all the values.
+ break;
+ }
+ }
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element) throws IOException {
+ elemQ.push(element);
+ if(element == ImageElement.INODE)
+ newLine();
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element,
+ ImageElement key, String value) throws IOException {
+ elemQ.push(element);
+ if(element == ImageElement.INODE)
+ newLine();
+ else if (element == ImageElement.BLOCKS)
+ numBlocks = Integer.valueOf(value);
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java?rev=1594440&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java (added)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java Wed May 14 01:15:48 2014
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * File name distribution visitor.
+ * <p>
+ * It analyzes file names in fsimage and prints the following information:
+ * <li>Number of unique file names</li>
+ * <li>Number file names and the corresponding number range of files that use
+ * these same names</li>
+ * <li>Heap saved if the file name objects are reused</li>
+ */
+@InterfaceAudience.Private
+public class NameDistributionVisitor extends TextWriterImageVisitor {
+ HashMap<String, Integer> counts = new HashMap<String, Integer>();
+
+ public NameDistributionVisitor(String filename, boolean printToScreen)
+ throws IOException {
+ super(filename, printToScreen);
+ }
+
+ @Override
+ void finish() throws IOException {
+ final int BYTEARRAY_OVERHEAD = 24;
+
+ write("Total unique file names " + counts.size());
+ // Columns: Frequency of file occurrence, savings in heap, total files using
+ // the name and number of file names
+ final long stats[][] = { { 100000, 0, 0, 0 },
+ { 10000, 0, 0, 0 },
+ { 1000, 0, 0, 0 },
+ { 100, 0, 0, 0 },
+ { 10, 0, 0, 0 },
+ { 5, 0, 0, 0 },
+ { 4, 0, 0, 0 },
+ { 3, 0, 0, 0 },
+ { 2, 0, 0, 0 }};
+
+ int highbound = Integer.MIN_VALUE;
+ for (Entry<String, Integer> entry : counts.entrySet()) {
+ highbound = Math.max(highbound, entry.getValue());
+ for (int i = 0; i < stats.length; i++) {
+ if (entry.getValue() >= stats[i][0]) {
+ stats[i][1] += (BYTEARRAY_OVERHEAD + entry.getKey().length())
+ * (entry.getValue() - 1);
+ stats[i][2] += entry.getValue();
+ stats[i][3]++;
+ break;
+ }
+ }
+ }
+
+ long lowbound = 0;
+ long totalsavings = 0;
+ for (long[] stat : stats) {
+ lowbound = stat[0];
+ totalsavings += stat[1];
+ String range = lowbound == highbound ? " " + lowbound :
+ " between " + lowbound + "-" + highbound;
+ write("\n" + stat[3] + " names are used by " + stat[2] + " files"
+ + range + " times. Heap savings ~" + stat[1] + " bytes.");
+ highbound = (int) stat[0] - 1;
+ }
+ write("\n\nTotal saved heap ~" + totalsavings + "bytes.\n");
+ super.finish();
+ }
+
+ @Override
+ void visit(ImageElement element, String value) throws IOException {
+ if (element == ImageElement.INODE_PATH) {
+ String filename = value.substring(value.lastIndexOf("/") + 1);
+ if (counts.containsKey(filename)) {
+ counts.put(filename, counts.get(filename) + 1);
+ } else {
+ counts.put(filename, 1);
+ }
+ }
+ }
+
+ @Override
+ void leaveEnclosingElement() throws IOException {
+ }
+
+ @Override
+ void start() throws IOException {
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element) throws IOException {
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element, ImageElement key,
+ String value) throws IOException {
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java?rev=1594440&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java (added)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java Wed May 14 01:15:48 2014
@@ -0,0 +1,274 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.IOException;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.IOUtils;
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogLoader.PositionTrackingInputStream;
+
+/**
+ * OfflineImageViewer to dump the contents of an Hadoop image file to XML
+ * or the console. Main entry point into utility, either via the
+ * command line or programatically.
+ */
+@InterfaceAudience.Private
+public class OfflineImageViewer {
+ public static final Log LOG = LogFactory.getLog(OfflineImageViewer.class);
+
+ private final static String usage =
+ "Usage: bin/hdfs oiv [OPTIONS] -i INPUTFILE -o OUTPUTFILE\n" +
+ "Offline Image Viewer\n" +
+ "View a Hadoop fsimage INPUTFILE using the specified PROCESSOR,\n" +
+ "saving the results in OUTPUTFILE.\n" +
+ "\n" +
+ "The oiv utility will attempt to parse correctly formed image files\n" +
+ "and will abort fail with mal-formed image files.\n" +
+ "\n" +
+ "The tool works offline and does not require a running cluster in\n" +
+ "order to process an image file.\n" +
+ "\n" +
+ "The following image processors are available:\n" +
+ " * Ls: The default image processor generates an lsr-style listing\n" +
+ " of the files in the namespace, with the same fields in the same\n" +
+ " order. Note that in order to correctly determine file sizes,\n" +
+ " this formatter cannot skip blocks and will override the\n" +
+ " -skipBlocks option.\n" +
+ " * Indented: This processor enumerates over all of the elements in\n" +
+ " the fsimage file, using levels of indentation to delineate\n" +
+ " sections within the file.\n" +
+ " * Delimited: Generate a text file with all of the elements common\n" +
+ " to both inodes and inodes-under-construction, separated by a\n" +
+ " delimiter. The default delimiter is \u0001, though this may be\n" +
+ " changed via the -delimiter argument. This processor also overrides\n" +
+ " the -skipBlocks option for the same reason as the Ls processor\n" +
+ " * XML: This processor creates an XML document with all elements of\n" +
+ " the fsimage enumerated, suitable for further analysis by XML\n" +
+ " tools.\n" +
+ " * FileDistribution: This processor analyzes the file size\n" +
+ " distribution in the image.\n" +
+ " -maxSize specifies the range [0, maxSize] of file sizes to be\n" +
+ " analyzed (128GB by default).\n" +
+ " -step defines the granularity of the distribution. (2MB by default)\n" +
+ " * NameDistribution: This processor analyzes the file names\n" +
+ " in the image and prints total number of file names and how frequently\n" +
+ " file names are reused.\n" +
+ "\n" +
+ "Required command line arguments:\n" +
+ "-i,--inputFile <arg> FSImage file to process.\n" +
+ "-o,--outputFile <arg> Name of output file. If the specified\n" +
+ " file exists, it will be overwritten.\n" +
+ "\n" +
+ "Optional command line arguments:\n" +
+ "-p,--processor <arg> Select which type of processor to apply\n" +
+ " against image file." +
+ " (Ls|XML|Delimited|Indented|FileDistribution).\n" +
+ "-h,--help Display usage information and exit\n" +
+ "-printToScreen For processors that write to a file, also\n" +
+ " output to screen. On large image files this\n" +
+ " will dramatically increase processing time.\n" +
+ "-skipBlocks Skip inodes' blocks information. May\n" +
+ " significantly decrease output.\n" +
+ " (default = false).\n" +
+ "-delimiter <arg> Delimiting string to use with Delimited processor\n";
+
+ private final boolean skipBlocks;
+ private final String inputFile;
+ private final ImageVisitor processor;
+
+ public OfflineImageViewer(String inputFile, ImageVisitor processor,
+ boolean skipBlocks) {
+ this.inputFile = inputFile;
+ this.processor = processor;
+ this.skipBlocks = skipBlocks;
+ }
+
+ /**
+ * Process image file.
+ */
+ public void go() throws IOException {
+ DataInputStream in = null;
+ PositionTrackingInputStream tracker = null;
+ ImageLoader fsip = null;
+ boolean done = false;
+ try {
+ tracker = new PositionTrackingInputStream(new BufferedInputStream(
+ new FileInputStream(new File(inputFile))));
+ in = new DataInputStream(tracker);
+
+ int imageVersionFile = findImageVersion(in);
+
+ fsip = ImageLoader.LoaderFactory.getLoader(imageVersionFile);
+
+ if(fsip == null)
+ throw new IOException("No image processor to read version " +
+ imageVersionFile + " is available.");
+ fsip.loadImage(in, processor, skipBlocks);
+ done = true;
+ } finally {
+ if (!done) {
+ LOG.error("image loading failed at offset " + tracker.getPos());
+ }
+ IOUtils.cleanup(LOG, in, tracker);
+ }
+ }
+
+ /**
+ * Check an fsimage datainputstream's version number.
+ *
+ * The datainput stream is returned at the same point as it was passed in;
+ * this method has no effect on the datainputstream's read pointer.
+ *
+ * @param in Datainputstream of fsimage
+ * @return Filesystem layout version of fsimage represented by stream
+ * @throws IOException If problem reading from in
+ */
+ private int findImageVersion(DataInputStream in) throws IOException {
+ in.mark(42); // arbitrary amount, resetting immediately
+
+ int version = in.readInt();
+ in.reset();
+
+ return version;
+ }
+
+ /**
+ * Build command-line options and descriptions
+ */
+ public static Options buildOptions() {
+ Options options = new Options();
+
+ // Build in/output file arguments, which are required, but there is no
+ // addOption method that can specify this
+ OptionBuilder.isRequired();
+ OptionBuilder.hasArgs();
+ OptionBuilder.withLongOpt("outputFile");
+ options.addOption(OptionBuilder.create("o"));
+
+ OptionBuilder.isRequired();
+ OptionBuilder.hasArgs();
+ OptionBuilder.withLongOpt("inputFile");
+ options.addOption(OptionBuilder.create("i"));
+
+ options.addOption("p", "processor", true, "");
+ options.addOption("h", "help", false, "");
+ options.addOption("skipBlocks", false, "");
+ options.addOption("printToScreen", false, "");
+ options.addOption("delimiter", true, "");
+
+ return options;
+ }
+
+ /**
+ * Entry point to command-line-driven operation. User may specify
+ * options and start fsimage viewer from the command line. Program
+ * will process image file and exit cleanly or, if an error is
+ * encountered, inform user and exit.
+ *
+ * @param args Command line options
+ * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+ Options options = buildOptions();
+ if(args.length == 0) {
+ printUsage();
+ return;
+ }
+
+ CommandLineParser parser = new PosixParser();
+ CommandLine cmd;
+
+ try {
+ cmd = parser.parse(options, args);
+ } catch (ParseException e) {
+ System.out.println("Error parsing command-line options: ");
+ printUsage();
+ return;
+ }
+
+ if(cmd.hasOption("h")) { // print help and exit
+ printUsage();
+ return;
+ }
+
+ boolean skipBlocks = cmd.hasOption("skipBlocks");
+ boolean printToScreen = cmd.hasOption("printToScreen");
+ String inputFile = cmd.getOptionValue("i");
+ String processor = cmd.getOptionValue("p", "Ls");
+ String outputFile = cmd.getOptionValue("o");
+ String delimiter = cmd.getOptionValue("delimiter");
+
+ if( !(delimiter == null || processor.equals("Delimited")) ) {
+ System.out.println("Can only specify -delimiter with Delimited processor");
+ printUsage();
+ return;
+ }
+
+ ImageVisitor v;
+ if(processor.equals("Indented")) {
+ v = new IndentedImageVisitor(outputFile, printToScreen);
+ } else if (processor.equals("XML")) {
+ v = new XmlImageVisitor(outputFile, printToScreen);
+ } else if (processor.equals("Delimited")) {
+ v = delimiter == null ?
+ new DelimitedImageVisitor(outputFile, printToScreen) :
+ new DelimitedImageVisitor(outputFile, printToScreen, delimiter);
+ skipBlocks = false;
+ } else if (processor.equals("FileDistribution")) {
+ long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
+ int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
+ v = new FileDistributionVisitor(outputFile, maxSize, step);
+ } else if (processor.equals("NameDistribution")) {
+ v = new NameDistributionVisitor(outputFile, printToScreen);
+ } else {
+ v = new LsImageVisitor(outputFile, printToScreen);
+ skipBlocks = false;
+ }
+
+ try {
+ OfflineImageViewer d = new OfflineImageViewer(inputFile, v, skipBlocks);
+ d.go();
+ } catch (EOFException e) {
+ System.err.println("Input file ended unexpectedly. Exiting");
+ } catch(IOException e) {
+ System.err.println("Encountered exception. Exiting: " + e.getMessage());
+ }
+ }
+
+ /**
+ * Print application usage instructions.
+ */
+ private static void printUsage() {
+ System.out.println(usage);
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java?rev=1594440&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java (added)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java Wed May 14 01:15:48 2014
@@ -0,0 +1,109 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.FileOutputStream;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+
+import com.google.common.base.Charsets;
+
+/**
+ * TextWriterImageProcessor mixes in the ability for ImageVisitor
+ * implementations to easily write their output to a text file.
+ *
+ * Implementing classes should be sure to call the super methods for the
+ * constructors, finish and finishAbnormally methods, in order that the
+ * underlying file may be opened and closed correctly.
+ *
+ * Note, this class does not add newlines to text written to file or (if
+ * enabled) screen. This is the implementing class' responsibility.
+ */
+abstract class TextWriterImageVisitor extends ImageVisitor {
+ private boolean printToScreen = false;
+ private boolean okToWrite = false;
+ final private OutputStreamWriter fw;
+
+ /**
+ * Create a processor that writes to the file named.
+ *
+ * @param filename Name of file to write output to
+ */
+ public TextWriterImageVisitor(String filename) throws IOException {
+ this(filename, false);
+ }
+
+ /**
+ * Create a processor that writes to the file named and may or may not
+ * also output to the screen, as specified.
+ *
+ * @param filename Name of file to write output to
+ * @param printToScreen Mirror output to screen?
+ */
+ public TextWriterImageVisitor(String filename, boolean printToScreen)
+ throws IOException {
+ super();
+ this.printToScreen = printToScreen;
+ fw = new OutputStreamWriter(new FileOutputStream(filename), Charsets.UTF_8);
+ okToWrite = true;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.hadoop.hdfs.tools.offlineImageViewer.ImageVisitor#finish()
+ */
+ @Override
+ void finish() throws IOException {
+ close();
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.hadoop.hdfs.tools.offlineImageViewer.ImageVisitor#finishAbnormally()
+ */
+ @Override
+ void finishAbnormally() throws IOException {
+ close();
+ }
+
+ /**
+ * Close output stream and prevent further writing
+ */
+ private void close() throws IOException {
+ fw.close();
+ okToWrite = false;
+ }
+
+ /**
+ * Write parameter to output file (and possibly screen).
+ *
+ * @param toWrite Text to write to file
+ */
+ protected void write(String toWrite) throws IOException {
+ if(!okToWrite)
+ throw new IOException("file not open for writing.");
+
+ if(printToScreen)
+ System.out.print(toWrite);
+
+ try {
+ fw.write(toWrite);
+ } catch (IOException e) {
+ okToWrite = false;
+ throw e;
+ }
+ }
+}
Added: hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java?rev=1594440&view=auto
==============================================================================
--- hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java (added)
+++ hadoop/common/branches/branch-2/hadoop-hdfs-project/hadoop-hdfs/src/main/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java Wed May 14 01:15:48 2014
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.IOException;
+import java.util.LinkedList;
+
+/**
+ * An XmlImageVisitor walks over an fsimage structure and writes out
+ * an equivalent XML document that contains the fsimage's components.
+ */
+public class XmlImageVisitor extends TextWriterImageVisitor {
+ final private LinkedList<ImageElement> tagQ =
+ new LinkedList<ImageElement>();
+
+ public XmlImageVisitor(String filename) throws IOException {
+ super(filename, false);
+ }
+
+ public XmlImageVisitor(String filename, boolean printToScreen)
+ throws IOException {
+ super(filename, printToScreen);
+ }
+
+ @Override
+ void finish() throws IOException {
+ super.finish();
+ }
+
+ @Override
+ void finishAbnormally() throws IOException {
+ write("\n<!-- Error processing image file. Exiting -->\n");
+ super.finishAbnormally();
+ }
+
+ @Override
+ void leaveEnclosingElement() throws IOException {
+ if(tagQ.size() == 0)
+ throw new IOException("Tried to exit non-existent enclosing element " +
+ "in FSImage file");
+
+ ImageElement element = tagQ.pop();
+ write("</" + element.toString() + ">\n");
+ }
+
+ @Override
+ void start() throws IOException {
+ write("<?xml version=\"1.0\" ?>\n");
+ }
+
+ @Override
+ void visit(ImageElement element, String value) throws IOException {
+ writeTag(element.toString(), value);
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element) throws IOException {
+ write("<" + element.toString() + ">\n");
+ tagQ.push(element);
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element,
+ ImageElement key, String value)
+ throws IOException {
+ write("<" + element.toString() + " " + key + "=\"" + value +"\">\n");
+ tagQ.push(element);
+ }
+
+ private void writeTag(String tag, String value) throws IOException {
+ write("<" + tag + ">" + value + "</" + tag + ">\n");
+ }
+}