You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2013/03/31 06:27:31 UTC
svn commit: r1462875 [2/2] - in /hadoop/common/branches/branch-1: ./ bin/
src/core/org/apache/hadoop/io/compress/
src/docs/src/documentation/content/xdocs/ src/hdfs/org/apache/hadoop/hdfs/
src/hdfs/org/apache/hadoop/hdfs/protocol/ src/hdfs/org/apache/h...
Added: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java?rev=1462875&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java (added)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/IndentedImageVisitor.java Sun Mar 31 04:27:29 2013
@@ -0,0 +1,111 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.IOException;
+import java.util.Date;
+
+/**
+ * IndentedImageVisitor walks over an FSImage and displays its structure
+ * using indenting to organize sections within the image file.
+ */
+class IndentedImageVisitor extends TextWriterImageVisitor {
+
+ public IndentedImageVisitor(String filename) throws IOException {
+ super(filename);
+ }
+
+ public IndentedImageVisitor(String filename, boolean printToScreen) throws IOException {
+ super(filename, printToScreen);
+ }
+
+ final private DepthCounter dc = new DepthCounter();// to track leading spacing
+
+ @Override
+ void start() throws IOException {}
+
+ @Override
+ void finish() throws IOException { super.finish(); }
+
+ @Override
+ void finishAbnormally() throws IOException {
+ System.out.println("*** Image processing finished abnormally. Ending ***");
+ super.finishAbnormally();
+ }
+
+ @Override
+ void leaveEnclosingElement() throws IOException {
+ dc.decLevel();
+ }
+
+ @Override
+ void visit(ImageElement element, String value) throws IOException {
+ printIndents();
+ write(element + " = " + value + "\n");
+ }
+
+ @Override
+ void visit(ImageElement element, long value) throws IOException {
+ if ((element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_EXPIRY_TIME) ||
+ (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_ISSUE_DATE) ||
+ (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_MAX_DATE)) {
+ visit(element, new Date(value).toString());
+ } else {
+ visit(element, Long.toString(value));
+ }
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element) throws IOException {
+ printIndents();
+ write(element + "\n");
+ dc.incLevel();
+ }
+
+ // Print element, along with associated key/value pair, in brackets
+ @Override
+ void visitEnclosingElement(ImageElement element,
+ ImageElement key, String value)
+ throws IOException {
+ printIndents();
+ write(element + " [" + key + " = " + value + "]\n");
+ dc.incLevel();
+ }
+
+ /**
+ * Print an appropriate number of spaces for the current level.
+ * FsImages can potentially be millions of lines long, so caching can
+ * significantly speed up output.
+ */
+ final private static String [] indents = { "",
+ " ",
+ " ",
+ " ",
+ " ",
+ " ",
+ " "};
+ private void printIndents() throws IOException {
+ try {
+ write(indents[dc.getLevel()]);
+ } catch (IndexOutOfBoundsException e) {
+ // There's no reason in an fsimage would need a deeper indent
+ for(int i = 0; i < dc.getLevel(); i++)
+ write(" ");
+ }
+ }
+}
Added: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java?rev=1462875&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java (added)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/LsImageVisitor.java Sun Mar 31 04:27:29 2013
@@ -0,0 +1,178 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.IOException;
+import java.util.Formatter;
+import java.util.LinkedList;
+
+/**
+ * LsImageVisitor displays the blocks of the namespace in a format very similar
+ * to the output of ls/lsr. Entries are marked as directories or not,
+ * permissions listed, replication, username and groupname, along with size,
+ * modification date and full path.
+ *
+ * Note: A significant difference between the output of the lsr command
+ * and this image visitor is that this class cannot sort the file entries;
+ * they are listed in the order they are stored within the fsimage file.
+ * Therefore, the output of this class cannot be directly compared to the
+ * output of the lsr command.
+ */
+class LsImageVisitor extends TextWriterImageVisitor {
+ final private LinkedList<ImageElement> elemQ = new LinkedList<ImageElement>();
+
+ private int numBlocks;
+ private String perms;
+ private int replication;
+ private String username;
+ private String group;
+ private long filesize;
+ private String modTime;
+ private String path;
+ private String linkTarget;
+
+ private boolean inInode = false;
+ final private StringBuilder sb = new StringBuilder();
+ final private Formatter formatter = new Formatter(sb);
+
+ public LsImageVisitor(String filename) throws IOException {
+ super(filename);
+ }
+
+ public LsImageVisitor(String filename, boolean printToScreen) throws IOException {
+ super(filename, printToScreen);
+ }
+
+ /**
+ * Start a new line of output, reset values.
+ */
+ private void newLine() {
+ numBlocks = 0;
+ perms = username = group = path = linkTarget = "";
+ filesize = 0l;
+ replication = 0;
+
+ inInode = true;
+ }
+
+ /**
+ * All the values have been gathered. Print them to the console in an
+ * ls-style format.
+ */
+ private final static int widthRepl = 2;
+ private final static int widthUser = 8;
+ private final static int widthGroup = 10;
+ private final static int widthSize = 10;
+ private final static int widthMod = 10;
+ private final static String lsStr = " %" + widthRepl + "s %" + widthUser +
+ "s %" + widthGroup + "s %" + widthSize +
+ "d %" + widthMod + "s %s";
+ private void printLine() throws IOException {
+ sb.append(numBlocks < 0 ? "d" : "-");
+ sb.append(perms);
+
+ if (0 != linkTarget.length()) {
+ path = path + " -> " + linkTarget;
+ }
+ formatter.format(lsStr, replication > 0 ? replication : "-",
+ username, group, filesize, modTime, path);
+ sb.append("\n");
+
+ write(sb.toString());
+ sb.setLength(0); // clear string builder
+
+ inInode = false;
+ }
+
+ @Override
+ void start() throws IOException {}
+
+ @Override
+ void finish() throws IOException {
+ super.finish();
+ }
+
+ @Override
+ void finishAbnormally() throws IOException {
+ System.out.println("Input ended unexpectedly.");
+ super.finishAbnormally();
+ }
+
+ @Override
+ void leaveEnclosingElement() throws IOException {
+ ImageElement elem = elemQ.pop();
+
+ if(elem == ImageElement.INODE)
+ printLine();
+ }
+
+ // Maintain state of location within the image tree and record
+ // values needed to display the inode in ls-style format.
+ @Override
+ void visit(ImageElement element, String value) throws IOException {
+ if(inInode) {
+ switch(element) {
+ case INODE_PATH:
+ if(value.equals("")) path = "/";
+ else path = value;
+ break;
+ case PERMISSION_STRING:
+ perms = value;
+ break;
+ case REPLICATION:
+ replication = Integer.valueOf(value);
+ break;
+ case USER_NAME:
+ username = value;
+ break;
+ case GROUP_NAME:
+ group = value;
+ break;
+ case NUM_BYTES:
+ filesize += Long.valueOf(value);
+ break;
+ case MODIFICATION_TIME:
+ modTime = value;
+ break;
+ case SYMLINK:
+ linkTarget = value;
+ break;
+ default:
+ // This is OK. We're not looking for all the values.
+ break;
+ }
+ }
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element) throws IOException {
+ elemQ.push(element);
+ if(element == ImageElement.INODE)
+ newLine();
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element,
+ ImageElement key, String value) throws IOException {
+ elemQ.push(element);
+ if(element == ImageElement.INODE)
+ newLine();
+ else if (element == ImageElement.BLOCKS)
+ numBlocks = Integer.valueOf(value);
+ }
+}
Added: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java?rev=1462875&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java (added)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/NameDistributionVisitor.java Sun Mar 31 04:27:29 2013
@@ -0,0 +1,118 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map.Entry;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+
+/**
+ * File name distribution visitor.
+ * <p>
+ * It analyzes file names in fsimage and prints the following information:
+ * <li>Number of unique file names</li>
+ * <li>Number file names and the corresponding number range of files that use
+ * these same names</li>
+ * <li>Heap saved if the file name objects are reused</li>
+ */
+@InterfaceAudience.Private
+public class NameDistributionVisitor extends TextWriterImageVisitor {
+ HashMap<String, Integer> counts = new HashMap<String, Integer>();
+
+ public NameDistributionVisitor(String filename, boolean printToScreen)
+ throws IOException {
+ super(filename, printToScreen);
+ }
+
+ @Override
+ void finish() throws IOException {
+ final int BYTEARRAY_OVERHEAD = 24;
+
+ write("Total unique file names " + counts.size());
+ // Columns: Frequency of file occurrence, savings in heap, total files using
+ // the name and number of file names
+ final long stats[][] = { { 100000, 0, 0, 0 },
+ { 10000, 0, 0, 0 },
+ { 1000, 0, 0, 0 },
+ { 100, 0, 0, 0 },
+ { 10, 0, 0, 0 },
+ { 5, 0, 0, 0 },
+ { 4, 0, 0, 0 },
+ { 3, 0, 0, 0 },
+ { 2, 0, 0, 0 }};
+
+ int highbound = Integer.MIN_VALUE;
+ for (Entry<String, Integer> entry : counts.entrySet()) {
+ highbound = Math.max(highbound, entry.getValue());
+ for (int i = 0; i < stats.length; i++) {
+ if (entry.getValue() >= stats[i][0]) {
+ stats[i][1] += (BYTEARRAY_OVERHEAD + entry.getKey().length())
+ * (entry.getValue() - 1);
+ stats[i][2] += entry.getValue();
+ stats[i][3]++;
+ break;
+ }
+ }
+ }
+
+ long lowbound = 0;
+ long totalsavings = 0;
+ for (long[] stat : stats) {
+ lowbound = stat[0];
+ totalsavings += stat[1];
+ String range = lowbound == highbound ? " " + lowbound :
+ " between " + lowbound + "-" + highbound;
+ write("\n" + stat[3] + " names are used by " + stat[2] + " files"
+ + range + " times. Heap savings ~" + stat[1] + " bytes.");
+ highbound = (int) stat[0] - 1;
+ }
+ write("\n\nTotal saved heap ~" + totalsavings + "bytes.\n");
+ super.finish();
+ }
+
+ @Override
+ void visit(ImageElement element, String value) throws IOException {
+ if (element == ImageElement.INODE_PATH) {
+ String filename = value.substring(value.lastIndexOf("/") + 1);
+ if (counts.containsKey(filename)) {
+ counts.put(filename, counts.get(filename) + 1);
+ } else {
+ counts.put(filename, 1);
+ }
+ }
+ }
+
+ @Override
+ void leaveEnclosingElement() throws IOException {
+ }
+
+ @Override
+ void start() throws IOException {
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element) throws IOException {
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element, ImageElement key,
+ String value) throws IOException {
+ }
+}
Added: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java?rev=1462875&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java (added)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/OfflineImageViewer.java Sun Mar 31 04:27:29 2013
@@ -0,0 +1,335 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.BufferedInputStream;
+import java.io.DataInputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FilterInputStream;
+import java.io.IOException;
+import java.io.InputStream;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.OptionBuilder;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.io.IOUtils;
+
+/**
+ * OfflineImageViewer to dump the contents of an Hadoop image file to XML
+ * or the console. Main entry point into utility, either via the
+ * command line or programatically.
+ */
+@InterfaceAudience.Private
+public class OfflineImageViewer {
+ public static final Log LOG = LogFactory.getLog(OfflineImageViewer.class);
+
+ private final static String usage =
+ "Usage: bin/hadoop oiv [OPTIONS] -i INPUTFILE -o OUTPUTFILE\n" +
+ "Offline Image Viewer\n" +
+ "View a Hadoop fsimage INPUTFILE using the specified PROCESSOR,\n" +
+ "saving the results in OUTPUTFILE.\n" +
+ "\n" +
+ "The oiv utility will attempt to parse correctly formed image files\n" +
+ "and will abort fail with mal-formed image files.\n" +
+ "\n" +
+ "The tool works offline and does not require a running cluster in\n" +
+ "order to process an image file.\n" +
+ "\n" +
+ "The following image processors are available:\n" +
+ " * Ls: The default image processor generates an lsr-style listing\n" +
+ " of the files in the namespace, with the same fields in the same\n" +
+ " order. Note that in order to correctly determine file sizes,\n" +
+ " this formatter cannot skip blocks and will override the\n" +
+ " -skipBlocks option.\n" +
+ " * Indented: This processor enumerates over all of the elements in\n" +
+ " the fsimage file, using levels of indentation to delineate\n" +
+ " sections within the file.\n" +
+ " * Delimited: Generate a text file with all of the elements common\n" +
+ " to both inodes and inodes-under-construction, separated by a\n" +
+ " delimiter. The default delimiter is \u0001, though this may be\n" +
+ " changed via the -delimiter argument. This processor also overrides\n" +
+ " the -skipBlocks option for the same reason as the Ls processor\n" +
+ " * XML: This processor creates an XML document with all elements of\n" +
+ " the fsimage enumerated, suitable for further analysis by XML\n" +
+ " tools.\n" +
+ " * FileDistribution: This processor analyzes the file size\n" +
+ " distribution in the image.\n" +
+ " -maxSize specifies the range [0, maxSize] of file sizes to be\n" +
+ " analyzed (128GB by default).\n" +
+ " -step defines the granularity of the distribution. (2MB by default)\n" +
+ " * NameDistribution: This processor analyzes the file names\n" +
+ " in the image and prints total number of file names and how frequently" +
+ " file names are reused.\n" +
+ "\n" +
+ "Required command line arguments:\n" +
+ "-i,--inputFile <arg> FSImage file to process.\n" +
+ "-o,--outputFile <arg> Name of output file. If the specified\n" +
+ " file exists, it will be overwritten.\n" +
+ "\n" +
+ "Optional command line arguments:\n" +
+ "-p,--processor <arg> Select which type of processor to apply\n" +
+ " against image file." +
+ " (Ls|XML|Delimited|Indented|FileDistribution).\n" +
+ "-h,--help Display usage information and exit\n" +
+ "-printToScreen For processors that write to a file, also\n" +
+ " output to screen. On large image files this\n" +
+ " will dramatically increase processing time.\n" +
+ "-skipBlocks Skip inodes' blocks information. May\n" +
+ " significantly decrease output.\n" +
+ " (default = false).\n" +
+ "-delimiter <arg> Delimiting string to use with Delimited processor\n";
+
+ private final boolean skipBlocks;
+ private final String inputFile;
+ private final ImageVisitor processor;
+
+ public OfflineImageViewer(String inputFile, ImageVisitor processor,
+ boolean skipBlocks) {
+ this.inputFile = inputFile;
+ this.processor = processor;
+ this.skipBlocks = skipBlocks;
+ }
+
+ /**
+ * Process image file.
+ */
+ public void go() throws IOException {
+ DataInputStream in = null;
+ PositionTrackingInputStream tracker = null;
+ ImageLoader fsip = null;
+ boolean done = false;
+ try {
+ tracker = new PositionTrackingInputStream(new BufferedInputStream(
+ new FileInputStream(new File(inputFile))));
+ in = new DataInputStream(tracker);
+
+ int imageVersionFile = findImageVersion(in);
+
+ fsip = ImageLoader.LoaderFactory.getLoader(imageVersionFile);
+
+ if(fsip == null)
+ throw new IOException("No image processor to read version " +
+ imageVersionFile + " is available.");
+ fsip.loadImage(in, processor, skipBlocks);
+ done = true;
+ } finally {
+ if (!done) {
+ LOG.error("image loading failed at offset " + tracker.getPos());
+ }
+ IOUtils.cleanup(LOG, in, tracker);
+ }
+ }
+
+ /**
+ * Check an fsimage datainputstream's version number.
+ *
+ * The datainput stream is returned at the same point as it was passed in;
+ * this method has no effect on the datainputstream's read pointer.
+ *
+ * @param in Datainputstream of fsimage
+ * @return Filesystem layout version of fsimage represented by stream
+ * @throws IOException If problem reading from in
+ */
+ private int findImageVersion(DataInputStream in) throws IOException {
+ in.mark(42); // arbitrary amount, resetting immediately
+
+ int version = in.readInt();
+ in.reset();
+
+ return version;
+ }
+
+ /**
+ * Build command-line options and descriptions
+ */
+ public static Options buildOptions() {
+ Options options = new Options();
+
+ // Build in/output file arguments, which are required, but there is no
+ // addOption method that can specify this
+ OptionBuilder.isRequired();
+ OptionBuilder.hasArgs();
+ OptionBuilder.withLongOpt("outputFile");
+ options.addOption(OptionBuilder.create("o"));
+
+ OptionBuilder.isRequired();
+ OptionBuilder.hasArgs();
+ OptionBuilder.withLongOpt("inputFile");
+ options.addOption(OptionBuilder.create("i"));
+
+ options.addOption("p", "processor", true, "");
+ options.addOption("h", "help", false, "");
+ options.addOption("skipBlocks", false, "");
+ options.addOption("printToScreen", false, "");
+ options.addOption("delimiter", true, "");
+
+ return options;
+ }
+
+ /**
+ * Entry point to command-line-driven operation. User may specify
+ * options and start fsimage viewer from the command line. Program
+ * will process image file and exit cleanly or, if an error is
+ * encountered, inform user and exit.
+ *
+ * @param args Command line options
+ * @throws IOException
+ */
+ public static void main(String[] args) throws IOException {
+ Options options = buildOptions();
+ if(args.length == 0) {
+ printUsage();
+ return;
+ }
+
+ CommandLineParser parser = new PosixParser();
+ CommandLine cmd;
+
+ try {
+ cmd = parser.parse(options, args);
+ } catch (ParseException e) {
+ System.out.println("Error parsing command-line options: ");
+ printUsage();
+ return;
+ }
+
+ if(cmd.hasOption("h")) { // print help and exit
+ printUsage();
+ return;
+ }
+
+ boolean skipBlocks = cmd.hasOption("skipBlocks");
+ boolean printToScreen = cmd.hasOption("printToScreen");
+ String inputFile = cmd.getOptionValue("i");
+ String processor = cmd.getOptionValue("p", "Ls");
+ String outputFile = cmd.getOptionValue("o");
+ String delimiter = cmd.getOptionValue("delimiter");
+
+ if( !(delimiter == null || processor.equals("Delimited")) ) {
+ System.out.println("Can only specify -delimiter with Delimited processor");
+ printUsage();
+ return;
+ }
+
+ ImageVisitor v;
+ if(processor.equals("Indented")) {
+ v = new IndentedImageVisitor(outputFile, printToScreen);
+ } else if (processor.equals("XML")) {
+ v = new XmlImageVisitor(outputFile, printToScreen);
+ } else if (processor.equals("Delimited")) {
+ v = delimiter == null ?
+ new DelimitedImageVisitor(outputFile, printToScreen) :
+ new DelimitedImageVisitor(outputFile, printToScreen, delimiter);
+ skipBlocks = false;
+ } else if (processor.equals("FileDistribution")) {
+ long maxSize = Long.parseLong(cmd.getOptionValue("maxSize", "0"));
+ int step = Integer.parseInt(cmd.getOptionValue("step", "0"));
+ v = new FileDistributionVisitor(outputFile, maxSize, step);
+ } else if (processor.equals("NameDistribution")) {
+ v = new NameDistributionVisitor(outputFile, printToScreen);
+ } else {
+ v = new LsImageVisitor(outputFile, printToScreen);
+ skipBlocks = false;
+ }
+
+ try {
+ OfflineImageViewer d = new OfflineImageViewer(inputFile, v, skipBlocks);
+ d.go();
+ } catch (EOFException e) {
+ System.err.println("Input file ended unexpectedly. Exiting");
+ } catch(IOException e) {
+ System.err.println("Encountered exception. Exiting: " + e.getMessage());
+ }
+ }
+
+ /**
+ * Print application usage instructions.
+ */
+ private static void printUsage() {
+ System.out.println(usage);
+ }
+
+ /**
+ * Stream wrapper that keeps track of the current stream position.
+ */
+ private static class PositionTrackingInputStream extends FilterInputStream {
+ private long curPos = 0;
+ private long markPos = -1;
+
+ public PositionTrackingInputStream(InputStream is) {
+ super(is);
+ }
+
+ @Override
+ public int read() throws IOException {
+ int ret = super.read();
+ if (ret != -1) curPos++;
+ return ret;
+ }
+
+ @Override
+ public int read(byte[] data) throws IOException {
+ int ret = super.read(data);
+ if (ret > 0) curPos += ret;
+ return ret;
+ }
+
+ @Override
+ public int read(byte[] data, int offset, int length) throws IOException {
+ int ret = super.read(data, offset, length);
+ if (ret > 0) curPos += ret;
+ return ret;
+ }
+
+ @Override
+ public void mark(int limit) {
+ super.mark(limit);
+ markPos = curPos;
+ }
+
+ @Override
+ public void reset() throws IOException {
+ if (markPos == -1) {
+ throw new IOException("Not marked!");
+ }
+ super.reset();
+ curPos = markPos;
+ markPos = -1;
+ }
+
+ public long getPos() {
+ return curPos;
+ }
+
+ @Override
+ public long skip(long amt) throws IOException {
+ long ret = super.skip(amt);
+ curPos += ret;
+ return ret;
+ }
+ }
+}
Added: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java?rev=1462875&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java (added)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/TextWriterImageVisitor.java Sun Mar 31 04:27:29 2013
@@ -0,0 +1,110 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.FileOutputStream;
+import java.io.FileWriter;
+import java.io.IOException;
+import java.io.OutputStreamWriter;
+import java.nio.charset.Charset;
+
+/**
+ * TextWriterImageProcessor mixes in the ability for ImageVisitor
+ * implementations to easily write their output to a text file.
+ *
+ * Implementing classes should be sure to call the super methods for the
+ * constructors, finish and finishAbnormally methods, in order that the
+ * underlying file may be opened and closed correctly.
+ *
+ * Note, this class does not add newlines to text written to file or (if
+ * enabled) screen. This is the implementing class' responsibility.
+ */
+abstract class TextWriterImageVisitor extends ImageVisitor {
+ private static final Charset UTF_8 = Charset.forName("UTF-8");
+ private boolean printToScreen = false;
+ private boolean okToWrite = false;
+ final private OutputStreamWriter fw;
+
+ /**
+ * Create a processor that writes to the file named.
+ *
+ * @param filename Name of file to write output to
+ */
+ public TextWriterImageVisitor(String filename) throws IOException {
+ this(filename, false);
+ }
+
+ /**
+ * Create a processor that writes to the file named and may or may not
+ * also output to the screen, as specified.
+ *
+ * @param filename Name of file to write output to
+ * @param printToScreen Mirror output to screen?
+ */
+ public TextWriterImageVisitor(String filename, boolean printToScreen)
+ throws IOException {
+ super();
+ this.printToScreen = printToScreen;
+ fw = new OutputStreamWriter(new FileOutputStream(filename), UTF_8);
+ okToWrite = true;
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.hadoop.hdfs.tools.offlineImageViewer.ImageVisitor#finish()
+ */
+ @Override
+ void finish() throws IOException {
+ close();
+ }
+
+ /* (non-Javadoc)
+ * @see org.apache.hadoop.hdfs.tools.offlineImageViewer.ImageVisitor#finishAbnormally()
+ */
+ @Override
+ void finishAbnormally() throws IOException {
+ close();
+ }
+
+ /**
+ * Close output stream and prevent further writing
+ */
+ private void close() throws IOException {
+ fw.close();
+ okToWrite = false;
+ }
+
+ /**
+ * Write parameter to output file (and possibly screen).
+ *
+ * @param toWrite Text to write to file
+ */
+ protected void write(String toWrite) throws IOException {
+ if(!okToWrite)
+ throw new IOException("file not open for writing.");
+
+ if(printToScreen)
+ System.out.print(toWrite);
+
+ try {
+ fw.write(toWrite);
+ } catch (IOException e) {
+ okToWrite = false;
+ throw e;
+ }
+ }
+}
Added: hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java?rev=1462875&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java (added)
+++ hadoop/common/branches/branch-1/src/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/XmlImageVisitor.java Sun Mar 31 04:27:29 2013
@@ -0,0 +1,88 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.IOException;
+import java.util.LinkedList;
+
+/**
+ * An XmlImageVisitor walks over an fsimage structure and writes out
+ * an equivalent XML document that contains the fsimage's components.
+ */
+class XmlImageVisitor extends TextWriterImageVisitor {
+ final private LinkedList<ImageElement> tagQ =
+ new LinkedList<ImageElement>();
+
+ public XmlImageVisitor(String filename) throws IOException {
+ super(filename, false);
+ }
+
+ public XmlImageVisitor(String filename, boolean printToScreen)
+ throws IOException {
+ super(filename, printToScreen);
+ }
+
+ @Override
+ void finish() throws IOException {
+ super.finish();
+ }
+
+ @Override
+ void finishAbnormally() throws IOException {
+ write("\n<!-- Error processing image file. Exiting -->\n");
+ super.finishAbnormally();
+ }
+
+ @Override
+ void leaveEnclosingElement() throws IOException {
+ if(tagQ.size() == 0)
+ throw new IOException("Tried to exit non-existent enclosing element " +
+ "in FSImage file");
+
+ ImageElement element = tagQ.pop();
+ write("</" + element.toString() + ">\n");
+ }
+
+ @Override
+ void start() throws IOException {
+ write("<?xml version=\"1.0\" ?>\n");
+ }
+
+ @Override
+ void visit(ImageElement element, String value) throws IOException {
+ writeTag(element.toString(), value);
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element) throws IOException {
+ write("<" + element.toString() + ">\n");
+ tagQ.push(element);
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element,
+ ImageElement key, String value)
+ throws IOException {
+ write("<" + element.toString() + " " + key + "=\"" + value +"\">\n");
+ tagQ.push(element);
+ }
+
+ private void writeTag(String tag, String value) throws IOException {
+ write("<" + tag + ">" + value + "</" + tag + ">\n");
+ }
+}
Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/SpotCheckImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/SpotCheckImageVisitor.java?rev=1462875&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/SpotCheckImageVisitor.java (added)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/SpotCheckImageVisitor.java Sun Mar 31 04:27:29 2013
@@ -0,0 +1,89 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import java.io.IOException;
+import java.util.HashSet;
+import java.util.Set;
+
+/**
+ * ImageVisitor to spot check an fsimage and generate several statistics
+ * about it that we can compare with known values to give a reasonable
+ * assertion that the image was processed correctly.
+ */
+class SpotCheckImageVisitor extends ImageVisitor {
+
+ // Statistics gathered by the visitor for Inodes and InodesUnderConstruction
+ static public class ImageInfo {
+ public long totalNumBlocks = 0; // Total number of blocks in section
+ public Set<String> pathNames = new HashSet<String>(); // All path names
+ public long totalFileSize = 0; // Total size of all the files
+ public long totalReplications = 0; // Sum of all the replications
+ }
+
+ final private ImageInfo inodes = new ImageInfo();
+ final private ImageInfo INUCs = new ImageInfo();
+ private ImageInfo current = null;
+
+ @Override
+ void visit(ImageElement element, String value) throws IOException {
+ if(element == ImageElement.NUM_BYTES)
+ current.totalFileSize += Long.valueOf(value);
+ else if (element == ImageElement.REPLICATION)
+ current.totalReplications += Long.valueOf(value);
+ else if (element == ImageElement.INODE_PATH)
+ current.pathNames.add(value);
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element, ImageElement key,
+ String value) throws IOException {
+ switch(element) {
+ case INODES:
+ current = inodes;
+ break;
+ case INODES_UNDER_CONSTRUCTION:
+ current = INUCs;
+ break;
+ case BLOCKS:
+ current.totalNumBlocks += Long.valueOf(value);
+ break;
+ // OK to not have a default, we're skipping most of the values
+ }
+ }
+
+ public ImageInfo getINodesInfo() { return inodes; }
+
+ public ImageInfo getINUCsInfo() { return INUCs; }
+
+ // Unnecessary visitor methods
+ @Override
+ void finish() throws IOException {}
+
+ @Override
+ void finishAbnormally() throws IOException {}
+
+ @Override
+ void leaveEnclosingElement() throws IOException {}
+
+ @Override
+ void start() throws IOException {}
+
+ @Override
+ void visitEnclosingElement(ImageElement element) throws IOException {}
+}
Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestDelimitedImageVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestDelimitedImageVisitor.java?rev=1462875&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestDelimitedImageVisitor.java (added)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestDelimitedImageVisitor.java Sun Mar 31 04:27:29 2013
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNull;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedReader;
+import java.io.File;
+import java.io.FileReader;
+import java.io.IOException;
+
+import org.apache.hadoop.hdfs.tools.offlineImageViewer.ImageVisitor.ImageElement;
+import org.junit.Test;
+
+/**
+ * Test that the DelimitedImageVisistor gives the expected output based
+ * on predetermined inputs
+ */
+public class TestDelimitedImageVisitor {
+ private static String ROOT = System.getProperty("test.build.data","/tmp");
+ private static final String delim = "--";
+
+ // Record an element in the visitor and build the expected line in the output
+ private void build(DelimitedImageVisitor div, ImageElement elem, String val,
+ StringBuilder sb, boolean includeDelim) throws IOException {
+ div.visit(elem, val);
+ sb.append(val);
+
+ if(includeDelim)
+ sb.append(delim);
+ }
+
+ @Test
+ public void testDelimitedImageVisistor() {
+ String filename = ROOT + "/testDIV";
+ File f = new File(filename);
+ BufferedReader br = null;
+ StringBuilder sb = new StringBuilder();
+
+ try {
+ DelimitedImageVisitor div = new DelimitedImageVisitor(filename, true, delim);
+
+ div.visit(ImageElement.FS_IMAGE, "Not in ouput");
+ div.visitEnclosingElement(ImageElement.INODE);
+ div.visit(ImageElement.LAYOUT_VERSION, "not in");
+ div.visit(ImageElement.LAYOUT_VERSION, "the output");
+
+ build(div, ImageElement.INODE_PATH, "hartnell", sb, true);
+ build(div, ImageElement.REPLICATION, "99", sb, true);
+ build(div, ImageElement.MODIFICATION_TIME, "troughton", sb, true);
+ build(div, ImageElement.ACCESS_TIME, "pertwee", sb, true);
+ build(div, ImageElement.BLOCK_SIZE, "baker", sb, true);
+ build(div, ImageElement.NUM_BLOCKS, "davison", sb, true);
+ build(div, ImageElement.NUM_BYTES, "55", sb, true);
+ build(div, ImageElement.NS_QUOTA, "baker2", sb, true);
+ build(div, ImageElement.DS_QUOTA, "mccoy", sb, true);
+ build(div, ImageElement.PERMISSION_STRING, "eccleston", sb, true);
+ build(div, ImageElement.USER_NAME, "tennant", sb, true);
+ build(div, ImageElement.GROUP_NAME, "smith", sb, false);
+
+ div.leaveEnclosingElement(); // INode
+ div.finish();
+
+ br = new BufferedReader(new FileReader(f));
+ String actual = br.readLine();
+
+ // Should only get one line
+ assertNull(br.readLine());
+ br.close();
+
+ String exepcted = sb.toString();
+ System.out.println("Expect to get: " + exepcted);
+ System.out.println("Actually got: " + actual);
+ assertEquals(exepcted, actual);
+
+ } catch (IOException e) {
+ fail("Error while testing delmitedImageVisitor" + e.getMessage());
+ } finally {
+ if(f.exists())
+ f.delete();
+ }
+ }
+}
Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOIVCanReadOldVersions.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOIVCanReadOldVersions.java?rev=1462875&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOIVCanReadOldVersions.java (added)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOIVCanReadOldVersions.java Sun Mar 31 04:27:29 2013
@@ -0,0 +1,135 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+import java.util.Collections;
+import java.util.HashSet;
+import java.util.Set;
+
+import org.apache.hadoop.hdfs.tools.offlineImageViewer.SpotCheckImageVisitor.ImageInfo;
+import org.junit.Test;
+
+public class TestOIVCanReadOldVersions {
+ // Location of fsimage files during testing.
+ public static final String TEST_CACHE_DATA_DIR =
+ System.getProperty("test.cache.data", "build/test/cache");
+
+ // Verify that the image processor can correctly process prior Hadoop
+ // layout versions. These fsimages were previously generated and stored
+ // with the test. Test success indicates that no changes have been made
+ // to the OIV that causes older fsimages to be incorrectly processed.
+ @Test
+ public void testOldFSImages() {
+ // Define the expected values from the prior versions, as they were created
+ // and verified at time of creation
+ Set<String> pathNames = new HashSet<String>();
+ Collections.addAll(pathNames, "", /* root */
+ "/bar",
+ "/bar/dir0",
+ "/bar/dir0/file0",
+ "/bar/dir0/file1",
+ "/bar/dir1",
+ "/bar/dir1/file0",
+ "/bar/dir1/file1",
+ "/bar/dir2",
+ "/bar/dir2/file0",
+ "/bar/dir2/file1",
+ "/foo",
+ "/foo/dir0",
+ "/foo/dir0/file0",
+ "/foo/dir0/file1",
+ "/foo/dir0/file2",
+ "/foo/dir0/file3",
+ "/foo/dir1",
+ "/foo/dir1/file0",
+ "/foo/dir1/file1",
+ "/foo/dir1/file2",
+ "/foo/dir1/file3");
+
+ Set<String> INUCpaths = new HashSet<String>();
+ Collections.addAll(INUCpaths, "/bar/dir0/file0",
+ "/bar/dir0/file1",
+ "/bar/dir1/file0",
+ "/bar/dir1/file1",
+ "/bar/dir2/file0",
+ "/bar/dir2/file1");
+
+ ImageInfo v18Inodes = new ImageInfo(); // Hadoop version 18 inodes
+ v18Inodes.totalNumBlocks = 12;
+ v18Inodes.totalFileSize = 1069548540l;
+ v18Inodes.pathNames = pathNames;
+ v18Inodes.totalReplications = 14;
+
+ ImageInfo v18INUCs = new ImageInfo(); // Hadoop version 18 inodes under construction
+ v18INUCs.totalNumBlocks = 0;
+ v18INUCs.totalFileSize = 0;
+ v18INUCs.pathNames = INUCpaths;
+ v18INUCs.totalReplications = 6;
+
+ ImageInfo v19Inodes = new ImageInfo(); // Hadoop version 19 inodes
+ v19Inodes.totalNumBlocks = 12;
+ v19Inodes.totalFileSize = 1069548540l;
+ v19Inodes.pathNames = pathNames;
+ v19Inodes.totalReplications = 14;
+
+ ImageInfo v19INUCs = new ImageInfo(); // Hadoop version 19 inodes under construction
+ v19INUCs.totalNumBlocks = 0;
+ v19INUCs.totalFileSize = 0;
+ v19INUCs.pathNames = INUCpaths;
+ v19INUCs.totalReplications = 6;
+
+
+ spotCheck("18", TEST_CACHE_DATA_DIR + "/fsimageV18", v18Inodes, v18INUCs);
+ spotCheck("19", TEST_CACHE_DATA_DIR + "/fsimageV19", v19Inodes, v19INUCs);
+ }
+
+ // Check that running the processor now gives us the same values as before
+ private void spotCheck(String hadoopVersion, String input,
+ ImageInfo inodes, ImageInfo INUCs) {
+ SpotCheckImageVisitor v = new SpotCheckImageVisitor();
+ OfflineImageViewer oiv = new OfflineImageViewer(input, v, false);
+ try {
+ oiv.go();
+ } catch (IOException e) {
+ fail("Error processing file: " + input);
+ }
+
+ compareSpotCheck(hadoopVersion, v.getINodesInfo(), inodes);
+ compareSpotCheck(hadoopVersion, v.getINUCsInfo(), INUCs);
+ System.out.println("Successfully processed fsimage file from Hadoop version " +
+ hadoopVersion);
+ }
+
+ // Compare the spot check results of what we generated from the image
+ // processor and what we expected to receive
+ private void compareSpotCheck(String hadoopVersion,
+ ImageInfo generated, ImageInfo expected) {
+ assertEquals("Version " + hadoopVersion + ": Same number of total blocks",
+ expected.totalNumBlocks, generated.totalNumBlocks);
+ assertEquals("Version " + hadoopVersion + ": Same total file size",
+ expected.totalFileSize, generated.totalFileSize);
+ assertEquals("Version " + hadoopVersion + ": Same total replication factor",
+ expected.totalReplications, generated.totalReplications);
+ assertEquals("Version " + hadoopVersion + ": One-to-one matching of path names",
+ expected.pathNames, generated.pathNames);
+ }
+}
Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java?rev=1462875&view=auto
==============================================================================
--- hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java (added)
+++ hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/TestOfflineImageViewer.java Sun Mar 31 04:27:29 2013
@@ -0,0 +1,470 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineImageViewer;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+import static org.junit.Assert.fail;
+
+import java.io.BufferedReader;
+import java.io.DataInputStream;
+import java.io.DataOutputStream;
+import java.io.EOFException;
+import java.io.File;
+import java.io.FileInputStream;
+import java.io.FileOutputStream;
+import java.io.FileReader;
+import java.io.IOException;
+import java.io.InputStream;
+import java.io.OutputStream;
+import java.util.HashMap;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.Set;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hdfs.DFSConfigKeys;
+import org.apache.hadoop.hdfs.MiniDFSCluster;
+import org.apache.hadoop.hdfs.protocol.FSConstants.SafeModeAction;
+import org.apache.hadoop.security.token.Token;
+import org.junit.AfterClass;
+import org.junit.BeforeClass;
+import org.junit.Test;
+
+
+/**
+ * Test function of OfflineImageViewer by:
+ * * confirming it can correctly process a valid fsimage file and that
+ * the processing generates a correct representation of the namespace
+ * * confirming it correctly fails to process an fsimage file with a layout
+ * version it shouldn't be able to handle
+ * * confirm it correctly bails on malformed image files, in particular, a
+ * file that ends suddenly.
+ */
+public class TestOfflineImageViewer {
+ private static final Log LOG = LogFactory.getLog(OfflineImageViewer.class);
+ private static final int NUM_DIRS = 3;
+ private static final int FILES_PER_DIR = 4;
+ private static final String TEST_RENEWER = "JobTracker";
+ private static File originalFsimage = null;
+
+ // Elements of lines of ls-file output to be compared to FileStatus instance
+ private static class LsElements {
+ public String perms;
+ public int replication;
+ public String username;
+ public String groupname;
+ public long filesize;
+ public char dir; // d if dir, - otherwise
+ }
+
+ // namespace as written to dfs, to be compared with viewer's output
+ final static HashMap<String, FileStatus> writtenFiles =
+ new HashMap<String, FileStatus>();
+
+ private static String ROOT = System.getProperty("test.build.data",
+ "build/test/data");
+
+ // Create a populated namespace for later testing. Save its contents to a
+ // data structure and store its fsimage location.
+ // We only want to generate the fsimage file once and use it for
+ // multiple tests.
+ @BeforeClass
+ public static void createOriginalFSImage() throws IOException {
+ MiniDFSCluster cluster = null;
+ try {
+ Configuration conf = new Configuration();
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_MAX_LIFETIME_KEY, 10000);
+ conf.setLong(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_RENEW_INTERVAL_KEY, 5000);
+ conf.setBoolean(DFSConfigKeys.DFS_NAMENODE_DELEGATION_TOKEN_ALWAYS_USE_KEY, true);
+ conf.set("hadoop.security.auth_to_local",
+ "RULE:[2:$1@$0](JobTracker@.*FOO.COM)s/@.*//" + "DEFAULT");
+ cluster = new MiniDFSCluster(conf, 4, true, null);
+ cluster.waitActive();
+ FileSystem hdfs = cluster.getFileSystem();
+
+ int filesize = 256;
+
+ // Create a reasonable namespace
+ for(int i = 0; i < NUM_DIRS; i++) {
+ Path dir = new Path("/dir" + i);
+ hdfs.mkdirs(dir);
+ writtenFiles.put(dir.toString(), pathToFileEntry(hdfs, dir.toString()));
+ for(int j = 0; j < FILES_PER_DIR; j++) {
+ Path file = new Path(dir, "file" + j);
+ FSDataOutputStream o = hdfs.create(file);
+ o.write(new byte[ filesize++ ]);
+ o.close();
+
+ writtenFiles.put(file.toString(), pathToFileEntry(hdfs, file.toString()));
+ }
+ }
+
+ // Get delegation tokens so we log the delegation token op
+ Token<?> delegationToken = hdfs.getDelegationToken(TEST_RENEWER);
+ LOG.debug("got token " + delegationToken);
+
+ // Write results to the fsimage file
+ cluster.getNameNode().setSafeMode(SafeModeAction.SAFEMODE_ENTER);
+ cluster.getNameNode().saveNamespace();
+
+ // Determine location of fsimage file
+ File [] files = cluster.getNameDirs().toArray(new File[0]);
+ originalFsimage = new File(files[0], "current/fsimage");
+
+ if(!originalFsimage.exists())
+ fail("Didn't generate or can't find fsimage.");
+
+ } finally {
+ if(cluster != null)
+ cluster.shutdown();
+ }
+ }
+
+ @AfterClass
+ public static void deleteOriginalFSImage() throws IOException {
+ if(originalFsimage != null && originalFsimage.exists()) {
+ originalFsimage.delete();
+ }
+ }
+
+ // Convenience method to generate a file status from file system for
+ // later comparison
+ private static FileStatus pathToFileEntry(FileSystem hdfs, String file)
+ throws IOException {
+ return hdfs.getFileStatus(new Path(file));
+ }
+
+ // Verify that we can correctly generate an ls-style output for a valid
+ // fsimage
+ @Test
+ public void outputOfLSVisitor() throws IOException {
+ File testFile = new File(ROOT, "/basicCheck");
+ File outputFile = new File(ROOT, "/basicCheckOutput");
+
+ try {
+ copyFile(originalFsimage, testFile);
+
+ ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true);
+ OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false);
+
+ oiv.go();
+
+ HashMap<String, LsElements> fileOutput = readLsfile(outputFile);
+
+ compareNamespaces(writtenFiles, fileOutput);
+ } finally {
+ if(testFile.exists()) testFile.delete();
+ if(outputFile.exists()) outputFile.delete();
+ }
+ LOG.debug("Correctly generated ls-style output.");
+ }
+
+ // Confirm that attempting to read an fsimage file with an unsupported
+ // layout results in an error
+ @Test
+ public void unsupportedFSLayoutVersion() throws IOException {
+ File testFile = new File(ROOT, "/invalidLayoutVersion");
+ File outputFile = new File(ROOT, "invalidLayoutVersionOutput");
+
+ try {
+ int badVersionNum = -432;
+ changeLayoutVersion(originalFsimage, testFile, badVersionNum);
+ ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true);
+ OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false);
+
+ try {
+ oiv.go();
+ fail("Shouldn't be able to read invalid laytout version");
+ } catch(IOException e) {
+ if(!e.getMessage().contains(Integer.toString(badVersionNum)))
+ throw e; // wasn't error we were expecting
+ LOG.debug("Correctly failed at reading bad image version.");
+ }
+ } finally {
+ if(testFile.exists()) testFile.delete();
+ if(outputFile.exists()) outputFile.delete();
+ }
+ }
+
+ // Verify that image viewer will bail on a file that ends unexpectedly
+ @Test
+ public void truncatedFSImage() throws IOException {
+ File testFile = new File(ROOT, "/truncatedFSImage");
+ File outputFile = new File(ROOT, "/trucnatedFSImageOutput");
+ try {
+ copyPartOfFile(originalFsimage, testFile);
+ assertTrue("Created truncated fsimage", testFile.exists());
+
+ ImageVisitor v = new LsImageVisitor(outputFile.getPath(), true);
+ OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, false);
+
+ try {
+ oiv.go();
+ fail("Managed to process a truncated fsimage file");
+ } catch (EOFException e) {
+ LOG.debug("Correctly handled EOF");
+ }
+
+ } finally {
+ if(testFile.exists()) testFile.delete();
+ if(outputFile.exists()) outputFile.delete();
+ }
+ }
+
+ // Test that our ls file has all the same compenents of the original namespace
+ private void compareNamespaces(HashMap<String, FileStatus> written,
+ HashMap<String, LsElements> fileOutput) {
+ assertEquals( "Should be the same number of files in both, plus one for root"
+ + " in fileoutput", fileOutput.keySet().size(),
+ written.keySet().size() + 1);
+ Set<String> inFile = fileOutput.keySet();
+
+ // For each line in the output file, verify that the namespace had a
+ // filestatus counterpart
+ for (String path : inFile) {
+ if (path.equals("/")) // root's not included in output from system call
+ continue;
+
+ assertTrue("Path in file (" + path + ") was written to fs", written
+ .containsKey(path));
+
+ compareFiles(written.get(path), fileOutput.get(path));
+
+ written.remove(path);
+ }
+
+ assertEquals("No more files were written to fs", 0, written.size());
+ }
+
+ // Compare two files as listed in the original namespace FileStatus and
+ // the output of the ls file from the image processor
+ private void compareFiles(FileStatus fs, LsElements elements) {
+ assertEquals("directory listed as such",
+ fs.isDir() ? 'd' : '-', elements.dir);
+ assertEquals("perms string equal",
+ fs.getPermission().toString(), elements.perms);
+ assertEquals("replication equal", fs.getReplication(), elements.replication);
+ assertEquals("owner equal", fs.getOwner(), elements.username);
+ assertEquals("group equal", fs.getGroup(), elements.groupname);
+ assertEquals("lengths equal", fs.getLen(), elements.filesize);
+ }
+
+ // Read the contents of the file created by the Ls processor
+ private HashMap<String, LsElements> readLsfile(File lsFile) throws IOException {
+ BufferedReader br = new BufferedReader(new FileReader(lsFile));
+ String line = null;
+ HashMap<String, LsElements> fileContents = new HashMap<String, LsElements>();
+
+ while((line = br.readLine()) != null)
+ readLsLine(line, fileContents);
+
+ return fileContents;
+ }
+
+ // Parse a line from the ls output. Store permissions, replication,
+ // username, groupname and filesize in hashmap keyed to the path name
+ private void readLsLine(String line, HashMap<String, LsElements> fileContents) {
+ String elements [] = line.split("\\s+");
+
+ assertEquals("Not enough elements in ls output", 8, elements.length);
+
+ LsElements lsLine = new LsElements();
+
+ lsLine.dir = elements[0].charAt(0);
+ lsLine.perms = elements[0].substring(1);
+ lsLine.replication = elements[1].equals("-")
+ ? 0 : Integer.valueOf(elements[1]);
+ lsLine.username = elements[2];
+ lsLine.groupname = elements[3];
+ lsLine.filesize = Long.valueOf(elements[4]);
+ // skipping date and time
+
+ String path = elements[7];
+
+ // Check that each file in the ls output was listed once
+ assertFalse("LS file had duplicate file entries",
+ fileContents.containsKey(path));
+
+ fileContents.put(path, lsLine);
+ }
+
+ // Copy one fsimage to another, changing the layout version in the process
+ private void changeLayoutVersion(File src, File dest, int newVersion)
+ throws IOException {
+ DataInputStream in = null;
+ DataOutputStream out = null;
+
+ try {
+ in = new DataInputStream(new FileInputStream(src));
+ out = new DataOutputStream(new FileOutputStream(dest));
+
+ in.readInt();
+ out.writeInt(newVersion);
+
+ byte [] b = new byte[1024];
+ while( in.read(b) > 0 ) {
+ out.write(b);
+ }
+ } finally {
+ if(in != null) in.close();
+ if(out != null) out.close();
+ }
+ }
+
+ // Only copy part of file into the other. Used for testing truncated fsimage
+ private void copyPartOfFile(File src, File dest) throws IOException {
+ InputStream in = null;
+ OutputStream out = null;
+
+ byte [] b = new byte[256];
+ int bytesWritten = 0;
+ int count;
+ int maxBytes = 700;
+
+ try {
+ in = new FileInputStream(src);
+ out = new FileOutputStream(dest);
+
+ while( (count = in.read(b)) > 0 && bytesWritten < maxBytes ) {
+ out.write(b);
+ bytesWritten += count;
+ }
+ } finally {
+ if(in != null) in.close();
+ if(out != null) out.close();
+ }
+ }
+
+ // Copy one file's contents into the other
+ private void copyFile(File src, File dest) throws IOException {
+ InputStream in = null;
+ OutputStream out = null;
+
+ try {
+ in = new FileInputStream(src);
+ out = new FileOutputStream(dest);
+
+ byte [] b = new byte[1024];
+ while( in.read(b) > 0 ) {
+ out.write(b);
+ }
+ } finally {
+ if(in != null) in.close();
+ if(out != null) out.close();
+ }
+ }
+
+ @Test
+ public void outputOfFileDistributionVisitor() throws IOException {
+ File testFile = new File(ROOT, "/basicCheck");
+ File outputFile = new File(ROOT, "/fileDistributionCheckOutput");
+
+ int totalFiles = 0;
+ try {
+ copyFile(originalFsimage, testFile);
+ ImageVisitor v = new FileDistributionVisitor(outputFile.getPath(), 0, 0);
+ OfflineImageViewer oiv =
+ new OfflineImageViewer(testFile.getPath(), v, false);
+
+ oiv.go();
+
+ BufferedReader reader = new BufferedReader(new FileReader(outputFile));
+ String line = reader.readLine();
+ assertEquals(line, "Size\tNumFiles");
+ while((line = reader.readLine()) != null) {
+ String[] row = line.split("\t");
+ assertEquals(row.length, 2);
+ totalFiles += Integer.parseInt(row[1]);
+ }
+ } finally {
+ if(testFile.exists()) testFile.delete();
+ if(outputFile.exists()) outputFile.delete();
+ }
+ assertEquals(totalFiles, NUM_DIRS * FILES_PER_DIR);
+ }
+
+ private static class TestImageVisitor extends ImageVisitor {
+ private List<String> delegationTokenRenewers = new LinkedList<String>();
+ TestImageVisitor() {
+ }
+
+ List<String> getDelegationTokenRenewers() {
+ return delegationTokenRenewers;
+ }
+
+ @Override
+ void start() throws IOException {
+ }
+
+ @Override
+ void finish() throws IOException {
+ }
+
+ @Override
+ void finishAbnormally() throws IOException {
+ }
+
+ @Override
+ void visit(ImageElement element, String value) throws IOException {
+ if (element == ImageElement.DELEGATION_TOKEN_IDENTIFIER_RENEWER) {
+ delegationTokenRenewers.add(value);
+ }
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element) throws IOException {
+ }
+
+ @Override
+ void visitEnclosingElement(ImageElement element, ImageElement key,
+ String value) throws IOException {
+ }
+
+ @Override
+ void leaveEnclosingElement() throws IOException {
+ }
+ }
+
+ @Test
+ public void outputOfTestVisitor() throws IOException {
+ File testFile = new File(ROOT, "/basicCheck");
+
+ try {
+ copyFile(originalFsimage, testFile);
+ TestImageVisitor v = new TestImageVisitor();
+ OfflineImageViewer oiv = new OfflineImageViewer(testFile.getPath(), v, true);
+ oiv.go();
+
+ // Validated stored delegation token identifiers.
+ List<String> dtrs = v.getDelegationTokenRenewers();
+ assertEquals(1, dtrs.size());
+ assertEquals(TEST_RENEWER, dtrs.get(0));
+ } finally {
+ if(testFile.exists()) testFile.delete();
+ }
+ LOG.debug("Passed TestVisitor validation.");
+ }
+}
Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18?rev=1462875&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream
Added: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19?rev=1462875&view=auto
==============================================================================
Binary file - no diff available.
Propchange: hadoop/common/branches/branch-1/src/test/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19
------------------------------------------------------------------------------
svn:mime-type = application/octet-stream