You are viewing a plain text version of this content. The canonical link for it is here.
Posted to hdfs-commits@hadoop.apache.org by sz...@apache.org on 2011/01/21 23:08:05 UTC
svn commit: r1062045 [1/2] - in /hadoop/hdfs/trunk: ./ bin/
src/docs/src/documentation/content/xdocs/
src/java/org/apache/hadoop/hdfs/server/namenode/
src/java/org/apache/hadoop/hdfs/server/protocol/
src/java/org/apache/hadoop/hdfs/tools/offlineEditsVi...
Author: szetszwo
Date: Fri Jan 21 22:08:04 2011
New Revision: 1062045
URL: http://svn.apache.org/viewvc?rev=1062045&view=rev
Log:
HDFS-1448. Add a new tool Offline Edits Viewer (oev). Contributed by Erik Steffl
Added:
hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/hdfs_editsviewer.xml
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryTokenizer.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoader.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitor.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitorFactory.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/OfflineEditsViewer.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/StatisticsEditsVisitor.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TextEditsVisitor.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/Tokenizer.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TokenizerFactory.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlEditsVisitor.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/XmlTokenizer.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/DepthCounter.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/server/namenode/OfflineEditsViewerHelper.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/TestOfflineEditsViewer.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored.xml
Modified:
hadoop/hdfs/trunk/CHANGES.txt
hadoop/hdfs/trunk/bin/hdfs
hadoop/hdfs/trunk/build.xml
hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/site.xml
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineImageViewer/ImageVisitor.java
hadoop/hdfs/trunk/src/test/hdfs/org/apache/hadoop/hdfs/DFSTestUtil.java
Modified: hadoop/hdfs/trunk/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/CHANGES.txt?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/CHANGES.txt (original)
+++ hadoop/hdfs/trunk/CHANGES.txt Fri Jan 21 22:08:04 2011
@@ -14,6 +14,9 @@ Trunk (unreleased changes)
HDFS-1482. Add listCorruptFileBlocks to DistributedFileSystem.
(Patrick Kling via hairong)
+ HDFS-1448. Add a new tool Offline Edits Viewer (oev). (Erik Steffl
+ via szetszwo)
+
IMPROVEMENTS
HDFS-1510. Added test-patch.properties required by test-patch.sh (nigel)
Modified: hadoop/hdfs/trunk/bin/hdfs
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/bin/hdfs?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/bin/hdfs (original)
+++ hadoop/hdfs/trunk/bin/hdfs Fri Jan 21 22:08:04 2011
@@ -32,6 +32,7 @@ function print_usage(){
echo " balancer run a cluster balancing utility"
echo " jmxget get JMX exported values from NameNode or DataNode."
echo " oiv apply the offline fsimage viewer to an fsimage"
+ echo " oev apply the offline edits viewer to an edits file"
echo " fetchdt fetch a delegation token from the NameNode"
echo " Use -help to see options"
echo ""
@@ -89,6 +90,8 @@ elif [ "$COMMAND" = "jmxget" ] ; then
CLASS=org.apache.hadoop.hdfs.tools.JMXGet
elif [ "$COMMAND" = "oiv" ] ; then
CLASS=org.apache.hadoop.hdfs.tools.offlineImageViewer.OfflineImageViewer
+elif [ "$COMMAND" = "oev" ] ; then
+ CLASS=org.apache.hadoop.hdfs.tools.offlineEditsViewer.OfflineEditsViewer
elif [ "$COMMAND" = "fetchdt" ] ; then
CLASS=org.apache.hadoop.hdfs.tools.DelegationTokenFetcher
else
Modified: hadoop/hdfs/trunk/build.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/build.xml?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/build.xml (original)
+++ hadoop/hdfs/trunk/build.xml Fri Jan 21 22:08:04 2011
@@ -422,6 +422,8 @@
<copy file="${test.src.dir}/hdfs/org/apache/hadoop/cli/clitest_data/data1k" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV18" todir="${test.cache.data}"/>
<copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineImageViewer/fsimageV19" todir="${test.cache.data}"/>
+ <copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored" todir="${test.cache.data}"/>
+ <copy file="${test.src.dir}/hdfs/org/apache/hadoop/hdfs/tools/offlineEditsViewer/editsStored.xml" todir="${test.cache.data}"/>
</target>
<macrodef name="macro-compile-hdfs-test">
Added: hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/hdfs_editsviewer.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/hdfs_editsviewer.xml?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/hdfs_editsviewer.xml (added)
+++ hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/hdfs_editsviewer.xml Fri Jan 21 22:08:04 2011
@@ -0,0 +1,157 @@
+<?xml version="1.0"?>
+<!--
+ Licensed to the Apache Software Foundation (ASF) under one or more
+ contributor license agreements. See the NOTICE file distributed with
+ this work for additional information regarding copyright ownership.
+ The ASF licenses this file to You under the Apache License, Version 2.0
+ (the "License"); you may not use this file except in compliance with
+ the License. You may obtain a copy of the License at
+
+ http://www.apache.org/licenses/LICENSE-2.0
+
+ Unless required by applicable law or agreed to in writing, software
+ distributed under the License is distributed on an "AS IS" BASIS,
+ WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ See the License for the specific language governing permissions and
+ limitations under the License.
+-->
+
+<!DOCTYPE document PUBLIC "-//APACHE//DTD Documentation V2.0//EN" "http://forrest.apache.org/dtd/document-v20.dtd">
+
+<document>
+
+ <header>
+ <title>Offline Edits Viewer Guide</title>
+ <authors>
+ <person name="Erik Steffl" email="steffl@yahoo-inc.com"/>
+ </authors>
+ </header>
+
+ <body>
+
+ <section>
+
+ <title>Overview</title>
+
+ <p>
+ Offline Edits Viewer is a tool to parse the Edits log file. The
+ current processors are mostly useful for conversion between
+ different formats, including XML which is human readable and
+ easier to edit than native binary format.
+ </p>
+
+ <p>
+ The tool can parse the edits formats -18 (roughly Hadoop 0.19)
+ and later. The tool operates on files only, it does not need
+ Hadoop cluster to be running.
+ </p>
+
+ <p>Input formats supported:</p>
+ <ol>
+ <li><strong>binary</strong>: native binary format that Hadoop uses internally</li>
+ <li>
+ <strong>xml</strong>: XML format, as produced by
+ <strong>xml</strong> processor, used if filename has xml
+ (case insensitive) extension
+ </li>
+ </ol>
+
+ <p>
+ The Offline Edits Viewer provides several output processors
+ (unless stated otherwise the output of the processor can be
+ converted back to original edits file):
+ </p>
+ <ol>
+ <li><strong>binary</strong>: native binary format that Hadoop uses internally</li>
+ <li><strong>xml</strong>: XML format</li>
+ <li><strong>stats</strong>: prints out statistics, this cannot be converted back to Edits file</li>
+ </ol>
+
+ </section> <!-- Overview -->
+
+ <section>
+
+ <title>Usage</title>
+
+ <p><code>bash$ bin/hdfs oev -i edits -o edits.xml</code></p>
+
+ <table>
+ <tr><th>Flag</th><th>Description</th></tr>
+ <tr>
+ <td><code>[-i|--inputFile] <input file></code></td>
+ <td>
+ Specify the input edits log file to process. Xml (case
+ insensitive) extension means XML format otherwise binary
+ format is assumed. Required.
+ </td>
+ </tr>
+ <tr>
+ <td><code>[-o|--outputFile] <output file></code></td>
+ <td>
+ Specify the output filename, if the specified output processor
+ generates one. If the specified file already exists, it is
+ silently overwritten. Required.
+ </td>
+ </tr>
+ <tr>
+ <td><code>[-p|--processor] <processor></code></td>
+ <td>
+ Specify the image processor to apply against the image
+ file. Currently valid options are <strong>binary</strong>,
+ <strong>xml</strong> (default) and <strong>stats</strong>.
+ </td>
+ </tr>
+ <tr>
+ <td><code>[-v|--verbose]-</code></td>
+ <td>
+ Print the input and output filenames and pipe output of
+ processor to console as well as specified file. On extremely
+ large files, this may increase processing time by an order
+ of magnitude.
+ </td>
+ </tr>
+ <tr>
+ <td><code>[-h|--help]</code></td>
+ <td>
+ Display the tool usage and help information and exit.
+ </td>
+ </tr>
+ </table>
+
+ </section> <!-- Usage -->
+
+ <section>
+
+ <title>Case study: Hadoop cluster recovery</title>
+
+ <p>
+ In case there is some problem with hadoop cluster and the edits
+ file is corrupted it is possible to save at least part of the
+ edits file that is correct. This can be done by converting the
+ binary edits to XML, edit it manually and then convert it back
+ to binary. The most common problem is that the edits file is
+ missing the closing record (record that has opCode -1). This
+ should be recognized by the tool and the XML format should be
+ properly closed.
+ </p>
+
+ <p>
+ If there is no closing record in the XML file you can add one
+ after last correct record. Anything after the record with opCode
+ -1 is ignored.
+ </p>
+
+ <p>Example of a closing record (with opCode -1):</p>
+<source>
+<RECORD>
+ <OPCODE>-1</OPCODE>
+ <DATA>
+ </DATA>
+</RECORD>
+</source>
+
+ </section> <!-- Case study: Hadoop cluster recovery -->
+
+ </body>
+
+</document>
Modified: hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/site.xml
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/site.xml?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/site.xml (original)
+++ hadoop/hdfs/trunk/src/docs/src/documentation/content/xdocs/site.xml Fri Jan 21 22:08:04 2011
@@ -42,6 +42,7 @@ See http://forrest.apache.org/docs/linki
<hdfs_quotas label="Quotas" href="hdfs_quota_admin_guide.html" />
<hdfs_SLG label="Synthetic Load Generator" href="SLG_user_guide.html" />
<hdfs_imageviewer label="Offline Image Viewer" href="hdfs_imageviewer.html" />
+ <hdfs_editsviewer label="Offline Edits Viewer" href="hdfs_editsviewer.html" />
<hdfsproxy label="HDFS Proxy" href="hdfsproxy.html"/>
<faultinject_framework label="Fault Injection" href="faultinject_framework.html" />
<hdfs_libhdfs label="C API libhdfs" href="libhdfs.html" />
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogBackupOutputStream.java Fri Jan 21 22:08:04 2011
@@ -147,7 +147,7 @@ class EditLogBackupOutputStream extends
JournalRecord jRec = null;
for(; idx < bufReadySize; idx++) {
jRec = bufReady.get(idx);
- if(jRec.op >= FSEditLog.Ops.OP_JSPOOL_START)
+ if(jRec.op >= FSEditLogOpCodes.OP_JSPOOL_START.getOpCode())
break; // special operation should be sent in a separate call to BN
jRec.write(out);
}
@@ -177,7 +177,7 @@ class EditLogBackupOutputStream extends
private void send(int ja) throws IOException {
try {
int length = out.getLength();
- out.write(FSEditLog.Ops.OP_INVALID);
+ out.write(FSEditLogOpCodes.OP_INVALID.getOpCode());
backupNode.journal(nnRegistration, ja, length, out.getData());
} finally {
out.reset();
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/EditLogFileOutputStream.java Fri Jan 21 22:08:04 2011
@@ -128,7 +128,7 @@ class EditLogFileOutputStream extends Ed
@Override
void setReadyToFlush() throws IOException {
assert bufReady.size() == 0 : "previous data is not flushed yet";
- write(FSEditLog.Ops.OP_INVALID); // insert end-of-file marker
+ write(FSEditLogOpCodes.OP_INVALID.getOpCode()); // insert eof marker
DataOutputBuffer tmp = bufReady;
bufReady = bufCurrent;
bufCurrent = tmp;
@@ -189,7 +189,7 @@ class EditLogFileOutputStream extends Ed
*/
@Override
boolean isOperationSupported(byte op) {
- return op < FSEditLog.Ops.OP_JSPOOL_START - 1;
+ return op < FSEditLogOpCodes.OP_JSPOOL_START.getOpCode() - 1;
}
/**
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLog.java Fri Jan 21 22:08:04 2011
@@ -48,6 +48,8 @@ import org.apache.hadoop.io.LongWritable
import org.apache.hadoop.io.Writable;
import org.apache.hadoop.security.token.delegation.DelegationKey;
+import static org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes.*;
+
/**
* FSEditLog maintains a log of the namespace modifications.
*
@@ -55,44 +57,6 @@ import org.apache.hadoop.security.token.
@InterfaceAudience.Private
@InterfaceStability.Evolving
public class FSEditLog {
-
- abstract static class Ops {
- public static final byte OP_INVALID = -1;
- public static final byte OP_ADD = 0;
- public static final byte OP_RENAME_OLD = 1; // rename
- public static final byte OP_DELETE = 2; // delete
- public static final byte OP_MKDIR = 3; // create directory
- public static final byte OP_SET_REPLICATION = 4; // set replication
- //the following two are used only for backward compatibility :
- @Deprecated public static final byte OP_DATANODE_ADD = 5;
- @Deprecated public static final byte OP_DATANODE_REMOVE = 6;
- public static final byte OP_SET_PERMISSIONS = 7;
- public static final byte OP_SET_OWNER = 8;
- public static final byte OP_CLOSE = 9; // close after write
- public static final byte OP_SET_GENSTAMP = 10; // store genstamp
- /* The following two are not used any more. Should be removed once
- * LAST_UPGRADABLE_LAYOUT_VERSION is -17 or newer. */
- public static final byte OP_SET_NS_QUOTA = 11; // set namespace quota
- public static final byte OP_CLEAR_NS_QUOTA = 12; // clear namespace quota
- public static final byte OP_TIMES = 13; // sets mod & access time on a file
- public static final byte OP_SET_QUOTA = 14; // sets name and disk quotas.
- public static final byte OP_RENAME = 15; // new rename
- public static final byte OP_CONCAT_DELETE = 16; // concat files.
- public static final byte OP_SYMLINK = 17; // a symbolic link
- public static final byte OP_GET_DELEGATION_TOKEN = 18; //new delegation token
- public static final byte OP_RENEW_DELEGATION_TOKEN = 19; //renew delegation token
- public static final byte OP_CANCEL_DELEGATION_TOKEN = 20; //cancel delegation token
- public static final byte OP_UPDATE_MASTER_KEY = 21; //update master key
-
- /*
- * The following operations are used to control remote edit log streams,
- * and not logged into file streams.
- */
- static final byte OP_JSPOOL_START = // start journal spool
- NamenodeProtocol.JA_JSPOOL_START;
- static final byte OP_CHECKPOINT_TIME = // incr checkpoint time
- NamenodeProtocol.JA_CHECKPOINT_TIME;
- }
static final String NO_JOURNAL_STREAMS_WARNING = "!!! WARNING !!!" +
" File system changes are not persistent. No journal streams.";
@@ -370,7 +334,7 @@ public class FSEditLog {
* Write an operation to the edit log. Do not sync to persistent
* store yet.
*/
- void logEdit(byte op, Writable ... writables) {
+ void logEdit(FSEditLogOpCodes opCode, Writable ... writables) {
synchronized (this) {
// wait if an automatic sync is scheduled
waitIfAutoSyncScheduled();
@@ -380,10 +344,10 @@ public class FSEditLog {
ArrayList<EditLogOutputStream> errorStreams = null;
long start = now();
for(EditLogOutputStream eStream : editStreams) {
- if(!eStream.isOperationSupported(op))
+ if(!eStream.isOperationSupported(opCode.getOpCode()))
continue;
try {
- eStream.write(op, writables);
+ eStream.write(opCode.getOpCode(), writables);
} catch (IOException ie) {
LOG.error("logEdit: removing "+ eStream.getName(), ie);
if(errorStreams == null)
@@ -643,7 +607,7 @@ public class FSEditLog {
FSEditLog.toLogLong(newNode.getModificationTime()),
FSEditLog.toLogLong(newNode.getAccessTime()),
FSEditLog.toLogLong(newNode.getPreferredBlockSize())};
- logEdit(Ops.OP_ADD,
+ logEdit(OP_ADD,
new ArrayWritable(DeprecatedUTF8.class, nameReplicationPair),
new ArrayWritable(Block.class, newNode.getBlocks()),
newNode.getPermissionStatus(),
@@ -661,7 +625,7 @@ public class FSEditLog {
FSEditLog.toLogLong(newNode.getModificationTime()),
FSEditLog.toLogLong(newNode.getAccessTime()),
FSEditLog.toLogLong(newNode.getPreferredBlockSize())};
- logEdit(Ops.OP_CLOSE,
+ logEdit(OP_CLOSE,
new ArrayWritable(DeprecatedUTF8.class, nameReplicationPair),
new ArrayWritable(Block.class, newNode.getBlocks()),
newNode.getPermissionStatus());
@@ -676,8 +640,9 @@ public class FSEditLog {
FSEditLog.toLogLong(newNode.getModificationTime()),
FSEditLog.toLogLong(newNode.getAccessTime())
};
- logEdit(Ops.OP_MKDIR, new ArrayWritable(DeprecatedUTF8.class, info),
- newNode.getPermissionStatus());
+ logEdit(OP_MKDIR,
+ new ArrayWritable(DeprecatedUTF8.class, info),
+ newNode.getPermissionStatus());
}
/**
@@ -689,7 +654,7 @@ public class FSEditLog {
new DeprecatedUTF8(src),
new DeprecatedUTF8(dst),
FSEditLog.toLogLong(timestamp)};
- logEdit(Ops.OP_RENAME_OLD, new ArrayWritable(DeprecatedUTF8.class, info));
+ logEdit(OP_RENAME_OLD, new ArrayWritable(DeprecatedUTF8.class, info));
}
/**
@@ -700,17 +665,18 @@ public class FSEditLog {
new DeprecatedUTF8(src),
new DeprecatedUTF8(dst),
FSEditLog.toLogLong(timestamp)};
- logEdit(Ops.OP_RENAME, new ArrayWritable(DeprecatedUTF8.class, info),
- toBytesWritable(options));
+ logEdit(OP_RENAME,
+ new ArrayWritable(DeprecatedUTF8.class, info),
+ toBytesWritable(options));
}
/**
* Add set replication record to edit log
*/
void logSetReplication(String src, short replication) {
- logEdit(Ops.OP_SET_REPLICATION,
- new DeprecatedUTF8(src),
- FSEditLog.toLogReplication(replication));
+ logEdit(OP_SET_REPLICATION,
+ new DeprecatedUTF8(src),
+ FSEditLog.toLogReplication(replication));
}
/** Add set namespace quota record to edit log
@@ -719,20 +685,21 @@ public class FSEditLog {
* @param quota the directory size limit
*/
void logSetQuota(String src, long nsQuota, long dsQuota) {
- logEdit(Ops.OP_SET_QUOTA, new DeprecatedUTF8(src),
- new LongWritable(nsQuota), new LongWritable(dsQuota));
+ logEdit(OP_SET_QUOTA,
+ new DeprecatedUTF8(src),
+ new LongWritable(nsQuota), new LongWritable(dsQuota));
}
/** Add set permissions record to edit log */
void logSetPermissions(String src, FsPermission permissions) {
- logEdit(Ops.OP_SET_PERMISSIONS, new DeprecatedUTF8(src), permissions);
+ logEdit(OP_SET_PERMISSIONS, new DeprecatedUTF8(src), permissions);
}
/** Add set owner record to edit log */
void logSetOwner(String src, String username, String groupname) {
DeprecatedUTF8 u = new DeprecatedUTF8(username == null? "": username);
DeprecatedUTF8 g = new DeprecatedUTF8(groupname == null? "": groupname);
- logEdit(Ops.OP_SET_OWNER, new DeprecatedUTF8(src), u, g);
+ logEdit(OP_SET_OWNER, new DeprecatedUTF8(src), u, g);
}
/**
@@ -747,7 +714,7 @@ public class FSEditLog {
info[idx++] = new DeprecatedUTF8(srcs[i]);
}
info[idx] = FSEditLog.toLogLong(timestamp);
- logEdit(Ops.OP_CONCAT_DELETE, new ArrayWritable(DeprecatedUTF8.class, info));
+ logEdit(OP_CONCAT_DELETE, new ArrayWritable(DeprecatedUTF8.class, info));
}
/**
@@ -757,14 +724,14 @@ public class FSEditLog {
DeprecatedUTF8 info[] = new DeprecatedUTF8[] {
new DeprecatedUTF8(src),
FSEditLog.toLogLong(timestamp)};
- logEdit(Ops.OP_DELETE, new ArrayWritable(DeprecatedUTF8.class, info));
+ logEdit(OP_DELETE, new ArrayWritable(DeprecatedUTF8.class, info));
}
/**
* Add generation stamp record to edit log
*/
void logGenerationStamp(long genstamp) {
- logEdit(Ops.OP_SET_GENSTAMP, new LongWritable(genstamp));
+ logEdit(OP_SET_GENSTAMP, new LongWritable(genstamp));
}
/**
@@ -775,7 +742,7 @@ public class FSEditLog {
new DeprecatedUTF8(src),
FSEditLog.toLogLong(mtime),
FSEditLog.toLogLong(atime)};
- logEdit(Ops.OP_TIMES, new ArrayWritable(DeprecatedUTF8.class, info));
+ logEdit(OP_TIMES, new ArrayWritable(DeprecatedUTF8.class, info));
}
/**
@@ -788,9 +755,9 @@ public class FSEditLog {
new DeprecatedUTF8(value),
FSEditLog.toLogLong(mtime),
FSEditLog.toLogLong(atime)};
- logEdit(Ops.OP_SYMLINK,
- new ArrayWritable(DeprecatedUTF8.class, info),
- node.getPermissionStatus());
+ logEdit(OP_SYMLINK,
+ new ArrayWritable(DeprecatedUTF8.class, info),
+ node.getPermissionStatus());
}
/**
@@ -801,20 +768,20 @@ public class FSEditLog {
*/
void logGetDelegationToken(DelegationTokenIdentifier id,
long expiryTime) {
- logEdit(Ops.OP_GET_DELEGATION_TOKEN, id, FSEditLog.toLogLong(expiryTime));
+ logEdit(OP_GET_DELEGATION_TOKEN, id, FSEditLog.toLogLong(expiryTime));
}
void logRenewDelegationToken(DelegationTokenIdentifier id,
long expiryTime) {
- logEdit(Ops.OP_RENEW_DELEGATION_TOKEN, id, FSEditLog.toLogLong(expiryTime));
+ logEdit(OP_RENEW_DELEGATION_TOKEN, id, FSEditLog.toLogLong(expiryTime));
}
void logCancelDelegationToken(DelegationTokenIdentifier id) {
- logEdit(Ops.OP_CANCEL_DELEGATION_TOKEN, id);
+ logEdit(OP_CANCEL_DELEGATION_TOKEN, id);
}
void logUpdateMasterKey(DelegationKey key) {
- logEdit(Ops.OP_UPDATE_MASTER_KEY, key);
+ logEdit(OP_UPDATE_MASTER_KEY, key);
}
static private DeprecatedUTF8 toLogReplication(short replication) {
@@ -1063,7 +1030,7 @@ public class FSEditLog {
boStream = new EditLogBackupOutputStream(bnReg, nnReg);
editStreams.add(boStream);
}
- logEdit(Ops.OP_JSPOOL_START, (Writable[])null);
+ logEdit(OP_JSPOOL_START, (Writable[])null);
}
/**
@@ -1161,7 +1128,7 @@ public class FSEditLog {
void incrementCheckpointTime() {
fsimage.incrementCheckpointTime();
Writable[] args = {new LongWritable(fsimage.getCheckpointTime())};
- logEdit(Ops.OP_CHECKPOINT_TIME, args);
+ logEdit(OP_CHECKPOINT_TIME, args);
}
synchronized void releaseBackupStream(NamenodeRegistration registration) {
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogLoader.java Fri Jan 21 22:08:04 2011
@@ -34,7 +34,6 @@ import org.apache.hadoop.hdfs.security.t
import static org.apache.hadoop.hdfs.server.common.Util.now;
import org.apache.hadoop.hdfs.server.common.GenerationStamp;
import org.apache.hadoop.hdfs.server.common.Storage;
-import org.apache.hadoop.hdfs.server.namenode.FSEditLog.Ops;
import org.apache.hadoop.io.BytesWritable;
import org.apache.hadoop.io.LongWritable;
import org.apache.hadoop.io.Writable;
@@ -122,11 +121,12 @@ public class FSEditLogLoader {
long mtime = 0;
long atime = 0;
long blockSize = 0;
- byte opcode = -1;
+ FSEditLogOpCodes opCode;
try {
in.mark(1);
- opcode = in.readByte();
- if (opcode == Ops.OP_INVALID) {
+ byte opCodeByte = in.readByte();
+ opCode = FSEditLogOpCodes.fromByte(opCodeByte);
+ if (opCode == FSEditLogOpCodes.OP_INVALID) {
in.reset(); // reset back to end of file if somebody reads it again
break; // no more transactions
}
@@ -134,9 +134,9 @@ public class FSEditLogLoader {
break; // no more transactions
}
numEdits++;
- switch (opcode) {
- case Ops.OP_ADD:
- case Ops.OP_CLOSE: {
+ switch (opCode) {
+ case OP_ADD:
+ case OP_CLOSE: {
// versions > 0 support per file replication
// get name and replication
int length = in.readInt();
@@ -158,7 +158,7 @@ public class FSEditLogLoader {
blockSize = readLong(in);
}
// get blocks
- boolean isFileUnderConstruction = (opcode == Ops.OP_ADD);
+ boolean isFileUnderConstruction = (opCode == FSEditLogOpCodes.OP_ADD);
BlockInfo blocks[] =
readBlocks(in, logVersion, isFileUnderConstruction, replication);
@@ -181,7 +181,7 @@ public class FSEditLogLoader {
}
// clientname, clientMachine and block locations of last block.
- if (opcode == Ops.OP_ADD && logVersion <= -12) {
+ if (opCode == FSEditLogOpCodes.OP_ADD && logVersion <= -12) {
clientName = FSImageSerialization.readString(in);
clientMachine = FSImageSerialization.readString(in);
if (-13 <= logVersion) {
@@ -195,7 +195,7 @@ public class FSEditLogLoader {
// The open lease transaction re-creates a file if necessary.
// Delete the file if it already exists.
if (FSNamesystem.LOG.isDebugEnabled()) {
- FSNamesystem.LOG.debug(opcode + ": " + path +
+ FSNamesystem.LOG.debug(opCode + ": " + path +
" numblocks : " + blocks.length +
" clientHolder " + clientName +
" clientMachine " + clientMachine);
@@ -229,16 +229,16 @@ public class FSEditLogLoader {
}
break;
}
- case Ops.OP_SET_REPLICATION: {
+ case OP_SET_REPLICATION: {
numOpSetRepl++;
path = FSImageSerialization.readString(in);
short replication = fsNamesys.adjustReplication(readShort(in));
fsDir.unprotectedSetReplication(path, replication, null);
break;
}
- case Ops.OP_CONCAT_DELETE: {
+ case OP_CONCAT_DELETE: {
if (logVersion > -22) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
numOpConcatDelete++;
@@ -257,7 +257,7 @@ public class FSEditLogLoader {
fsDir.unprotectedConcat(trg, srcs);
break;
}
- case Ops.OP_RENAME_OLD: {
+ case OP_RENAME_OLD: {
numOpRenameOld++;
int length = in.readInt();
if (length != 3) {
@@ -272,7 +272,7 @@ public class FSEditLogLoader {
fsNamesys.changeLease(s, d, dinfo);
break;
}
- case Ops.OP_DELETE: {
+ case OP_DELETE: {
numOpDelete++;
int length = in.readInt();
if (length != 2) {
@@ -284,7 +284,7 @@ public class FSEditLogLoader {
fsDir.unprotectedDelete(path, timestamp);
break;
}
- case Ops.OP_MKDIR: {
+ case OP_MKDIR: {
numOpMkDir++;
PermissionStatus permissions = fsNamesys.getUpgradePermission();
int length = in.readInt();
@@ -309,47 +309,47 @@ public class FSEditLogLoader {
fsDir.unprotectedMkdir(path, permissions, timestamp);
break;
}
- case Ops.OP_SET_GENSTAMP: {
+ case OP_SET_GENSTAMP: {
numOpSetGenStamp++;
long lw = in.readLong();
fsNamesys.setGenerationStamp(lw);
break;
}
- case Ops.OP_DATANODE_ADD: {
+ case OP_DATANODE_ADD: {
numOpOther++;
//Datanodes are not persistent any more.
FSImageSerialization.DatanodeImage.skipOne(in);
break;
}
- case Ops.OP_DATANODE_REMOVE: {
+ case OP_DATANODE_REMOVE: {
numOpOther++;
DatanodeID nodeID = new DatanodeID();
nodeID.readFields(in);
//Datanodes are not persistent any more.
break;
}
- case Ops.OP_SET_PERMISSIONS: {
+ case OP_SET_PERMISSIONS: {
numOpSetPerm++;
if (logVersion > -11)
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
fsDir.unprotectedSetPermission(
FSImageSerialization.readString(in), FsPermission.read(in));
break;
}
- case Ops.OP_SET_OWNER: {
+ case OP_SET_OWNER: {
numOpSetOwner++;
if (logVersion > -11)
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
fsDir.unprotectedSetOwner(FSImageSerialization.readString(in),
FSImageSerialization.readString_EmptyAsNull(in),
FSImageSerialization.readString_EmptyAsNull(in));
break;
}
- case Ops.OP_SET_NS_QUOTA: {
+ case OP_SET_NS_QUOTA: {
if (logVersion > -16) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
fsDir.unprotectedSetQuota(FSImageSerialization.readString(in),
@@ -357,9 +357,9 @@ public class FSEditLogLoader {
FSConstants.QUOTA_DONT_SET);
break;
}
- case Ops.OP_CLEAR_NS_QUOTA: {
+ case OP_CLEAR_NS_QUOTA: {
if (logVersion > -16) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
fsDir.unprotectedSetQuota(FSImageSerialization.readString(in),
@@ -368,14 +368,14 @@ public class FSEditLogLoader {
break;
}
- case Ops.OP_SET_QUOTA:
+ case OP_SET_QUOTA:
fsDir.unprotectedSetQuota(FSImageSerialization.readString(in),
readLongWritable(in),
readLongWritable(in));
break;
- case Ops.OP_TIMES: {
+ case OP_TIMES: {
numOpTimes++;
int length = in.readInt();
if (length != 3) {
@@ -388,7 +388,7 @@ public class FSEditLogLoader {
fsDir.unprotectedSetTimes(path, mtime, atime, true);
break;
}
- case Ops.OP_SYMLINK: {
+ case OP_SYMLINK: {
numOpSymlink++;
int length = in.readInt();
if (length != 4) {
@@ -403,9 +403,9 @@ public class FSEditLogLoader {
fsDir.unprotectedSymlink(path, value, mtime, atime, perm);
break;
}
- case Ops.OP_RENAME: {
+ case OP_RENAME: {
if (logVersion > -21) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
numOpRename++;
@@ -423,9 +423,9 @@ public class FSEditLogLoader {
fsNamesys.changeLease(s, d, dinfo);
break;
}
- case Ops.OP_GET_DELEGATION_TOKEN: {
+ case OP_GET_DELEGATION_TOKEN: {
if (logVersion > -24) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
numOpGetDelegationToken++;
@@ -437,9 +437,9 @@ public class FSEditLogLoader {
.addPersistedDelegationToken(delegationTokenId, expiryTime);
break;
}
- case Ops.OP_RENEW_DELEGATION_TOKEN: {
+ case OP_RENEW_DELEGATION_TOKEN: {
if (logVersion > -24) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
numOpRenewDelegationToken++;
@@ -451,9 +451,9 @@ public class FSEditLogLoader {
.updatePersistedTokenRenewal(delegationTokenId, expiryTime);
break;
}
- case Ops.OP_CANCEL_DELEGATION_TOKEN: {
+ case OP_CANCEL_DELEGATION_TOKEN: {
if (logVersion > -24) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
numOpCancelDelegationToken++;
@@ -464,9 +464,9 @@ public class FSEditLogLoader {
.updatePersistedTokenCancellation(delegationTokenId);
break;
}
- case Ops.OP_UPDATE_MASTER_KEY: {
+ case OP_UPDATE_MASTER_KEY: {
if (logVersion > -24) {
- throw new IOException("Unexpected opcode " + opcode
+ throw new IOException("Unexpected opCode " + opCode
+ " for version " + logVersion);
}
numOpUpdateMasterKey++;
@@ -477,7 +477,7 @@ public class FSEditLogLoader {
break;
}
default: {
- throw new IOException("Never seen opcode " + opcode);
+ throw new IOException("Never seen opCode " + opCode);
}
}
}
Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/namenode/FSEditLogOpCodes.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,99 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.server.namenode;
+
+import java.util.Map;
+import java.util.HashMap;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Op codes for edits file
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public enum FSEditLogOpCodes {
+ // last op code in file
+ OP_INVALID ((byte) -1),
+ OP_ADD ((byte) 0),
+ OP_RENAME_OLD ((byte) 1), // deprecated operation
+ OP_DELETE ((byte) 2),
+ OP_MKDIR ((byte) 3),
+ OP_SET_REPLICATION ((byte) 4),
+ @Deprecated OP_DATANODE_ADD ((byte) 5),
+ @Deprecated OP_DATANODE_REMOVE((byte) 6),
+ OP_SET_PERMISSIONS ((byte) 7),
+ OP_SET_OWNER ((byte) 8),
+ OP_CLOSE ((byte) 9),
+ OP_SET_GENSTAMP ((byte) 10),
+ OP_SET_NS_QUOTA ((byte) 11), // obsolete
+ OP_CLEAR_NS_QUOTA ((byte) 12), // obsolete
+ OP_TIMES ((byte) 13), // set atime, mtime
+ OP_SET_QUOTA ((byte) 14),
+ OP_RENAME ((byte) 15), // filecontext rename
+ OP_CONCAT_DELETE ((byte) 16), // concat files
+ OP_SYMLINK ((byte) 17),
+ OP_GET_DELEGATION_TOKEN ((byte) 18),
+ OP_RENEW_DELEGATION_TOKEN ((byte) 19),
+ OP_CANCEL_DELEGATION_TOKEN ((byte) 20),
+ OP_UPDATE_MASTER_KEY ((byte) 21),
+ // must be same as NamenodeProtocol.JA_JSPOOL_START
+ OP_JSPOOL_START ((byte)102),
+ // must be same as NamenodeProtocol.JA_CHECKPOINT_TIME
+ OP_CHECKPOINT_TIME ((byte)103);
+
+ private byte opCode;
+
+ /**
+ * Constructor
+ *
+ * @param opCode byte value of constructed enum
+ */
+ FSEditLogOpCodes(byte opCode) {
+ this.opCode = opCode;
+ }
+
+ /**
+ * return the byte value of the enum
+ *
+ * @return the byte value of the enum
+ */
+ public byte getOpCode() {
+ return opCode;
+ }
+
+ private static final Map<Byte, FSEditLogOpCodes> byteToEnum =
+ new HashMap<Byte, FSEditLogOpCodes>();
+
+ static {
+ // initialize byte to enum map
+ for(FSEditLogOpCodes opCode : values())
+ byteToEnum.put(opCode.getOpCode(), opCode);
+ }
+
+ /**
+ * Converts byte to FSEditLogOpCodes enum value
+ *
+ * @param opCode get enum for this opCode
+ * @return enum with byte value of opCode
+ */
+ public static FSEditLogOpCodes fromByte(byte opCode) {
+ return byteToEnum.get(opCode);
+ }
+}
Modified: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java?rev=1062045&r1=1062044&r2=1062045&view=diff
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java (original)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/server/protocol/NamenodeProtocol.java Fri Jan 21 22:08:04 2011
@@ -54,8 +54,8 @@ public interface NamenodeProtocol extend
// Journal action codes. See journal().
public static byte JA_IS_ALIVE = 100; // check whether the journal is alive
public static byte JA_JOURNAL = 101; // just journal
- public static byte JA_JSPOOL_START = 102; // = FSEditLog.OP_JSPOOL_START
- public static byte JA_CHECKPOINT_TIME = 103; // = FSEditLog.OP_CHECKPOINT_TIME
+ public static byte JA_JSPOOL_START = 102; // = FSEditLogOpCodes.OP_JSPOOL_START
+ public static byte JA_CHECKPOINT_TIME = 103; // = FSEditLogOpCodes.OP_CHECKPOINT_TIME
public final static int ACT_UNKNOWN = 0; // unknown action
public final static int ACT_SHUTDOWN = 50; // shutdown node
Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryEditsVisitor.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,120 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import java.io.FileOutputStream;
+import java.io.DataOutputStream;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * BinaryEditsVisitor implements a binary EditsVisitor
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class BinaryEditsVisitor extends EditsVisitor {
+ final private DataOutputStream out;
+
+ /**
+ * Create a processor that writes to a given file and
+ * reads using a given Tokenizer
+ *
+ * @param filename Name of file to write output to
+ * @param tokenizer Input tokenizer
+ */
+ public BinaryEditsVisitor(String filename, Tokenizer tokenizer)
+ throws IOException {
+
+ this(filename, tokenizer, false);
+ }
+
+ /**
+ * Create a processor that writes to a given file and reads using
+ * a given Tokenizer, may also print to screen
+ *
+ * @param filename Name of file to write output to
+ * @param tokenizer Input tokenizer
+ * @param printToScreen Mirror output to screen? (ignored for binary)
+ */
+ public BinaryEditsVisitor(String filename,
+ Tokenizer tokenizer,
+ boolean printToScreen) throws IOException {
+
+ super(tokenizer);
+ out = new DataOutputStream(new FileOutputStream(filename));
+ }
+
+ /**
+ * Start the visitor (initialization)
+ */
+ @Override
+ void start() throws IOException {
+ // nothing to do for binary format
+ }
+
+ /**
+ * Finish the visitor
+ */
+ @Override
+ void finish() throws IOException {
+ close();
+ }
+
+ /**
+ * Finish the visitor and indicate an error
+ */
+ @Override
+ void finishAbnormally() throws IOException {
+ System.err.println("Error processing EditLog file. Exiting.");
+ close();
+ }
+
+ /**
+ * Close output stream and prevent further writing
+ */
+ private void close() throws IOException {
+ out.close();
+ }
+
+ /**
+ * Visit a enclosing element (element that has other elements in it)
+ */
+ @Override
+ void visitEnclosingElement(Tokenizer.Token value) throws IOException {
+ // nothing to do for binary format
+ }
+
+ /**
+ * End of eclosing element
+ */
+ @Override
+ void leaveEnclosingElement() throws IOException {
+ // nothing to do for binary format
+ }
+
+ /**
+ * Visit a Token
+ */
+ @Override
+ Tokenizer.Token visit(Tokenizer.Token value) throws IOException {
+ value.toBinary(out);
+ return value;
+ }
+}
Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryTokenizer.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryTokenizer.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryTokenizer.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/BinaryTokenizer.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,67 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import java.io.FileInputStream;
+import java.io.DataInputStream;
+import java.io.FileNotFoundException;
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Tokenizer that reads tokens from a binary file
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class BinaryTokenizer implements Tokenizer {
+
+ private DataInputStream in;
+
+ /**
+ * BinaryTokenizer constructor
+ *
+ * @param filename input filename
+ */
+ public BinaryTokenizer(String filename) throws FileNotFoundException {
+ in = new DataInputStream(new FileInputStream(filename));
+ }
+
+ /**
+ * BinaryTokenizer constructor
+ *
+ * @param in input stream
+ */
+ public BinaryTokenizer(DataInputStream in) throws IOException {
+ this.in = in;
+ }
+
+ /**
+ * @see org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer#read
+ *
+ * @param t a Token to read
+ * @return token that was just read
+ */
+ @Override
+ public Token read(Token t) throws IOException {
+ t.fromBinary(in);
+ return t;
+ }
+}
Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsElement.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,84 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * Structural elements of an EditLog that may be encountered within the
+ * file. EditsVisitor is able to process these elements.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public enum EditsElement {
+ EDITS,
+ EDITS_VERSION,
+ RECORD,
+ OPCODE,
+ DATA,
+ // elements in the data part of the editLog records
+ LENGTH,
+ // for OP_SET_GENSTAMP
+ GENERATION_STAMP,
+ // for OP_ADD, OP_CLOSE
+ PATH,
+ REPLICATION,
+ MTIME,
+ ATIME,
+ BLOCKSIZE,
+ NUMBLOCKS,
+ BLOCK,
+ BLOCK_ID,
+ BLOCK_NUM_BYTES,
+ BLOCK_GENERATION_STAMP,
+ PERMISSION_STATUS,
+ FS_PERMISSIONS,
+ CLIENT_NAME,
+ CLIENT_MACHINE,
+ // for OP_RENAME_OLD
+ SOURCE,
+ DESTINATION,
+ TIMESTAMP,
+ // for OP_SET_OWNER
+ USERNAME,
+ GROUPNAME,
+ // for OP_SET_QUOTA
+ NS_QUOTA,
+ DS_QUOTA,
+ // for OP_RENAME
+ RENAME_OPTIONS,
+ // for OP_CONCAT_DELETE
+ CONCAT_TARGET,
+ CONCAT_SOURCE,
+ // for OP_GET_DELEGATION_TOKEN
+ T_VERSION,
+ T_OWNER,
+ T_RENEWER,
+ T_REAL_USER,
+ T_ISSUE_DATE,
+ T_MAX_DATE,
+ T_SEQUENCE_NUMBER,
+ T_MASTER_KEY_ID,
+ T_EXPIRY_TIME,
+ // for OP_UPDATE_MASTER_KEY
+ KEY_ID,
+ KEY_EXPIRY_DATE,
+ KEY_LENGTH,
+ KEY_BLOB
+}
Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoader.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoader.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoader.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoader.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,76 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import java.io.IOException;
+import java.util.Map;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * An EditsLoader can read a Hadoop EditLog file and walk over its
+ * structure using the supplied EditsVisitor.
+ *
+ * Each implementation of EditsLoader is designed to rapidly process an
+ * edits log file. As long as minor changes are made from one layout version
+ * to another, it is acceptable to tweak one implementation to read the next.
+ * However, if the layout version changes enough that it would make a
+ * processor slow or difficult to read, another processor should be created.
+ * This allows each processor to quickly read an edits log without getting
+ * bogged down in dealing with significant differences between layout versions.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+interface EditsLoader {
+
+ /**
+ * Loads the edits file
+ */
+ public void loadEdits() throws IOException;
+
+ /**
+ * Can this processor handle the specified version of EditLog file?
+ *
+ * @param version EditLog version file
+ * @return True if this instance can process the file
+ */
+ public boolean canLoadVersion(int version);
+
+ /**
+ * Factory for obtaining version of edits log loader that can read
+ * a particular edits log format.
+ */
+ @InterfaceAudience.Private
+ @InterfaceStability.Unstable
+ public class LoaderFactory {
+ // Java doesn't support static methods on interfaces, which necessitates
+ // this factory class
+
+ /**
+ * Create an edits log loader, at this point we only have one,
+ * we might need to add more later
+ *
+ * @param v an instance of EditsVisitor (binary, XML etc.)
+ * @return EditsLoader that can interpret specified version, or null
+ */
+ static public EditsLoader getLoader(EditsVisitor v) {
+ return new EditsLoaderCurrent(v);
+ }
+ }
+}
Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsLoaderCurrent.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,478 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import java.io.DataInputStream;
+import java.io.IOException;
+import java.io.EOFException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import org.apache.hadoop.hdfs.server.namenode.FSEditLogOpCodes;
+
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.ByteToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.ShortToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.IntToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.VIntToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.LongToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.VLongToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.StringUTF8Token;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.StringTextToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.BlobToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.BytesWritableToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.EmptyToken;
+
+/**
+ * EditsLoaderCurrent processes Hadoop EditLogs files and walks over
+ * them using a provided EditsVisitor, calling the visitor at each element
+ * enumerated below.
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+class EditsLoaderCurrent implements EditsLoader {
+
+ private static int [] supportedVersions = {
+ -18, -19, -20, -21, -22, -23, -24, -25, -26, -27 };
+
+ private EditsVisitor v;
+ private int editsVersion = 0;
+
+ /**
+ * Constructor
+ */
+ public EditsLoaderCurrent(EditsVisitor visitor) {
+ v = visitor;
+ }
+
+ /**
+ * Checks if this EditsLoader can load given version of edits
+ *
+ * @param version version to load
+ * @return true if this EditsLoader can load given version of edits
+ */
+ @Override
+ public boolean canLoadVersion(int version) {
+ for(int v : supportedVersions) { if(v == version) return true; }
+ return false;
+ }
+
+ /**
+ * Visit OP_INVALID
+ */
+ private void visit_OP_INVALID() throws IOException {
+ ; // nothing to do, this op code has no data
+ }
+
+ /**
+ * Visit OP_ADD
+ */
+ private void visit_OP_ADD() throws IOException {
+ visit_OP_ADD_or_OP_CLOSE(FSEditLogOpCodes.OP_ADD);
+ }
+
+ /**
+ * Visit OP_CLOSE
+ */
+ private void visit_OP_CLOSE() throws IOException {
+ visit_OP_ADD_or_OP_CLOSE(FSEditLogOpCodes.OP_CLOSE);
+ }
+
+ /**
+ * Visit OP_ADD and OP_CLOSE, they are almost the same
+ *
+ * @param editsOpCode op code to visit
+ */
+ private void visit_OP_ADD_or_OP_CLOSE(FSEditLogOpCodes editsOpCode)
+ throws IOException {
+
+ IntToken opAddLength = v.visitInt(EditsElement.LENGTH);
+ // this happens if the edits is not properly ended (-1 op code),
+ // it is padded at the end with all zeros, OP_ADD is zero so
+ // without this check we would treat all zeros as empty OP_ADD)
+ if(opAddLength.value == 0) {
+ throw new IOException("OpCode " + editsOpCode +
+ " has zero length (corrupted edits)");
+ }
+ v.visitStringUTF8(EditsElement.PATH);
+ v.visitStringUTF8(EditsElement.REPLICATION);
+ v.visitStringUTF8(EditsElement.MTIME);
+ v.visitStringUTF8(EditsElement.ATIME);
+ v.visitStringUTF8(EditsElement.BLOCKSIZE);
+ // now read blocks
+ IntToken numBlocksToken = v.visitInt(EditsElement.NUMBLOCKS);
+ for (int i = 0; i < numBlocksToken.value; i++) {
+ v.visitEnclosingElement(EditsElement.BLOCK);
+
+ v.visitLong(EditsElement.BLOCK_ID);
+ v.visitLong(EditsElement.BLOCK_NUM_BYTES);
+ v.visitLong(EditsElement.BLOCK_GENERATION_STAMP);
+
+ v.leaveEnclosingElement();
+ }
+ // PERMISSION_STATUS
+ v.visitEnclosingElement(EditsElement.PERMISSION_STATUS);
+
+ v.visitStringText( EditsElement.USERNAME);
+ v.visitStringText( EditsElement.GROUPNAME);
+ v.visitShort( EditsElement.FS_PERMISSIONS);
+
+ v.leaveEnclosingElement();
+ if(editsOpCode == FSEditLogOpCodes.OP_ADD) {
+ v.visitStringUTF8(EditsElement.CLIENT_NAME);
+ v.visitStringUTF8(EditsElement.CLIENT_MACHINE);
+ }
+ }
+
+ /**
+ * Visit OP_RENAME_OLD
+ */
+ private void visit_OP_RENAME_OLD() throws IOException {
+ v.visitInt( EditsElement.LENGTH);
+ v.visitStringUTF8( EditsElement.SOURCE);
+ v.visitStringUTF8( EditsElement.DESTINATION);
+ v.visitStringUTF8( EditsElement.TIMESTAMP);
+ }
+
+ /**
+ * Visit OP_DELETE
+ */
+ private void visit_OP_DELETE() throws IOException {
+ v.visitInt( EditsElement.LENGTH);
+ v.visitStringUTF8( EditsElement.PATH);
+ v.visitStringUTF8( EditsElement.TIMESTAMP);
+ }
+
+ /**
+ * Visit OP_MKDIR
+ */
+ private void visit_OP_MKDIR() throws IOException {
+ v.visitInt( EditsElement.LENGTH);
+ v.visitStringUTF8( EditsElement.PATH);
+ v.visitStringUTF8( EditsElement.TIMESTAMP);
+ v.visitStringUTF8( EditsElement.ATIME);
+ // PERMISSION_STATUS
+ v.visitEnclosingElement( EditsElement.PERMISSION_STATUS);
+
+ v.visitStringText( EditsElement.USERNAME);
+ v.visitStringText( EditsElement.GROUPNAME);
+ v.visitShort( EditsElement.FS_PERMISSIONS);
+
+ v.leaveEnclosingElement();
+ }
+
+ /**
+ * Visit OP_SET_REPLICATION
+ */
+ private void visit_OP_SET_REPLICATION() throws IOException {
+ v.visitStringUTF8(EditsElement.PATH);
+ v.visitStringUTF8(EditsElement.REPLICATION);
+ }
+
+ /**
+ * Visit OP_SET_PERMISSIONS
+ */
+ private void visit_OP_SET_PERMISSIONS() throws IOException {
+ v.visitStringUTF8( EditsElement.PATH);
+ v.visitShort( EditsElement.FS_PERMISSIONS);
+ }
+
+ /**
+ * Visit OP_SET_OWNER
+ */
+ private void visit_OP_SET_OWNER() throws IOException {
+ v.visitStringUTF8(EditsElement.PATH);
+ v.visitStringUTF8(EditsElement.USERNAME);
+ v.visitStringUTF8(EditsElement.GROUPNAME);
+ }
+
+ /**
+ * Visit OP_SET_GENSTAMP
+ */
+ private void visit_OP_SET_GENSTAMP() throws IOException {
+ v.visitLong(EditsElement.GENERATION_STAMP);
+ }
+
+ /**
+ * Visit OP_TIMES
+ */
+ private void visit_OP_TIMES() throws IOException {
+ v.visitInt( EditsElement.LENGTH);
+ v.visitStringUTF8( EditsElement.PATH);
+ v.visitStringUTF8( EditsElement.MTIME);
+ v.visitStringUTF8( EditsElement.ATIME);
+ }
+
+ /**
+ * Visit OP_SET_QUOTA
+ */
+ private void visit_OP_SET_QUOTA() throws IOException {
+ v.visitStringUTF8( EditsElement.PATH);
+ v.visitLong( EditsElement.NS_QUOTA);
+ v.visitLong( EditsElement.DS_QUOTA);
+ }
+
+ /**
+ * Visit OP_RENAME
+ */
+ private void visit_OP_RENAME() throws IOException {
+ if(editsVersion > -21) {
+ throw new IOException("Unexpected op code " + FSEditLogOpCodes.OP_RENAME
+ + " for edit log version " + editsVersion
+ + " (op code 15 only expected for 21 and later)");
+ }
+ v.visitInt( EditsElement.LENGTH);
+ v.visitStringUTF8( EditsElement.SOURCE);
+ v.visitStringUTF8( EditsElement.DESTINATION);
+ v.visitStringUTF8( EditsElement.TIMESTAMP);
+ v.visitBytesWritable( EditsElement.RENAME_OPTIONS);
+ }
+
+ /**
+ * Visit OP_CONCAT_DELETE
+ */
+ private void visit_OP_CONCAT_DELETE() throws IOException {
+ if(editsVersion > -22) {
+ throw new IOException("Unexpected op code "
+ + FSEditLogOpCodes.OP_CONCAT_DELETE
+ + " for edit log version " + editsVersion
+ + " (op code 16 only expected for 22 and later)");
+ }
+ IntToken lengthToken = v.visitInt(EditsElement.LENGTH);
+ v.visitStringUTF8(EditsElement.CONCAT_TARGET);
+ // all except of CONCAT_TARGET and TIMESTAMP
+ int sourceCount = lengthToken.value - 2;
+ for(int i = 0; i < sourceCount; i++) {
+ v.visitStringUTF8(EditsElement.CONCAT_SOURCE);
+ }
+ v.visitStringUTF8(EditsElement.TIMESTAMP);
+ }
+
+ /**
+ * Visit OP_SYMLINK
+ */
+ private void visit_OP_SYMLINK() throws IOException {
+ v.visitInt( EditsElement.LENGTH);
+ v.visitStringUTF8( EditsElement.SOURCE);
+ v.visitStringUTF8( EditsElement.DESTINATION);
+ v.visitStringUTF8( EditsElement.MTIME);
+ v.visitStringUTF8( EditsElement.ATIME);
+ // PERMISSION_STATUS
+ v.visitEnclosingElement(EditsElement.PERMISSION_STATUS);
+
+ v.visitStringText( EditsElement.USERNAME);
+ v.visitStringText( EditsElement.GROUPNAME);
+ v.visitShort( EditsElement.FS_PERMISSIONS);
+
+ v.leaveEnclosingElement();
+ }
+
+ /**
+ * Visit OP_GET_DELEGATION_TOKEN
+ */
+ private void visit_OP_GET_DELEGATION_TOKEN() throws IOException {
+ if(editsVersion > -24) {
+ throw new IOException("Unexpected op code "
+ + FSEditLogOpCodes.OP_GET_DELEGATION_TOKEN
+ + " for edit log version " + editsVersion
+ + " (op code 18 only expected for 24 and later)");
+ }
+ v.visitByte( EditsElement.T_VERSION);
+ v.visitStringText( EditsElement.T_OWNER);
+ v.visitStringText( EditsElement.T_RENEWER);
+ v.visitStringText( EditsElement.T_REAL_USER);
+ v.visitVLong( EditsElement.T_ISSUE_DATE);
+ v.visitVLong( EditsElement.T_MAX_DATE);
+ v.visitVInt( EditsElement.T_SEQUENCE_NUMBER);
+ v.visitVInt( EditsElement.T_MASTER_KEY_ID);
+ v.visitStringUTF8( EditsElement.T_EXPIRY_TIME);
+ }
+
+ /**
+ * Visit OP_RENEW_DELEGATION_TOKEN
+ */
+ private void visit_OP_RENEW_DELEGATION_TOKEN()
+ throws IOException {
+
+ if(editsVersion > -24) {
+ throw new IOException("Unexpected op code "
+ + FSEditLogOpCodes.OP_RENEW_DELEGATION_TOKEN
+ + " for edit log version " + editsVersion
+ + " (op code 19 only expected for 24 and later)");
+ }
+ v.visitByte( EditsElement.T_VERSION);
+ v.visitStringText( EditsElement.T_OWNER);
+ v.visitStringText( EditsElement.T_RENEWER);
+ v.visitStringText( EditsElement.T_REAL_USER);
+ v.visitVLong( EditsElement.T_ISSUE_DATE);
+ v.visitVLong( EditsElement.T_MAX_DATE);
+ v.visitVInt( EditsElement.T_SEQUENCE_NUMBER);
+ v.visitVInt( EditsElement.T_MASTER_KEY_ID);
+ v.visitStringUTF8( EditsElement.T_EXPIRY_TIME);
+ }
+
+ /**
+ * Visit OP_CANCEL_DELEGATION_TOKEN
+ */
+ private void visit_OP_CANCEL_DELEGATION_TOKEN()
+ throws IOException {
+
+ if(editsVersion > -24) {
+ throw new IOException("Unexpected op code "
+ + FSEditLogOpCodes.OP_CANCEL_DELEGATION_TOKEN
+ + " for edit log version " + editsVersion
+ + " (op code 20 only expected for 24 and later)");
+ }
+ v.visitByte( EditsElement.T_VERSION);
+ v.visitStringText( EditsElement.T_OWNER);
+ v.visitStringText( EditsElement.T_RENEWER);
+ v.visitStringText( EditsElement.T_REAL_USER);
+ v.visitVLong( EditsElement.T_ISSUE_DATE);
+ v.visitVLong( EditsElement.T_MAX_DATE);
+ v.visitVInt( EditsElement.T_SEQUENCE_NUMBER);
+ v.visitVInt( EditsElement.T_MASTER_KEY_ID);
+ }
+
+ /**
+ * Visit OP_UPDATE_MASTER_KEY
+ */
+ private void visit_OP_UPDATE_MASTER_KEY()
+ throws IOException {
+
+ if(editsVersion > -24) {
+ throw new IOException("Unexpected op code "
+ + FSEditLogOpCodes.OP_UPDATE_MASTER_KEY
+ + " for edit log version " + editsVersion
+ + "(op code 21 only expected for 24 and later)");
+ }
+ v.visitVInt( EditsElement.KEY_ID);
+ v.visitVLong( EditsElement.KEY_EXPIRY_DATE);
+ VIntToken blobLengthToken = v.visitVInt(EditsElement.KEY_LENGTH);
+ v.visitBlob(EditsElement.KEY_BLOB, blobLengthToken.value);
+ }
+
+ private void visitOpCode(FSEditLogOpCodes editsOpCode)
+ throws IOException {
+
+ switch(editsOpCode) {
+ case OP_INVALID: // -1
+ visit_OP_INVALID();
+ break;
+ case OP_ADD: // 0
+ visit_OP_ADD();
+ break;
+ case OP_CLOSE: // 9
+ visit_OP_CLOSE();
+ break;
+ case OP_RENAME_OLD: // 1
+ visit_OP_RENAME_OLD();
+ break;
+ case OP_DELETE: // 2
+ visit_OP_DELETE();
+ break;
+ case OP_MKDIR: // 3
+ visit_OP_MKDIR();
+ break;
+ case OP_SET_REPLICATION: // 4
+ visit_OP_SET_REPLICATION();
+ break;
+ case OP_SET_PERMISSIONS: // 7
+ visit_OP_SET_PERMISSIONS();
+ break;
+ case OP_SET_OWNER: // 8
+ visit_OP_SET_OWNER();
+ break;
+ case OP_SET_GENSTAMP: // 10
+ visit_OP_SET_GENSTAMP();
+ break;
+ case OP_TIMES: // 13
+ visit_OP_TIMES();
+ break;
+ case OP_SET_QUOTA: // 14
+ visit_OP_SET_QUOTA();
+ break;
+ case OP_RENAME: // 15
+ visit_OP_RENAME();
+ break;
+ case OP_CONCAT_DELETE: // 16
+ visit_OP_CONCAT_DELETE();
+ break;
+ case OP_SYMLINK: // 17
+ visit_OP_SYMLINK();
+ break;
+ case OP_GET_DELEGATION_TOKEN: // 18
+ visit_OP_GET_DELEGATION_TOKEN();
+ break;
+ case OP_RENEW_DELEGATION_TOKEN: // 19
+ visit_OP_RENEW_DELEGATION_TOKEN();
+ break;
+ case OP_CANCEL_DELEGATION_TOKEN: // 20
+ visit_OP_CANCEL_DELEGATION_TOKEN();
+ break;
+ case OP_UPDATE_MASTER_KEY: // 21
+ visit_OP_UPDATE_MASTER_KEY();
+ break;
+ default:
+ {
+ throw new IOException("Unknown op code " + editsOpCode);
+ }
+ }
+ }
+
+ /**
+ * Loads edits file, uses visitor to process all elements
+ */
+ @Override
+ public void loadEdits() throws IOException {
+
+ try {
+ v.start();
+ v.visitEnclosingElement(EditsElement.EDITS);
+
+ IntToken editsVersionToken = v.visitInt(EditsElement.EDITS_VERSION);
+ editsVersion = editsVersionToken.value;
+ if(!canLoadVersion(editsVersion)) {
+ throw new IOException("Cannot process editLog version " +
+ editsVersionToken.value);
+ }
+
+ FSEditLogOpCodes editsOpCode;
+ do {
+ v.visitEnclosingElement(EditsElement.RECORD);
+
+ ByteToken opCodeToken = v.visitByte(EditsElement.OPCODE);
+ editsOpCode = FSEditLogOpCodes.fromByte(opCodeToken.value);
+
+ v.visitEnclosingElement(EditsElement.DATA);
+
+ visitOpCode(editsOpCode);
+
+ v.leaveEnclosingElement(); // DATA
+ v.leaveEnclosingElement(); // RECORD
+ } while(editsOpCode != FSEditLogOpCodes.OP_INVALID);
+
+ v.leaveEnclosingElement(); // EDITS
+ v.finish();
+ } catch(IOException e) {
+ // Tell the visitor to clean up, then re-throw the exception
+ v.finishAbnormally();
+ throw e;
+ }
+ }
+}
Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitor.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitor.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitor.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitor.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,177 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import java.io.IOException;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+import org.apache.hadoop.hdfs.DeprecatedUTF8;
+
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.ByteToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.ShortToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.IntToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.VIntToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.LongToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.VLongToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.StringUTF8Token;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.StringTextToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.BlobToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.BytesWritableToken;
+import static org.apache.hadoop.hdfs.tools.offlineEditsViewer.Tokenizer.EmptyToken;
+
+/**
+ * An implementation of EditsVisitor can traverse the structure of an
+ * Hadoop edits log and respond to each of the structures within the file.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+abstract public class EditsVisitor {
+
+ private Tokenizer tokenizer;
+
+ public EditsVisitor(Tokenizer tokenizer) {
+ this.tokenizer = tokenizer;
+ }
+
+ /**
+ * Begin visiting the edits log structure. Opportunity to perform
+ * any initialization necessary for the implementing visitor.
+ */
+ abstract void start() throws IOException;
+
+ /**
+ * Finish visiting the edits log structure. Opportunity to perform any
+ * clean up necessary for the implementing visitor.
+ */
+ abstract void finish() throws IOException;
+
+ /**
+ * Finish visiting the edits log structure after an error has occurred
+ * during the processing. Opportunity to perform any clean up necessary
+ * for the implementing visitor.
+ */
+ abstract void finishAbnormally() throws IOException;
+
+ /**
+ * Visit non enclosing element of edits log with specified value.
+ *
+ * @param value a token to visit
+ */
+ abstract Tokenizer.Token visit(Tokenizer.Token value) throws IOException;
+
+ /**
+ * Convenience shortcut method to parse a specific token type
+ */
+ public ByteToken visitByte(EditsElement e) throws IOException {
+ return (ByteToken)visit(tokenizer.read(new ByteToken(e)));
+ }
+
+ /**
+ * Convenience shortcut method to parse a specific token type
+ */
+ public ShortToken visitShort(EditsElement e) throws IOException {
+ return (ShortToken)visit(tokenizer.read(new ShortToken(e)));
+ }
+
+ /**
+ * Convenience shortcut method to parse a specific token type
+ */
+ public IntToken visitInt(EditsElement e) throws IOException {
+ return (IntToken)visit(tokenizer.read(new IntToken(e)));
+ }
+
+ /**
+ * Convenience shortcut method to parse a specific token type
+ */
+ public VIntToken visitVInt(EditsElement e) throws IOException {
+ return (VIntToken)visit(tokenizer.read(new VIntToken(e)));
+ }
+
+ /**
+ * Convenience shortcut method to parse a specific token type
+ */
+ public LongToken visitLong(EditsElement e) throws IOException {
+ return (LongToken)visit(tokenizer.read(new LongToken(e)));
+ }
+
+ /**
+ * Convenience shortcut method to parse a specific token type
+ */
+ public VLongToken visitVLong(EditsElement e) throws IOException {
+ return (VLongToken)visit(tokenizer.read(new VLongToken(e)));
+ }
+
+ /**
+ * Convenience shortcut method to parse a specific token type
+ */
+ public StringUTF8Token visitStringUTF8(EditsElement e) throws IOException {
+ return (StringUTF8Token)visit(tokenizer.read(new StringUTF8Token(e)));
+ }
+
+ /**
+ * Convenience shortcut method to parse a specific token type
+ */
+ public StringTextToken visitStringText(EditsElement e) throws IOException {
+ return (StringTextToken)visit(tokenizer.read(new StringTextToken(e)));
+ }
+
+ /**
+ * Convenience shortcut method to parse a specific token type
+ */
+ public BlobToken visitBlob(EditsElement e, int length) throws IOException {
+ return (BlobToken)visit(tokenizer.read(new BlobToken(e, length)));
+ }
+
+ /**
+ * Convenience shortcut method to parse a specific token type
+ */
+ public BytesWritableToken visitBytesWritable(EditsElement e) throws IOException {
+ return (BytesWritableToken)visit(tokenizer.read(new BytesWritableToken(e)));
+ }
+
+ /**
+ * Convenience shortcut method to parse a specific token type
+ */
+ public EmptyToken visitEmpty(EditsElement e) throws IOException {
+ return (EmptyToken)visit(tokenizer.read(new EmptyToken(e)));
+ }
+
+ /**
+ * Begin visiting an element that encloses another element, such as
+ * the beginning of the list of blocks that comprise a file.
+ *
+ * @param value Token being visited
+ */
+ abstract void visitEnclosingElement(Tokenizer.Token value)
+ throws IOException;
+
+ /**
+ * Convenience shortcut method (it virutally always uses EmptyToken)
+ */
+ void visitEnclosingElement(EditsElement e) throws IOException {
+ visitEnclosingElement(tokenizer.read(new EmptyToken(e)));
+ }
+
+ /**
+ * Leave current enclosing element. Called, for instance, at the end of
+ * processing the blocks that compromise a file.
+ */
+ abstract void leaveEnclosingElement() throws IOException;
+}
Added: hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitorFactory.java
URL: http://svn.apache.org/viewvc/hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitorFactory.java?rev=1062045&view=auto
==============================================================================
--- hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitorFactory.java (added)
+++ hadoop/hdfs/trunk/src/java/org/apache/hadoop/hdfs/tools/offlineEditsViewer/EditsVisitorFactory.java Fri Jan 21 22:08:04 2011
@@ -0,0 +1,57 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hdfs.tools.offlineEditsViewer;
+
+import java.io.IOException;
+import java.util.regex.Pattern;
+
+import org.apache.hadoop.classification.InterfaceAudience;
+import org.apache.hadoop.classification.InterfaceStability;
+
+/**
+ * EditsVisitorFactory for different implementations of EditsVisitor
+ *
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Unstable
+public class EditsVisitorFactory {
+
+ /**
+ * Factory function that creates an EditsVisitor object
+ *
+ * @param filename output filename
+ * @param tokenizer input tokenizer
+ * @return EditsVisitor for appropriate output format (binary, XML etc.)
+ */
+ static public EditsVisitor getEditsVisitor(String filename,
+ String processor,
+ Tokenizer tokenizer,
+ boolean printToScreen) throws IOException {
+
+ if(processor.toLowerCase().equals("xml")) {
+ return new XmlEditsVisitor(filename, tokenizer, printToScreen);
+ } else if(processor.toLowerCase().equals("stats")) {
+ return new StatisticsEditsVisitor(filename, tokenizer, printToScreen);
+ } else if(processor.toLowerCase().equals("binary")) {
+ return new BinaryEditsVisitor(filename, tokenizer, printToScreen);
+ } else {
+ throw new IOException("Unknown proccesor " + processor +
+ " (valid processors: xml, binary, stats)");
+ }
+ }
+}