You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 04:03:55 UTC
svn commit: r1181382 - in
/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal:
HLog.java SequenceFileLogReader.java
Author: nspiegelberg
Date: Tue Oct 11 02:03:54 2011
New Revision: 1181382
URL: http://svn.apache.org/viewvc?rev=1181382&view=rev
Log:
HBASE-2889 : HFile Pretty Printer
Summary:
Added to help us debug EOF problems that we've been seeing
Test Plan:
mvn clean install
bin/hbase org.apache.hadoop.hbase.regionserver.wal.HLog --dump /debug/1.log >
badfile.out 2>&1
DiffCamp Revision: 161534
Reviewed By: kannan
CC: kannan
Revert Plan:
OK
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1181382&r1=1181381&r2=1181382&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Tue Oct 11 02:03:54 2011
@@ -1775,9 +1775,54 @@ public class HLog implements Syncable {
return this.actionListeners.remove(list);
}
+ public static final long FIXED_OVERHEAD = ClassSize.align(
+ ClassSize.OBJECT + (5 * ClassSize.REFERENCE) +
+ ClassSize.ATOMIC_INTEGER + Bytes.SIZEOF_INT + (3 * Bytes.SIZEOF_LONG));
+
private static void usage() {
- System.err.println("Usage: java org.apache.hbase.HLog" +
- " {--dump <logfile>... | --split <logdir>...}");
+ System.err.println("Usage: HLog <ARGS>");
+ System.err.println("Arguments:");
+ System.err.println(" --dump Dump textual representation of passed one or more files");
+ System.err.println(" For example: HLog --dump hdfs://example.com:9000/hbase/.logs/MACHINE/LOGFILE");
+ System.err.println(" --split Split the passed directory of WAL logs");
+ System.err.println(" For example: HLog --split hdfs://example.com:9000/hbase/.logs/DIR");
+ }
+
+ private static void dump(final Configuration conf, final Path p)
+ throws IOException {
+ FileSystem fs = FileSystem.get(conf);
+ if (!fs.exists(p)) {
+ throw new FileNotFoundException(p.toString());
+ }
+ if (!fs.isFile(p)) {
+ throw new IOException(p + " is not a file");
+ }
+ Reader log = getReader(fs, p, conf);
+ try {
+ int count = 0;
+ HLog.Entry entry;
+ while ((entry = log.next()) != null) {
+ System.out.println("#" + count + ", pos=" + log.getPosition() + " " +
+ entry.toString());
+ count++;
+ }
+ } finally {
+ log.close();
+ }
+ }
+
+ private static void split(final Configuration conf, final Path p)
+ throws IOException {
+ FileSystem fs = FileSystem.get(conf);
+ if (!fs.exists(p)) {
+ throw new FileNotFoundException(p.toString());
+ }
+ final Path baseDir = new Path(conf.get(HConstants.HBASE_DIR));
+ final Path oldLogDir = new Path(baseDir, HConstants.HREGION_OLDLOGDIR_NAME);
+ if (!fs.getFileStatus(p).isDir()) {
+ throw new IOException(p + " is not a directory");
+ }
+ splitLog(baseDir, p, oldLogDir, fs, conf);
}
/**
@@ -1803,38 +1848,19 @@ public class HLog implements Syncable {
}
}
Configuration conf = HBaseConfiguration.create();
- FileSystem fs = FileSystem.get(conf);
- final Path baseDir = new Path(conf.get(HConstants.HBASE_DIR));
- final Path oldLogDir = new Path(baseDir, HConstants.HREGION_OLDLOGDIR_NAME);
for (int i = 1; i < args.length; i++) {
+ try {
Path logPath = new Path(args[i]);
- if (!fs.exists(logPath)) {
- throw new FileNotFoundException(args[i] + " does not exist");
- }
if (dump) {
- if (!fs.isFile(logPath)) {
- throw new IOException(args[i] + " is not a file");
- }
- Reader log = getReader(fs, logPath, conf);
- try {
- HLog.Entry entry;
- while ((entry = log.next()) != null) {
- System.out.println(entry.toString());
- }
- } finally {
- log.close();
- }
+ dump(conf, logPath);
} else {
- if (!fs.getFileStatus(logPath).isDir()) {
- throw new IOException(args[i] + " is not a directory");
- }
- splitLog(baseDir, logPath, oldLogDir, fs, conf);
+ split(conf, logPath);
+ }
+ } catch (Throwable t) {
+ t.printStackTrace();
+ System.exit(-1);
}
}
}
- public static final long FIXED_OVERHEAD = ClassSize.align(
- ClassSize.OBJECT + (5 * ClassSize.REFERENCE) +
- ClassSize.ATOMIC_INTEGER + Bytes.SIZEOF_INT + (3 * Bytes.SIZEOF_LONG));
-
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java?rev=1181382&r1=1181381&r2=1181382&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/SequenceFileLogReader.java Tue Oct 11 02:03:54 2011
@@ -21,6 +21,9 @@
package org.apache.hadoop.hbase.regionserver.wal;
import java.io.IOException;
+import java.lang.Class;
+import java.lang.reflect.Constructor;
+import java.lang.reflect.Field;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.FSDataInputStream;
@@ -29,6 +32,7 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.hbase.regionserver.wal.HLog;
import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
import org.apache.hadoop.io.SequenceFile;
+import org.mortbay.log.Log;
public class SequenceFileLogReader implements HLog.Reader {
@@ -93,6 +97,10 @@ public class SequenceFileLogReader imple
Configuration conf;
WALReader reader;
+ // Needed logging exceptions
+ Path path;
+ int edit = 0;
+ long entryStart = 0;
public SequenceFileLogReader() { }
@@ -100,12 +108,17 @@ public class SequenceFileLogReader imple
public void init(FileSystem fs, Path path, Configuration conf)
throws IOException {
this.conf = conf;
+ this.path = path;
reader = new WALReader(fs, path, conf);
}
@Override
public void close() throws IOException {
- reader.close();
+ try {
+ reader.close();
+ } catch (IOException ioe) {
+ throw addFileInfoToException(ioe);
+ }
}
@Override
@@ -115,21 +128,30 @@ public class SequenceFileLogReader imple
@Override
public HLog.Entry next(HLog.Entry reuse) throws IOException {
- if (reuse == null) {
+ this.entryStart = this.reader.getPosition();
+ HLog.Entry e = reuse;
+ if (e == null) {
HLogKey key = HLog.newKey(conf);
WALEdit val = new WALEdit();
- if (reader.next(key, val)) {
- return new HLog.Entry(key, val);
- }
- } else if (reader.next(reuse.getKey(), reuse.getEdit())) {
- return reuse;
+ e = new HLog.Entry(key, val);
}
- return null;
+ boolean b = false;
+ try {
+ b = this.reader.next(e.getKey(), e.getEdit());
+ } catch (IOException ioe) {
+ throw addFileInfoToException(ioe);
+ }
+ edit++;
+ return b? e: null;
}
@Override
public void seek(long pos) throws IOException {
- reader.seek(pos);
+ try {
+ reader.seek(pos);
+ } catch (IOException ioe) {
+ throw addFileInfoToException(ioe);
+ }
}
@Override
@@ -137,4 +159,36 @@ public class SequenceFileLogReader imple
return reader.getPosition();
}
+ private IOException addFileInfoToException(final IOException ioe)
+ throws IOException {
+ long pos = -1;
+ try {
+ pos = getPosition();
+ } catch (IOException e) {
+ Log.warn("Failed getting position to add to throw", e);
+ }
+
+ // See what SequenceFile.Reader thinks is the end of the file
+ long end = Long.MAX_VALUE;
+ try {
+ Field fEnd = SequenceFile.Reader.class.getDeclaredField("end");
+ fEnd.setAccessible(true);
+ end = fEnd.getLong(this.reader);
+ } catch(Exception e) { /* reflection fail. keep going */ }
+
+ String msg = (this.path == null? "": this.path.toString()) +
+ ", entryStart=" + entryStart + ", pos=" + pos +
+ ((end == Long.MAX_VALUE) ? "" : ", end=" + end) +
+ ", edit=" + this.edit;
+
+ // Enhance via reflection so we don't change the original class type
+ try {
+ return (IOException) ioe.getClass()
+ .getConstructor(String.class)
+ .newInstance(msg)
+ .initCause(ioe);
+ } catch(Exception e) { /* reflection fail. keep going */ }
+
+ return ioe;
+ }
}