You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 04:23:58 UTC
svn commit: r1181596 - in
/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase: KeyValue.java
regionserver/wal/HLog.java regionserver/wal/HLogKey.java
regionserver/wal/HLogPrettyPrinter.java
Author: nspiegelberg
Date: Tue Oct 11 02:23:58 2011
New Revision: 1181596
URL: http://svn.apache.org/viewvc?rev=1181596&view=rev
Log:
HLog Pretty Printer Implementation (HBASE-3968)
Summary:
A pretty printer for write-ahead-logs in HBase, targeted for debugging
purposes. Includes options for output to JSON as well as pretty formatting.
Also implements filtering by region, sequence, or row, all handled by command
line switches.
Can be used on the command line or from within HBase itself.
Test Plan:
1. Build HBase including the pretty printer.
2. Start HBase and create a table with multiple rows.
3. Run `HBasePrettyPrinter -p <logfilepath>` and ensure sanity of output.
4. Try filtering by region using '-r'
5. Try filtering by sequence using '-s'
6. Try filtering by row using '-w'
7. Get JSON output for a desired query using '-j' and pipe this output to a PHP
script that runs 'json_decode' on its input. Sanity check that the script
correctly decoded valid JSON.
Reviewed By: nspiegelberg
Reviewers: nspiegelberg
CC: nspiegelberg, riley, hbase@lists
Revert Plan: Simple removal of the new class is safe; nothing depends on it.
Differential Revision: 274539
Task ID: 592510
Added:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java
Modified:
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java?rev=1181596&r1=1181595&r2=1181596&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/KeyValue.java Tue Oct 11 02:23:58 2011
@@ -24,6 +24,8 @@ import java.io.DataOutput;
import java.io.IOException;
import java.nio.ByteBuffer;
import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Map;
import com.google.common.primitives.Longs;
import org.apache.commons.logging.Log;
@@ -626,6 +628,23 @@ public class KeyValue implements Writabl
qualifier + "/" + timestamp + "/" + Type.codeToType(type);
}
+ /**
+ * Produces a string map for this key/value pair. Useful for programmatic use
+ * and manipulation of the data stored in an HLogKey, for example, printing
+ * as JSON. Values are left out due to their tendency to be large. If needed,
+ * they can be added manually.
+ *
+ * @return the Map<String,?> containing data from this key
+ */
+ public Map<String, Object> toStringMap() {
+ Map<String, Object> stringMap = new HashMap<String, Object>();
+ stringMap.put("row", Bytes.toStringBinary(getRow()));
+ stringMap.put("family", Bytes.toStringBinary(getFamily()));
+ stringMap.put("qualifier", Bytes.toStringBinary(getQualifier()));
+ stringMap.put("timestamp", getTimestamp());
+ return stringMap;
+ }
+
//---------------------------------------------------------------------------
//
// Public Member Accessors
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java?rev=1181596&r1=1181595&r2=1181596&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLog.java Tue Oct 11 02:23:58 2011
@@ -33,6 +33,7 @@ import java.lang.reflect.Method;
import java.net.URLEncoder;
import java.text.ParseException;
import java.util.ArrayList;
+import java.util.Arrays;
import java.util.Collections;
import java.util.HashMap;
import java.util.LinkedList;
@@ -1922,30 +1923,25 @@ public class HLog implements Syncable {
usage();
System.exit(-1);
}
- boolean dump = true;
- if (args[0].compareTo("--dump") != 0) {
- if (args[0].compareTo("--split") == 0) {
- dump = false;
-
- } else {
- usage();
- System.exit(-1);
- }
- }
- Configuration conf = HBaseConfiguration.create();
- for (int i = 1; i < args.length; i++) {
- try {
- Path logPath = new Path(args[i]);
- if (dump) {
- dump(conf, logPath);
- } else {
- split(conf, logPath);
- }
- } catch (Throwable t) {
- t.printStackTrace();
- System.exit(-1);
+ // either dump using the HLogPrettyPrinter or split, depending on args
+ if (args[0].compareTo("--dump") == 0) {
+ HLogPrettyPrinter.run(Arrays.copyOfRange(args, 1, args.length));
+ } else if (args[0].compareTo("--split") == 0) {
+ Configuration conf = HBaseConfiguration.create();
+ for (int i = 1; i < args.length; i++) {
+ try {
+ conf.set("fs.default.name", args[i]);
+ conf.set("fs.defaultFS", args[i]);
+ Path logPath = new Path(args[i]);
+ split(conf, logPath);
+ } catch (Throwable t) {
+ t.printStackTrace(System.err);
+ System.exit(-1);
+ }
}
+ } else {
+ usage();
+ System.exit(-1);
}
}
-
}
Modified: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java?rev=1181596&r1=1181595&r2=1181596&view=diff
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java (original)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogKey.java Tue Oct 11 02:23:58 2011
@@ -23,6 +23,8 @@ import java.io.DataInput;
import java.io.DataOutput;
import java.io.EOFException;
import java.io.IOException;
+import java.util.HashMap;
+import java.util.Map;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.util.Bytes;
@@ -123,6 +125,21 @@ public class HLogKey implements Writable
logSeqNum;
}
+ /**
+ * Produces a string map for this key. Useful for programmatic use and
+ * manipulation of the data stored in an HLogKey, for example, printing
+ * as JSON.
+ *
+ * @return a Map containing data from this key
+ */
+ public Map<String, Object> toStringMap() {
+ Map<String, Object> stringMap = new HashMap<String, Object>();
+ stringMap.put("table", Bytes.toStringBinary(tablename));
+ stringMap.put("region", Bytes.toStringBinary(regionName));
+ stringMap.put("sequence", logSeqNum);
+ return stringMap;
+ }
+
@Override
public boolean equals(Object obj) {
if (this == obj) {
Added: hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java?rev=1181596&view=auto
==============================================================================
--- hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java (added)
+++ hbase/branches/0.89/src/main/java/org/apache/hadoop/hbase/regionserver/wal/HLogPrettyPrinter.java Tue Oct 11 02:23:58 2011
@@ -0,0 +1,349 @@
+package org.apache.hadoop.hbase.regionserver.wal;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.io.PrintStream;
+import java.util.Date;
+import java.util.List;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.CommandLineParser;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.cli.ParseException;
+import org.apache.commons.cli.PosixParser;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.regionserver.wal.HLog.Reader;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.codehaus.jettison.json.JSONArray;
+import org.codehaus.jettison.json.JSONException;
+import org.codehaus.jettison.json.JSONObject;
+
+/**
+ * HLogPrettyPrinter prints the contents of a given HLog with a variety of
+ * options affecting formatting and extent of content.
+ *
+ * It targets two usage cases: pretty printing for ease of debugging directly by
+ * humans, and JSON output for consumption by monitoring and/or maintenance
+ * scripts.
+ *
+ * It can filter by row, region, or sequence id.
+ *
+ * It can also toggle output of values.
+ *
+ */
+public class HLogPrettyPrinter {
+ private boolean outputValues;
+ private boolean outputJSON;
+ // The following enable filtering by sequence, region, and row, respectively
+ private long sequence;
+ private String region;
+ private String row;
+ // enable in order to output a single list of transactions from several files
+ private boolean persistentOutput;
+ private boolean firstTxn;
+ // useful for programatic capture of JSON output
+ private PrintStream out;
+
+ /**
+ * Basic constructor that simply initializes values to reasonable defaults.
+ */
+ public HLogPrettyPrinter() {
+ outputValues = false;
+ outputJSON = false;
+ sequence = -1;
+ region = null;
+ row = null;
+ persistentOutput = false;
+ firstTxn = true;
+ out = System.out;
+ }
+
+ /**
+ * Fully specified constructor.
+ *
+ * @param outputValues
+ * when true, enables output of values along with other log
+ * information
+ * @param outputJSON
+ * when true, enables output in JSON format rather than a
+ * "pretty string"
+ * @param sequence
+ * when nonnegative, serves as a filter; only log entries with this
+ * sequence id will be printed
+ * @param region
+ * when not null, serves as a filter; only log entries from this
+ * region will be printed
+ * @param row
+ * when not null, serves as a filter; only log entries from this row
+ * will be printed
+ * @param persistentOutput
+ * keeps a single list running for multiple files. if enabled, the
+ * endPersistentOutput() method must be used!
+ * @param out
+ * Specifies an alternative to stdout for the destination of this
+ * PrettyPrinter's output.
+ */
+ public HLogPrettyPrinter(boolean outputValues, boolean outputJSON,
+ long sequence, String region, String row, boolean persistentOutput,
+ PrintStream out) {
+ this.outputValues = outputValues;
+ this.outputJSON = outputJSON;
+ this.sequence = sequence;
+ this.region = region;
+ this.row = row;
+ this.persistentOutput = persistentOutput;
+ if (persistentOutput) {
+ beginPersistentOutput();
+ }
+ this.out = out;
+ this.firstTxn = true;
+ }
+
+ /**
+ * turns value output on
+ */
+ public void enableValues() {
+ outputValues = true;
+ }
+
+ /**
+ * turns value output off
+ */
+ public void disableValues() {
+ outputValues = false;
+ }
+
+ /**
+ * turns JSON output on
+ */
+ public void enableJSON() {
+ outputJSON = true;
+ }
+
+ /**
+ * turns JSON output off, and turns on "pretty strings" for human consumption
+ */
+ public void disableJSON() {
+ outputJSON = false;
+ }
+
+ /**
+ * sets the region by which output will be filtered
+ *
+ * @param sequence
+ * when nonnegative, serves as a filter; only log entries with this
+ * sequence id will be printed
+ */
+ public void setSequenceFilter(long sequence) {
+ this.sequence = sequence;
+ }
+
+ /**
+ * sets the region by which output will be filtered
+ *
+ * @param region
+ * when not null, serves as a filter; only log entries from this
+ * region will be printed
+ */
+ public void setRegionFilter(String region) {
+ this.region = region;
+ }
+
+ /**
+ * sets the region by which output will be filtered
+ *
+ * @param row
+ * when not null, serves as a filter; only log entries from this row
+ * will be printed
+ */
+ public void setRowFilter(String row) {
+ this.row = row;
+ }
+
+ /**
+ * enables output as a single, persistent list. at present, only relevant in
+ * the case of JSON output.
+ */
+ public void beginPersistentOutput() {
+ if (persistentOutput)
+ return;
+ persistentOutput = true;
+ firstTxn = true;
+ if (outputJSON)
+ out.print("[");
+ }
+
+ /**
+ * ends output of a single, persistent list. at present, only relevant in the
+ * case of JSON output.
+ */
+ public void endPersistentOutput() {
+ if (!persistentOutput)
+ return;
+ persistentOutput = false;
+ if (outputJSON)
+ out.print("]");
+ }
+
+ /**
+ * reads a log file and outputs its contents, one transaction at a time, as
+ * specified by the currently configured options
+ *
+ * @param conf
+ * the HBase configuration relevant to this log file
+ * @param p
+ * the path of the log file to be read
+ * @throws IOException
+ * may be unable to access the configured filesystem or requested
+ * file.
+ */
+ public void processFile(final Configuration conf, final Path p)
+ throws IOException {
+ FileSystem fs = FileSystem.get(conf);
+ if (!fs.exists(p)) {
+ throw new FileNotFoundException(p.toString());
+ }
+ if (!fs.isFile(p)) {
+ throw new IOException(p + " is not a file");
+ }
+ if (outputJSON && !persistentOutput) {
+ out.print("[");
+ firstTxn = true;
+ }
+ Reader log = HLog.getReader(fs, p, conf);
+ try {
+ HLog.Entry entry;
+ while ((entry = log.next()) != null) {
+ HLogKey key = entry.getKey();
+ WALEdit edit = entry.getEdit();
+ // begin building a transaction structure
+ JSONObject txn = new JSONObject(key.toStringMap());
+ // check output filters
+ if (sequence >= 0 && ((Long) txn.get("sequence")) != sequence)
+ continue;
+ if (region != null && !((String) txn.get("region")).equals(region))
+ continue;
+ // initialize list into which we will store atomic actions
+ JSONArray actions = new JSONArray();
+ for (KeyValue kv : edit.getKeyValues()) {
+ // add atomic operation to txn
+ JSONObject op = new JSONObject(kv.toStringMap());
+ if (outputValues)
+ op.put("value", Bytes.toStringBinary(kv.getValue()));
+ if (row == null || ((String) op.get("row")).equals(row))
+ actions.put(op);
+ }
+ if (actions.length() == 0)
+ continue;
+ txn.put("actions", actions);
+ if (outputJSON) {
+ // JSON output is a straightforward "toString" on the txn object
+ if (firstTxn)
+ firstTxn = false;
+ else
+ out.print(",");
+ out.print(txn);
+ } else {
+ // Pretty output, complete with indentation by atomic action
+ out.println("Sequence " + txn.getLong("sequence") + " "
+ + "from region " + txn.getString("region") + " " + "in table "
+ + txn.getString("table"));
+ for (int i = 0; i < actions.length(); i++) {
+ JSONObject op = actions.getJSONObject(i);
+ out.println(" Put action:");
+ out.println(" row: " + op.getString("row"));
+ out.println(" column: " + op.getString("family") + ":"
+ + op.getString("qualifier"));
+ out.println(" at time: "
+ + (new Date(op.getLong("timestamp"))));
+ if (outputValues)
+ out.println(" value: " + op.get("value"));
+ }
+ }
+ }
+ } catch (JSONException e) {
+ e.printStackTrace();
+ } finally {
+ log.close();
+ }
+ if (outputJSON && !persistentOutput) {
+ out.print("]");
+ }
+ }
+
+ /**
+ * Pass one or more log file names and formatting options and it will dump out
+ * a text version of the contents on <code>stdout</code>.
+ *
+ * @param args
+ * Command line arguments
+ * @throws IOException
+ * Thrown upon file system errors etc.
+ * @throws ParseException
+ * Thrown if command-line parsing fails.
+ */
+ public static void run(String[] args) throws IOException {
+ // create options
+ Options options = new Options();
+ options.addOption("h", "help", false, "Output help message");
+ options.addOption("j", "json", false, "Output JSON");
+ options.addOption("p", "printvals", false, "Print values");
+ options.addOption("r", "region", true,
+ "Region to filter by. Pass region name; e.g. '.META.,,1'");
+ options.addOption("s", "sequence", true,
+ "Sequence to filter by. Pass sequence number.");
+ options.addOption("w", "row", true, "Row to filter by. Pass row name.");
+
+ HLogPrettyPrinter printer = new HLogPrettyPrinter();
+ CommandLineParser parser = new PosixParser();
+ List files = null;
+ try {
+ CommandLine cmd = parser.parse(options, args);
+ files = cmd.getArgList();
+ if (files.size() == 0 || cmd.hasOption("h")) {
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("HFile filename(s) ", options, true);
+ System.exit(-1);
+ }
+ // configure the pretty printer using command line options
+ if (cmd.hasOption("p"))
+ printer.enableValues();
+ if (cmd.hasOption("j"))
+ printer.enableJSON();
+ if (cmd.hasOption("r"))
+ printer.setRegionFilter(cmd.getOptionValue("r"));
+ if (cmd.hasOption("s"))
+ printer.setSequenceFilter(Long.parseLong(cmd.getOptionValue("s")));
+ if (cmd.hasOption("w"))
+ printer.setRowFilter(cmd.getOptionValue("w"));
+ } catch (ParseException e) {
+ e.printStackTrace();
+ HelpFormatter formatter = new HelpFormatter();
+ formatter.printHelp("HFile filename(s) ", options, true);
+ System.exit(-1);
+ }
+ // get configuration, file system, and process the given files
+ Configuration conf = HBaseConfiguration.create();
+ conf.set("fs.defaultFS",
+ conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
+ conf.set("fs.default.name",
+ conf.get(org.apache.hadoop.hbase.HConstants.HBASE_DIR));
+ // begin output
+ printer.beginPersistentOutput();
+ for (Object f : files) {
+ Path file = new Path((String) f);
+ FileSystem fs = file.getFileSystem(conf);
+ if (!fs.exists(file)) {
+ System.err.println("ERROR, file doesnt exist: " + file);
+ return;
+ }
+ printer.processFile(conf, file);
+ }
+ printer.endPersistentOutput();
+ }
+}