You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by vj...@apache.org on 2020/10/23 07:22:39 UTC
[hbase] branch branch-2.3 updated: HBASE-25193: Add support for row
prefix and type in the WAL Pretty Printer
This is an automated email from the ASF dual-hosted git repository.
vjasani pushed a commit to branch branch-2.3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2.3 by this push:
new 033b402 HBASE-25193: Add support for row prefix and type in the WAL Pretty Printer
033b402 is described below
commit 033b402524d777e7f58c45ec79e04bdca934bea8
Author: Sandeep Pal <sa...@salesforce.com>
AuthorDate: Fri Oct 23 12:23:36 2020 +0530
HBASE-25193: Add support for row prefix and type in the WAL Pretty Printer
Closes #2556
Signed-off-by: Wellington Chevreuil <wc...@apache.org>
Signed-off-by: Bharath Vissapragada <bh...@apache.org>
Signed-off-by: Duo Zhang <zh...@apache.org>
Signed-off-by: Viraj Jasani <vj...@apache.org>
---
.../store/region/WALProcedurePrettyPrinter.java | 2 +-
.../apache/hadoop/hbase/wal/WALPrettyPrinter.java | 96 +++++++++++++++++-----
2 files changed, 75 insertions(+), 23 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/WALProcedurePrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/WALProcedurePrettyPrinter.java
index a4ed733..0e60709 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/WALProcedurePrettyPrinter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure2/store/region/WALProcedurePrettyPrinter.java
@@ -105,7 +105,7 @@ public class WALProcedurePrettyPrinter extends AbstractHBaseTool {
if (!Bytes.equals(PROC_FAMILY, 0, PROC_FAMILY.length, cell.getFamilyArray(),
cell.getFamilyOffset(), cell.getFamilyLength())) {
// We could have cells other than procedure edits, for example, a flush marker
- WALPrettyPrinter.printCell(out, op, false);
+ WALPrettyPrinter.printCell(out, op, false, false);
continue;
}
long procId = Bytes.toLong(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength());
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java
index fe2e9d4..d274d8b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/wal/WALPrettyPrinter.java
@@ -46,7 +46,7 @@ import org.apache.yetus.audience.InterfaceAudience;
import org.apache.yetus.audience.InterfaceStability;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
-
+import org.apache.hbase.thirdparty.com.google.common.base.Strings;
import org.apache.hbase.thirdparty.com.google.gson.Gson;
import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLine;
import org.apache.hbase.thirdparty.org.apache.commons.cli.CommandLineParser;
@@ -85,7 +85,12 @@ public class WALPrettyPrinter {
// List of tables for filter
private final Set<String> tableSet;
private String region;
+
+ // exact row which needs to be filtered
private String row;
+ // prefix of rows which needs to be filtered
+ private String rowPrefix;
+
private boolean outputOnlyRowKey;
// enable in order to output a single list of transactions from several files
private boolean persistentOutput;
@@ -102,7 +107,7 @@ public class WALPrettyPrinter {
*/
public WALPrettyPrinter() {
this(false, false, -1, new HashSet<>(), null,
- null, false, false, System.out);
+ null, null, false, false, System.out);
}
/**
@@ -126,6 +131,9 @@ public class WALPrettyPrinter {
* @param row
* when not null, serves as a filter; only log entries from this row
* will be printed
+ * @param rowPrefix
+ * when not null, serves as a filter; only log entries with row key
+ * having this prefix will be printed
* @param persistentOutput
* keeps a single list running for multiple files. if enabled, the
* endPersistentOutput() method must be used!
@@ -134,7 +142,7 @@ public class WALPrettyPrinter {
* PrettyPrinter's output.
*/
public WALPrettyPrinter(boolean outputValues, boolean outputJSON, long sequence,
- Set<String> tableSet, String region, String row, boolean outputOnlyRowKey,
+ Set<String> tableSet, String region, String row, String rowPrefix, boolean outputOnlyRowKey,
boolean persistentOutput, PrintStream out) {
this.outputValues = outputValues;
this.outputJSON = outputJSON;
@@ -142,6 +150,7 @@ public class WALPrettyPrinter {
this.tableSet = tableSet;
this.region = region;
this.row = row;
+ this.rowPrefix = rowPrefix;
this.outputOnlyRowKey = outputOnlyRowKey;
this.persistentOutput = persistentOutput;
if (persistentOutput) {
@@ -220,6 +229,17 @@ public class WALPrettyPrinter {
}
/**
+ * sets the rowPrefix key prefix by which output will be filtered
+ *
+ * @param rowPrefix
+ * when not null, serves as a filter; only log entries with rows
+ * having this prefix will be printed
+ */
+ public void setRowPrefixFilter(String rowPrefix) {
+ this.rowPrefix = rowPrefix;
+ }
+
+ /**
* Option to print the row key only in case you just need the row keys from the WAL
*/
public void setOutputOnlyRowKey() {
@@ -339,15 +359,12 @@ public class WALPrettyPrinter {
List<Map<String, Object>> actions = new ArrayList<>();
for (Cell cell : edit.getCells()) {
// add atomic operation to txn
- Map<String, Object> op = new HashMap<>(toStringMap(cell, outputOnlyRowKey));
- if (outputValues) {
- op.put("value", Bytes.toStringBinary(CellUtil.cloneValue(cell)));
- }
- // check row output filter
- if (row == null || ((String) op.get("row")).equals(row)) {
- actions.add(op);
+ Map<String, Object> op =
+ new HashMap<>(toStringMap(cell, outputOnlyRowKey, rowPrefix, row, outputValues));
+ if (op.isEmpty()) {
+ continue;
}
- op.put("total_size_sum", cell.heapSize());
+ actions.add(op);
}
if (actions.isEmpty()) {
continue;
@@ -364,15 +381,19 @@ public class WALPrettyPrinter {
out.print(GSON.toJson(txn));
} else {
// Pretty output, complete with indentation by atomic action
- out.println(String.format(outputTmpl,
+ if (!outputOnlyRowKey) {
+ out.println(String.format(outputTmpl,
txn.get("sequence"), txn.get("table"), txn.get("region"), new Date(writeTime)));
+ }
for (int i = 0; i < actions.size(); i++) {
Map<String, Object> op = actions.get(i);
- printCell(out, op, outputValues);
+ printCell(out, op, outputValues, outputOnlyRowKey);
}
}
- out.println("edit heap size: " + entry.getEdit().heapSize());
- out.println("position: " + log.getPosition());
+ if (!outputOnlyRowKey) {
+ out.println("edit heap size: " + entry.getEdit().heapSize());
+ out.println("position: " + log.getPosition());
+ }
}
} finally {
log.close();
@@ -382,9 +403,17 @@ public class WALPrettyPrinter {
}
}
- public static void printCell(PrintStream out, Map<String, Object> op, boolean outputValues) {
- out.println("row=" + op.get("row") + ", type=" + op.get("type") + ", column=" +
- op.get("family") + ":" + op.get("qualifier"));
+ public static void printCell(PrintStream out, Map<String, Object> op,
+ boolean outputValues, boolean outputOnlyRowKey) {
+ String rowDetails = "row=" + op.get("row");
+ if (outputOnlyRowKey) {
+ out.println(rowDetails);
+ return;
+ }
+
+ rowDetails += ", column=" + op.get("family") + ":" + op.get("qualifier");
+ rowDetails += ", type=" + op.get("type");
+ out.println(rowDetails);
if (op.get("tag") != null) {
out.println(" tag: " + op.get("tag"));
}
@@ -394,11 +423,20 @@ public class WALPrettyPrinter {
out.println("cell total size sum: " + op.get("total_size_sum"));
}
- public static Map<String, Object> toStringMap(Cell cell, boolean printRowKeyOnly) {
+ public static Map<String, Object> toStringMap(Cell cell,
+ boolean printRowKeyOnly, String rowPrefix, String row, boolean outputValues) {
Map<String, Object> stringMap = new HashMap<>();
- stringMap.put("row",
- Bytes.toStringBinary(cell.getRowArray(), cell.getRowOffset(), cell.getRowLength()));
+ String rowKey = Bytes.toStringBinary(cell.getRowArray(),
+ cell.getRowOffset(), cell.getRowLength());
+ // Row and row prefix are mutually options so both cannot be true at the
+ // same time. We can include checks in the same condition
+ // Check if any of the filters are satisfied by the row, if not return empty map
+ if ((!Strings.isNullOrEmpty(rowPrefix) && !rowKey.startsWith(rowPrefix)) ||
+ (!Strings.isNullOrEmpty(row) && !rowKey.equals(row))) {
+ return stringMap;
+ }
+ stringMap.put("row", rowKey);
if (printRowKeyOnly) {
return stringMap;
}
@@ -410,6 +448,7 @@ public class WALPrettyPrinter {
cell.getQualifierLength()));
stringMap.put("timestamp", cell.getTimestamp());
stringMap.put("vlen", cell.getValueLength());
+ stringMap.put("total_size_sum", cell.heapSize());
if (cell.getTagsLength() > 0) {
List<String> tagsString = new ArrayList<>();
Iterator<Tag> tagsIterator = PrivateCellUtil.tagsIterator(cell);
@@ -420,11 +459,14 @@ public class WALPrettyPrinter {
}
stringMap.put("tag", tagsString);
}
+ if (outputValues) {
+ stringMap.put("value", Bytes.toStringBinary(CellUtil.cloneValue(cell)));
+ }
return stringMap;
}
public static Map<String, Object> toStringMap(Cell cell) {
- return toStringMap(cell, false);
+ return toStringMap(cell, false, null, null, false);
}
public static void main(String[] args) throws IOException {
@@ -455,6 +497,7 @@ public class WALPrettyPrinter {
options.addOption("k", "outputOnlyRowKey", false,
"Print only row keys");
options.addOption("w", "row", true, "Row to filter by. Pass row name.");
+ options.addOption("f", "rowPrefix", true, "Row prefix to filter by.");
options.addOption("g", "goto", true, "Position to seek to in the file");
WALPrettyPrinter printer = new WALPrettyPrinter();
@@ -488,8 +531,17 @@ public class WALPrettyPrinter {
printer.setSequenceFilter(Long.parseLong(cmd.getOptionValue("s")));
}
if (cmd.hasOption("w")) {
+ if (cmd.hasOption("f")) {
+ throw new ParseException("Row and Row-prefix cannot be supplied together");
+ }
printer.setRowFilter(cmd.getOptionValue("w"));
}
+ if (cmd.hasOption("f")) {
+ if (cmd.hasOption("w")) {
+ throw new ParseException("Row and Row-prefix cannot be supplied together");
+ }
+ printer.setRowPrefixFilter(cmd.getOptionValue("f"));
+ }
if (cmd.hasOption("g")) {
printer.setPosition(Long.parseLong(cmd.getOptionValue("g")));
}