You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2014/08/08 02:52:27 UTC
git commit: HBASE-11693 Backport HBASE-11026 (Provide option to
filter out all rows in PerformanceEvaluation tool) to 0.94
Repository: hbase
Updated Branches:
refs/heads/0.94 9a8142d11 -> 18ce81375
HBASE-11693 Backport HBASE-11026 (Provide option to filter out all rows in PerformanceEvaluation tool) to 0.94
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/18ce8137
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/18ce8137
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/18ce8137
Branch: refs/heads/0.94
Commit: 18ce81375d875461431c8868600301a6304f48b1
Parents: 9a8142d
Author: Andrew Purtell <ap...@apache.org>
Authored: Thu Aug 7 15:25:05 2014 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Aug 7 15:25:05 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 93 ++++++++++++++++----
.../hadoop/hbase/filter/FilterAllFilter.java | 49 +++++++++++
2 files changed, 123 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/18ce8137/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index cca8210..6d73e45 100644
--- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -54,9 +54,10 @@ import org.apache.hadoop.hbase.client.Put;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.Scan;
+import org.apache.hadoop.hbase.filter.FilterAllFilter;
+import org.apache.hadoop.hbase.filter.FilterList;
import org.apache.hadoop.hbase.filter.PageFilter;
import org.apache.hadoop.hbase.filter.WhileMatchFilter;
-import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.BinaryComparator;
@@ -132,8 +133,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
private DataBlockEncoding blockEncoding = DataBlockEncoding.NONE;
private float sampleRate = 1.0f;
private boolean flushCommits = true;
- private boolean reportLatency = false;
private boolean writeToWAL = true;
+ private boolean reportLatency = false;
+ private boolean filterAll = false;
private int presplitRegions = 0;
private static final Path PERF_EVAL_DIR = new Path("performance_evaluation");
@@ -147,7 +149,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
"clients=(\\d+),\\s+" +
"flushCommits=(\\w+),\\s+" +
"writeToWAL=(\\w+),\\s+" +
- "reportLatency=(\\w+)");
+ "reportLatency=(\\w+),\\s+" +
+ "filterAll=(\\w+)");
/**
* Enum for map metrics. Keep it out here rather than inside in the Map
@@ -225,11 +228,13 @@ public class PerformanceEvaluation extends Configured implements Tool {
private boolean flushCommits = false;
private boolean writeToWAL = true;
private boolean reportLatency = false;
+ private boolean filterAll = false;
public PeInputSplit() {}
public PeInputSplit(int startRow, int rows, int totalRows, float sampleRate,
- int clients, boolean flushCommits, boolean writeToWAL, boolean reportLatency) {
+ int clients, boolean flushCommits, boolean writeToWAL, boolean reportLatency,
+ boolean filterAll) {
this.startRow = startRow;
this.rows = rows;
this.totalRows = totalRows;
@@ -238,6 +243,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.flushCommits = flushCommits;
this.writeToWAL = writeToWAL;
this.reportLatency = reportLatency;
+ this.filterAll = filterAll;
}
@Override
@@ -250,6 +256,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.flushCommits = in.readBoolean();
this.writeToWAL = in.readBoolean();
this.reportLatency = in.readBoolean();
+ this.filterAll = in.readBoolean();
}
@Override
@@ -262,6 +269,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
out.writeBoolean(flushCommits);
out.writeBoolean(writeToWAL);
out.writeBoolean(reportLatency);
+ out.writeBoolean(filterAll);
}
@Override
@@ -305,6 +313,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
public boolean isReportLatency() {
return reportLatency;
}
+
+ public boolean isFilterAll() {
+ return filterAll;
+ }
}
/**
@@ -340,6 +352,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
boolean flushCommits = Boolean.parseBoolean(m.group(6));
boolean writeToWAL = Boolean.parseBoolean(m.group(7));
boolean reportLatency = Boolean.parseBoolean(m.group(8));
+ boolean filterAll = Boolean.parseBoolean(m.group(9));
LOG.debug("split["+ splitList.size() + "] " +
" startRow=" + startRow +
@@ -349,11 +362,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
" clients=" + clients +
" flushCommits=" + flushCommits +
" writeToWAL=" + writeToWAL +
- " reportLatency=" + reportLatency);
+ " reportLatency=" + reportLatency +
+ " filterAll=" + filterAll);
PeInputSplit newSplit =
new PeInputSplit(startRow, rows, totalRows, sampleRate, clients,
- flushCommits, writeToWAL, reportLatency);
+ flushCommits, writeToWAL, reportLatency, filterAll);
splitList.add(newSplit);
}
}
@@ -475,7 +489,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
long elapsedTime = this.pe.runOneClient(this.cmd, value.getStartRow(),
value.getRows(), value.getTotalRows(), value.getSampleRate(),
value.isFlushCommits(), value.isWriteToWAL(), value.isReportLatency(),
- status);
+ value.isFilterAll(), status);
// Collect how much time the thing took. Report as map output and
// to the ELAPSED_TIME counter.
context.getCounter(Counter.ELAPSED_TIME).increment(elapsedTime);
@@ -580,7 +594,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
try {
long elapsedTime = pe.runOneClient(cmd, index * perClientRows,
perClientRows, R, sampleRate, flushCommits, writeToWAL,
- reportLatency, new Status() {
+ reportLatency, filterAll, new Status() {
public void setStatus(final String msg) throws IOException {
LOG.info("client-" + getName() + " " + msg);
}
@@ -679,7 +693,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
", clients=" + this.N +
", flushCommits=" + this.flushCommits +
", writeToWAL=" + this.writeToWAL +
- ", reportLatency=" + this.reportLatency;
+ ", reportLatency=" + this.reportLatency +
+ ", filterAll=" + this.filterAll;
int hash = h.hash(Bytes.toBytes(s));
m.put(hash, s);
}
@@ -733,11 +748,13 @@ public class PerformanceEvaluation extends Configured implements Tool {
private boolean flushCommits;
private boolean writeToWAL = true;
private boolean reportLatency;
+ private boolean filterAll;
TestOptions() {}
TestOptions(int startRow, int perClientRunRows, int totalRows, float sampleRate,
- byte[] tableName, boolean flushCommits, boolean writeToWAL, boolean reportLatency) {
+ byte[] tableName, boolean flushCommits, boolean writeToWAL, boolean reportLatency,
+ boolean filterAll) {
this.startRow = startRow;
this.perClientRunRows = perClientRunRows;
this.totalRows = totalRows;
@@ -746,6 +763,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.flushCommits = flushCommits;
this.writeToWAL = writeToWAL;
this.reportLatency = reportLatency;
+ this.filterAll = filterAll;
}
public int getStartRow() {
@@ -779,6 +797,10 @@ public class PerformanceEvaluation extends Configured implements Tool {
public boolean isReportLatency() {
return reportLatency;
}
+
+ public boolean isFilterAll() {
+ return filterAll;
+ }
}
/*
@@ -807,6 +829,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
protected boolean flushCommits;
protected boolean writeToWAL;
protected boolean reportlatency;
+ protected boolean filterAll;
/**
* Note that all subclasses of this class must provide a public contructor
@@ -825,6 +848,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
this.flushCommits = options.isFlushCommits();
this.writeToWAL = options.isWriteToWAL();
this.reportlatency = options.isReportLatency();
+ this.filterAll = options.isFilterAll();
}
private String generateStatus(final int sr, final int i, final int lr) {
@@ -900,7 +924,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
void testRow(final int i) throws IOException {
Scan scan = new Scan(getRandomRow(this.rand, this.totalRows));
scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
- scan.setFilter(new WhileMatchFilter(new PageFilter(120)));
+ FilterList list = new FilterList();
+ if (this.filterAll) {
+ list.addFilter(new FilterAllFilter());
+ }
+ list.addFilter(new WhileMatchFilter(new PageFilter(120)));
+ scan.setFilter(list);
ResultScanner s = this.table.getScanner(scan);
for (Result rr; (rr = s.next()) != null;) ;
s.close();
@@ -925,6 +954,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
Pair<byte[], byte[]> startAndStopRow = getStartAndStopRow();
Scan scan = new Scan(startAndStopRow.getFirst(), startAndStopRow.getSecond());
scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
+ if (this.filterAll) {
+ scan.setFilter(new FilterAllFilter());
+ }
ResultScanner s = this.table.getScanner(scan);
int count = 0;
for (Result rr; (rr = s.next()) != null;) {
@@ -1022,6 +1054,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
if (i % everyN == 0) {
Get get = new Get(getRandomRow(this.rand, this.totalRows));
get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
+ if (this.filterAll) {
+ get.setFilter(new FilterAllFilter());
+ }
long start = System.nanoTime();
this.table.get(get);
if (this.reportLatency) {
@@ -1087,6 +1122,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
if (this.testScanner == null) {
Scan scan = new Scan(format(this.startRow));
scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
+ if (this.filterAll) {
+ scan.setFilter(new FilterAllFilter());
+ }
this.testScanner = table.getScanner(scan);
}
testScanner.next();
@@ -1103,6 +1141,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
void testRow(final int i) throws IOException {
Get get = new Get(format(i));
get.addColumn(FAMILY_NAME, QUALIFIER_NAME);
+ if (this.filterAll) {
+ get.setFilter(new FilterAllFilter());
+ }
table.get(get);
}
}
@@ -1144,13 +1185,17 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
protected Scan constructScan(byte[] valuePrefix) throws IOException {
- Filter filter = new SingleColumnValueFilter(
- FAMILY_NAME, QUALIFIER_NAME, CompareFilter.CompareOp.EQUAL,
- new BinaryComparator(valuePrefix)
- );
Scan scan = new Scan();
scan.addColumn(FAMILY_NAME, QUALIFIER_NAME);
- scan.setFilter(filter);
+ FilterList list = new FilterList();
+ list.addFilter(new SingleColumnValueFilter(
+ FAMILY_NAME, QUALIFIER_NAME, CompareFilter.CompareOp.EQUAL,
+ new BinaryComparator(valuePrefix)
+ ));
+ if (this.filterAll) {
+ list.addFilter(new FilterAllFilter());
+ }
+ scan.setFilter(list);
return scan;
}
}
@@ -1223,7 +1268,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
long runOneClient(final Class<? extends Test> cmd, final int startRow,
final int perClientRunRows, final int totalRows, final float sampleRate,
boolean flushCommits, boolean writeToWAL, boolean reportLatency,
- final Status status)
+ boolean filterAll, final Status status)
throws IOException {
status.setStatus("Start " + cmd + " at offset " + startRow + " for " +
perClientRunRows + " rows");
@@ -1231,7 +1276,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
Test t = null;
TestOptions options = new TestOptions(startRow, perClientRunRows, totalRows,
- sampleRate, getTableDescriptor().getName(), flushCommits, writeToWAL, reportLatency);
+ sampleRate, getTableDescriptor().getName(), flushCommits, writeToWAL, reportLatency,
+ filterAll);
try {
Constructor<? extends Test> constructor = cmd.getDeclaredConstructor(
Configuration.class, TestOptions.class, Status.class);
@@ -1264,7 +1310,7 @@ public class PerformanceEvaluation extends Configured implements Tool {
admin = new HBaseAdmin(getConf());
checkTable(admin);
runOneClient(cmd, 0, this.R, this.R, this.sampleRate, this.flushCommits,
- this.writeToWAL, this.writeToWAL, status);
+ this.writeToWAL, this.reportLatency, this.filterAll, status);
} catch (Exception e) {
LOG.error("Failed", e);
} finally {
@@ -1339,6 +1385,9 @@ public class PerformanceEvaluation extends Configured implements Tool {
System.err.println(" writeToWAL Set writeToWAL on puts. Default: True");
System.err.println(" presplit Create presplit table. Recommended for accurate perf " +
"analysis (see guide). Default: disabled");
+ System.err.println(" filterAll Helps to filter out all the rows on the server side"
+ + " there by not returning any thing back to the client. Helps to check the server side"
+ + " performance. Uses FilterAllFilter internally. ");
System.err.println(" latency Set to report operation latencies. " +
"Currently only supported by randomRead test. Default: False");
System.err.println();
@@ -1458,6 +1507,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
continue;
}
+ final String filterOutAll = "--filterAll";
+ if (cmd.startsWith(filterOutAll)) {
+ this.filterAll = true;
+ continue;
+ }
+
Class<? extends Test> cmdClass = determineCommandClass(cmd);
if (cmdClass != null) {
getArgs(i + 1, args);
http://git-wip-us.apache.org/repos/asf/hbase/blob/18ce8137/src/test/java/org/apache/hadoop/hbase/filter/FilterAllFilter.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/filter/FilterAllFilter.java b/src/test/java/org/apache/hadoop/hbase/filter/FilterAllFilter.java
new file mode 100644
index 0000000..e883bd0
--- /dev/null
+++ b/src/test/java/org/apache/hadoop/hbase/filter/FilterAllFilter.java
@@ -0,0 +1,49 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.filter;
+
+import java.io.DataInput;
+import java.io.DataOutput;
+import java.io.IOException;
+
+import org.apache.hadoop.hbase.KeyValue;
+
+public class FilterAllFilter extends FilterBase {
+
+ public FilterAllFilter() {
+ }
+
+ @Override
+ public ReturnCode filterKeyValue(KeyValue ignored) {
+ return ReturnCode.SKIP;
+ }
+
+ @Override
+ public boolean filterRow() {
+ return true;
+ }
+
+ @Override
+ public void readFields(DataInput in) throws IOException {
+ }
+
+ @Override
+ public void write(DataOutput out) throws IOException {
+ }
+
+}
\ No newline at end of file