You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2014/08/08 00:24:35 UTC
[1/2] git commit: HBASE-11690 Backport HBASE-5934 (Add the ability
for Performance Evaluation to set the table compression) to 0.94
Repository: hbase
Updated Branches:
refs/heads/0.94 c51e19afe -> 9a8142d11
HBASE-11690 Backport HBASE-5934 (Add the ability for Performance Evaluation to set the table compression) to 0.94
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/46c26949
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/46c26949
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/46c26949
Branch: refs/heads/0.94
Commit: 46c26949ca5aa7b1a34f06778b058f1e3ad72364
Parents: c51e19a
Author: Andrew Purtell <ap...@apache.org>
Authored: Thu Aug 7 15:20:02 2014 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Aug 7 15:20:02 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 42 ++++++++++++++++++--
1 file changed, 39 insertions(+), 3 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/46c26949/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 9284b4e..37ad8e2 100644
--- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -59,6 +59,7 @@ import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -101,6 +102,7 @@ import org.apache.hadoop.util.LineReader;
public class PerformanceEvaluation {
protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName());
+ public static final byte[] COMPRESSION = Bytes.toBytes("NONE");
public static final byte[] TABLE_NAME = Bytes.toBytes("TestTable");
public static final byte[] FAMILY_NAME = Bytes.toBytes("info");
public static final byte[] QUALIFIER_NAME = Bytes.toBytes("data");
@@ -122,6 +124,8 @@ public class PerformanceEvaluation {
private boolean nomapred = false;
private int N = 1;
private int R = ROWS_PER_GB;
+ private byte[] tableName = TABLE_NAME;
+ private Compression.Algorithm compression = Compression.Algorithm.NONE;
private float sampleRate = 1.0f;
private boolean flushCommits = true;
private boolean reportLatency = false;
@@ -511,8 +515,9 @@ public class PerformanceEvaluation {
protected HTableDescriptor getTableDescriptor() {
if (TABLE_DESCRIPTOR == null) {
- TABLE_DESCRIPTOR = new HTableDescriptor(TABLE_NAME);
+ TABLE_DESCRIPTOR = new HTableDescriptor(tableName);
HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME);
+ family.setCompressionType(compression);
TABLE_DESCRIPTOR.addFamily(family);
}
return TABLE_DESCRIPTOR;
@@ -1185,7 +1190,23 @@ public class PerformanceEvaluation {
*/
public static byte[] generateValue(final Random r) {
byte [] b = new byte [VALUE_LENGTH];
- r.nextBytes(b);
+ int i = 0;
+
+ for(i = 0; i < (ROW_LENGTH-8); i += 8) {
+ b[i] = (byte) (65 + r.nextInt(26));
+ b[i+1] = b[i];
+ b[i+2] = b[i];
+ b[i+3] = b[i];
+ b[i+4] = b[i];
+ b[i+5] = b[i];
+ b[i+6] = b[i];
+ b[i+7] = b[i];
+ }
+
+ byte a = (byte) (65 + r.nextInt(26));
+ for(; i < ROW_LENGTH; i++) {
+ b[i] = a;
+ }
return b;
}
@@ -1294,13 +1315,16 @@ public class PerformanceEvaluation {
System.err.println(message);
}
System.err.println("Usage: java " + this.getClass().getName() + " \\");
- System.err.println(" [--miniCluster] [--nomapred] [--rows=ROWS] <command> <nclients>");
+ System.err.println(" [--miniCluster] [--nomapred] [--rows=ROWS] [--table=NAME] \\");
+ System.err.println(" [--compress=TYPE] <command> <nclients>");
System.err.println();
System.err.println("Options:");
System.err.println(" miniCluster Run the test on an HBaseMiniCluster");
System.err.println(" nomapred Run multiple clients using threads " +
"(rather than use mapreduce)");
System.err.println(" rows Rows each client runs. Default: One million");
+ System.err.println(" table Alternate table name. Default: 'TestTable'");
+ System.err.println(" compress Compression type to use (GZ, LZO, ...). Default: 'NONE'");
System.err.println(" sampleRate Execute test on a sample of total " +
"rows. Only supported by randomRead. Default: 1.0");
System.err.println(" flushCommits Used to determine if the test should flush the table. " +
@@ -1380,6 +1404,18 @@ public class PerformanceEvaluation {
continue;
}
+ final String table = "--table=";
+ if (cmd.startsWith(table)) {
+ this.tableName = Bytes.toBytes(cmd.substring(table.length()));
+ continue;
+ }
+
+ final String compress = "--compress=";
+ if (cmd.startsWith(compress)) {
+ this.compression = Compression.Algorithm.valueOf(cmd.substring(compress.length()));
+ continue;
+ }
+
final String flushCommits = "--flushCommits=";
if (cmd.startsWith(flushCommits)) {
this.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length()));
[2/2] git commit: HBASE-11691 Backport HBASE-7156 (Add Data Block
Encoding and -D opts to Performance Evaluation) to 0.94
Posted by ap...@apache.org.
HBASE-11691 Backport HBASE-7156 (Add Data Block Encoding and -D opts to Performance Evaluation) to 0.94
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9a8142d1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9a8142d1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9a8142d1
Branch: refs/heads/0.94
Commit: 9a8142d11f39f37a24486f77ac989a483643a1ba
Parents: 46c2694
Author: Andrew Purtell <ap...@apache.org>
Authored: Thu Aug 7 15:22:42 2014 -0700
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Aug 7 15:22:42 2014 -0700
----------------------------------------------------------------------
.../hadoop/hbase/PerformanceEvaluation.java | 56 +++++++++++++-------
1 file changed, 37 insertions(+), 19 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9a8142d1/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
----------------------------------------------------------------------
diff --git a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 37ad8e2..cca8210 100644
--- a/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -42,6 +42,7 @@ import java.lang.reflect.Constructor;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
import org.apache.hadoop.fs.FSDataInputStream;
import org.apache.hadoop.fs.FileStatus;
import org.apache.hadoop.fs.FileSystem;
@@ -59,6 +60,7 @@ import org.apache.hadoop.hbase.filter.Filter;
import org.apache.hadoop.hbase.filter.SingleColumnValueFilter;
import org.apache.hadoop.hbase.filter.CompareFilter;
import org.apache.hadoop.hbase.filter.BinaryComparator;
+import org.apache.hadoop.hbase.io.encoding.DataBlockEncoding;
import org.apache.hadoop.hbase.io.hfile.Compression;
import org.apache.hadoop.hbase.mapreduce.TableMapReduceUtil;
import org.apache.hadoop.hbase.util.Bytes;
@@ -82,6 +84,8 @@ import org.apache.hadoop.mapreduce.lib.input.FileInputFormat;
import org.apache.hadoop.mapreduce.lib.output.TextOutputFormat;
import org.apache.hadoop.mapreduce.lib.reduce.LongSumReducer;
import org.apache.hadoop.util.LineReader;
+import org.apache.hadoop.util.Tool;
+import org.apache.hadoop.util.ToolRunner;
/**
* Script used evaluating HBase performance and scalability. Runs a HBase
@@ -99,7 +103,7 @@ import org.apache.hadoop.util.LineReader;
* <p>If number of clients > 1, we start up a MapReduce job. Each map task
* runs an individual client. Each client does about 1GB of data.
*/
-public class PerformanceEvaluation {
+public class PerformanceEvaluation extends Configured implements Tool {
protected static final Log LOG = LogFactory.getLog(PerformanceEvaluation.class.getName());
public static final byte[] COMPRESSION = Bytes.toBytes("NONE");
@@ -119,13 +123,13 @@ public class PerformanceEvaluation {
protected static HTableDescriptor TABLE_DESCRIPTOR;
protected Map<String, CmdDescriptor> commands = new TreeMap<String, CmdDescriptor>();
- volatile Configuration conf;
private boolean miniCluster = false;
private boolean nomapred = false;
private int N = 1;
private int R = ROWS_PER_GB;
private byte[] tableName = TABLE_NAME;
private Compression.Algorithm compression = Compression.Algorithm.NONE;
+ private DataBlockEncoding blockEncoding = DataBlockEncoding.NONE;
private float sampleRate = 1.0f;
private boolean flushCommits = true;
private boolean reportLatency = false;
@@ -158,10 +162,10 @@ public class PerformanceEvaluation {
/**
* Constructor
- * @param c Configuration object
+ * @param conf Configuration object
*/
- public PerformanceEvaluation(final Configuration c) {
- this.conf = c;
+ public PerformanceEvaluation(final Configuration conf) {
+ super(conf);
addCommandDescriptor(RandomReadTest.class, "randomRead",
"Run random read test");
@@ -517,6 +521,7 @@ public class PerformanceEvaluation {
if (TABLE_DESCRIPTOR == null) {
TABLE_DESCRIPTOR = new HTableDescriptor(tableName);
HColumnDescriptor family = new HColumnDescriptor(FAMILY_NAME);
+ family.setDataBlockEncoding(blockEncoding);
family.setCompressionType(compression);
TABLE_DESCRIPTOR.addFamily(family);
}
@@ -549,7 +554,7 @@ public class PerformanceEvaluation {
*/
private void runNIsMoreThanOne(final Class<? extends Test> cmd)
throws IOException, InterruptedException, ClassNotFoundException {
- checkTable(new HBaseAdmin(conf));
+ checkTable(new HBaseAdmin(getConf()));
if (this.nomapred) {
doMultipleClients(cmd);
} else {
@@ -570,7 +575,7 @@ public class PerformanceEvaluation {
@Override
public void run() {
super.run();
- PerformanceEvaluation pe = new PerformanceEvaluation(conf);
+ PerformanceEvaluation pe = new PerformanceEvaluation(getConf());
int index = Integer.parseInt(getName());
try {
long elapsedTime = pe.runOneClient(cmd, index * perClientRows,
@@ -612,10 +617,11 @@ public class PerformanceEvaluation {
*/
private void doMapReduce(final Class<? extends Test> cmd) throws IOException,
InterruptedException, ClassNotFoundException {
- Path inputDir = writeInputFile(this.conf);
- this.conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
- this.conf.set(EvaluationMapTask.PE_KEY, getClass().getName());
- Job job = new Job(this.conf);
+ Configuration conf = getConf();
+ Path inputDir = writeInputFile(conf);
+ conf.set(EvaluationMapTask.CMD_KEY, cmd.getName());
+ conf.set(EvaluationMapTask.PE_KEY, getClass().getName());
+ Job job = new Job(conf);
job.setJarByClass(PerformanceEvaluation.class);
job.setJobName("HBase Performance Evaluation");
@@ -1229,7 +1235,7 @@ public class PerformanceEvaluation {
try {
Constructor<? extends Test> constructor = cmd.getDeclaredConstructor(
Configuration.class, TestOptions.class, Status.class);
- t = constructor.newInstance(this.conf, options, status);
+ t = constructor.newInstance(getConf(), options, status);
} catch (NoSuchMethodException e) {
throw new IllegalArgumentException("Invalid command class: " +
cmd.getName() + ". It does not provide a constructor as described by" +
@@ -1255,7 +1261,7 @@ public class PerformanceEvaluation {
HBaseAdmin admin = null;
try {
- admin = new HBaseAdmin(this.conf);
+ admin = new HBaseAdmin(getConf());
checkTable(admin);
runOneClient(cmd, 0, this.R, this.R, this.sampleRate, this.flushCommits,
this.writeToWAL, this.writeToWAL, status);
@@ -1271,6 +1277,7 @@ public class PerformanceEvaluation {
MiniHBaseCluster hbaseMiniCluster = null;
MiniDFSCluster dfsCluster = null;
MiniZooKeeperCluster zooKeeperCluster = null;
+ Configuration conf = getConf();
if (this.miniCluster) {
dfsCluster = new MiniDFSCluster(conf, 2, true, (String[])null);
zooKeeperCluster = new MiniZooKeeperCluster();
@@ -1285,7 +1292,7 @@ public class PerformanceEvaluation {
conf.set(HConstants.HBASE_DIR, parentdir.toString());
fs.mkdirs(parentdir);
FSUtils.setVersion(fs, parentdir);
- hbaseMiniCluster = new MiniHBaseCluster(this.conf, N);
+ hbaseMiniCluster = new MiniHBaseCluster(conf, N);
}
try {
@@ -1316,7 +1323,7 @@ public class PerformanceEvaluation {
}
System.err.println("Usage: java " + this.getClass().getName() + " \\");
System.err.println(" [--miniCluster] [--nomapred] [--rows=ROWS] [--table=NAME] \\");
- System.err.println(" [--compress=TYPE] <command> <nclients>");
+ System.err.println(" [--compress=TYPE] [--blockEncoding=TYPE] [-D<property=value>]* <command> <nclients>");
System.err.println();
System.err.println("Options:");
System.err.println(" miniCluster Run the test on an HBaseMiniCluster");
@@ -1335,6 +1342,11 @@ public class PerformanceEvaluation {
System.err.println(" latency Set to report operation latencies. " +
"Currently only supported by randomRead test. Default: False");
System.err.println();
+ System.err.println(" Note: -D properties will be applied to the conf used. ");
+ System.err.println(" For example: ");
+ System.err.println(" -Dmapred.output.compress=true");
+ System.err.println(" -Dmapreduce.task.timeout=60000");
+ System.err.println();
System.err.println("Command:");
for (CmdDescriptor command : commands.values()) {
System.err.println(String.format(" %-15s %s", command.getName(), command.getDescription()));
@@ -1362,7 +1374,7 @@ public class PerformanceEvaluation {
this.R = this.R * N;
}
- public int doCommandLine(final String[] args) {
+ public int run(String[] args) throws Exception {
// Process command-line args. TODO: Better cmd-line processing
// (but hopefully something not as painful as cli options).
int errCode = -1;
@@ -1416,6 +1428,12 @@ public class PerformanceEvaluation {
continue;
}
+ final String blockEncoding = "--blockEncoding=";
+ if (cmd.startsWith(blockEncoding)) {
+ this.blockEncoding = DataBlockEncoding.valueOf(cmd.substring(blockEncoding.length()));
+ continue;
+ }
+
final String flushCommits = "--flushCommits=";
if (cmd.startsWith(flushCommits)) {
this.flushCommits = Boolean.parseBoolean(cmd.substring(flushCommits.length()));
@@ -1463,8 +1481,8 @@ public class PerformanceEvaluation {
return descriptor != null ? descriptor.getCmdClass() : null;
}
- public static void main(final String[] args) {
- Configuration c = HBaseConfiguration.create();
- System.exit(new PerformanceEvaluation(c).doCommandLine(args));
+ public static void main(final String[] args) throws Exception {
+ int res = ToolRunner.run(new PerformanceEvaluation(HBaseConfiguration.create()), args);
+ System.exit(res);
}
}