You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ns...@apache.org on 2011/10/11 04:06:10 UTC
svn commit: r1181411 - in
/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual:
HBaseTest.java utils/HdfsAppender.java
Author: nspiegelberg
Date: Tue Oct 11 02:06:09 2011
New Revision: 1181411
URL: http://svn.apache.org/viewvc?rev=1181411&view=rev
Log:
HBaseTest: Append Latency Tests
Summary:
A simple addition to test append latencies on a full cluster.
Test Plan:
test
DiffCamp Revision: 172803
Reviewed By: kannan
CC: nspiegelberg, achao, kannan
Revert Plan:
OK
Added:
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/HdfsAppender.java
Modified:
hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/HBaseTest.java
Modified: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/HBaseTest.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/HBaseTest.java?rev=1181411&r1=1181410&r2=1181411&view=diff
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/HBaseTest.java (original)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/HBaseTest.java Tue Oct 11 02:06:09 2011
@@ -33,6 +33,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.manual.utils.HBaseUtils;
+import org.apache.hadoop.hbase.manual.utils.HdfsAppender;
import org.apache.hadoop.hbase.manual.utils.KillProcessesAndVerify;
import org.apache.hadoop.hbase.manual.utils.MultiThreadedReader;
import org.apache.hadoop.hbase.manual.utils.MultiThreadedWriter;
@@ -158,6 +159,20 @@ public class HBaseTest
System.out.printf("Started kill test...");
}
+ static final String OPT_USAGE_APPEND = " <MIN Block Size>:<MAX Block Size>";
+
+ /* test the performance of HDFS.append() using a range of block sizes */
+ public void appendTest() {
+ String[] cols = cmd_.getOptionValue(OPT_APPEND).split(":");
+ int minSize = Integer.parseInt(cols[0]);
+ int maxSize = Integer.parseInt(cols[1]);
+
+ System.out.printf("Block Range: %d -> %d\n", minSize, maxSize);
+
+ (new HdfsAppender(configList_.get(0), minSize, maxSize)).start();
+ System.out.printf("Started append test...");
+ }
+
public static void main(String[] args) {
try {
// parse the command line args
@@ -192,6 +207,10 @@ public class HBaseTest
if(cmd_.hasOption(OPT_READ)) {
hBaseTest.readData();
}
+ // call HDFS.append() in an infinite loop
+ if(cmd_.hasOption(OPT_APPEND)) {
+ hBaseTest.appendTest();
+ }
} catch(Exception e) {
e.printStackTrace();
printUsage();
@@ -205,6 +224,7 @@ public class HBaseTest
private static final String OPT_LOAD = "load";
private static final String OPT_READ = "read";
private static final String OPT_KILL = "kill";
+ private static final String OPT_APPEND = "append";
private static final String OPT_TABLE_NAME = "tn";
static void initAndParseArgs(String[] args) throws ParseException {
// set the usage object
@@ -220,6 +240,7 @@ public class HBaseTest
options_.addOption(OPT_LOAD , true, OPT_USAGE_LOAD);
options_.addOption(OPT_READ , true, OPT_USAGE_READ);
options_.addOption(OPT_KILL , true, OPT_USAGE_KILL);
+ options_.addOption(OPT_APPEND , true, OPT_USAGE_APPEND);
// parse the passed in options
CommandLineParser parser = new BasicParser();
cmd_ = parser.parse(options_, args);
Added: hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/HdfsAppender.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/HdfsAppender.java?rev=1181411&view=auto
==============================================================================
--- hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/HdfsAppender.java (added)
+++ hbase/branches/0.89/src/test/java/org/apache/hadoop/hbase/manual/utils/HdfsAppender.java Tue Oct 11 02:06:09 2011
@@ -0,0 +1,97 @@
+/**
+ * Copyright 2010 The Apache Software Foundation
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.manual.utils;
+
+import java.io.IOException;
+import java.util.Random;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.KeyValue;
+import org.apache.hadoop.hbase.regionserver.wal.HLog;
+import org.apache.hadoop.hbase.regionserver.wal.HLogKey;
+import org.apache.hadoop.hbase.regionserver.wal.WALEdit;
+
+/**
+ * Runs a continuous loop, appending files to measure perf
+ */
+public class HdfsAppender extends Thread
+{
+ private static final Log LOG = LogFactory.getLog(HdfsAppender.class);
+
+ final int minSize;
+ final int maxSize;
+ final HBaseConfiguration conf;
+
+ public HdfsAppender(HBaseConfiguration conf, int minSize, int maxSize) {
+ this.conf = conf;
+ this.minSize = minSize;
+ this.maxSize = maxSize;
+ }
+
+ @Override
+ public void run() {
+ Random r = new Random();
+ byte[] t = new byte[] {'t','e','s','t'};
+ HLogKey key = new HLogKey(t, t, 0, 0);
+
+ try {
+ FileSystem fs = FileSystem.get(conf);
+ int fileCount = 0;
+
+ long blocksize = conf.getLong("hbase.regionserver.hlog.blocksize",
+ fs.getDefaultBlockSize());
+ // Roll at 95% of block size.
+ float multi = conf.getFloat("hbase.regionserver.logroll.multiplier", 0.95f);
+ long logrollsize = (long)(blocksize * multi);
+
+ while(true) {
+ Path p = new Path("/appendtest", Integer.toString(fileCount));
+ HLog.Writer writer = HLog.createWriter(fs, p, conf);
+
+ while (writer.getLength() > logrollsize) {
+ int rSize = r.nextInt(maxSize-minSize) + minSize;
+ WALEdit value = new WALEdit();
+ value.add(new KeyValue(t, t, t, 0, new byte[rSize]));
+
+ long now = System.currentTimeMillis();
+ writer.append(new HLog.Entry(key, value));
+ long took = System.currentTimeMillis() - now;
+ LOG.info(String.format("append time (%d) = %d ms", rSize, took));
+
+ now = System.currentTimeMillis();
+ writer.sync();
+ took = System.currentTimeMillis() - now;
+ LOG.info(String.format("sync time = %d ms", took));
+ }
+ LOG.info(String.format("rolling logs to %d", fileCount));
+
+ writer.close();
+ fs.delete(p, false);
+ ++fileCount;
+ }
+ } catch (IOException ioe) {
+ LOG.error("ioe encountered! stopping append thread", ioe);
+ }
+ }
+}
\ No newline at end of file