You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2013/03/29 20:29:33 UTC
svn commit: r1462618 - in
/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase: ./ coprocessor/
master/ util/
Author: apurtell
Date: Fri Mar 29 19:29:32 2013
New Revision: 1462618
URL: http://svn.apache.org/r1462618
Log:
HBASE-8209. Improve LoadTest extensibility
Modified:
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java?rev=1462618&r1=1462617&r2=1462618&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java Fri Mar 29 19:29:32 2013
@@ -1835,29 +1835,45 @@ public class HBaseTestingUtility {
* @param countOfRegions How many regions in .META.
* @throws IOException
*/
- public void waitUntilAllRegionsAssigned(final int countOfRegions)
+ public void waitUntilAllRegionsAssigned(final byte[] tableName, final int countOfRegions)
throws IOException {
+ int retries = 30; // We may wait up to 30 seconds
+ int rows = 0;
HTable meta = new HTable(getConfiguration(), HConstants.META_TABLE_NAME);
- while (true) {
- int rows = 0;
- Scan scan = new Scan();
- scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
- ResultScanner s = meta.getScanner(scan);
- for (Result r = null; (r = s.next()) != null;) {
- byte [] b =
- r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
- if (b == null || b.length <= 0) {
+ try {
+ do {
+ Scan scan = new Scan();
+ scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+ scan.addColumn(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
+ ResultScanner s = meta.getScanner(scan);
+ try {
+ for (Result r = null; (r = s.next()) != null;) {
+ byte[] b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.REGIONINFO_QUALIFIER);
+ HRegionInfo hri = Writables.getHRegionInfoOrNull(b);
+ if (hri != null && Bytes.equals(hri.getTableName(), tableName)) {
+ b = r.getValue(HConstants.CATALOG_FAMILY, HConstants.SERVER_QUALIFIER);
+ if (b == null || b.length <= 0) {
+ continue;
+ }
+ rows++;
+ }
+ }
+ } finally {
+ s.close();
+ }
+ // If I get to here and all rows have a Server, then all have been assigned.
+ if (rows == countOfRegions) {
break;
}
- rows++;
- }
- s.close();
- // If I get to here and all rows have a Server, then all have been assigned.
- if (rows == countOfRegions) {
- break;
- }
- LOG.info("Found=" + rows);
- Threads.sleep(200);
+ LOG.info("Found=" + rows);
+ Threads.sleep(1000);
+ } while (--retries > 0);
+ } finally {
+ meta.close();
+ }
+ if (rows != countOfRegions) {
+ throw new IOException("Timed out waiting for " + countOfRegions + " regions of " +
+ Bytes.toStringBinary(tableName) + " to come online");
}
}
@@ -2138,12 +2154,23 @@ public class HBaseTestingUtility {
HColumnDescriptor hcd = new HColumnDescriptor(columnFamily);
hcd.setDataBlockEncoding(dataBlockEncoding);
hcd.setCompressionType(compression);
- desc.addFamily(hcd);
+ return createPreSplitLoadTestTable(conf, desc, hcd);
+ }
+
+ /**
+ * Creates a pre-split table for load testing. If the table already exists,
+ * logs a warning and continues.
+ * @return the number of regions the table was split into
+ */
+ public static int createPreSplitLoadTestTable(Configuration conf,
+ HTableDescriptor desc, HColumnDescriptor hcd) throws IOException {
+ if (!desc.hasFamily(hcd.getName())) {
+ desc.addFamily(hcd);
+ }
int totalNumberOfRegions = 0;
+ HBaseAdmin admin = new HBaseAdmin(conf);
try {
- HBaseAdmin admin = new HBaseAdmin(conf);
-
// create a table a pre-splits regions.
// The number of splits is set as:
// region servers * regions per region server).
@@ -2166,8 +2193,10 @@ public class HBaseTestingUtility {
LOG.error("Master not running", e);
throw new IOException(e);
} catch (TableExistsException e) {
- LOG.warn("Table " + Bytes.toStringBinary(tableName) +
+ LOG.warn("Table " + Bytes.toStringBinary(desc.getName()) +
" already exists, continuing");
+ } finally {
+ admin.close();
}
return totalNumberOfRegions;
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java?rev=1462618&r1=1462617&r2=1462618&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterObserver.java Fri Mar 29 19:29:32 2013
@@ -748,7 +748,7 @@ public class TestMasterObserver {
HTable table = UTIL.createTable(TEST_TABLE, TEST_FAMILY);
int countOfRegions = UTIL.createMultiRegions(table, TEST_FAMILY);
- UTIL.waitUntilAllRegionsAssigned(countOfRegions);
+ UTIL.waitUntilAllRegionsAssigned(TEST_TABLE, countOfRegions);
NavigableMap<HRegionInfo, ServerName> regions = table.getRegionLocations();
Map.Entry<HRegionInfo, ServerName> firstGoodPair = null;
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java?rev=1462618&r1=1462617&r2=1462618&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithAbort.java Fri Mar 29 19:29:32 2013
@@ -109,7 +109,7 @@ public class TestRegionServerCoprocessor
byte[] TEST_FAMILY = Bytes.toBytes("aaa");
HTable table = TEST_UTIL.createTable(TEST_TABLE, TEST_FAMILY);
- TEST_UTIL.waitUntilAllRegionsAssigned(
+ TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE,
TEST_UTIL.createMultiRegions(table, TEST_FAMILY));
// Note which regionServer will abort (after put is attempted).
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java?rev=1462618&r1=1462617&r2=1462618&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/coprocessor/TestRegionServerCoprocessorExceptionWithRemove.java Fri Mar 29 19:29:32 2013
@@ -92,7 +92,7 @@ public class TestRegionServerCoprocessor
byte[] TEST_FAMILY = Bytes.toBytes("aaa");
HTable table = TEST_UTIL.createTable(TEST_TABLE, TEST_FAMILY);
- TEST_UTIL.waitUntilAllRegionsAssigned(
+ TEST_UTIL.waitUntilAllRegionsAssigned(TEST_TABLE,
TEST_UTIL.createMultiRegions(table, TEST_FAMILY));
// Note which regionServer that should survive the buggy coprocessor's
// prePut().
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java?rev=1462618&r1=1462617&r2=1462618&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestMasterTransitions.java Fri Mar 29 19:29:32 2013
@@ -62,7 +62,7 @@ public class TestMasterTransitions {
TEST_UTIL.createTable(Bytes.toBytes(TABLENAME), FAMILIES);
HTable t = new HTable(TEST_UTIL.getConfiguration(), TABLENAME);
int countOfRegions = TEST_UTIL.createMultiRegions(t, getTestFamily());
- TEST_UTIL.waitUntilAllRegionsAssigned(countOfRegions);
+ TEST_UTIL.waitUntilAllRegionsAssigned(Bytes.toBytes(TABLENAME), countOfRegions);
addToEachStartKey(countOfRegions);
t.close();
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java?rev=1462618&r1=1462617&r2=1462618&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedAction.java Fri Mar 29 19:29:32 2013
@@ -265,6 +265,14 @@ public abstract class MultiThreadedActio
sb.append(v);
}
+ protected static void appendToStatus(StringBuilder sb, String desc,
+ String v) {
+ sb.append(", ");
+ sb.append(desc);
+ sb.append("=");
+ sb.append(v);
+ }
+
/**
* See {@link #verifyResultAgainstDataGenerator(Result, boolean, boolean)}.
* Does not verify cf/column integrity.
@@ -338,4 +346,5 @@ public abstract class MultiThreadedActio
}
return true;
}
+
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java?rev=1462618&r1=1462617&r2=1462618&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/MultiThreadedWriter.java Fri Mar 29 19:29:32 2013
@@ -132,7 +132,7 @@ public class MultiThreadedWriter extends
put.add(cf, column, value);
++columnCount;
if (!isMultiPut) {
- insert(put, rowKeyBase);
+ insert(table, put, rowKeyBase);
numCols.addAndGet(1);
put = new Put(rowKey);
}
@@ -142,7 +142,7 @@ public class MultiThreadedWriter extends
if (verbose) {
LOG.debug("Preparing put for key = [" + rowKey + "], " + columnCount + " columns");
}
- insert(put, rowKeyBase);
+ insert(table, put, rowKeyBase);
numCols.addAndGet(columnCount);
}
if (trackInsertedKeys) {
@@ -158,17 +158,17 @@ public class MultiThreadedWriter extends
numThreadsWorking.decrementAndGet();
}
}
+ }
- public void insert(Put put, long keyBase) {
- try {
- long start = System.currentTimeMillis();
- table.put(put);
- totalOpTimeMs.addAndGet(System.currentTimeMillis() - start);
- } catch (IOException e) {
- failedKeySet.add(keyBase);
- LOG.error("Failed to insert: " + keyBase);
- e.printStackTrace();
- }
+ public void insert(HTable table, Put put, long keyBase) {
+ try {
+ long start = System.currentTimeMillis();
+ table.put(put);
+ totalOpTimeMs.addAndGet(System.currentTimeMillis() - start);
+ } catch (IOException e) {
+ failedKeySet.add(keyBase);
+ LOG.error("Failed to insert: " + keyBase);
+ e.printStackTrace();
}
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java?rev=1462618&r1=1462617&r2=1462618&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/TestLoadTestKVGenerator.java Fri Mar 29 19:29:32 2013
@@ -64,7 +64,7 @@ public class TestLoadTestKVGenerator {
public void testCorrectAndUniqueKeys() {
Set<String> keys = new HashSet<String>();
for (int i = 0; i < 1000; ++i) {
- String k = LoadTestKVGenerator.md5PrefixedKey(i);
+ String k = gen.md5PrefixedKey(i);
assertFalse(keys.contains(k));
assertTrue(k.endsWith("-" + i));
keys.add(k);
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java?rev=1462618&r1=1462617&r2=1462618&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/util/TestMiniClusterLoadSequential.java Fri Mar 29 19:29:32 2013
@@ -29,6 +29,7 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.LargeTests;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.client.HBaseAdmin;
@@ -102,6 +103,19 @@ public class TestMiniClusterLoadSequenti
TEST_UTIL.shutdownMiniCluster();
}
+ protected MultiThreadedReader prepareReaderThreads(LoadTestDataGenerator dataGen,
+ Configuration conf, byte[] tableName, double verifyPercent) {
+ MultiThreadedReader reader = new MultiThreadedReader(dataGen, conf, tableName, verifyPercent);
+ return reader;
+ }
+
+ protected MultiThreadedWriter prepareWriterThreads(LoadTestDataGenerator dataGen,
+ Configuration conf, byte[] tableName) {
+ MultiThreadedWriter writer = new MultiThreadedWriter(dataGen, conf, tableName);
+ writer.setMultiPut(isMultiPut);
+ return writer;
+ }
+
@Test(timeout=TIMEOUT_MS)
public void loadTest() throws Exception {
prepareForLoadTest();
@@ -120,6 +134,12 @@ public class TestMiniClusterLoadSequenti
assertEquals(numKeys, readerThreads.getNumKeysVerified());
}
+ protected void createPreSplitLoadTestTable(HTableDescriptor htd, HColumnDescriptor hcd)
+ throws IOException {
+ int numRegions = HBaseTestingUtility.createPreSplitLoadTestTable(conf, htd, hcd);
+ TEST_UTIL.waitUntilAllRegionsAssigned(htd.getName(), numRegions);
+ }
+
protected void prepareForLoadTest() throws IOException {
LOG.info("Starting load test: dataBlockEncoding=" + dataBlockEncoding +
", isMultiPut=" + isMultiPut);
@@ -131,15 +151,15 @@ public class TestMiniClusterLoadSequenti
}
admin.close();
- int numRegions = HBaseTestingUtility.createPreSplitLoadTestTable(conf,
- TABLE, CF, compression, dataBlockEncoding);
-
- TEST_UTIL.waitUntilAllRegionsAssigned(numRegions);
+ HTableDescriptor htd = new HTableDescriptor(TABLE);
+ HColumnDescriptor hcd = new HColumnDescriptor(CF)
+ .setCompressionType(compression)
+ .setDataBlockEncoding(dataBlockEncoding);
+ createPreSplitLoadTestTable(htd, hcd);
LoadTestDataGenerator dataGen = new MultiThreadedAction.DefaultDataGenerator(CF);
- writerThreads = new MultiThreadedWriter(dataGen, conf, TABLE);
- writerThreads.setMultiPut(isMultiPut);
- readerThreads = new MultiThreadedReader(dataGen, conf, TABLE, 100);
+ writerThreads = prepareWriterThreads(dataGen, conf, TABLE);
+ readerThreads = prepareReaderThreads(dataGen, conf, TABLE, 100);
}
protected int numKeys() {