You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by mb...@apache.org on 2016/09/16 02:16:18 UTC
[2/3] hbase git commit: HBASE-16634 Speedup TestExportSnapshot
HBASE-16634 Speedup TestExportSnapshot
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e19632a1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e19632a1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e19632a1
Branch: refs/heads/master
Commit: e19632a19abc52a983d444811852bf0f11cd0d0b
Parents: 216e847
Author: Matteo Bertozzi <ma...@cloudera.com>
Authored: Thu Sep 15 18:29:09 2016 -0700
Committer: Matteo Bertozzi <ma...@cloudera.com>
Committed: Thu Sep 15 18:29:09 2016 -0700
----------------------------------------------------------------------
.../hbase/snapshot/MobSnapshotTestingUtils.java | 15 ++++++++-
.../hbase/snapshot/SnapshotTestingUtils.java | 34 ++++++++++++++++----
.../hbase/snapshot/TestExportSnapshot.java | 8 ++---
.../snapshot/TestFlushSnapshotFromClient.java | 8 ++---
.../hbase/snapshot/TestMobExportSnapshot.java | 8 ++---
.../snapshot/TestMobSecureExportSnapshot.java | 4 +--
.../snapshot/TestSecureExportSnapshot.java | 1 -
7 files changed, 54 insertions(+), 24 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java
index 9026115..107f487 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/MobSnapshotTestingUtils.java
@@ -38,6 +38,20 @@ public class MobSnapshotTestingUtils {
public static void createMobTable(final HBaseTestingUtility util,
final TableName tableName, int regionReplication,
final byte[]... families) throws IOException, InterruptedException {
+ createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(),
+ regionReplication, families);
+ }
+
+ public static void createPreSplitMobTable(final HBaseTestingUtility util,
+ final TableName tableName, int nRegions, final byte[]... families)
+ throws IOException, InterruptedException {
+ createMobTable(util, tableName, SnapshotTestingUtils.getSplitKeys(nRegions),
+ 1, families);
+ }
+
+ private static void createMobTable(final HBaseTestingUtility util,
+ final TableName tableName, final byte[][] splitKeys, int regionReplication,
+ final byte[]... families) throws IOException, InterruptedException {
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.setRegionReplication(regionReplication);
for (byte[] family : families) {
@@ -46,7 +60,6 @@ public class MobSnapshotTestingUtils {
hcd.setMobThreshold(0L);
htd.addFamily(hcd);
}
- byte[][] splitKeys = SnapshotTestingUtils.getSplitKeys();
util.getHBaseAdmin().createTable(htd, splitKeys);
SnapshotTestingUtils.waitForTableToBeOnline(util, tableName);
assertEquals((splitKeys.length + 1) * regionReplication, util
http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
index 770bb00..dfd00b3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/SnapshotTestingUtils.java
@@ -82,9 +82,10 @@ import com.google.protobuf.ServiceException;
*/
@InterfaceAudience.Private
public final class SnapshotTestingUtils {
-
private static final Log LOG = LogFactory.getLog(SnapshotTestingUtils.class);
- private static byte[] KEYS = Bytes.toBytes("0123456789");
+
+ // default number of regions (and keys) given by getSplitKeys() and createTable()
+ private static byte[] KEYS = Bytes.toBytes("0123456");
private SnapshotTestingUtils() {
// private constructor for utility class
@@ -750,23 +751,32 @@ public final class SnapshotTestingUtils {
}
public static void createTable(final HBaseTestingUtility util, final TableName tableName,
- int regionReplication, final byte[]... families) throws IOException, InterruptedException {
+ int regionReplication, int nRegions, final byte[]... families)
+ throws IOException, InterruptedException {
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.setRegionReplication(regionReplication);
for (byte[] family : families) {
HColumnDescriptor hcd = new HColumnDescriptor(family);
htd.addFamily(hcd);
}
- byte[][] splitKeys = getSplitKeys();
+ byte[][] splitKeys = getSplitKeys(nRegions);
util.createTable(htd, splitKeys);
assertEquals((splitKeys.length + 1) * regionReplication,
util.getHBaseAdmin().getTableRegions(tableName).size());
}
public static byte[][] getSplitKeys() {
- byte[][] splitKeys = new byte[KEYS.length-2][];
+ return getSplitKeys(KEYS.length);
+ }
+
+ public static byte[][] getSplitKeys(int nRegions) {
+ nRegions = nRegions < KEYS.length ? nRegions : (KEYS.length - 1);
+ final byte[][] splitKeys = new byte[nRegions-1][];
+ final int step = KEYS.length / nRegions;
+ int keyIndex = 1;
for (int i = 0; i < splitKeys.length; ++i) {
- splitKeys[i] = new byte[] { KEYS[i+1] };
+ splitKeys[i] = new byte[] { KEYS[keyIndex] };
+ keyIndex += step;
}
return splitKeys;
}
@@ -776,6 +786,16 @@ public final class SnapshotTestingUtils {
createTable(util, tableName, 1, families);
}
+ public static void createTable(final HBaseTestingUtility util, final TableName tableName,
+ final int regionReplication, final byte[]... families) throws IOException, InterruptedException {
+ createTable(util, tableName, regionReplication, KEYS.length, families);
+ }
+
+ public static void createPreSplitTable(final HBaseTestingUtility util, final TableName tableName,
+ final int nRegions, final byte[]... families) throws IOException, InterruptedException {
+ createTable(util, tableName, 1, nRegions, families);
+ }
+
public static void loadData(final HBaseTestingUtility util, final TableName tableName, int rows,
byte[]... families) throws IOException, InterruptedException {
BufferedMutator mutator = util.getConnection().getBufferedMutator(tableName);
@@ -861,6 +881,6 @@ public final class SnapshotTestingUtils {
}
}
}
- assert(set.size() == getSplitKeys().length + 1);
+ assertEquals(getSplitKeys().length + 1, set.size());
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
index d0b32f8..ee0d0e2 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestExportSnapshot.java
@@ -45,7 +45,7 @@ import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescriptio
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotFileInfo;
import org.apache.hadoop.hbase.protobuf.generated.SnapshotProtos.SnapshotRegionManifest;
import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils.SnapshotMock;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
import org.apache.hadoop.hbase.util.Bytes;
import org.apache.hadoop.hbase.util.FSUtils;
@@ -62,7 +62,7 @@ import org.junit.rules.TestRule;
/**
* Test Export Snapshot Tool
*/
-@Category({VerySlowRegionServerTests.class, MediumTests.class})
+@Category({VerySlowRegionServerTests.class, LargeTests.class})
public class TestExportSnapshot {
@Rule public final TestRule timeout = CategoryBasedTimeout.builder().
withTimeout(this.getClass()).withLookingForStuckThread(true).build();
@@ -91,12 +91,10 @@ public class TestExportSnapshot {
public static void setUpBeforeClass() throws Exception {
setUpBaseConf(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
- TEST_UTIL.startMiniMapReduceCluster();
}
@AfterClass
public static void tearDownAfterClass() throws Exception {
- TEST_UTIL.shutdownMiniMapReduceCluster();
TEST_UTIL.shutdownMiniCluster();
}
@@ -127,7 +125,7 @@ public class TestExportSnapshot {
}
protected void createTable() throws Exception {
- SnapshotTestingUtils.createTable(TEST_UTIL, tableName, FAMILY);
+ SnapshotTestingUtils.createPreSplitTable(TEST_UTIL, tableName, 2, FAMILY);
}
@After
http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
index 8efc2d7..a852547 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestFlushSnapshotFromClient.java
@@ -325,8 +325,8 @@ public class TestFlushSnapshotFromClient {
int numRegionsAfterMerge = numRegions - 2;
admin.mergeRegionsAsync(regions.get(1).getEncodedNameAsBytes(),
regions.get(2).getEncodedNameAsBytes(), true);
- admin.mergeRegionsAsync(regions.get(5).getEncodedNameAsBytes(),
- regions.get(6).getEncodedNameAsBytes(), true);
+ admin.mergeRegionsAsync(regions.get(4).getEncodedNameAsBytes(),
+ regions.get(5).getEncodedNameAsBytes(), true);
// Verify that there's one region less
waitRegionsAfterMerge(numRegionsAfterMerge);
@@ -366,8 +366,8 @@ public class TestFlushSnapshotFromClient {
int numRegionsAfterMerge = numRegions - 2;
admin.mergeRegionsAsync(regions.get(1).getEncodedNameAsBytes(),
regions.get(2).getEncodedNameAsBytes(), true);
- admin.mergeRegionsAsync(regions.get(5).getEncodedNameAsBytes(),
- regions.get(6).getEncodedNameAsBytes(), true);
+ admin.mergeRegionsAsync(regions.get(4).getEncodedNameAsBytes(),
+ regions.get(5).getEncodedNameAsBytes(), true);
waitRegionsAfterMerge(numRegionsAfterMerge);
assertEquals(numRegionsAfterMerge, admin.getTableRegions(TABLE_NAME).size());
http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
index b6cb339..d755961 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobExportSnapshot.java
@@ -25,14 +25,15 @@ import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.mob.MobConstants;
import org.apache.hadoop.hbase.mob.MobUtils;
-import org.apache.hadoop.hbase.testclassification.MediumTests;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
import org.junit.BeforeClass;
import org.junit.experimental.categories.Category;
/**
* Test Export Snapshot Tool
*/
-@Category(MediumTests.class)
+@Category({VerySlowRegionServerTests.class, LargeTests.class})
public class TestMobExportSnapshot extends TestExportSnapshot {
private final Log LOG = LogFactory.getLog(getClass());
@@ -45,12 +46,11 @@ public class TestMobExportSnapshot extends TestExportSnapshot {
public static void setUpBeforeClass() throws Exception {
setUpBaseConf(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
- TEST_UTIL.startMiniMapReduceCluster();
}
@Override
protected void createTable() throws Exception {
- MobSnapshotTestingUtils.createMobTable(TEST_UTIL, tableName, 1, FAMILY);
+ MobSnapshotTestingUtils.createPreSplitMobTable(TEST_UTIL, tableName, 2, FAMILY);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
index 4a89db8..8154995 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestMobSecureExportSnapshot.java
@@ -20,6 +20,7 @@
package org.apache.hadoop.hbase.snapshot;
import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.VerySlowRegionServerTests;
import org.apache.hadoop.hbase.mapreduce.HadoopSecurityEnabledUserProviderForTesting;
import org.apache.hadoop.hbase.security.UserProvider;
import org.apache.hadoop.hbase.security.access.AccessControlLists;
@@ -31,7 +32,7 @@ import org.junit.experimental.categories.Category;
/**
* Reruns TestMobExportSnapshot using MobExportSnapshot in secure mode.
*/
-@Category(LargeTests.class)
+@Category({VerySlowRegionServerTests.class, LargeTests.class})
public class TestMobSecureExportSnapshot extends TestMobExportSnapshot {
@BeforeClass
public static void setUpBeforeClass() throws Exception {
@@ -45,7 +46,6 @@ public class TestMobSecureExportSnapshot extends TestMobExportSnapshot {
SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
- TEST_UTIL.startMiniMapReduceCluster();
// Wait for the ACL table to become available
TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);
http://git-wip-us.apache.org/repos/asf/hbase/blob/e19632a1/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
index cf55c4d..f335e4e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/snapshot/TestSecureExportSnapshot.java
@@ -51,7 +51,6 @@ public class TestSecureExportSnapshot extends TestExportSnapshot {
SecureTestUtil.enableSecurity(TEST_UTIL.getConfiguration());
TEST_UTIL.startMiniCluster(3);
- TEST_UTIL.startMiniMapReduceCluster();
// Wait for the ACL table to become available
TEST_UTIL.waitTableEnabled(AccessControlLists.ACL_TABLE_NAME);