You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2020/12/01 06:52:32 UTC
[hbase] branch branch-2 updated: HBASE-25127 Enhance
PerformanceEvaluation to profile meta replica performance. (#2655)
This is an automated email from the ASF dual-hosted git repository.
stack pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new 4c8fa47 HBASE-25127 Enhance PerformanceEvaluation to profile meta replica performance. (#2655)
4c8fa47 is described below
commit 4c8fa47b282e8538564d7f0d7402f40a80ba86c7
Author: clarax <cl...@gmail.com>
AuthorDate: Mon Nov 30 22:51:57 2020 -0800
HBASE-25127 Enhance PerformanceEvaluation to profile meta replica performance. (#2655)
* HBASE-25127 Enhance PerformanceEvaluation to profile meta replica performance. (#2644)
Signed-off-by: Duo Zhang <zh...@apache.org>
Signed-off-by: Huaxiang Sun <hu...@apache.com>
Signed-off-by: stack <st...@apache.com>
---
.../org/apache/hadoop/hbase/MetaTableAccessor.java | 2 +-
.../apache/hadoop/hbase/PerformanceEvaluation.java | 148 +++++++++++++++++++--
2 files changed, 139 insertions(+), 11 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 669b227..235d120 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -1306,7 +1306,7 @@ public class MetaTableAccessor {
/**
* Generates and returns a Delete containing the region info for the catalog table
*/
- private static Delete makeDeleteFromRegionInfo(RegionInfo regionInfo, long ts) {
+ public static Delete makeDeleteFromRegionInfo(RegionInfo regionInfo, long ts) {
if (regionInfo == null) {
throw new IllegalArgumentException("Can't make a delete for null region");
}
diff --git a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
index 03d8ee6..e6df491 100644
--- a/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
+++ b/hbase-mapreduce/src/test/java/org/apache/hadoop/hbase/PerformanceEvaluation.java
@@ -31,6 +31,7 @@ import java.util.ArrayList;
import java.util.Arrays;
import java.util.Date;
import java.util.LinkedList;
+import java.util.List;
import java.util.Locale;
import java.util.Map;
import java.util.NoSuchElementException;
@@ -61,6 +62,9 @@ import org.apache.hadoop.hbase.client.Durability;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.Increment;
import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionInfoBuilder;
+import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.ResultScanner;
import org.apache.hadoop.hbase.client.RowMutations;
@@ -157,17 +161,18 @@ public class PerformanceEvaluation extends Configured implements Tool {
static {
addCommandDescriptor(AsyncRandomReadTest.class, "asyncRandomRead",
- "Run async random read test");
+ "Run async random read test");
addCommandDescriptor(AsyncRandomWriteTest.class, "asyncRandomWrite",
- "Run async random write test");
+ "Run async random write test");
addCommandDescriptor(AsyncSequentialReadTest.class, "asyncSequentialRead",
- "Run async sequential read test");
+ "Run async sequential read test");
addCommandDescriptor(AsyncSequentialWriteTest.class, "asyncSequentialWrite",
- "Run async sequential write test");
+ "Run async sequential write test");
addCommandDescriptor(AsyncScanTest.class, "asyncScan",
- "Run async scan test (read every row)");
- addCommandDescriptor(RandomReadTest.class, RANDOM_READ,
- "Run random read test");
+ "Run async scan test (read every row)");
+ addCommandDescriptor(RandomReadTest.class, RANDOM_READ, "Run random read test");
+ addCommandDescriptor(MetaRandomReadTest.class, "metaRandomRead",
+ "Run getRegionLocation test");
addCommandDescriptor(RandomSeekScanTest.class, RANDOM_SEEK_SCAN,
"Run random seek and scan 100 test");
addCommandDescriptor(RandomScanWithRange10Test.class, "scanRange10",
@@ -184,11 +189,12 @@ public class PerformanceEvaluation extends Configured implements Tool {
"Run sequential read test");
addCommandDescriptor(SequentialWriteTest.class, "sequentialWrite",
"Run sequential write test");
- addCommandDescriptor(ScanTest.class, "scan",
- "Run scan test (read every row)");
+ addCommandDescriptor(MetaWriteTest.class, "metaWrite",
+ "Populate meta table;used with 1 thread; to be cleaned up by cleanMeta");
+ addCommandDescriptor(ScanTest.class, "scan", "Run scan test (read every row)");
addCommandDescriptor(FilteredScanTest.class, "filterScan",
"Run scan test using a filter to find a specific row based on it's value " +
- "(make sure to use --rows=20)");
+ "(make sure to use --rows=20)");
addCommandDescriptor(IncrementTest.class, "increment",
"Increment on each row; clients overlap on keyspace so some concurrent operations");
addCommandDescriptor(AppendTest.class, "append",
@@ -199,6 +205,8 @@ public class PerformanceEvaluation extends Configured implements Tool {
"CheckAndPut on each row; clients overlap on keyspace so some concurrent operations");
addCommandDescriptor(CheckAndDeleteTest.class, "checkAndDelete",
"CheckAndDelete on each row; clients overlap on keyspace so some concurrent operations");
+ addCommandDescriptor(CleanMetaTest.class, "cleanMeta",
+ "Remove fake region entries on meta table inserted by metaWrite; used with 1 thread");
}
/**
@@ -1471,6 +1479,31 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
}
+ /*
+ Parent class for all meta tests: MetaWriteTest, MetaRandomReadTest and CleanMetaTest
+ */
+ static abstract class MetaTest extends TableTest {
+ protected int keyLength;
+
+ MetaTest(Connection con, TestOptions options, Status status) {
+ super(con, options, status);
+ keyLength = Integer.toString(opts.perClientRunRows).length();
+ }
+
+ @Override
+ void onTakedown() throws IOException {
+ // No clean up
+ }
+
+ /*
+ Generates Lexicographically ascending strings
+ */
+ protected byte[] getSplitKey(final int i) {
+ return Bytes.toBytes(String.format("%0" + keyLength + "d", i));
+ }
+
+ }
+
static abstract class AsyncTableTest extends AsyncTest {
protected AsyncTable<?> table;
@@ -1989,6 +2022,47 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
}
+ /*
+ Send random reads against fake regions inserted by MetaWriteTest
+ */
+ static class MetaRandomReadTest extends MetaTest {
+ private Random rd = new Random();
+ private RegionLocator regionLocator;
+
+ MetaRandomReadTest(Connection con, TestOptions options, Status status) {
+ super(con, options, status);
+ LOG.info("call getRegionLocation");
+ }
+
+ @Override
+ void onStartup() throws IOException {
+ super.onStartup();
+ this.regionLocator = connection.getRegionLocator(table.getName());
+ }
+
+ @Override
+ boolean testRow(final int i, final long startTime) throws IOException, InterruptedException {
+ if (opts.randomSleep > 0) {
+ Thread.sleep(rd.nextInt(opts.randomSleep));
+ }
+ HRegionLocation hRegionLocation = regionLocator.getRegionLocation(
+ getSplitKey(rd.nextInt(opts.perClientRunRows)), true);
+ LOG.debug("get location for region: " + hRegionLocation);
+ return true;
+ }
+
+ @Override
+ protected int getReportingPeriod() {
+ int period = opts.perClientRunRows / 10;
+ return period == 0 ? opts.perClientRunRows : period;
+ }
+
+ @Override
+ protected void testTakedown() throws IOException {
+ super.testTakedown();
+ }
+ }
+
static class RandomWriteTest extends SequentialWriteTest {
RandomWriteTest(Connection con, TestOptions options, Status status) {
super(con, options, status);
@@ -2178,6 +2252,34 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
}
+ /*
+ Delete all fake regions inserted to meta table by MetaWriteTest.
+ */
+ static class CleanMetaTest extends MetaTest {
+ CleanMetaTest(Connection con, TestOptions options, Status status) {
+ super(con, options, status);
+ }
+
+ @Override
+ boolean testRow(final int i, final long startTime) throws IOException {
+ try {
+ RegionInfo regionInfo = connection.getRegionLocator(table.getName())
+ .getRegionLocation(getSplitKey(i), false).getRegion();
+ LOG.debug("deleting region from meta: " + regionInfo);
+
+ Delete delete = MetaTableAccessor
+ .makeDeleteFromRegionInfo(regionInfo, HConstants.LATEST_TIMESTAMP);
+ try (Table t = MetaTableAccessor.getMetaHTable(connection)) {
+ t.delete(delete);
+ }
+ } catch (IOException ie) {
+ // Log and continue
+ LOG.error("cannot find region with start key: " + i);
+ }
+ return true;
+ }
+ }
+
static class SequentialReadTest extends TableTest {
SequentialReadTest(Connection con, TestOptions options, Status status) {
super(con, options, status);
@@ -2267,6 +2369,32 @@ public class PerformanceEvaluation extends Configured implements Tool {
}
}
+ /*
+ Insert fake regions into meta table with contiguous split keys.
+ */
+ static class MetaWriteTest extends MetaTest {
+
+ MetaWriteTest(Connection con, TestOptions options, Status status) {
+ super(con, options, status);
+ }
+
+ @Override
+ boolean testRow(final int i, final long startTime) throws IOException {
+ List<RegionInfo> regionInfos = new ArrayList<RegionInfo>();
+ RegionInfo regionInfo = (RegionInfoBuilder.newBuilder(TableName.valueOf(TABLE_NAME))
+ .setStartKey(getSplitKey(i))
+ .setEndKey(getSplitKey(i + 1))
+ .build());
+ regionInfos.add(regionInfo);
+ MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 1);
+
+ // write the serverName columns
+ MetaTableAccessor.updateRegionLocation(connection,
+ regionInfo, ServerName.valueOf("localhost", 60010, rand.nextLong()), i,
+ System.currentTimeMillis());
+ return true;
+ }
+ }
static class FilteredScanTest extends TableTest {
protected static final Logger LOG = LoggerFactory.getLogger(FilteredScanTest.class.getName());