You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by se...@apache.org on 2019/05/06 22:22:55 UTC
[hbase] branch HBASE-22346 created (now 0780b84)
This is an automated email from the ASF dual-hosted git repository.
sershe pushed a change to branch HBASE-22346
in repository https://gitbox.apache.org/repos/asf/hbase.git.
at 0780b84 HBASE-22346 scanner priorities/deadline units are invalid for non-huge scanners - CR feedback
This branch includes the following new commits:
new 791e8bd Merge branch 'master' into HBASE-22346
new 0780b84 HBASE-22346 scanner priorities/deadline units are invalid for non-huge scanners - CR feedback
The 2 revisions listed above as "new" are entirely new to this
repository and will be described in separate emails. The revisions
listed as "add" were already present in the repository and have only
been added to this reference.
[hbase] 01/02: Merge branch 'master' into HBASE-22346
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sershe pushed a commit to branch HBASE-22346
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 791e8bd56b396f4e59c87eee20ef2e329afc72eb
Merge: 39ad070 67c937f
Author: Sergey Shelukhin <se...@apache.org>
AuthorDate: Mon May 6 15:21:18 2019 -0700
Merge branch 'master' into HBASE-22346
hbase-assembly/src/main/assembly/client.xml | 1 -
.../src/main/assembly/hadoop-two-compat.xml | 1 -
.../java/org/apache/hadoop/hbase/KeyValueUtil.java | 167 +++++++++++-------
.../hbase/regionserver/wal/MetricsWALSource.java | 19 ++-
.../regionserver/wal/MetricsWALSourceImpl.java | 30 +++-
.../regionserver/MetricsTableWrapperStub.java | 4 +-
.../hadoop/hbase/procedure2/ProcedureUtil.java | 47 ++++--
.../hadoop/hbase/procedure2/TestProcedureUtil.java | 20 ---
hbase-protocol-shaded/pom.xml | 3 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 1 +
.../assignment/RegionRemoteProcedureBase.java | 8 +-
.../assignment/TransitRegionStateProcedure.java | 14 +-
.../hbase/master/procedure/InitMetaProcedure.java | 8 +-
.../procedure/ReopenTableRegionsProcedure.java | 10 +-
.../hbase/master/procedure/SplitWALProcedure.java | 25 ++-
.../procedure/SwitchRpcThrottleProcedure.java | 16 +-
.../replication/AbstractPeerNoLockProcedure.java | 20 ++-
.../master/replication/ModifyPeerProcedure.java | 54 +++---
.../SyncReplicationReplayWALProcedure.java | 8 +-
.../TransitPeerSyncReplicationStateProcedure.java | 78 ++++-----
.../hbase/master/snapshot/SnapshotFileCache.java | 130 ++++++--------
.../hadoop/hbase/regionserver/HRegionServer.java | 6 +-
.../hadoop/hbase/regionserver/LogRoller.java | 2 +-
.../hbase/regionserver/wal/AbstractFSWAL.java | 127 +++++++++++---
.../hadoop/hbase/regionserver/wal/AsyncFSWAL.java | 22 +--
.../hadoop/hbase/regionserver/wal/FSHLog.java | 65 ++++---
.../hadoop/hbase/regionserver/wal/MetricsWAL.java | 19 ++-
.../hbase/regionserver/wal/ProtobufLogReader.java | 34 ++--
.../hbase/regionserver/wal/WALActionsListener.java | 14 +-
.../hadoop/hbase/wal/DisabledWALProvider.java | 2 +-
.../hadoop/hbase/TestClientOperationTimeout.java | 73 ++++----
.../assignment/TestAssignmentManagerBase.java | 4 +
.../master/snapshot/TestSnapshotFileCache.java | 64 ++++---
.../hbase/regionserver/wal/TestLogRolling.java | 186 ++++++++++++++++++++-
.../hbase/regionserver/wal/TestMetricsWAL.java | 16 +-
hbase-shaded/hbase-shaded-client/pom.xml | 6 +-
hbase-shaded/pom.xml | 6 +-
hbase-shell/src/main/ruby/hbase/table.rb | 12 +-
pom.xml | 10 +-
39 files changed, 873 insertions(+), 459 deletions(-)
[hbase] 02/02: HBASE-22346 scanner priorities/deadline units are
invalid for non-huge scanners - CR feedback
Posted by se...@apache.org.
This is an automated email from the ASF dual-hosted git repository.
sershe pushed a commit to branch HBASE-22346
in repository https://gitbox.apache.org/repos/asf/hbase.git
commit 0780b8421ea66219f757ce0dec59846967a2b7df
Author: Sergey Shelukhin <se...@apache.org>
AuthorDate: Mon May 6 15:22:30 2019 -0700
HBASE-22346 scanner priorities/deadline units are invalid for non-huge scanners - CR feedback
---
.../AnnotationReadingPriorityFunction.java | 34 ++++++++++++----------
.../hadoop/hbase/regionserver/TestPriorityRpc.java | 28 +++++++++---------
2 files changed, 32 insertions(+), 30 deletions(-)
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java
index 65a9d51..aba5e8a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/AnnotationReadingPriorityFunction.java
@@ -80,11 +80,11 @@ public class AnnotationReadingPriorityFunction implements PriorityFunction {
/** When to use the actual time-based deadline for scanners */
public static final String SCAN_DEADLINE_PRIORITY = "hbase.ipc.server.scan.deadline.only";
- private static final ScanDeadlineOnly SCAN_DEADLINE_PRIORITY_DEFAULT = ScanDeadlineOnly.NEVER;
+ private static final ScanDeadlineOnly SCAN_DEADLINE_PRIORITY_DEFAULT = ScanDeadlineOnly.NONE;
enum ScanDeadlineOnly {
- ALWAYS,
- META,
- NEVER
+ ALL,
+ META_ONLY,
+ NONE
}
private static final ByteString META_PREFIX = ByteString.copyFrom(
@@ -178,11 +178,11 @@ public class AnnotationReadingPriorityFunction implements PriorityFunction {
} catch (IllegalArgumentException ex) {
LOG.warn("Invalid value for {} ({}); using the default", SCAN_DEADLINE_PRIORITY, val);
}
- if (result == ScanDeadlineOnly.META && LoadBalancer.isTablesOnMaster(conf)
+ if (result == ScanDeadlineOnly.META_ONLY && LoadBalancer.isTablesOnMaster(conf)
&& LoadBalancer.isSystemTablesOnlyOnMaster(conf) && rpcServices.isMaster()) {
- result = ScanDeadlineOnly.ALWAYS;
+ result = ScanDeadlineOnly.ALL;
}
- if (result != ScanDeadlineOnly.NEVER) {
+ if (result != ScanDeadlineOnly.NONE) {
LOG.info("Using deadline-based scanner priority {}", result);
}
return result;
@@ -301,15 +301,17 @@ public class AnnotationReadingPriorityFunction implements PriorityFunction {
if (param instanceof ScanRequest) {
ScanRequest request = (ScanRequest)param;
boolean useDeadline;
- long addToVTime = 0;
+ long baseTime = 0;
switch (scanDeadlineOnly) {
- case ALWAYS: useDeadline = true; break;
- case NEVER: useDeadline = false; break;
- case META: {
+ case ALL: useDeadline = true; break;
+ case NONE: useDeadline = false; break;
+ case META_ONLY: {
useDeadline = isMetaScan(request);
- // Make sure non-meta scans are generally after meta scans; add the default scanner delay.
+ // If meta regions use real time and non-meta use vtime, make sure they are comparable
+ // and that non-meta scans are generally after meta scans; add the default scanner
+ // delay to map the former to real time.
if (!useDeadline) {
- addToVTime = rpcServices.getScannerExpirationDelayMs(null);
+ baseTime = rpcServices.getScannerExpirationDelayMs(null);
}
break;
}
@@ -319,13 +321,13 @@ public class AnnotationReadingPriorityFunction implements PriorityFunction {
return rpcServices.getScannerExpirationDelayMs(
request.hasScannerId() ? request.getScannerId() : null);
} else if (!request.hasScannerId()) {
- return addToVTime;
+ return baseTime;
} else {
// get the 'virtual time' of the scanner, and applies sqrt() to get a
- // nice curve for the delay. More a scanner is used the less priority it gets.
+ // nice curve for the delay. The more a scanner is used the less priority it gets.
// The weight is used to have more control on the delay.
long vtime = rpcServices.getScannerVirtualTime(request.getScannerId());
- return addToVTime + Math.round(Math.sqrt(vtime * scanVirtualTimeWeight));
+ return baseTime + Math.round(Math.sqrt(vtime * scanVirtualTimeWeight));
}
}
return 0;
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java
index 5bfaafb..5420190 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestPriorityRpc.java
@@ -250,15 +250,15 @@ public class TestPriorityRpc {
// Test new reqs and non-meta scan here, as well as old non-meta scanner.
AnnotationReadingPriorityFunction fn = prepareDeadlineTestFn(
- ScanDeadlineOnly.NEVER, mockRS.getRSRpcServices());
+ ScanDeadlineOnly.NONE, mockRS.getRSRpcServices());
assertEquals(0L, fn.getDeadline(header, newReqNoMeta));
assertEquals(0L, fn.getDeadline(header, newReqMeta));
assertEquals(VTIMESQRT, fn.getDeadline(header, oldReq));
- fn = prepareDeadlineTestFn(ScanDeadlineOnly.META, mockRS.getRSRpcServices());
+ fn = prepareDeadlineTestFn(ScanDeadlineOnly.META_ONLY, mockRS.getRSRpcServices());
assertEquals(DEFAULT_DELAY, fn.getDeadline(header, newReqNoMeta));
assertEquals(DEFAULT_DELAY, fn.getDeadline(header, newReqMeta));
assertEquals(DEFAULT_DELAY + VTIMESQRT, fn.getDeadline(header, oldReq));
- fn = prepareDeadlineTestFn(ScanDeadlineOnly.ALWAYS, mockRS.getRSRpcServices());
+ fn = prepareDeadlineTestFn(ScanDeadlineOnly.ALL, mockRS.getRSRpcServices());
assertEquals(DEFAULT_DELAY, fn.getDeadline(header, newReqNoMeta));
assertEquals(DEFAULT_DELAY, fn.getDeadline(header, newReqMeta));
assertEquals(ACTUAL_DELAY, fn.getDeadline(header, oldReq));
@@ -267,28 +267,28 @@ public class TestPriorityRpc {
mockRS = prepareDeadlineTest(RegionInfoBuilder.FIRST_META_REGIONINFO.getTable(),
true, DEFAULT_DELAY);
addMockScanner(mockRS.getRSRpcServices(), SCANNER_ID, ACTUAL_DELAY, VTIMESQRT * VTIMESQRT);
- fn = prepareDeadlineTestFn(ScanDeadlineOnly.NEVER, mockRS.getRSRpcServices());
+ fn = prepareDeadlineTestFn(ScanDeadlineOnly.NONE, mockRS.getRSRpcServices());
assertEquals(VTIMESQRT, fn.getDeadline(header, oldReq));
- fn = prepareDeadlineTestFn(ScanDeadlineOnly.META, mockRS.getRSRpcServices());
+ fn = prepareDeadlineTestFn(ScanDeadlineOnly.META_ONLY, mockRS.getRSRpcServices());
assertEquals(ACTUAL_DELAY, fn.getDeadline(header, oldReq));
- fn = prepareDeadlineTestFn(ScanDeadlineOnly.ALWAYS, mockRS.getRSRpcServices());
+ fn = prepareDeadlineTestFn(ScanDeadlineOnly.ALL, mockRS.getRSRpcServices());
assertEquals(ACTUAL_DELAY, fn.getDeadline(header, oldReq));
// Meta on master shortcut - just check the config.
mockRS = prepareDeadlineTest(RegionInfoBuilder.FIRST_META_REGIONINFO.getTable(),
true, DEFAULT_DELAY);
- fn = prepareDeadlineTestFn(ScanDeadlineOnly.META, mockRS.getRSRpcServices());
- assertEquals(ScanDeadlineOnly.META, fn.getScanDeadlineOnly());
+ fn = prepareDeadlineTestFn(ScanDeadlineOnly.META_ONLY, mockRS.getRSRpcServices());
+ assertEquals(ScanDeadlineOnly.META_ONLY, fn.getScanDeadlineOnly());
Configuration conf = mockRS.getRSRpcServices().getConfiguration();
conf.setBoolean(LoadBalancer.TABLES_ON_MASTER, true);
conf.setBoolean(LoadBalancer.SYSTEM_TABLES_ON_MASTER, true);
- fn = prepareDeadlineTestFn(ScanDeadlineOnly.META, mockRS.getRSRpcServices());
- assertEquals(ScanDeadlineOnly.META, fn.getScanDeadlineOnly());
+ fn = prepareDeadlineTestFn(ScanDeadlineOnly.META_ONLY, mockRS.getRSRpcServices());
+ assertEquals(ScanDeadlineOnly.META_ONLY, fn.getScanDeadlineOnly());
Mockito.when(mockRS.getRSRpcServices().isMaster()).thenReturn(true);
- fn = prepareDeadlineTestFn(ScanDeadlineOnly.META, mockRS.getRSRpcServices());
- assertEquals(ScanDeadlineOnly.ALWAYS, fn.getScanDeadlineOnly());
+ fn = prepareDeadlineTestFn(ScanDeadlineOnly.META_ONLY, mockRS.getRSRpcServices());
+ assertEquals(ScanDeadlineOnly.ALL, fn.getScanDeadlineOnly());
conf.setBoolean(LoadBalancer.TABLES_ON_MASTER, false);
- fn = prepareDeadlineTestFn(ScanDeadlineOnly.META, mockRS.getRSRpcServices());
- assertEquals(ScanDeadlineOnly.META, fn.getScanDeadlineOnly());
+ fn = prepareDeadlineTestFn(ScanDeadlineOnly.META_ONLY, mockRS.getRSRpcServices());
+ assertEquals(ScanDeadlineOnly.META_ONLY, fn.getScanDeadlineOnly());
}
}