You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by ap...@apache.org on 2015/02/12 20:00:30 UTC
[1/4] hbase git commit: HBASE-13008 Better default for
hbase.regionserver.regionSplitLimit parameter (Srikanth Srungarapu)
Repository: hbase
Updated Branches:
refs/heads/0.98 4ee823b3f -> 7871c9305
refs/heads/branch-1 38de2d227 -> 7841bf73b
refs/heads/branch-1.0 497a604e4 -> bdb938ba2
refs/heads/master 02759f2d8 -> fc2e849cd
HBASE-13008 Better default for hbase.regionserver.regionSplitLimit parameter (Srikanth Srungarapu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/fc2e849c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/fc2e849c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/fc2e849c
Branch: refs/heads/master
Commit: fc2e849cd35ffef0cf91b80d42c983702e456199
Parents: 02759f2
Author: Andrew Purtell <ap...@apache.org>
Authored: Thu Feb 12 10:13:38 2015 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Feb 12 10:13:38 2015 -0800
----------------------------------------------------------------------
hbase-common/src/main/resources/hbase-default.xml | 9 +++++++++
.../hadoop/hbase/regionserver/CompactSplitThread.java | 12 ++++++++++--
2 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/fc2e849c/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 2985685..6dcd80d 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -333,6 +333,15 @@ possible configurations would overwhelm and obscure the important.
DelimitedKeyPrefixRegionSplitPolicy, KeyPrefixRegionSplitPolicy etc.
</description>
</property>
+ <property>
+ <name>hbase.regionserver.regionSplitLimit</name>
+ <value>1000</value>
+ <description>
+ Limit for the number of regions after which no more region splitting should take place.
+ This is not hard limit for the number of regions but acts as a guideline for the regionserver
+ to stop splitting after a certain limit. Default is set to 1000.
+ </description>
+ </property>
<!--ZooKeeper configuration-->
<property>
http://git-wip-us.apache.org/repos/asf/hbase/blob/fc2e849c/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index 3e50036..e891d00 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -75,6 +75,10 @@ public class CompactSplitThread implements CompactionRequestor, PropagatingConfi
// Configuration keys for merge threads
public final static String MERGE_THREADS = "hbase.regionserver.thread.merge";
public final static int MERGE_THREADS_DEFAULT = 1;
+
+ public static final String REGION_SERVER_REGION_SPLIT_LIMIT =
+ "hbase.regionserver.regionSplitLimit";
+ public static final int DEFAULT_REGION_SERVER_REGION_SPLIT_LIMIT= 1000;
private final HRegionServer server;
private final Configuration conf;
@@ -98,8 +102,8 @@ public class CompactSplitThread implements CompactionRequestor, PropagatingConfi
super();
this.server = server;
this.conf = server.getConfiguration();
- this.regionSplitLimit = conf.getInt("hbase.regionserver.regionSplitLimit",
- Integer.MAX_VALUE);
+ this.regionSplitLimit = conf.getInt(REGION_SERVER_REGION_SPLIT_LIMIT,
+ DEFAULT_REGION_SERVER_REGION_SPLIT_LIMIT);
int largeThreads = Math.max(1, conf.getInt(
LARGE_COMPACTION_THREADS, LARGE_COMPACTION_THREADS_DEFAULT));
@@ -427,6 +431,10 @@ public class CompactSplitThread implements CompactionRequestor, PropagatingConfi
}
private boolean shouldSplitRegion() {
+ if(server.getNumberOfOnlineRegions() > 0.9*regionSplitLimit) {
+ LOG.warn("Total number of regions is approaching the upper limit " + regionSplitLimit + ". "
+ + "Please consider taking a look at http://hbase.apache.org/book.html#ops.regionmgt");
+ }
return (regionSplitLimit > server.getNumberOfOnlineRegions());
}
[4/4] hbase git commit: HBASE-13008 Better default for
hbase.regionserver.regionSplitLimit parameter (Srikanth Srungarapu)
Posted by ap...@apache.org.
HBASE-13008 Better default for hbase.regionserver.regionSplitLimit parameter (Srikanth Srungarapu)
Conflicts:
hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7871c930
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7871c930
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7871c930
Branch: refs/heads/0.98
Commit: 7871c930547b078c7d63569231d646b396f40115
Parents: 4ee823b
Author: Andrew Purtell <ap...@apache.org>
Authored: Thu Feb 12 10:13:38 2015 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Feb 12 10:18:20 2015 -0800
----------------------------------------------------------------------
hbase-common/src/main/resources/hbase-default.xml | 9 +++++++++
.../hadoop/hbase/regionserver/CompactSplitThread.java | 12 ++++++++++--
2 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7871c930/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 8bda312..4a3442f 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -303,6 +303,15 @@ possible configurations would overwhelm and obscure the important.
DelimitedKeyPrefixRegionSplitPolicy, KeyPrefixRegionSplitPolicy etc.
</description>
</property>
+ <property>
+ <name>hbase.regionserver.regionSplitLimit</name>
+ <value>1000</value>
+ <description>
+ Limit for the number of regions after which no more region splitting should take place.
+ This is not hard limit for the number of regions but acts as a guideline for the regionserver
+ to stop splitting after a certain limit. Default is set to 1000.
+ </description>
+ </property>
<!--ZooKeeper configuration-->
<property>
http://git-wip-us.apache.org/repos/asf/hbase/blob/7871c930/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index dec4c93..f9b12f7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -56,6 +56,10 @@ import com.google.common.base.Preconditions;
public class CompactSplitThread implements CompactionRequestor {
static final Log LOG = LogFactory.getLog(CompactSplitThread.class);
+ public static final String REGION_SERVER_REGION_SPLIT_LIMIT =
+ "hbase.regionserver.regionSplitLimit";
+ public static final int DEFAULT_REGION_SERVER_REGION_SPLIT_LIMIT= 1000;
+
private final HRegionServer server;
private final Configuration conf;
@@ -78,8 +82,8 @@ public class CompactSplitThread implements CompactionRequestor {
super();
this.server = server;
this.conf = server.getConfiguration();
- this.regionSplitLimit = conf.getInt("hbase.regionserver.regionSplitLimit",
- Integer.MAX_VALUE);
+ this.regionSplitLimit = conf.getInt(REGION_SERVER_REGION_SPLIT_LIMIT,
+ DEFAULT_REGION_SERVER_REGION_SPLIT_LIMIT);
int largeThreads = Math.max(1, conf.getInt(
"hbase.regionserver.thread.compaction.large", 1));
@@ -407,6 +411,10 @@ public class CompactSplitThread implements CompactionRequestor {
}
private boolean shouldSplitRegion() {
+ if(server.getNumberOfOnlineRegions() > 0.9*regionSplitLimit) {
+ LOG.warn("Total number of regions is approaching the upper limit " + regionSplitLimit + ". "
+ + "Please consider taking a look at http://hbase.apache.org/book.html#ops.regionmgt");
+ }
return (regionSplitLimit > server.getNumberOfOnlineRegions());
}
[2/4] hbase git commit: HBASE-13008 Better default for
hbase.regionserver.regionSplitLimit parameter (Srikanth Srungarapu)
Posted by ap...@apache.org.
HBASE-13008 Better default for hbase.regionserver.regionSplitLimit parameter (Srikanth Srungarapu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/7841bf73
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/7841bf73
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/7841bf73
Branch: refs/heads/branch-1
Commit: 7841bf73be5a4170713b3f67f1315c617f30c326
Parents: 38de2d2
Author: Andrew Purtell <ap...@apache.org>
Authored: Thu Feb 12 10:13:38 2015 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Feb 12 10:17:23 2015 -0800
----------------------------------------------------------------------
hbase-common/src/main/resources/hbase-default.xml | 9 +++++++++
.../hadoop/hbase/regionserver/CompactSplitThread.java | 12 ++++++++++--
2 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/7841bf73/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index c14bb8b..f7b0bc3 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -333,6 +333,15 @@ possible configurations would overwhelm and obscure the important.
DelimitedKeyPrefixRegionSplitPolicy, KeyPrefixRegionSplitPolicy etc.
</description>
</property>
+ <property>
+ <name>hbase.regionserver.regionSplitLimit</name>
+ <value>1000</value>
+ <description>
+ Limit for the number of regions after which no more region splitting should take place.
+ This is not hard limit for the number of regions but acts as a guideline for the regionserver
+ to stop splitting after a certain limit. Default is set to 1000.
+ </description>
+ </property>
<!--ZooKeeper configuration-->
<property>
http://git-wip-us.apache.org/repos/asf/hbase/blob/7841bf73/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index 0ff52b5..acf92b3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -75,6 +75,10 @@ public class CompactSplitThread implements CompactionRequestor, PropagatingConfi
// Configuration keys for merge threads
public final static String MERGE_THREADS = "hbase.regionserver.thread.merge";
public final static int MERGE_THREADS_DEFAULT = 1;
+
+ public static final String REGION_SERVER_REGION_SPLIT_LIMIT =
+ "hbase.regionserver.regionSplitLimit";
+ public static final int DEFAULT_REGION_SERVER_REGION_SPLIT_LIMIT= 1000;
private final HRegionServer server;
private final Configuration conf;
@@ -98,8 +102,8 @@ public class CompactSplitThread implements CompactionRequestor, PropagatingConfi
super();
this.server = server;
this.conf = server.getConfiguration();
- this.regionSplitLimit = conf.getInt("hbase.regionserver.regionSplitLimit",
- Integer.MAX_VALUE);
+ this.regionSplitLimit = conf.getInt(REGION_SERVER_REGION_SPLIT_LIMIT,
+ DEFAULT_REGION_SERVER_REGION_SPLIT_LIMIT);
int largeThreads = Math.max(1, conf.getInt(
LARGE_COMPACTION_THREADS, LARGE_COMPACTION_THREADS_DEFAULT));
@@ -427,6 +431,10 @@ public class CompactSplitThread implements CompactionRequestor, PropagatingConfi
}
private boolean shouldSplitRegion() {
+ if(server.getNumberOfOnlineRegions() > 0.9*regionSplitLimit) {
+ LOG.warn("Total number of regions is approaching the upper limit " + regionSplitLimit + ". "
+ + "Please consider taking a look at http://hbase.apache.org/book.html#ops.regionmgt");
+ }
return (regionSplitLimit > server.getNumberOfOnlineRegions());
}
[3/4] hbase git commit: HBASE-13008 Better default for
hbase.regionserver.regionSplitLimit parameter (Srikanth Srungarapu)
Posted by ap...@apache.org.
HBASE-13008 Better default for hbase.regionserver.regionSplitLimit parameter (Srikanth Srungarapu)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/bdb938ba
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/bdb938ba
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/bdb938ba
Branch: refs/heads/branch-1.0
Commit: bdb938ba21d9e2a0ef083f8c764b81763f344ab3
Parents: 497a604
Author: Andrew Purtell <ap...@apache.org>
Authored: Thu Feb 12 10:13:38 2015 -0800
Committer: Andrew Purtell <ap...@apache.org>
Committed: Thu Feb 12 10:17:27 2015 -0800
----------------------------------------------------------------------
hbase-common/src/main/resources/hbase-default.xml | 9 +++++++++
.../hadoop/hbase/regionserver/CompactSplitThread.java | 12 ++++++++++--
2 files changed, 19 insertions(+), 2 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/bdb938ba/hbase-common/src/main/resources/hbase-default.xml
----------------------------------------------------------------------
diff --git a/hbase-common/src/main/resources/hbase-default.xml b/hbase-common/src/main/resources/hbase-default.xml
index 10f6232..c0f3f9a 100644
--- a/hbase-common/src/main/resources/hbase-default.xml
+++ b/hbase-common/src/main/resources/hbase-default.xml
@@ -330,6 +330,15 @@ possible configurations would overwhelm and obscure the important.
DelimitedKeyPrefixRegionSplitPolicy, KeyPrefixRegionSplitPolicy etc.
</description>
</property>
+ <property>
+ <name>hbase.regionserver.regionSplitLimit</name>
+ <value>1000</value>
+ <description>
+ Limit for the number of regions after which no more region splitting should take place.
+ This is not hard limit for the number of regions but acts as a guideline for the regionserver
+ to stop splitting after a certain limit. Default is set to 1000.
+ </description>
+ </property>
<!--ZooKeeper configuration-->
<property>
http://git-wip-us.apache.org/repos/asf/hbase/blob/bdb938ba/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
index ea9f971..d070fe7 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
@@ -72,6 +72,10 @@ public class CompactSplitThread implements CompactionRequestor, PropagatingConfi
// Configuration keys for merge threads
public final static String MERGE_THREADS = "hbase.regionserver.thread.merge";
public final static int MERGE_THREADS_DEFAULT = 1;
+
+ public static final String REGION_SERVER_REGION_SPLIT_LIMIT =
+ "hbase.regionserver.regionSplitLimit";
+ public static final int DEFAULT_REGION_SERVER_REGION_SPLIT_LIMIT= 1000;
private final HRegionServer server;
private final Configuration conf;
@@ -93,8 +97,8 @@ public class CompactSplitThread implements CompactionRequestor, PropagatingConfi
super();
this.server = server;
this.conf = server.getConfiguration();
- this.regionSplitLimit = conf.getInt("hbase.regionserver.regionSplitLimit",
- Integer.MAX_VALUE);
+ this.regionSplitLimit = conf.getInt(REGION_SERVER_REGION_SPLIT_LIMIT,
+ DEFAULT_REGION_SERVER_REGION_SPLIT_LIMIT);
int largeThreads = Math.max(1, conf.getInt(
LARGE_COMPACTION_THREADS, LARGE_COMPACTION_THREADS_DEFAULT));
@@ -418,6 +422,10 @@ public class CompactSplitThread implements CompactionRequestor, PropagatingConfi
}
private boolean shouldSplitRegion() {
+ if(server.getNumberOfOnlineRegions() > 0.9*regionSplitLimit) {
+ LOG.warn("Total number of regions is approaching the upper limit " + regionSplitLimit + ". "
+ + "Please consider taking a look at http://hbase.apache.org/book.html#ops.regionmgt");
+ }
return (regionSplitLimit > server.getNumberOfOnlineRegions());
}