You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2014/12/15 21:04:02 UTC
hbase git commit: HBASE-12651 Backport HBASE-12559 'Provide
LoadBalancer with online configuration capability' to branch-1
Repository: hbase
Updated Branches:
refs/heads/branch-1 8b85feceb -> 677153ad6
HBASE-12651 Backport HBASE-12559 'Provide LoadBalancer with online configuration capability' to branch-1
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/677153ad
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/677153ad
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/677153ad
Branch: refs/heads/branch-1
Commit: 677153ad61a260f14cbaa6ec6cb899e5428c94d8
Parents: 8b85fec
Author: tedyu <yu...@gmail.com>
Authored: Mon Dec 15 12:03:55 2014 -0800
Committer: tedyu <yu...@gmail.com>
Committed: Mon Dec 15 12:03:55 2014 -0800
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/master/HMaster.java | 1 +
.../hadoop/hbase/master/LoadBalancer.java | 10 +-
.../hbase/master/balancer/BaseLoadBalancer.java | 5 +
.../master/balancer/StochasticLoadBalancer.java | 35 +++--
.../hbase/regionserver/HRegionServer.java | 2 +-
.../hbase/client/TestUpdateConfiguration.java | 28 ++++
hbase-server/src/test/resources/hbase-site2.xml | 154 +++++++++++++++++++
7 files changed, 221 insertions(+), 14 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index f966b12..738be01 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -729,6 +729,7 @@ public class HMaster extends HRegionServer implements MasterServices, Server {
status.markComplete("Initialization successful");
LOG.info("Master has completed initialization");
+ configurationManager.registerObserver(this.balancer);
initialized = true;
// clear the dead servers with same host name and port of online server because we are not
// removing dead server with same hostname and port of rs which is trying to check in before
http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
index e24d745..f979403 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/LoadBalancer.java
@@ -22,7 +22,9 @@ import java.util.List;
import java.util.Map;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.conf.ConfigurationObserver;
import org.apache.hadoop.conf.Configurable;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.ClusterStatus;
import org.apache.hadoop.hbase.HBaseIOException;
import org.apache.hadoop.hbase.HRegionInfo;
@@ -46,7 +48,7 @@ import org.apache.hadoop.hbase.Stoppable;
* <p>This classes produces plans for the {@link AssignmentManager} to execute.
*/
@InterfaceAudience.Private
-public interface LoadBalancer extends Configurable, Stoppable {
+public interface LoadBalancer extends Configurable, Stoppable, ConfigurationObserver {
/**
* Set the current cluster status. This allows a LoadBalancer to map host name to a server
@@ -130,4 +132,10 @@ public interface LoadBalancer extends Configurable, Stoppable {
* @param regionInfo
*/
void regionOffline(HRegionInfo regionInfo);
+
+ /*
+ * Notification that config has changed
+ * @param conf
+ */
+ void onConfigurationChange(Configuration conf);
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
index 21f25ee..7aca88e 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/BaseLoadBalancer.java
@@ -44,6 +44,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.RegionLoad;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.client.RegionReplicaUtil;
+import org.apache.hadoop.hbase.conf.ConfigurationObserver;
import org.apache.hadoop.hbase.master.LoadBalancer;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.RackManager;
@@ -1373,4 +1374,8 @@ public abstract class BaseLoadBalancer implements LoadBalancer {
return new HashMap<ServerName, List<HRegionInfo>>();
}
}
+
+ @Override
+ public void onConfigurationChange(Configuration conf) {
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
index 8187630..9d8c7cb 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/balancer/StochasticLoadBalancer.java
@@ -125,8 +125,14 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
private RegionReplicaRackCostFunction regionReplicaRackCostFunction;
@Override
- public void setConf(Configuration conf) {
+ public void onConfigurationChange(Configuration conf) {
+ setConf(conf);
+ }
+
+ @Override
+ public synchronized void setConf(Configuration conf) {
super.setConf(conf);
+ LOG.info("loading config");
maxSteps = conf.getInt(MAX_STEPS_KEY, maxSteps);
@@ -135,15 +141,19 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
numRegionLoadsToRemember = conf.getInt(KEEP_REGION_LOADS, numRegionLoadsToRemember);
- localityCandidateGenerator = new LocalityBasedCandidateGenerator(services);
+ if (localityCandidateGenerator == null) {
+ localityCandidateGenerator = new LocalityBasedCandidateGenerator(services);
+ }
localityCost = new LocalityCostFunction(conf, services);
- candidateGenerators = new CandidateGenerator[] {
- new RandomCandidateGenerator(),
- new LoadCandidateGenerator(),
- localityCandidateGenerator,
- new RegionReplicaRackCandidateGenerator(),
- };
+ if (candidateGenerators == null) {
+ candidateGenerators = new CandidateGenerator[] {
+ new RandomCandidateGenerator(),
+ new LoadCandidateGenerator(),
+ localityCandidateGenerator,
+ new RegionReplicaRackCandidateGenerator(),
+ };
+ }
regionLoadFunctions = new CostFromRegionLoadFunction[] {
new ReadRequestCostFunction(conf),
@@ -175,7 +185,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
}
@Override
- public void setClusterStatus(ClusterStatus st) {
+ public synchronized void setClusterStatus(ClusterStatus st) {
super.setClusterStatus(st);
updateRegionLoad();
for(CostFromRegionLoadFunction cost : regionLoadFunctions) {
@@ -184,7 +194,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
}
@Override
- public void setMasterServices(MasterServices masterServices) {
+ public synchronized void setMasterServices(MasterServices masterServices) {
super.setMasterServices(masterServices);
this.localityCost.setServices(masterServices);
this.localityCandidateGenerator.setServices(masterServices);
@@ -192,7 +202,7 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
}
@Override
- protected boolean areSomeRegionReplicasColocated(Cluster c) {
+ protected synchronized boolean areSomeRegionReplicasColocated(Cluster c) {
regionReplicaHostCostFunction.init(c);
if (regionReplicaHostCostFunction.cost() > 0) return true;
regionReplicaRackCostFunction.init(c);
@@ -205,7 +215,8 @@ public class StochasticLoadBalancer extends BaseLoadBalancer {
* should always approach the optimal state given enough steps.
*/
@Override
- public List<RegionPlan> balanceCluster(Map<ServerName, List<HRegionInfo>> clusterState) {
+ public synchronized List<RegionPlan> balanceCluster(Map<ServerName,
+ List<HRegionInfo>> clusterState) {
List<RegionPlan> plans = balanceMasterRegions(clusterState);
if (plans != null || clusterState == null || clusterState.size() <= 1) {
return plans;
http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
index b52065d..ddeacd3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
@@ -440,7 +440,7 @@ public class HRegionServer extends HasThread implements
* Configuration manager is used to register/deregister and notify the configuration observers
* when the regionserver is notified that there was a change in the on disk configs.
*/
- private final ConfigurationManager configurationManager;
+ protected final ConfigurationManager configurationManager;
/**
* Starts a HRegionServer at the default location.
http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
index 74590e0..e2af1ab 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestUpdateConfiguration.java
@@ -18,10 +18,17 @@
*/
package org.apache.hadoop.hbase.client;
+import static org.junit.Assert.assertEquals;
+
import java.io.IOException;
+import java.nio.file.FileSystems;
+import java.nio.file.Files;
+import java.nio.file.Path;
+import java.nio.file.StandardCopyOption;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.testclassification.MediumTests;
@@ -46,4 +53,25 @@ public class TestUpdateConfiguration {
ServerName server = TEST_UTIL.getHBaseCluster().getRegionServer(0).getServerName();
admin.updateConfiguration(server);
}
+
+ @Test
+ public void testMasterOnlineConfigChange() throws IOException {
+ LOG.debug("Starting the test");
+ Path cnfPath = FileSystems.getDefault().getPath("target/test-classes/hbase-site.xml");
+ Path cnf2Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site2.xml");
+ Path cnf3Path = FileSystems.getDefault().getPath("target/test-classes/hbase-site3.xml");
+ // make a backup of hbase-site.xml
+ Files.copy(cnfPath, cnf3Path, StandardCopyOption.REPLACE_EXISTING);
+ // update hbase-site.xml by overwriting it
+ Files.copy(cnf2Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
+
+ Admin admin = TEST_UTIL.getHBaseAdmin();
+ ServerName server = TEST_UTIL.getHBaseCluster().getMaster().getServerName();
+ admin.updateConfiguration(server);
+ Configuration conf = TEST_UTIL.getMiniHBaseCluster().getMaster().getConfiguration();
+ int custom = conf.getInt("hbase.custom.config", 0);
+ assertEquals(custom, 1000);
+ // restore hbase-site.xml
+ Files.copy(cnf3Path, cnfPath, StandardCopyOption.REPLACE_EXISTING);
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/677153ad/hbase-server/src/test/resources/hbase-site2.xml
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/resources/hbase-site2.xml b/hbase-server/src/test/resources/hbase-site2.xml
new file mode 100644
index 0000000..87d340e
--- /dev/null
+++ b/hbase-server/src/test/resources/hbase-site2.xml
@@ -0,0 +1,154 @@
+<?xml version="1.0"?>
+<?xml-stylesheet type="text/xsl" href="configuration.xsl"?>
+<!--
+/**
+ *
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+-->
+<configuration>
+ <property>
+ <name>hbase.custom.config</name>
+ <value>1000</value>
+ </property>
+ <property>
+ <name>hbase.regionserver.msginterval</name>
+ <value>1000</value>
+ <description>Interval between messages from the RegionServer to HMaster
+ in milliseconds. Default is 15. Set this value low if you want unit
+ tests to be responsive.
+ </description>
+ </property>
+ <property>
+ <name>hbase.defaults.for.version.skip</name>
+ <value>true</value>
+ </property>
+ <property>
+ <name>hbase.server.thread.wakefrequency</name>
+ <value>1000</value>
+ <description>Time to sleep in between searches for work (in milliseconds).
+ Used as sleep interval by service threads such as hbase:meta scanner and log roller.
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.event.waiting.time</name>
+ <value>50</value>
+ <description>Time to sleep between checks to see if a table event took place.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.handler.count</name>
+ <value>5</value>
+ </property>
+ <property>
+ <name>hbase.master.info.port</name>
+ <value>-1</value>
+ <description>The port for the hbase master web UI
+ Set to -1 if you do not want the info server to run.
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.port</name>
+ <value>0</value>
+ <description>Always have masters and regionservers come up on port '0' so we don't clash over
+ default ports.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.port</name>
+ <value>0</value>
+ <description>Always have masters and regionservers come up on port '0' so we don't clash over
+ default ports.
+ </description>
+ </property>
+ <property>
+ <name>hbase.ipc.client.fallback-to-simple-auth-allowed</name>
+ <value>true</value>
+ </property>
+
+ <property>
+ <name>hbase.regionserver.info.port</name>
+ <value>-1</value>
+ <description>The port for the hbase regionserver web UI
+ Set to -1 if you do not want the info server to run.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.info.port.auto</name>
+ <value>true</value>
+ <description>Info server auto port bind. Enables automatic port
+ search if hbase.regionserver.info.port is already in use.
+ Enabled for testing to run multiple tests on one machine.
+ </description>
+ </property>
+ <property>
+ <name>hbase.master.lease.thread.wakefrequency</name>
+ <value>3000</value>
+ <description>The interval between checks for expired region server leases.
+ This value has been reduced due to the other reduced values above so that
+ the master will notice a dead region server sooner. The default is 15 seconds.
+ </description>
+ </property>
+ <property>
+ <name>hbase.regionserver.safemode</name>
+ <value>false</value>
+ <description>
+ Turn on/off safe mode in region server. Always on for production, always off
+ for tests.
+ </description>
+ </property>
+ <property>
+ <name>hbase.hregion.max.filesize</name>
+ <value>67108864</value>
+ <description>
+ Maximum desired file size for an HRegion. If filesize exceeds
+ value + (value / 2), the HRegion is split in two. Default: 256M.
+
+ Keep the maximum filesize small so we split more often in tests.
+ </description>
+ </property>
+ <property>
+ <name>hadoop.log.dir</name>
+ <value>${user.dir}/../logs</value>
+ </property>
+ <property>
+ <name>hbase.zookeeper.property.clientPort</name>
+ <value>21818</value>
+ <description>Property from ZooKeeper's config zoo.cfg.
+ The port at which the clients will connect.
+ </description>
+ </property>
+ <property>
+ <name>hbase.defaults.for.version.skip</name>
+ <value>true</value>
+ <description>
+ Set to true to skip the 'hbase.defaults.for.version'.
+ Setting this to true can be useful in contexts other than
+ the other side of a maven generation; i.e. running in an
+ ide. You'll want to set this boolean to true to avoid
+ seeing the RuntimException complaint: "hbase-default.xml file
+ seems to be for and old version of HBase (@@@VERSION@@@), this
+ version is X.X.X-SNAPSHOT"
+ </description>
+ </property>
+ <property>
+ <name>hbase.table.sanity.checks</name>
+ <value>false</value>
+ <description>Skip sanity checks in tests
+ </description>
+ </property>
+</configuration>