You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dl...@apache.org on 2016/04/14 22:13:50 UTC
[1/2] accumulo git commit: ACCUMULO-4184: Changes to the
HostRegexTableLoadBalancer
Repository: accumulo
Updated Branches:
refs/heads/master 584b81212 -> ef39c1aae
ACCUMULO-4184: Changes to the HostRegexTableLoadBalancer
- Balancer now watches for configuration changes so the master does not have to be restarted
- Fixed a bug where pools overlapped the same hosts and tablets for those tables were constantly reassigned.
- Added a property to control the number of migrations that can be created by this balancer in one pass
- Fix issue where matching properties were being parsed as table regex properties
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/c37ba87c
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/c37ba87c
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/c37ba87c
Branch: refs/heads/master
Commit: c37ba87c032b1411decd034672ac41409144b3cf
Parents: d9e9d66
Author: Dave Marion <dl...@apache.org>
Authored: Thu Apr 14 15:56:02 2016 -0400
Committer: Dave Marion <dl...@apache.org>
Committed: Thu Apr 14 15:56:02 2016 -0400
----------------------------------------------------------------------
.../balancer/HostRegexTableLoadBalancer.java | 136 ++++++---
.../BaseHostRegexTableLoadBalancerTest.java | 289 +++++++++++++++++++
...gexTableLoadBalancerReconfigurationTest.java | 106 +++++++
.../HostRegexTableLoadBalancerTest.java | 259 +----------------
4 files changed, 498 insertions(+), 292 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c37ba87c/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
index baf33c3..29bc1cf 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
@@ -21,9 +21,11 @@ import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Random;
import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
@@ -31,6 +33,7 @@ import java.util.regex.Pattern;
import org.apache.accumulo.core.client.admin.TableOperations;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationObserver;
import org.apache.accumulo.core.conf.Property;
import org.apache.accumulo.core.data.impl.KeyExtent;
import org.apache.accumulo.core.master.thrift.TabletServerStatus;
@@ -38,6 +41,8 @@ import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
import org.apache.accumulo.server.conf.ServerConfiguration;
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.commons.lang.builder.ToStringBuilder;
+import org.apache.commons.lang.builder.ToStringStyle;
import org.apache.thrift.TException;
import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
@@ -56,18 +61,24 @@ import org.slf4j.LoggerFactory;
* <b>table.custom.balancer.host.regex.pool.check=5m</b><br>
* Regex matching can be based on either the host name (default) or host ip address. To set this balancer to match the regular expressions to the tablet server
* IP address, then set the following property:<br>
- * <b>table.custom.balancer.host.regex.is.ip=true</b>
+ * <b>table.custom.balancer.host.regex.is.ip=true</b><br>
+ * It's possible that this balancer may create a lot of migrations. To limit the number of migrations that are created during a balance call, set the following
+ * property (default 250):<br>
+ * <b>table.custom.balancer.host.regex.concurrent.migrations</b>
*
*/
-public class HostRegexTableLoadBalancer extends TableLoadBalancer {
+public class HostRegexTableLoadBalancer extends TableLoadBalancer implements ConfigurationObserver {
private static final Logger LOG = LoggerFactory.getLogger(HostRegexTableLoadBalancer.class);
public static final String HOST_BALANCER_PREFIX = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + "balancer.host.regex.";
- public static final String HOST_BALANCER_OOB_CHECK = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + "balancer.host.regex.oob.period";
+ public static final String HOST_BALANCER_OOB_CHECK_KEY = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + "balancer.host.regex.oob.period";
private static final String HOST_BALANCER_OOB_DEFAULT = "5m";
public static final String HOST_BALANCER_POOL_RECHECK_KEY = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + "balancer.host.regex.pool.check";
private static final String HOST_BALANCER_POOL_RECHECK_DEFAULT = "1m";
- public static final String HOST_BALANCER_REGEX_USING_IPS = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + "balancer.host.regex.is.ip";
+ public static final String HOST_BALANCER_REGEX_USING_IPS_KEY = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey() + "balancer.host.regex.is.ip";
+ public static final String HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY = Property.TABLE_ARBITRARY_PROP_PREFIX.getKey()
+ + "balancer.host.regex.concurrent.migrations";
+ private static final int HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT = 250;
protected static final String DEFAULT_POOL = "HostTableLoadBalancer.ALL";
protected long oobCheckMillis = AccumuloConfiguration.getTimeInMillis(HOST_BALANCER_OOB_DEFAULT);
@@ -79,6 +90,7 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
private volatile long lastPoolRecheck = 0;
private boolean isIpBasedRegex = false;
private Map<String,SortedMap<TServerInstance,TabletServerStatus>> pools = new HashMap<String,SortedMap<TServerInstance,TabletServerStatus>>();
+ private int maxTServerMigrations = HOST_BALANCER_REGEX_MAX_MIGRATIONS_DEFAULT;
/**
* Group the set of current tservers by pool name. Tservers that don't match a regex are put into a default pool. This could be expensive in the terms of the
@@ -171,10 +183,15 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
poolNameToRegexPattern = new HashMap<>();
for (Entry<String,String> table : t.tableIdMap().entrySet()) {
tableIdToTableName.put(table.getValue(), table.getKey());
+ conf.getTableConfiguration(table.getValue()).addObserver(this);
Map<String,String> customProps = conf.getTableConfiguration(table.getValue()).getAllPropertiesWithPrefix(Property.TABLE_ARBITRARY_PROP_PREFIX);
if (null != customProps && customProps.size() > 0) {
for (Entry<String,String> customProp : customProps.entrySet()) {
if (customProp.getKey().startsWith(HOST_BALANCER_PREFIX)) {
+ if (customProp.getKey().equals(HOST_BALANCER_OOB_CHECK_KEY) || customProp.getKey().equals(HOST_BALANCER_POOL_RECHECK_KEY)
+ || customProp.getKey().equals(HOST_BALANCER_REGEX_USING_IPS_KEY) || customProp.getKey().equals(HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY)) {
+ continue;
+ }
String tableName = customProp.getKey().substring(HOST_BALANCER_PREFIX.length());
String regex = customProp.getValue();
poolNameToRegexPattern.put(tableName, Pattern.compile(regex));
@@ -182,7 +199,7 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
}
}
}
- String oobProperty = conf.getConfiguration().get(HOST_BALANCER_OOB_CHECK);
+ String oobProperty = conf.getConfiguration().get(HOST_BALANCER_OOB_CHECK_KEY);
if (null != oobProperty) {
oobCheckMillis = AccumuloConfiguration.getTimeInMillis(oobProperty);
}
@@ -190,10 +207,26 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
if (null != poolRecheckProperty) {
poolRecheckMillis = AccumuloConfiguration.getTimeInMillis(poolRecheckProperty);
}
- String ipBased = conf.getConfiguration().get(HOST_BALANCER_REGEX_USING_IPS);
+ String ipBased = conf.getConfiguration().get(HOST_BALANCER_REGEX_USING_IPS_KEY);
if (null != ipBased) {
isIpBasedRegex = Boolean.parseBoolean(ipBased);
}
+ String migrations = conf.getConfiguration().get(HOST_BALANCER_REGEX_MAX_MIGRATIONS_KEY);
+ if (null != migrations) {
+ maxTServerMigrations = Integer.parseInt(migrations);
+ }
+ LOG.info("{}", this);
+ }
+
+ @Override
+ public String toString() {
+ ToStringBuilder buf = new ToStringBuilder(this, ToStringStyle.SHORT_PREFIX_STYLE);
+ buf.append("Pool Recheck Interval", this.poolRecheckMillis);
+ buf.append("Tablet Out Of Bounds Check Interval", this.oobCheckMillis);
+ buf.append("Max Tablet Server Migrations", this.maxTServerMigrations);
+ buf.append("Regular Expressions use IPs", this.isIpBasedRegex);
+ buf.append("Pools", this.poolNameToRegexPattern);
+ return buf.toString();
}
public Map<String,String> getTableIdToTableName() {
@@ -216,6 +249,10 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
return isIpBasedRegex;
}
+ public int getMaxConcurrentMigrations() {
+ return maxTServerMigrations;
+ }
+
@Override
public void init(ServerConfiguration conf) {
super.init(conf);
@@ -270,40 +307,46 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
try {
// Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate it.
for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
- for (String assignedPool : getPoolNamesForHost(e.getKey().host())) {
- for (String table : poolNameToRegexPattern.keySet()) {
- // pool names are the same as table names, except in the DEFAULT case.
- if (assignedPool.equals(table)) {
- // If this tserver is assigned to a regex pool, then we can skip checking tablets for this table on this host.
- continue;
- }
- String tid = t.tableIdMap().get(table);
- if (null == tid) {
- LOG.warn("Unable to check for out of bounds tablets for table {}, it may have been deleted or renamed.", table);
- continue;
- }
- try {
- List<TabletStats> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(), tid);
- for (TabletStats ts : outOfBoundsTablets) {
- KeyExtent ke = new KeyExtent(ts.getExtent());
- if (migrations.contains(ke)) {
- LOG.debug("Migration for out of bounds tablet {} has already been requested", ke);
- ;
- continue;
+ for (String table : poolNameToRegexPattern.keySet()) {
+ // pool names are the same as table names, except in the DEFAULT case.
+ // If this table is assigned to a pool for this host, then move on.
+ if (getPoolNamesForHost(e.getKey().host()).contains(table)) {
+ continue;
+ }
+ String tid = t.tableIdMap().get(table);
+ if (null == tid) {
+ LOG.warn("Unable to check for out of bounds tablets for table {}, it may have been deleted or renamed.", table);
+ continue;
+ }
+ try {
+ List<TabletStats> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(), tid);
+ Random random = new Random();
+ for (TabletStats ts : outOfBoundsTablets) {
+ KeyExtent ke = new KeyExtent(ts.getExtent());
+ if (migrations.contains(ke)) {
+ LOG.debug("Migration for out of bounds tablet {} has already been requested", ke);
+ continue;
+ }
+ String poolName = getPoolNameForTable(table);
+ SortedMap<TServerInstance,TabletServerStatus> currentView = currentGrouped.get(poolName);
+ if (null != currentView) {
+ int skip = random.nextInt(currentView.size());
+ Iterator<TServerInstance> iter = currentView.keySet().iterator();
+ for (int i = 0; i < skip; i++) {
+ iter.next();
}
- String poolName = getPoolNameForTable(table);
- SortedMap<TServerInstance,TabletServerStatus> currentView = currentGrouped.get(poolName);
- if (null != currentView) {
- TServerInstance nextTS = currentView.firstKey();
- LOG.info("Tablet {} is currently outside the bounds of the regex, migrating from {} to {}", ke, e.getKey(), nextTS);
- migrationsOut.add(new TabletMigration(ke, e.getKey(), nextTS));
- } else {
- LOG.warn("No tablet servers online for pool {}, unable to migrate out of bounds tablets", poolName);
+ TServerInstance nextTS = iter.next();
+ LOG.info("Tablet {} is currently outside the bounds of the regex, migrating from {} to {}", ke, e.getKey(), nextTS);
+ migrationsOut.add(new TabletMigration(ke, e.getKey(), nextTS));
+ if (migrationsOut.size() > this.maxTServerMigrations) {
+ break;
}
+ } else {
+ LOG.warn("No tablet servers online for pool {}, unable to migrate out of bounds tablets", poolName);
}
- } catch (TException e1) {
- LOG.error("Error in OOB check getting tablets for table {} from server {}", tid, e.getKey().host(), e);
}
+ } catch (TException e1) {
+ LOG.error("Error in OOB check getting tablets for table {} from server {}", tid, e.getKey().host(), e);
}
}
}
@@ -312,6 +355,11 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
}
}
+ if (migrationsOut.size() > 0) {
+ LOG.warn("Not balancing tables due to moving {} out of bounds tablets", migrationsOut.size());
+ return minBalanceTime;
+ }
+
if (migrations != null && migrations.size() > 0) {
LOG.warn("Not balancing tables due to {} outstanding migrations", migrations.size());
return minBalanceTime;
@@ -332,9 +380,25 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
minBalanceTime = tableBalanceTime;
}
migrationsOut.addAll(newMigrations);
+ if (migrationsOut.size() > this.maxTServerMigrations) {
+ break;
+ }
}
return minBalanceTime;
}
+ @Override
+ public void propertyChanged(String key) {
+ parseConfiguration(this.configuration);
+ }
+
+ @Override
+ public void propertiesChanged() {
+ parseConfiguration(this.configuration);
+ }
+
+ @Override
+ public void sessionExpired() {}
+
}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c37ba87c/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
new file mode 100644
index 0000000..aa1480f
--- /dev/null
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
@@ -0,0 +1,289 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.balancer;
+
+import java.net.UnknownHostException;
+import java.nio.ByteBuffer;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.SortedMap;
+import java.util.TreeMap;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.AccumuloException;
+import org.apache.accumulo.core.client.AccumuloSecurityException;
+import org.apache.accumulo.core.client.Connector;
+import org.apache.accumulo.core.client.Instance;
+import org.apache.accumulo.core.client.admin.TableOperations;
+import org.apache.accumulo.core.client.impl.ClientContext;
+import org.apache.accumulo.core.client.impl.TableOperationsImpl;
+import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+import org.apache.accumulo.core.conf.AccumuloConfiguration;
+import org.apache.accumulo.core.conf.ConfigurationCopy;
+import org.apache.accumulo.core.conf.Property;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+import org.apache.accumulo.server.conf.TableConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.hadoop.io.Text;
+import org.easymock.EasyMock;
+
+import com.google.common.base.Predicate;
+
+public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer {
+
+ protected static class TestInstance implements Instance {
+
+ @Override
+ public String getRootTabletLocation() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<String> getMasterLocations() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getInstanceID() {
+ return "1111";
+ }
+
+ @Override
+ public String getInstanceName() {
+ return "test";
+ }
+
+ @Override
+ public String getZooKeepers() {
+ return "";
+ }
+
+ @Override
+ public int getZooKeepersSessionTimeOut() {
+ return 30;
+ }
+
+ @Override
+ public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public AccumuloConfiguration getConfiguration() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setConfiguration(AccumuloConfiguration conf) {}
+
+ @Override
+ public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ protected static class Table {
+ private String tableName;
+ private String id;
+
+ Table(String tableName, String id) {
+ this.tableName = tableName;
+ this.id = id;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public String getId() {
+ return id;
+ }
+ }
+
+ protected static final HashMap<String,String> DEFAULT_TABLE_PROPERTIES = new HashMap<>();
+ {
+ DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY, "10s");
+ DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_POOL_RECHECK_KEY, "30s");
+ DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(), "r01.*");
+ DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r02.*");
+ }
+
+ protected static class TestServerConfigurationFactory extends ServerConfigurationFactory {
+
+ public TestServerConfigurationFactory(Instance instance) {
+ super(instance);
+ }
+
+ @Override
+ public synchronized AccumuloConfiguration getConfiguration() {
+ return new ConfigurationCopy(DEFAULT_TABLE_PROPERTIES);
+ }
+
+ @Override
+ public TableConfiguration getTableConfiguration(String tableId) {
+ return new TableConfiguration(getInstance(), tableId, null) {
+ @Override
+ public String get(Property property) {
+ return DEFAULT_TABLE_PROPERTIES.get(property.name());
+ }
+
+ @Override
+ public void getProperties(Map<String,String> props, Predicate<String> filter) {
+ for (Entry<String,String> e : DEFAULT_TABLE_PROPERTIES.entrySet()) {
+ if (filter.apply(e.getKey())) {
+ props.put(e.getKey(), e.getValue());
+ }
+ }
+ }
+ };
+ }
+ }
+
+ protected static final Table FOO = new Table("foo", "1");
+ protected static final Table BAR = new Table("bar", "2");
+ protected static final Table BAZ = new Table("baz", "3");
+
+ protected final TestInstance instance = new TestInstance();
+ protected final TestServerConfigurationFactory factory = new TestServerConfigurationFactory(instance);
+ protected final Map<String,String> servers = new HashMap<>(15);
+ protected final SortedMap<TServerInstance,TabletServerStatus> allTabletServers = new TreeMap<>();
+ protected final Map<String,List<KeyExtent>> tableExtents = new HashMap<>(3);
+
+ {
+ servers.put("192.168.0.1", "r01s01");
+ servers.put("192.168.0.2", "r01s02");
+ servers.put("192.168.0.3", "r01s03");
+ servers.put("192.168.0.4", "r01s04");
+ servers.put("192.168.0.5", "r01s05");
+ servers.put("192.168.0.6", "r02s01");
+ servers.put("192.168.0.7", "r02s02");
+ servers.put("192.168.0.8", "r02s03");
+ servers.put("192.168.0.9", "r02s04");
+ servers.put("192.168.0.10", "r02s05");
+ servers.put("192.168.0.11", "r03s01");
+ servers.put("192.168.0.12", "r03s02");
+ servers.put("192.168.0.13", "r03s03");
+ servers.put("192.168.0.14", "r03s04");
+ servers.put("192.168.0.15", "r03s05");
+
+ allTabletServers.put(new TServerInstance("192.168.0.1:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.2:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.3:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.4:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.5:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.6:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.7:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.8:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.9:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.10:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.11:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.12:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.13:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.14:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.15:9997", 1), new TabletServerStatus());
+
+ tableExtents.put(FOO.getTableName(), new ArrayList<KeyExtent>());
+ tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("1"), new Text("0")));
+ tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("2"), new Text("1")));
+ tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("3"), new Text("2")));
+ tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("4"), new Text("3")));
+ tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("5"), new Text("4")));
+ tableExtents.put(BAR.getTableName(), new ArrayList<KeyExtent>());
+ tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("11"), new Text("10")));
+ tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("12"), new Text("11")));
+ tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("13"), new Text("12")));
+ tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("14"), new Text("13")));
+ tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("15"), new Text("14")));
+ tableExtents.put(BAZ.getTableName(), new ArrayList<KeyExtent>());
+ tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("21"), new Text("20")));
+ tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("22"), new Text("21")));
+ tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("23"), new Text("22")));
+ tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("24"), new Text("23")));
+ tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("25"), new Text("24")));
+ }
+
+ protected boolean tabletInBounds(KeyExtent ke, TServerInstance tsi) {
+ String tid = ke.getTableId().toString();
+ String host = tsi.host();
+ if (tid.equals("1")
+ && (host.equals("192.168.0.1") || host.equals("192.168.0.2") || host.equals("192.168.0.3") || host.equals("192.168.0.4") || host.equals("192.168.0.5"))) {
+ return true;
+ } else if (tid.equals("2")
+ && (host.equals("192.168.0.6") || host.equals("192.168.0.7") || host.equals("192.168.0.8") || host.equals("192.168.0.9") || host.equals("192.168.0.10"))) {
+ return true;
+ } else if (tid.equals("3")
+ && (host.equals("192.168.0.11") || host.equals("192.168.0.12") || host.equals("192.168.0.13") || host.equals("192.168.0.14") || host
+ .equals("192.168.0.15"))) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ protected TableOperations getTableOperations() {
+ return new TableOperationsImpl(EasyMock.createMock(ClientContext.class)) {
+ @Override
+ public Map<String,String> tableIdMap() {
+ HashMap<String,String> tables = new HashMap<>();
+ tables.put(FOO.getTableName(), FOO.getId());
+ tables.put(BAR.getTableName(), BAR.getId());
+ tables.put(BAZ.getTableName(), BAZ.getId());
+ return tables;
+ }
+ };
+ }
+
+ @Override
+ protected TabletBalancer getBalancerForTable(String table) {
+ return new DefaultLoadBalancer();
+ }
+
+ @Override
+ protected String getNameFromIp(String hostIp) throws UnknownHostException {
+ if (servers.containsKey(hostIp)) {
+ return servers.get(hostIp);
+ } else {
+ throw new UnknownHostException();
+ }
+ }
+
+ protected SortedMap<TServerInstance,TabletServerStatus> createCurrent(int numTservers) {
+ String base = "192.168.0.";
+ TreeMap<TServerInstance,TabletServerStatus> current = new TreeMap<>();
+ for (int i = 1; i <= numTservers; i++) {
+ current.put(new TServerInstance(base + i + ":9997", 1), new TabletServerStatus());
+ }
+ return current;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c37ba87c/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java
new file mode 100644
index 0000000..eff9a11
--- /dev/null
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerReconfigurationTest.java
@@ -0,0 +1,106 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.accumulo.server.master.balancer;
+
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.HashSet;
+import java.util.List;
+import java.util.Map;
+import java.util.Set;
+import java.util.Map.Entry;
+
+import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
+import org.apache.accumulo.core.data.impl.KeyExtent;
+import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
+import org.apache.accumulo.fate.util.UtilWaitThread;
+import org.apache.accumulo.server.conf.ServerConfiguration;
+import org.apache.accumulo.server.master.state.TServerInstance;
+import org.apache.accumulo.server.master.state.TabletMigration;
+import org.apache.thrift.TException;
+import org.junit.Assert;
+import org.junit.Test;
+
+public class HostRegexTableLoadBalancerReconfigurationTest extends BaseHostRegexTableLoadBalancerTest {
+
+ private Map<KeyExtent,TServerInstance> assignments = new HashMap<>();
+
+ @Test
+ public void testConfigurationChanges() {
+ DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_POOL_RECHECK_KEY, "10s");
+
+ init((ServerConfiguration) factory);
+ Map<KeyExtent,TServerInstance> unassigned = new HashMap<>();
+ for (List<KeyExtent> extents : tableExtents.values()) {
+ for (KeyExtent ke : extents) {
+ unassigned.put(ke, null);
+ }
+ }
+ this.getAssignments(Collections.unmodifiableSortedMap(allTabletServers), Collections.unmodifiableMap(unassigned), assignments);
+ Assert.assertEquals(15, assignments.size());
+ // Ensure unique tservers
+ for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
+ for (Entry<KeyExtent,TServerInstance> e2 : assignments.entrySet()) {
+ if (e.getKey().equals(e2.getKey())) {
+ continue;
+ }
+ if (e.getValue().equals(e2.getValue())) {
+ Assert.fail("Assignment failure. " + e.getKey() + " and " + e2.getKey() + " are assigned to the same host: " + e.getValue());
+ }
+ }
+ }
+ // Ensure assignments are correct
+ for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
+ if (!tabletInBounds(e.getKey(), e.getValue())) {
+ Assert.fail("tablet not in bounds: " + e.getKey() + " -> " + e.getValue().host());
+ }
+ }
+ Set<KeyExtent> migrations = new HashSet<KeyExtent>();
+ List<TabletMigration> migrationsOut = new ArrayList<TabletMigration>();
+ // Wait to trigger the out of bounds check which will call our version of getOnlineTabletsForTable
+ UtilWaitThread.sleep(11000);
+ this.balance(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut);
+ Assert.assertEquals(0, migrationsOut.size());
+ // Change property, simulate call by TableConfWatcher
+ DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r01.*");
+ this.propertiesChanged();
+ // Wait to trigger the out of bounds check and the repool check
+ UtilWaitThread.sleep(11000);
+ this.balance(Collections.unmodifiableSortedMap(allTabletServers), migrations, migrationsOut);
+ Assert.assertEquals(5, migrationsOut.size());
+ for (TabletMigration migration : migrationsOut) {
+ Assert.assertTrue(migration.newServer.host().startsWith("192.168.0.1") || migration.newServer.host().startsWith("192.168.0.2")
+ || migration.newServer.host().startsWith("192.168.0.3") || migration.newServer.host().startsWith("192.168.0.4")
+ || migration.newServer.host().startsWith("192.168.0.5"));
+ }
+ }
+
+ @Override
+ public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, String tableId) throws ThriftSecurityException, TException {
+ List<TabletStats> tablets = new ArrayList<>();
+ // Report assignment information
+ for (Entry<KeyExtent,TServerInstance> e : this.assignments.entrySet()) {
+ if (e.getValue().equals(tserver) && e.getKey().getTableId().toString().equals(tableId)) {
+ TabletStats ts = new TabletStats();
+ ts.setExtent(e.getKey().toThrift());
+ tablets.add(ts);
+ }
+ }
+ return tablets;
+ }
+}
http://git-wip-us.apache.org/repos/asf/accumulo/blob/c37ba87c/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
index 266a09b..868ac0a 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
@@ -16,8 +16,6 @@
*/
package org.apache.accumulo.server.master.balancer;
-import java.net.UnknownHostException;
-import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
@@ -27,18 +25,9 @@ import java.util.Map;
import java.util.Map.Entry;
import java.util.Set;
import java.util.SortedMap;
-import java.util.TreeMap;
import java.util.regex.Pattern;
-import org.apache.accumulo.core.client.AccumuloException;
-import org.apache.accumulo.core.client.AccumuloSecurityException;
-import org.apache.accumulo.core.client.Connector;
-import org.apache.accumulo.core.client.Instance;
-import org.apache.accumulo.core.client.admin.TableOperations;
-import org.apache.accumulo.core.client.impl.ClientContext;
-import org.apache.accumulo.core.client.impl.TableOperationsImpl;
import org.apache.accumulo.core.client.impl.thrift.ThriftSecurityException;
-import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
import org.apache.accumulo.core.conf.AccumuloConfiguration;
import org.apache.accumulo.core.conf.ConfigurationCopy;
import org.apache.accumulo.core.conf.Property;
@@ -48,239 +37,16 @@ import org.apache.accumulo.core.master.thrift.TabletServerStatus;
import org.apache.accumulo.core.tabletserver.thrift.TabletStats;
import org.apache.accumulo.fate.util.UtilWaitThread;
import org.apache.accumulo.server.conf.ServerConfiguration;
-import org.apache.accumulo.server.conf.ServerConfigurationFactory;
import org.apache.accumulo.server.conf.TableConfiguration;
import org.apache.accumulo.server.master.state.TServerInstance;
import org.apache.accumulo.server.master.state.TabletMigration;
-import org.apache.hadoop.io.Text;
import org.apache.thrift.TException;
-import org.easymock.EasyMock;
import org.junit.Assert;
import org.junit.Test;
import com.google.common.base.Predicate;
-public class HostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer {
-
- static class TestInstance implements Instance {
-
- @Override
- public String getRootTabletLocation() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public List<String> getMasterLocations() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public String getInstanceID() {
- return "1111";
- }
-
- @Override
- public String getInstanceName() {
- return "test";
- }
-
- @Override
- public String getZooKeepers() {
- return "";
- }
-
- @Override
- public int getZooKeepersSessionTimeOut() {
- return 30;
- }
-
- @Override
- public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public AccumuloConfiguration getConfiguration() {
- throw new UnsupportedOperationException();
- }
-
- @Override
- public void setConfiguration(AccumuloConfiguration conf) {}
-
- @Override
- public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
- throw new UnsupportedOperationException();
- }
-
- }
-
- private static class Table {
- private String tableName;
- private String id;
-
- Table(String tableName, String id) {
- this.tableName = tableName;
- this.id = id;
- }
-
- public String getTableName() {
- return tableName;
- }
-
- public String getId() {
- return id;
- }
- }
-
- static class TestServerConfigurationFactory extends ServerConfigurationFactory {
-
- public TestServerConfigurationFactory(Instance instance) {
- super(instance);
- }
-
- @Override
- public synchronized AccumuloConfiguration getConfiguration() {
- HashMap<String,String> props = new HashMap<>();
- props.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK, "10s");
- props.put(HostRegexTableLoadBalancer.HOST_BALANCER_POOL_RECHECK_KEY, "30s");
- return new ConfigurationCopy(props);
- }
-
- @Override
- public TableConfiguration getTableConfiguration(String tableId) {
- return new TableConfiguration(getInstance(), tableId, null) {
- HashMap<String,String> tableProperties = new HashMap<>();
- {
- tableProperties.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(), "r01.*");
- tableProperties.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r02.*");
- }
-
- @Override
- public String get(Property property) {
- return tableProperties.get(property.name());
- }
-
- @Override
- public void getProperties(Map<String,String> props, Predicate<String> filter) {
- for (Entry<String,String> e : tableProperties.entrySet()) {
- if (filter.apply(e.getKey())) {
- props.put(e.getKey(), e.getValue());
- }
- }
- }
- };
- }
- }
-
- private static final Table FOO = new Table("foo", "1");
- private static final Table BAR = new Table("bar", "2");
- private static final Table BAZ = new Table("baz", "3");
-
- private final TestInstance instance = new TestInstance();
- private final TestServerConfigurationFactory factory = new TestServerConfigurationFactory(instance);
- private final Map<String,String> servers = new HashMap<>(15);
- private final SortedMap<TServerInstance,TabletServerStatus> allTabletServers = new TreeMap<>();
- private final Map<String,List<KeyExtent>> tableExtents = new HashMap<>(3);
-
- {
- servers.put("192.168.0.1", "r01s01");
- servers.put("192.168.0.2", "r01s02");
- servers.put("192.168.0.3", "r01s03");
- servers.put("192.168.0.4", "r01s04");
- servers.put("192.168.0.5", "r01s05");
- servers.put("192.168.0.6", "r02s01");
- servers.put("192.168.0.7", "r02s02");
- servers.put("192.168.0.8", "r02s03");
- servers.put("192.168.0.9", "r02s04");
- servers.put("192.168.0.10", "r02s05");
- servers.put("192.168.0.11", "r03s01");
- servers.put("192.168.0.12", "r03s02");
- servers.put("192.168.0.13", "r03s03");
- servers.put("192.168.0.14", "r03s04");
- servers.put("192.168.0.15", "r03s05");
-
- allTabletServers.put(new TServerInstance("192.168.0.1:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.2:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.3:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.4:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.5:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.6:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.7:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.8:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.9:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.10:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.11:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.12:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.13:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.14:9997", 1), new TabletServerStatus());
- allTabletServers.put(new TServerInstance("192.168.0.15:9997", 1), new TabletServerStatus());
-
- tableExtents.put(FOO.getTableName(), new ArrayList<KeyExtent>());
- tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("1"), new Text("0")));
- tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("2"), new Text("1")));
- tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("3"), new Text("2")));
- tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("4"), new Text("3")));
- tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("5"), new Text("4")));
- tableExtents.put(BAR.getTableName(), new ArrayList<KeyExtent>());
- tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("11"), new Text("10")));
- tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("12"), new Text("11")));
- tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("13"), new Text("12")));
- tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("14"), new Text("13")));
- tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("15"), new Text("14")));
- tableExtents.put(BAZ.getTableName(), new ArrayList<KeyExtent>());
- tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("21"), new Text("20")));
- tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("22"), new Text("21")));
- tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("23"), new Text("22")));
- tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("24"), new Text("23")));
- tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("25"), new Text("24")));
- }
-
- @Override
- protected String getNameFromIp(String hostIp) throws UnknownHostException {
- if (servers.containsKey(hostIp)) {
- return servers.get(hostIp);
- } else {
- throw new UnknownHostException();
- }
- }
-
- @Override
- protected TableOperations getTableOperations() {
- return new TableOperationsImpl(EasyMock.createMock(ClientContext.class)) {
- @Override
- public Map<String,String> tableIdMap() {
- HashMap<String,String> tables = new HashMap<>();
- tables.put(FOO.getTableName(), FOO.getId());
- tables.put(BAR.getTableName(), BAR.getId());
- tables.put(BAZ.getTableName(), BAZ.getId());
- return tables;
- }
- };
- }
-
- @Override
- protected TabletBalancer getBalancerForTable(String table) {
- return new DefaultLoadBalancer();
- }
-
- private SortedMap<TServerInstance,TabletServerStatus> createCurrent(int numTservers) {
- String base = "192.168.0.";
- TreeMap<TServerInstance,TabletServerStatus> current = new TreeMap<>();
- for (int i = 1; i <= numTservers; i++) {
- current.put(new TServerInstance(base + i + ":9997", 1), new TabletServerStatus());
- }
- return current;
- }
+public class HostRegexTableLoadBalancerTest extends BaseHostRegexTableLoadBalancerTest {
@Test
public void testInit() {
@@ -414,9 +180,9 @@ public class HostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer {
@Override
public synchronized AccumuloConfiguration getConfiguration() {
HashMap<String,String> props = new HashMap<>();
- props.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK, "30s");
+ props.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY, "30s");
props.put(HostRegexTableLoadBalancer.HOST_BALANCER_POOL_RECHECK_KEY, "30s");
- props.put(HostRegexTableLoadBalancer.HOST_BALANCER_REGEX_USING_IPS, "true");
+ props.put(HostRegexTableLoadBalancer.HOST_BALANCER_REGEX_USING_IPS_KEY, "true");
return new ConfigurationCopy(props);
}
@@ -612,27 +378,8 @@ public class HostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer {
TabletStats ts = new TabletStats();
ts.setExtent(tke);
tablets.add(ts);
-
}
return tablets;
}
- private boolean tabletInBounds(KeyExtent ke, TServerInstance tsi) {
- String tid = ke.getTableId().toString();
- String host = tsi.host();
- if (tid.equals("1")
- && (host.equals("192.168.0.1") || host.equals("192.168.0.2") || host.equals("192.168.0.3") || host.equals("192.168.0.4") || host.equals("192.168.0.5"))) {
- return true;
- } else if (tid.equals("2")
- && (host.equals("192.168.0.6") || host.equals("192.168.0.7") || host.equals("192.168.0.8") || host.equals("192.168.0.9") || host.equals("192.168.0.10"))) {
- return true;
- } else if (tid.equals("3")
- && (host.equals("192.168.0.11") || host.equals("192.168.0.12") || host.equals("192.168.0.13") || host.equals("192.168.0.14") || host
- .equals("192.168.0.15"))) {
- return true;
- } else {
- return false;
- }
- }
-
}
[2/2] accumulo git commit: Merge branch '1.7'
Posted by dl...@apache.org.
Merge branch '1.7'
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/ef39c1aa
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/ef39c1aa
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/ef39c1aa
Branch: refs/heads/master
Commit: ef39c1aaeadb3b12cd90262f5c6f85e2d02a719d
Parents: 584b812 c37ba87
Author: Dave Marion <dl...@apache.org>
Authored: Thu Apr 14 16:12:42 2016 -0400
Committer: Dave Marion <dl...@apache.org>
Committed: Thu Apr 14 16:12:42 2016 -0400
----------------------------------------------------------------------
.../balancer/HostRegexTableLoadBalancer.java | 136 ++++++---
.../BaseHostRegexTableLoadBalancerTest.java | 289 +++++++++++++++++++
...gexTableLoadBalancerReconfigurationTest.java | 106 +++++++
.../HostRegexTableLoadBalancerTest.java | 259 +----------------
4 files changed, 498 insertions(+), 292 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ef39c1aa/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/ef39c1aa/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
----------------------------------------------------------------------
diff --cc server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
index 0000000,aa1480f..4ac08d2
mode 000000,100644..100644
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/BaseHostRegexTableLoadBalancerTest.java
@@@ -1,0 -1,289 +1,289 @@@
+ /*
+ * Licensed to the Apache Software Foundation (ASF) under one or more
+ * contributor license agreements. See the NOTICE file distributed with
+ * this work for additional information regarding copyright ownership.
+ * The ASF licenses this file to You under the Apache License, Version 2.0
+ * (the "License"); you may not use this file except in compliance with
+ * the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+ package org.apache.accumulo.server.master.balancer;
+
+ import java.net.UnknownHostException;
+ import java.nio.ByteBuffer;
+ import java.util.ArrayList;
+ import java.util.HashMap;
+ import java.util.List;
+ import java.util.Map;
+ import java.util.SortedMap;
+ import java.util.TreeMap;
+ import java.util.Map.Entry;
+
+ import org.apache.accumulo.core.client.AccumuloException;
+ import org.apache.accumulo.core.client.AccumuloSecurityException;
+ import org.apache.accumulo.core.client.Connector;
+ import org.apache.accumulo.core.client.Instance;
+ import org.apache.accumulo.core.client.admin.TableOperations;
+ import org.apache.accumulo.core.client.impl.ClientContext;
+ import org.apache.accumulo.core.client.impl.TableOperationsImpl;
+ import org.apache.accumulo.core.client.security.tokens.AuthenticationToken;
+ import org.apache.accumulo.core.conf.AccumuloConfiguration;
+ import org.apache.accumulo.core.conf.ConfigurationCopy;
+ import org.apache.accumulo.core.conf.Property;
+ import org.apache.accumulo.core.data.impl.KeyExtent;
+ import org.apache.accumulo.core.master.thrift.TabletServerStatus;
+ import org.apache.accumulo.server.conf.ServerConfigurationFactory;
+ import org.apache.accumulo.server.conf.TableConfiguration;
+ import org.apache.accumulo.server.master.state.TServerInstance;
+ import org.apache.hadoop.io.Text;
+ import org.easymock.EasyMock;
+
+ import com.google.common.base.Predicate;
+
+ public abstract class BaseHostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer {
+
+ protected static class TestInstance implements Instance {
+
+ @Override
+ public String getRootTabletLocation() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public List<String> getMasterLocations() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public String getInstanceID() {
+ return "1111";
+ }
+
+ @Override
+ public String getInstanceName() {
+ return "test";
+ }
+
+ @Override
+ public String getZooKeepers() {
+ return "";
+ }
+
+ @Override
+ public int getZooKeepersSessionTimeOut() {
+ return 30;
+ }
+
+ @Override
+ public Connector getConnector(String user, byte[] pass) throws AccumuloException, AccumuloSecurityException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Connector getConnector(String user, ByteBuffer pass) throws AccumuloException, AccumuloSecurityException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public Connector getConnector(String user, CharSequence pass) throws AccumuloException, AccumuloSecurityException {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public AccumuloConfiguration getConfiguration() {
+ throw new UnsupportedOperationException();
+ }
+
+ @Override
+ public void setConfiguration(AccumuloConfiguration conf) {}
+
+ @Override
+ public Connector getConnector(String principal, AuthenticationToken token) throws AccumuloException, AccumuloSecurityException {
+ throw new UnsupportedOperationException();
+ }
+
+ }
+
+ protected static class Table {
+ private String tableName;
+ private String id;
+
+ Table(String tableName, String id) {
+ this.tableName = tableName;
+ this.id = id;
+ }
+
+ public String getTableName() {
+ return tableName;
+ }
+
+ public String getId() {
+ return id;
+ }
+ }
+
+ protected static final HashMap<String,String> DEFAULT_TABLE_PROPERTIES = new HashMap<>();
+ {
+ DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_OOB_CHECK_KEY, "10s");
+ DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_POOL_RECHECK_KEY, "30s");
+ DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + FOO.getTableName(), "r01.*");
+ DEFAULT_TABLE_PROPERTIES.put(HostRegexTableLoadBalancer.HOST_BALANCER_PREFIX + BAR.getTableName(), "r02.*");
+ }
+
+ protected static class TestServerConfigurationFactory extends ServerConfigurationFactory {
+
+ public TestServerConfigurationFactory(Instance instance) {
+ super(instance);
+ }
+
+ @Override
+ public synchronized AccumuloConfiguration getConfiguration() {
+ return new ConfigurationCopy(DEFAULT_TABLE_PROPERTIES);
+ }
+
+ @Override
+ public TableConfiguration getTableConfiguration(String tableId) {
+ return new TableConfiguration(getInstance(), tableId, null) {
+ @Override
+ public String get(Property property) {
+ return DEFAULT_TABLE_PROPERTIES.get(property.name());
+ }
+
+ @Override
+ public void getProperties(Map<String,String> props, Predicate<String> filter) {
+ for (Entry<String,String> e : DEFAULT_TABLE_PROPERTIES.entrySet()) {
+ if (filter.apply(e.getKey())) {
+ props.put(e.getKey(), e.getValue());
+ }
+ }
+ }
+ };
+ }
+ }
+
+ protected static final Table FOO = new Table("foo", "1");
+ protected static final Table BAR = new Table("bar", "2");
+ protected static final Table BAZ = new Table("baz", "3");
+
+ protected final TestInstance instance = new TestInstance();
+ protected final TestServerConfigurationFactory factory = new TestServerConfigurationFactory(instance);
+ protected final Map<String,String> servers = new HashMap<>(15);
+ protected final SortedMap<TServerInstance,TabletServerStatus> allTabletServers = new TreeMap<>();
+ protected final Map<String,List<KeyExtent>> tableExtents = new HashMap<>(3);
+
+ {
+ servers.put("192.168.0.1", "r01s01");
+ servers.put("192.168.0.2", "r01s02");
+ servers.put("192.168.0.3", "r01s03");
+ servers.put("192.168.0.4", "r01s04");
+ servers.put("192.168.0.5", "r01s05");
+ servers.put("192.168.0.6", "r02s01");
+ servers.put("192.168.0.7", "r02s02");
+ servers.put("192.168.0.8", "r02s03");
+ servers.put("192.168.0.9", "r02s04");
+ servers.put("192.168.0.10", "r02s05");
+ servers.put("192.168.0.11", "r03s01");
+ servers.put("192.168.0.12", "r03s02");
+ servers.put("192.168.0.13", "r03s03");
+ servers.put("192.168.0.14", "r03s04");
+ servers.put("192.168.0.15", "r03s05");
+
+ allTabletServers.put(new TServerInstance("192.168.0.1:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.2:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.3:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.4:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.5:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.6:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.7:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.8:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.9:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.10:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.11:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.12:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.13:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.14:9997", 1), new TabletServerStatus());
+ allTabletServers.put(new TServerInstance("192.168.0.15:9997", 1), new TabletServerStatus());
+
+ tableExtents.put(FOO.getTableName(), new ArrayList<KeyExtent>());
- tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("1"), new Text("0")));
- tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("2"), new Text("1")));
- tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("3"), new Text("2")));
- tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("4"), new Text("3")));
- tableExtents.get(FOO.getTableName()).add(new KeyExtent(new Text(FOO.getId()), new Text("5"), new Text("4")));
++ tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("1"), new Text("0")));
++ tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("2"), new Text("1")));
++ tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("3"), new Text("2")));
++ tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("4"), new Text("3")));
++ tableExtents.get(FOO.getTableName()).add(new KeyExtent(FOO.getId(), new Text("5"), new Text("4")));
+ tableExtents.put(BAR.getTableName(), new ArrayList<KeyExtent>());
- tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("11"), new Text("10")));
- tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("12"), new Text("11")));
- tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("13"), new Text("12")));
- tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("14"), new Text("13")));
- tableExtents.get(BAR.getTableName()).add(new KeyExtent(new Text(BAR.getId()), new Text("15"), new Text("14")));
++ tableExtents.get(BAR.getTableName()).add(new KeyExtent(BAR.getId(), new Text("11"), new Text("10")));
++ tableExtents.get(BAR.getTableName()).add(new KeyExtent(BAR.getId(), new Text("12"), new Text("11")));
++ tableExtents.get(BAR.getTableName()).add(new KeyExtent(BAR.getId(), new Text("13"), new Text("12")));
++ tableExtents.get(BAR.getTableName()).add(new KeyExtent(BAR.getId(), new Text("14"), new Text("13")));
++ tableExtents.get(BAR.getTableName()).add(new KeyExtent(BAR.getId(), new Text("15"), new Text("14")));
+ tableExtents.put(BAZ.getTableName(), new ArrayList<KeyExtent>());
- tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("21"), new Text("20")));
- tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("22"), new Text("21")));
- tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("23"), new Text("22")));
- tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("24"), new Text("23")));
- tableExtents.get(BAZ.getTableName()).add(new KeyExtent(new Text(BAZ.getId()), new Text("25"), new Text("24")));
++ tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("21"), new Text("20")));
++ tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("22"), new Text("21")));
++ tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("23"), new Text("22")));
++ tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("24"), new Text("23")));
++ tableExtents.get(BAZ.getTableName()).add(new KeyExtent(BAZ.getId(), new Text("25"), new Text("24")));
+ }
+
+ protected boolean tabletInBounds(KeyExtent ke, TServerInstance tsi) {
+ String tid = ke.getTableId().toString();
+ String host = tsi.host();
+ if (tid.equals("1")
+ && (host.equals("192.168.0.1") || host.equals("192.168.0.2") || host.equals("192.168.0.3") || host.equals("192.168.0.4") || host.equals("192.168.0.5"))) {
+ return true;
+ } else if (tid.equals("2")
+ && (host.equals("192.168.0.6") || host.equals("192.168.0.7") || host.equals("192.168.0.8") || host.equals("192.168.0.9") || host.equals("192.168.0.10"))) {
+ return true;
+ } else if (tid.equals("3")
+ && (host.equals("192.168.0.11") || host.equals("192.168.0.12") || host.equals("192.168.0.13") || host.equals("192.168.0.14") || host
+ .equals("192.168.0.15"))) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+
+ @Override
+ protected TableOperations getTableOperations() {
+ return new TableOperationsImpl(EasyMock.createMock(ClientContext.class)) {
+ @Override
+ public Map<String,String> tableIdMap() {
+ HashMap<String,String> tables = new HashMap<>();
+ tables.put(FOO.getTableName(), FOO.getId());
+ tables.put(BAR.getTableName(), BAR.getId());
+ tables.put(BAZ.getTableName(), BAZ.getId());
+ return tables;
+ }
+ };
+ }
+
+ @Override
+ protected TabletBalancer getBalancerForTable(String table) {
+ return new DefaultLoadBalancer();
+ }
+
+ @Override
+ protected String getNameFromIp(String hostIp) throws UnknownHostException {
+ if (servers.containsKey(hostIp)) {
+ return servers.get(hostIp);
+ } else {
+ throw new UnknownHostException();
+ }
+ }
+
+ protected SortedMap<TServerInstance,TabletServerStatus> createCurrent(int numTservers) {
+ String base = "192.168.0.";
+ TreeMap<TServerInstance,TabletServerStatus> current = new TreeMap<>();
+ for (int i = 1; i <= numTservers; i++) {
+ current.put(new TServerInstance(base + i + ":9997", 1), new TabletServerStatus());
+ }
+ return current;
+ }
+
+ }