You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@accumulo.apache.org by dl...@apache.org on 2016/04/12 15:58:55 UTC
[38/39] accumulo git commit: ACCUMULO-4173: Updated with changes
based on testing
ACCUMULO-4173: Updated with changes based on testing
Project: http://git-wip-us.apache.org/repos/asf/accumulo/repo
Commit: http://git-wip-us.apache.org/repos/asf/accumulo/commit/d745e7c1
Tree: http://git-wip-us.apache.org/repos/asf/accumulo/tree/d745e7c1
Diff: http://git-wip-us.apache.org/repos/asf/accumulo/diff/d745e7c1
Branch: refs/heads/ACCUMULO-4173
Commit: d745e7c1e9baf333f4a54f5ceeac22a588a927d5
Parents: c77a577
Author: Dave Marion <dl...@apache.org>
Authored: Tue Apr 12 09:47:44 2016 -0400
Committer: Dave Marion <dl...@apache.org>
Committed: Tue Apr 12 09:47:44 2016 -0400
----------------------------------------------------------------------
.../balancer/HostRegexTableLoadBalancer.java | 129 +++++++++++--------
.../HostRegexTableLoadBalancerTest.java | 48 ++++---
2 files changed, 101 insertions(+), 76 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d745e7c1/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
----------------------------------------------------------------------
diff --git a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
index 677e4ba..9d92361 100644
--- a/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
+++ b/server/base/src/main/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancer.java
@@ -44,7 +44,7 @@ import org.slf4j.LoggerFactory;
/**
* This balancer will create pools of tablet servers by grouping tablet servers that match a regex into the same pool and calling the balancer set on the table
- * to balance within the set of matching tablet servers. <br>
+ * to balance within the set of matching tablet servers. All tablet servers that do not match a regex are grouped into a default pool.<br>
* Regex properties for this balancer are specified as:<br>
* <b>table.custom.balancer.host.regex.<tablename>=<regex></b><br>
* Periodically (default 5m) this balancer will check to see if a tablet server is hosting tablets that it should not be according to the regex configuration.
@@ -75,8 +75,8 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
private Map<String,String> tableIdToTableName = null;
private Map<String,Pattern> poolNameToRegexPattern = null;
- private long lastOOBCheck = System.currentTimeMillis();
- private long lastPoolRecheck = 0;
+ private volatile long lastOOBCheck = System.currentTimeMillis();
+ private volatile long lastPoolRecheck = 0;
private boolean isIpBasedRegex = false;
private Map<String,SortedMap<TServerInstance,TabletServerStatus>> pools = new HashMap<String,SortedMap<TServerInstance,TabletServerStatus>>();
@@ -89,6 +89,7 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
*/
protected synchronized Map<String,SortedMap<TServerInstance,TabletServerStatus>> splitCurrentByRegex(SortedMap<TServerInstance,TabletServerStatus> current) {
if ((System.currentTimeMillis() - lastPoolRecheck) > poolRecheckMillis) {
+ LOG.debug("Performing pool recheck - regrouping tablet servers into pools based on regex");
Map<String,SortedMap<TServerInstance,TabletServerStatus>> newPools = new HashMap<String,SortedMap<TServerInstance,TabletServerStatus>>();
for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
List<String> poolNames = getPoolNamesForHost(e.getKey().host());
@@ -168,9 +169,11 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
Map<String,String> customProps = conf.getTableConfiguration(table.getValue()).getAllPropertiesWithPrefix(Property.TABLE_ARBITRARY_PROP_PREFIX);
if (null != customProps && customProps.size() > 0) {
for (Entry<String,String> customProp : customProps.entrySet()) {
- String tableName = customProp.getKey().substring(HOST_BALANCER_PREFIX.length());
- String regex = customProp.getValue();
- poolNameToRegexPattern.put(tableName, Pattern.compile(regex));
+ if (customProp.getKey().startsWith(HOST_BALANCER_PREFIX)) {
+ String tableName = customProp.getKey().substring(HOST_BALANCER_PREFIX.length());
+ String regex = customProp.getValue();
+ poolNameToRegexPattern.put(tableName, Pattern.compile(regex));
+ }
}
}
}
@@ -218,40 +221,10 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
public void getAssignments(SortedMap<TServerInstance,TabletServerStatus> current, Map<KeyExtent,TServerInstance> unassigned,
Map<KeyExtent,TServerInstance> assignments) {
- Map<KeyExtent,TServerInstance> unassignedClone = new TreeMap<>(unassigned);
- if ((System.currentTimeMillis() - this.lastOOBCheck) > this.oobCheckMillis) {
- // Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate it.
- for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
- for (String assignedPool : getPoolNamesForHost(e.getKey().host())) {
- for (String table : poolNameToRegexPattern.keySet()) {
- // pool names are the same as table names, except in the DEFAULT case.
- if (assignedPool.equals(table) || table.equals(DEFAULT_POOL)) {
- continue;
- }
- String tid = getTableOperations().tableIdMap().get(table);
- if (null == tid) {
- LOG.warn("Unable to check for out of bounds tablets for table {}, it may have been deleted or renamed.", table);
- continue;
- }
- try {
- List<TabletStats> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(), tid);
- for (TabletStats ts : outOfBoundsTablets) {
- LOG.info("Tablet {} is currently outside the bounds of the regex, reassigning", ts.toString());
- unassignedClone.put(new KeyExtent(ts.getExtent()), e.getKey());
- }
- } catch (TException e1) {
- LOG.error("Error in OOB check getting tablets for table {} from server {}", tid, e.getKey().host(), e);
- }
- }
- }
- }
- this.oobCheckMillis = System.currentTimeMillis();
- }
-
Map<String,SortedMap<TServerInstance,TabletServerStatus>> pools = splitCurrentByRegex(current);
- // separate the unassigned into tables
+ // group the unassigned into tables
Map<String,Map<KeyExtent,TServerInstance>> groupedUnassigned = new HashMap<String,Map<KeyExtent,TServerInstance>>();
- for (Entry<KeyExtent,TServerInstance> e : unassignedClone.entrySet()) {
+ for (Entry<KeyExtent,TServerInstance> e : unassigned.entrySet()) {
Map<KeyExtent,TServerInstance> tableUnassigned = groupedUnassigned.get(e.getKey().getTableId());
if (tableUnassigned == null) {
tableUnassigned = new HashMap<KeyExtent,TServerInstance>();
@@ -259,29 +232,20 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
}
tableUnassigned.put(e.getKey(), e.getValue());
}
- // Validate that the last server in the unassigned map is actually in the pool, if not set to null.
- Map<String,KeyExtent> overwrites = new TreeMap<>();
- for (Entry<String,Map<KeyExtent,TServerInstance>> e : groupedUnassigned.entrySet()) {
- String poolName = tableIdToTableName.get(e.getKey());
- for (Entry<KeyExtent,TServerInstance> e2 : e.getValue().entrySet()) {
- if (null != e2.getValue()) {
- List<String> hostPools = getPoolNamesForHost(e2.getValue().host());
- if (!hostPools.contains(poolName)) {
- overwrites.put(e.getKey(), e2.getKey());
- }
- }
- }
- }
- for (Entry<String,KeyExtent> e : overwrites.entrySet()) {
- groupedUnassigned.get(e.getKey()).remove(e.getValue());
- groupedUnassigned.get(e.getKey()).put(e.getValue(), null);
- }
// Send a view of the current servers to the tables tablet balancer
for (Entry<String,Map<KeyExtent,TServerInstance>> e : groupedUnassigned.entrySet()) {
Map<KeyExtent,TServerInstance> newAssignments = new HashMap<KeyExtent,TServerInstance>();
String tableName = tableIdToTableName.get(e.getKey());
String poolName = getPoolNameForTable(tableName);
SortedMap<TServerInstance,TabletServerStatus> currentView = pools.get(poolName);
+ if (null == currentView || currentView.size() == 0) {
+ LOG.warn("No tablet servers online for table {}, assigning within default pool", tableName);
+ currentView = pools.get(DEFAULT_POOL);
+ if (null == currentView) {
+ LOG.error("No tablet servers exist in the default pool, unable to assign tablets for table {}", tableName);
+ }
+ }
+ LOG.debug("Sending {} tablets to balancer for table {} for assignment within tservers {}", e.getValue().size(), tableName, currentView.keySet());
getBalancerForTable(e.getKey()).getAssignments(currentView, e.getValue(), newAssignments);
assignments.putAll(newAssignments);
}
@@ -295,16 +259,67 @@ public class HostRegexTableLoadBalancer extends TableLoadBalancer {
if (t == null)
return minBalanceTime;
+ Map<String,SortedMap<TServerInstance,TabletServerStatus>> currentGrouped = splitCurrentByRegex(current);
+ if ((System.currentTimeMillis() - this.lastOOBCheck) > this.oobCheckMillis) {
+ try {
+ // Check to see if a tablet is assigned outside the bounds of the pool. If so, migrate it.
+ for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
+ for (String assignedPool : getPoolNamesForHost(e.getKey().host())) {
+ for (String table : poolNameToRegexPattern.keySet()) {
+ // pool names are the same as table names, except in the DEFAULT case.
+ if (assignedPool.equals(table)) {
+ // If this tserver is assigned to a regex pool, then we can skip checking tablets for this table on this host.
+ continue;
+ }
+ String tid = getTableOperations().tableIdMap().get(table);
+ if (null == tid) {
+ LOG.warn("Unable to check for out of bounds tablets for table {}, it may have been deleted or renamed.", table);
+ continue;
+ }
+ try {
+ List<TabletStats> outOfBoundsTablets = getOnlineTabletsForTable(e.getKey(), tid);
+ for (TabletStats ts : outOfBoundsTablets) {
+ KeyExtent ke = new KeyExtent(ts.getExtent());
+ if (migrations.contains(ke)) {
+ LOG.debug("Migration for out of bounds tablet {} has already been requested", ke);
+ ;
+ continue;
+ }
+ String poolName = getPoolNameForTable(table);
+ SortedMap<TServerInstance,TabletServerStatus> currentView = currentGrouped.get(poolName);
+ if (null != currentView) {
+ TServerInstance nextTS = currentView.firstKey();
+ LOG.info("Tablet {} is currently outside the bounds of the regex, migrating from {} to {}", ke, e.getKey(), nextTS);
+ migrationsOut.add(new TabletMigration(ke, e.getKey(), nextTS));
+ } else {
+ LOG.warn("No tablet servers online for pool {}, unable to migrate out of bounds tablets", poolName);
+ }
+ }
+ } catch (TException e1) {
+ LOG.error("Error in OOB check getting tablets for table {} from server {}", tid, e.getKey().host(), e);
+ }
+ }
+ }
+ }
+ } finally {
+ this.lastOOBCheck = System.currentTimeMillis();
+ }
+ }
+
if (migrations != null && migrations.size() > 0) {
- LOG.warn("Not balancing tables due to outstanding migrations");
+ LOG.warn("Not balancing tables due to {} outstanding migrations", migrations.size());
return minBalanceTime;
}
- Map<String,SortedMap<TServerInstance,TabletServerStatus>> currentGrouped = splitCurrentByRegex(current);
for (String s : t.tableIdMap().values()) {
String tableName = tableIdToTableName.get(s);
String regexTableName = getPoolNameForTable(tableName);
SortedMap<TServerInstance,TabletServerStatus> currentView = currentGrouped.get(regexTableName);
+ if (null == currentView) {
+ LOG.warn("Skipping balance for table {} as no tablet servers are online, will recheck for online tservers at {} ms intervals", tableName,
+ this.poolRecheckMillis);
+ continue;
+ }
ArrayList<TabletMigration> newMigrations = new ArrayList<TabletMigration>();
long tableBalanceTime = getBalancerForTable(s).balance(currentView, migrations, newMigrations);
if (tableBalanceTime < minBalanceTime) {
http://git-wip-us.apache.org/repos/asf/accumulo/blob/d745e7c1/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
----------------------------------------------------------------------
diff --git a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
index 22bda0b..b273c5b 100644
--- a/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
+++ b/server/base/src/test/java/org/apache/accumulo/server/master/balancer/HostRegexTableLoadBalancerTest.java
@@ -21,9 +21,11 @@ import java.nio.ByteBuffer;
import java.util.ArrayList;
import java.util.Collections;
import java.util.HashMap;
+import java.util.HashSet;
import java.util.List;
import java.util.Map;
import java.util.Map.Entry;
+import java.util.Set;
import java.util.SortedMap;
import java.util.TreeMap;
import java.util.regex.Pattern;
@@ -548,30 +550,27 @@ public class HostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer {
}
@Test
- public void testOutOfBoundsTablets() {
+ public void testUnassignedWithNoTServers() {
init((ServerConfiguration) factory);
- // Wait to trigger the out of bounds check which will call our version of getOnlineTabletsForTable
- UtilWaitThread.sleep(11000);
Map<KeyExtent,TServerInstance> assignments = new HashMap<>();
Map<KeyExtent,TServerInstance> unassigned = new HashMap<>();
- for (List<KeyExtent> extents : tableExtents.values()) {
- for (KeyExtent ke : extents) {
- unassigned.put(ke, null);
- }
+ for (KeyExtent ke : tableExtents.get(BAR.getTableName())) {
+ unassigned.put(ke, null);
}
- this.getAssignments(Collections.unmodifiableSortedMap(allTabletServers), Collections.unmodifiableMap(unassigned), assignments);
- Assert.assertEquals(15, assignments.size());
- // Ensure unique tservers
- for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
- for (Entry<KeyExtent,TServerInstance> e2 : assignments.entrySet()) {
- if (e.getKey().equals(e2.getKey())) {
- continue;
- }
- if (e.getValue().equals(e2.getValue())) {
- Assert.fail("Assignment failure");
- }
+ SortedMap<TServerInstance,TabletServerStatus> current = createCurrent(15);
+ // Remove the BAR tablet servers from current
+ List<TServerInstance> removals = new ArrayList<TServerInstance>();
+ for (Entry<TServerInstance,TabletServerStatus> e : current.entrySet()) {
+ if (e.getKey().host().equals("192.168.0.6") || e.getKey().host().equals("192.168.0.7") || e.getKey().host().equals("192.168.0.8")
+ || e.getKey().host().equals("192.168.0.9") || e.getKey().host().equals("192.168.0.10")) {
+ removals.add(e.getKey());
}
}
+ for (TServerInstance r : removals) {
+ current.remove(r);
+ }
+ this.getAssignments(Collections.unmodifiableSortedMap(allTabletServers), Collections.unmodifiableMap(unassigned), assignments);
+ Assert.assertEquals(unassigned.size(), assignments.size());
// Ensure assignments are correct
for (Entry<KeyExtent,TServerInstance> e : assignments.entrySet()) {
if (!tabletInBounds(e.getKey(), e.getValue())) {
@@ -580,9 +579,20 @@ public class HostRegexTableLoadBalancerTest extends HostRegexTableLoadBalancer {
}
}
+ @Test
+ public void testOutOfBoundsTablets() {
+ init((ServerConfiguration) factory);
+ // Wait to trigger the out of bounds check which will call our version of getOnlineTabletsForTable
+ UtilWaitThread.sleep(11000);
+ Set<KeyExtent> migrations = new HashSet<KeyExtent>();
+ List<TabletMigration> migrationsOut = new ArrayList<TabletMigration>();
+ this.balance(createCurrent(15), migrations, migrationsOut);
+ Assert.assertEquals(2, migrationsOut.size());
+ }
+
@Override
public List<TabletStats> getOnlineTabletsForTable(TServerInstance tserver, String tableId) throws ThriftSecurityException, TException {
- // Report incorrent information so that getAssignments will create n assignment
+ // Report incorrect information so that balance will create an assignment
List<TabletStats> tablets = new ArrayList<>();
if (tableId.equals(BAR.getId()) && tserver.host().equals("192.168.0.1")) {
// Report that we have a bar tablet on this server