You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2012/12/09 23:12:54 UTC
svn commit: r1419174 - in /hbase/branches/0.94/src:
main/java/org/apache/hadoop/hbase/master/
main/java/org/apache/hadoop/hbase/master/handler/
test/java/org/apache/hadoop/hbase/master/
Author: tedyu
Date: Sun Dec 9 22:12:53 2012
New Revision: 1419174
URL: http://svn.apache.org/viewvc?rev=1419174&view=rev
Log:
HBASE-6317 Master clean start up and Partially enabled tables make region assignment inconsistent (Rajesh)
Modified:
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java?rev=1419174&r1=1419173&r2=1419174&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java Sun Dec 9 22:12:53 2012
@@ -142,9 +142,10 @@ public class AssignmentManager extends Z
// store all the table names in disabling state
Set<String> disablingTables = new HashSet<String>(1);
- // store all the enabling state tablenames.
- Set<String> enablingTables = new HashSet<String>(1);
-
+ // store all the enabling state table names and corresponding online servers' regions.
+ // This may be needed to avoid calling assign twice for the regions of the ENABLING table
+ // that could have been assigned through processRIT.
+ Map<String, List<HRegionInfo>> enablingTables = new HashMap<String, List<HRegionInfo>>(1);
/**
* Server to regions assignment map.
* Contains the set of regions currently assigned to a given server.
@@ -274,6 +275,16 @@ public class AssignmentManager extends Z
}
/**
+ * Gives enabling table regions.
+ *
+ * @param tableName
+ * @return list of regionInfos
+ */
+ public List<HRegionInfo> getEnablingTableRegions(String tableName){
+ return this.enablingTables.get(tableName);
+ }
+
+ /**
* Add a regionPlan for the specified region.
* @param encodedName
* @param plan
@@ -364,7 +375,9 @@ public class AssignmentManager extends Z
// Recover the tables that were not fully moved to DISABLED state.
// These tables are in DISABLING state when the master restarted/switched.
boolean isWatcherCreated = recoverTableInDisablingState(this.disablingTables);
- recoverTableInEnablingState(this.enablingTables, isWatcherCreated);
+ recoverTableInEnablingState(this.enablingTables.keySet(), isWatcherCreated);
+ this.enablingTables.clear();
+ this.disablingTables.clear();
}
/**
@@ -509,6 +522,10 @@ public class AssignmentManager extends Z
String encodedRegionName = regionInfo.getEncodedName();
LOG.info("Processing region " + regionInfo.getRegionNameAsString() +
" in state " + data.getEventType());
+ List<HRegionInfo> hris = this.enablingTables.get(regionInfo.getTableNameAsString());
+ if (hris != null && !hris.isEmpty()) {
+ hris.remove(regionInfo);
+ }
synchronized (regionsInTransition) {
RegionState regionState = regionsInTransition.get(encodedRegionName);
if (regionState != null ||
@@ -2306,11 +2323,12 @@ public class AssignmentManager extends Z
// Skip assignment for regions of tables in DISABLING state also because
// during clean cluster startup no RS is alive and regions map also doesn't
// have any information about the regions. See HBASE-6281.
- Set<String> disablingAndDisabledTables = new HashSet<String>(this.disablingTables);
- disablingAndDisabledTables.addAll(this.zkTable.getDisabledTables());
+ Set<String> disablingDisabledAndEnablingTables = new HashSet<String>(this.disablingTables);
+ disablingDisabledAndEnablingTables.addAll(this.zkTable.getDisabledTables());
+ disablingDisabledAndEnablingTables.addAll(this.enablingTables.keySet());
// Scan META for all user regions, skipping any disabled tables
Map<HRegionInfo, ServerName> allRegions = MetaReader.fullScan(catalogTracker,
- disablingAndDisabledTables, true);
+ disablingDisabledAndEnablingTables, true);
if (allRegions == null || allRegions.isEmpty()) return;
// Get all available servers
@@ -2558,13 +2576,14 @@ public class AssignmentManager extends Z
// from ENABLED state when application calls disableTable.
// It can't be in DISABLED state, because DISABLED states transitions
// from DISABLING state.
- if (false == checkIfRegionsBelongsToEnabling(regionInfo)) {
- LOG.warn("Region " + regionInfo.getEncodedName() +
- " has null regionLocation." + " But its table " + tableName +
- " isn't in ENABLING state.");
+ boolean enabling = checkIfRegionsBelongsToEnabling(regionInfo);
+ addTheTablesInPartialState(regionInfo);
+ if (enabling) {
+ addToEnablingTableRegions(regionInfo);
+ } else {
+ LOG.warn("Region " + regionInfo.getEncodedName() + " has null regionLocation."
+ + " But its table " + tableName + " isn't in ENABLING state.");
}
- addTheTablesInPartialState(this.disablingTables, this.enablingTables, regionInfo,
- tableName);
} else if (!onlineServers.contains(regionLocation)) {
// Region is located on a server that isn't online
List<Pair<HRegionInfo, Result>> offlineRegions =
@@ -2575,8 +2594,7 @@ public class AssignmentManager extends Z
}
offlineRegions.add(new Pair<HRegionInfo,Result>(regionInfo, result));
disabled = checkIfRegionBelongsToDisabled(regionInfo);
- disablingOrEnabling = addTheTablesInPartialState(this.disablingTables,
- this.enablingTables, regionInfo, tableName);
+ disablingOrEnabling = addTheTablesInPartialState(regionInfo);
// need to enable the table if not disabled or disabling or enabling
// this will be used in rolling restarts
enableTableIfNotDisabledOrDisablingOrEnabling(disabled,
@@ -2597,16 +2615,18 @@ public class AssignmentManager extends Z
}
// Region is being served and on an active server
// add only if region not in disabled and enabling table
- if (false == checkIfRegionBelongsToDisabled(regionInfo)
- && false == checkIfRegionsBelongsToEnabling(regionInfo)) {
+ boolean enabling = checkIfRegionsBelongsToEnabling(regionInfo);
+ disabled = checkIfRegionBelongsToDisabled(regionInfo);
+ if (!enabling && !disabled) {
synchronized (this.regions) {
regions.put(regionInfo, regionLocation);
addToServers(regionLocation, regionInfo);
}
}
- disablingOrEnabling = addTheTablesInPartialState(this.disablingTables,
- this.enablingTables, regionInfo, tableName);
- disabled = checkIfRegionBelongsToDisabled(regionInfo);
+ disablingOrEnabling = addTheTablesInPartialState(regionInfo);
+ if (enabling) {
+ addToEnablingTableRegions(regionInfo);
+ }
// need to enable the table if not disabled or disabling or enabling
// this will be used in rolling restarts
enableTableIfNotDisabledOrDisablingOrEnabling(disabled,
@@ -2616,6 +2636,18 @@ public class AssignmentManager extends Z
return offlineServers;
}
+ private void addToEnablingTableRegions(HRegionInfo regionInfo) {
+ String tableName = regionInfo.getTableNameAsString();
+ List<HRegionInfo> hris = this.enablingTables.get(tableName);
+ if (!hris.contains(regionInfo)) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Adding region" + regionInfo.getRegionNameAsString()
+ + " to enabling table " + tableName + ".");
+ }
+ hris.add(regionInfo);
+ }
+ }
+
private void enableTableIfNotDisabledOrDisablingOrEnabling(boolean disabled,
boolean disablingOrEnabling, String tableName) {
if (!disabled && !disablingOrEnabling
@@ -2624,14 +2656,15 @@ public class AssignmentManager extends Z
}
}
- private Boolean addTheTablesInPartialState(Set<String> disablingTables,
- Set<String> enablingTables, HRegionInfo regionInfo,
- String disablingTableName) {
+ private Boolean addTheTablesInPartialState(HRegionInfo regionInfo) {
+ String tableName = regionInfo.getTableNameAsString();
if (checkIfRegionBelongsToDisabling(regionInfo)) {
- disablingTables.add(disablingTableName);
+ this.disablingTables.add(tableName);
return true;
} else if (checkIfRegionsBelongsToEnabling(regionInfo)) {
- enablingTables.add(disablingTableName);
+ if (!this.enablingTables.containsKey(tableName)) {
+ this.enablingTables.put(tableName, new ArrayList<HRegionInfo>());
+ }
return true;
}
return false;
Modified: hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java?rev=1419174&r1=1419173&r2=1419174&view=diff
==============================================================================
--- hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java (original)
+++ hbase/branches/0.94/src/main/java/org/apache/hadoop/hbase/master/handler/EnableTableHandler.java Sun Dec 9 22:12:53 2012
@@ -20,6 +20,7 @@
package org.apache.hadoop.hbase.master.handler;
import java.io.IOException;
+import java.util.ArrayList;
import java.util.List;
import java.util.concurrent.ExecutorService;
@@ -27,6 +28,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.Server;
+import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.catalog.CatalogTracker;
@@ -34,7 +36,11 @@ import org.apache.hadoop.hbase.catalog.M
import org.apache.hadoop.hbase.executor.EventHandler;
import org.apache.hadoop.hbase.master.AssignmentManager;
import org.apache.hadoop.hbase.master.BulkAssigner;
+import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.RegionPlan;
+import org.apache.hadoop.hbase.master.ServerManager;
import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.Pair;
import org.apache.zookeeper.KeeperException;
/**
@@ -46,6 +52,7 @@ public class EnableTableHandler extends
private final String tableNameStr;
private final AssignmentManager assignmentManager;
private final CatalogTracker ct;
+ private boolean retainAssignment = false;
public EnableTableHandler(Server server, byte [] tableName,
CatalogTracker catalogTracker, AssignmentManager assignmentManager,
@@ -56,6 +63,7 @@ public class EnableTableHandler extends
this.tableNameStr = Bytes.toString(tableName);
this.ct = catalogTracker;
this.assignmentManager = assignmentManager;
+ this.retainAssignment = skipTableStateCheck;
// Check if table exists
if (!MetaReader.tableExists(catalogTracker, this.tableNameStr)) {
throw new TableNotFoundException(Bytes.toString(tableName));
@@ -99,10 +107,12 @@ public class EnableTableHandler extends
LOG.error("Error trying to enable the table " + this.tableNameStr, e);
} catch (KeeperException e) {
LOG.error("Error trying to enable the table " + this.tableNameStr, e);
+ } catch (InterruptedException e) {
+ LOG.error("Error trying to enable the table " + this.tableNameStr, e);
}
}
- private void handleEnableTable() throws IOException, KeeperException {
+ private void handleEnableTable() throws IOException, KeeperException, InterruptedException {
// I could check table is disabling and if so, not enable but require
// that user first finish disabling but that might be obnoxious.
@@ -111,18 +121,18 @@ public class EnableTableHandler extends
boolean done = false;
// Get the regions of this table. We're done when all listed
// tables are onlined.
- List<HRegionInfo> regionsInMeta;
- regionsInMeta = MetaReader.getTableRegions(this.ct, tableName, true);
- int countOfRegionsInTable = regionsInMeta.size();
- List<HRegionInfo> regions = regionsToAssign(regionsInMeta);
+ List<Pair<HRegionInfo, ServerName>> tableRegionsAndLocations = MetaReader
+ .getTableRegionsAndLocations(this.ct, tableName, true);
+ int countOfRegionsInTable = tableRegionsAndLocations.size();
+ List<HRegionInfo> regions = regionsToAssignWithServerName(tableRegionsAndLocations);
int regionsCount = regions.size();
if (regionsCount == 0) {
done = true;
}
LOG.info("Table has " + countOfRegionsInTable + " regions of which " +
regionsCount + " are offline.");
- BulkEnabler bd = new BulkEnabler(this.server, regions,
- countOfRegionsInTable);
+ BulkEnabler bd = new BulkEnabler(this.server, regions, countOfRegionsInTable,
+ this.retainAssignment);
try {
if (bd.bulkAssign()) {
done = true;
@@ -140,17 +150,34 @@ public class EnableTableHandler extends
/**
* @param regionsInMeta This datastructure is edited by this method.
- * @return The <code>regionsInMeta</code> list minus the regions that have
- * been onlined; i.e. List of regions that need onlining.
+ * @return List of regions neither in transition nor assigned.
* @throws IOException
*/
- private List<HRegionInfo> regionsToAssign(
- final List<HRegionInfo> regionsInMeta)
- throws IOException {
- final List<HRegionInfo> onlineRegions =
- this.assignmentManager.getRegionsOfTable(tableName);
- regionsInMeta.removeAll(onlineRegions);
- return regionsInMeta;
+ private List<HRegionInfo> regionsToAssignWithServerName(
+ final List<Pair<HRegionInfo, ServerName>> regionsInMeta) throws IOException {
+ ServerManager serverManager = ((HMaster) this.server).getServerManager();
+ List<HRegionInfo> regions = new ArrayList<HRegionInfo>();
+ List<HRegionInfo> enablingTableRegions = this.assignmentManager
+ .getEnablingTableRegions(this.tableNameStr);
+ final List<HRegionInfo> onlineRegions = this.assignmentManager.getRegionsOfTable(tableName);
+ for (Pair<HRegionInfo, ServerName> regionLocation : regionsInMeta) {
+ HRegionInfo hri = regionLocation.getFirst();
+ ServerName sn = regionLocation.getSecond();
+ if (this.retainAssignment) {
+ // Region may be available in enablingTableRegions during master startup only.
+ if (enablingTableRegions != null && enablingTableRegions.contains(hri)) {
+ regions.add(hri);
+ if (sn != null && serverManager.isServerOnline(sn)) {
+ this.assignmentManager.addPlan(hri.getEncodedName(), new RegionPlan(hri, null, sn));
+ }
+ }
+ } else if (onlineRegions.contains(hri)) {
+ continue;
+ } else {
+ regions.add(hri);
+ }
+ }
+ return regions;
}
/**
@@ -160,12 +187,14 @@ public class EnableTableHandler extends
private final List<HRegionInfo> regions;
// Count of regions in table at time this assign was launched.
private final int countOfRegionsInTable;
+ private final boolean retainAssignment;
BulkEnabler(final Server server, final List<HRegionInfo> regions,
- final int countOfRegionsInTable) {
+ final int countOfRegionsInTable,final boolean retainAssignment) {
super(server);
this.regions = regions;
this.countOfRegionsInTable = countOfRegionsInTable;
+ this.retainAssignment = retainAssignment;
}
@Override
@@ -173,7 +202,7 @@ public class EnableTableHandler extends
boolean roundRobinAssignment = this.server.getConfiguration().getBoolean(
"hbase.master.enabletable.roundrobin", false);
- if (!roundRobinAssignment) {
+ if (retainAssignment || !roundRobinAssignment) {
for (HRegionInfo region : regions) {
if (assignmentManager.isRegionInTransition(region) != null) {
continue;
@@ -181,7 +210,11 @@ public class EnableTableHandler extends
final HRegionInfo hri = region;
pool.execute(new Runnable() {
public void run() {
- assignmentManager.assign(hri, true);
+ if (retainAssignment) {
+ assignmentManager.assign(hri, true, false, false);
+ } else {
+ assignmentManager.assign(hri, true);
+ }
}
});
}
Modified: hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java
URL: http://svn.apache.org/viewvc/hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java?rev=1419174&r1=1419173&r2=1419174&view=diff
==============================================================================
--- hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java (original)
+++ hbase/branches/0.94/src/test/java/org/apache/hadoop/hbase/master/TestAssignmentManager.java Sun Dec 9 22:12:53 2012
@@ -74,6 +74,7 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.mockito.Mockito;
+import org.mockito.internal.util.reflection.Whitebox;
import com.google.protobuf.ServiceException;
@@ -91,6 +92,10 @@ public class TestAssignmentManager {
private static final HRegionInfo REGIONINFO =
new HRegionInfo(Bytes.toBytes("t"),
HConstants.EMPTY_START_ROW, HConstants.EMPTY_START_ROW);
+ private static final HRegionInfo REGIONINFO_2 = new HRegionInfo(Bytes.toBytes("t"),
+ Bytes.toBytes("a"),Bytes.toBytes( "b"));
+ private static int assignmentCount;
+ private static boolean enabling = false;
// Mocked objects or; get redone for each test.
private Server server;
@@ -768,14 +773,27 @@ public class TestAssignmentManager {
// with an encoded name by doing a Get on .META.
HRegionInterface ri = Mockito.mock(HRegionInterface.class);
// Get a meta row result that has region up on SERVERNAME_A for REGIONINFO
+ Result[] result = null;
+ if (enabling) {
+ result = new Result[2];
+ result[0] = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
+ result[1] = getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A);
+ }
Result r = getMetaTableRowResult(REGIONINFO, SERVERNAME_A);
Mockito.when(ri .openScanner((byte[]) Mockito.any(), (Scan) Mockito.any())).
thenReturn(System.currentTimeMillis());
- // Return good result 'r' first and then return null to indicate end of scan
- Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(new Result[] { r });
- // If a get, return the above result too for REGIONINFO
- Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).
- thenReturn(r);
+ if (enabling) {
+ Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(result, result, result,
+ (Result[]) null);
+ // If a get, return the above result too for REGIONINFO_2
+ Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(
+ getMetaTableRowResult(REGIONINFO_2, SERVERNAME_A));
+ } else {
+ // Return good result 'r' first and then return null to indicate end of scan
+ Mockito.when(ri.next(Mockito.anyLong(), Mockito.anyInt())).thenReturn(new Result[] { r });
+ // If a get, return the above result too for REGIONINFO
+ Mockito.when(ri.get((byte[]) Mockito.any(), (Get) Mockito.any())).thenReturn(r);
+ }
// Get a connection w/ mocked up common methods.
HConnection connection = HConnectionTestingUtility.
getMockedConnectionAndDecorate(HTU.getConfiguration(), ri, SERVERNAME_B,
@@ -892,6 +910,53 @@ public class TestAssignmentManager {
}
/**
+ * Test verifies whether all the enabling table regions assigned only once during master startup.
+ *
+ * @throws KeeperException
+ * @throws IOException
+ * @throws Exception
+ */
+ @Test
+ public void testMasterRestartWhenTableInEnabling() throws KeeperException, IOException, Exception {
+ enabling = true;
+ this.server.getConfiguration().setClass(HConstants.HBASE_MASTER_LOADBALANCER_CLASS,
+ DefaultLoadBalancer.class, LoadBalancer.class);
+ Map<ServerName, HServerLoad> serverAndLoad = new HashMap<ServerName, HServerLoad>();
+ serverAndLoad.put(SERVERNAME_A, null);
+ Mockito.when(this.serverManager.getOnlineServers()).thenReturn(serverAndLoad);
+ Mockito.when(this.serverManager.isServerOnline(SERVERNAME_B)).thenReturn(false);
+ Mockito.when(this.serverManager.isServerOnline(SERVERNAME_A)).thenReturn(true);
+ HTU.getConfiguration().setInt(HConstants.MASTER_PORT, 0);
+ Server server = new HMaster(HTU.getConfiguration());
+ Whitebox.setInternalState(server, "serverManager", this.serverManager);
+ assignmentCount = 0;
+ AssignmentManagerWithExtrasForTesting am = setUpMockedAssignmentManager(server,
+ this.serverManager);
+ am.regionOnline(new HRegionInfo("t1".getBytes(), HConstants.EMPTY_START_ROW,
+ HConstants.EMPTY_END_ROW), SERVERNAME_A);
+ am.gate.set(false);
+ try {
+ // set table in enabling state.
+ am.getZKTable().setEnablingTable(REGIONINFO.getTableNameAsString());
+ ZKAssign.createNodeOffline(this.watcher, REGIONINFO_2, SERVERNAME_B);
+
+ am.joinCluster();
+ while (!am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString())) {
+ Thread.sleep(10);
+ }
+ assertEquals("Number of assignments should be equal.", 2, assignmentCount);
+ assertTrue("Table should be enabled.",
+ am.getZKTable().isEnabledTable(REGIONINFO.getTableNameAsString()));
+ } finally {
+ enabling = false;
+ am.getZKTable().setEnabledTable(REGIONINFO.getTableNameAsString());
+ am.shutdown();
+ ZKAssign.deleteAllNodes(this.watcher);
+ assignmentCount = 0;
+ }
+ }
+
+ /**
* Mocked load balancer class used in the testcase to make sure that the testcase waits until
* random assignment is called and the gate variable is set to true.
*/
@@ -960,8 +1025,13 @@ public class TestAssignmentManager {
@Override
public void assign(HRegionInfo region, boolean setOfflineInZK, boolean forceNewPlan,
boolean hijack) {
- assignInvoked = true;
- super.assign(region, setOfflineInZK, forceNewPlan, hijack);
+ if (enabling) {
+ assignmentCount++;
+ this.regionOnline(region, SERVERNAME_A);
+ } else {
+ assignInvoked = true;
+ super.assign(region, setOfflineInZK, forceNewPlan, hijack);
+ }
}
@Override