You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by md...@apache.org on 2018/07/20 19:51:17 UTC
[25/29] hbase git commit: HBASE-20901 Reducing region replica has no
effect
HBASE-20901 Reducing region replica has no effect
Signed-off-by: tedyu <yu...@gmail.com>
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b4eacdab
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b4eacdab
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b4eacdab
Branch: refs/heads/HBASE-20749
Commit: b4eacdabd6b231748716896fe2cc99868a58e8f4
Parents: c92cda8
Author: Ankit Singhal <an...@gmail.com>
Authored: Thu Jul 19 12:37:13 2018 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu Jul 19 13:38:32 2018 -0700
----------------------------------------------------------------------
.../apache/hadoop/hbase/MetaTableAccessor.java | 27 +++++
.../master/procedure/EnableTableProcedure.java | 105 +++++++++----------
.../hadoop/hbase/TestMetaTableAccessor.java | 41 +++++++-
3 files changed, 118 insertions(+), 55 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b4eacdab/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
index 60afaca..77aeb20 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/MetaTableAccessor.java
@@ -847,6 +847,30 @@ public class MetaTableAccessor {
}
/**
+ * Returns the column qualifier for serialized region state
+ * @param replicaId the replicaId of the region
+ * @return a byte[] for state qualifier
+ */
+ @VisibleForTesting
+ public static byte[] getRegionStateColumn(int replicaId) {
+ return replicaId == 0 ? HConstants.STATE_QUALIFIER
+ : Bytes.toBytes(HConstants.STATE_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
+ + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId));
+ }
+
+ /**
+ * Returns the column qualifier for serialized region state
+ * @param replicaId the replicaId of the region
+ * @return a byte[] for sn column qualifier
+ */
+ @VisibleForTesting
+ public static byte[] getServerNameColumn(int replicaId) {
+ return replicaId == 0 ? HConstants.SERVERNAME_QUALIFIER
+ : Bytes.toBytes(HConstants.SERVERNAME_QUALIFIER_STR + META_REPLICA_ID_DELIMITER
+ + String.format(RegionInfo.REPLICA_ID_FORMAT, replicaId));
+ }
+
+ /**
* Returns the column qualifier for server column for replicaId
* @param replicaId the replicaId of the region
* @return a byte[] for server column qualifier
@@ -1406,7 +1430,10 @@ public class MetaTableAccessor {
getSeqNumColumn(i), now);
deleteReplicaLocations.addColumns(getCatalogFamily(),
getStartCodeColumn(i), now);
+ deleteReplicaLocations.addColumns(getCatalogFamily(), getServerNameColumn(i), now);
+ deleteReplicaLocations.addColumns(getCatalogFamily(), getRegionStateColumn(i), now);
}
+
deleteFromMetaTable(connection, deleteReplicaLocations);
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b4eacdab/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
index c46070c..f2fbb7a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/EnableTableProcedure.java
@@ -21,16 +21,12 @@ package org.apache.hadoop.hbase.master.procedure;
import java.io.IOException;
import java.util.ArrayList;
import java.util.List;
-
import org.apache.hadoop.hbase.Cell;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotDisabledException;
import org.apache.hadoop.hbase.TableNotFoundException;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.slf4j.Logger;
-import org.slf4j.LoggerFactory;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.Get;
import org.apache.hadoop.hbase.client.RegionInfo;
@@ -42,6 +38,10 @@ import org.apache.hadoop.hbase.client.TableState;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
import org.apache.hadoop.hbase.master.TableStateManager;
import org.apache.hadoop.hbase.procedure2.ProcedureStateSerializer;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.EnableTableState;
@@ -94,37 +94,37 @@ public class EnableTableProcedure
try {
switch (state) {
- case ENABLE_TABLE_PREPARE:
- if (prepareEnable(env)) {
- setNextState(EnableTableState.ENABLE_TABLE_PRE_OPERATION);
- } else {
- assert isFailed() : "enable should have an exception here";
- return Flow.NO_MORE_STATE;
- }
- break;
- case ENABLE_TABLE_PRE_OPERATION:
- preEnable(env, state);
- setNextState(EnableTableState.ENABLE_TABLE_SET_ENABLING_TABLE_STATE);
- break;
- case ENABLE_TABLE_SET_ENABLING_TABLE_STATE:
- setTableStateToEnabling(env, tableName);
- setNextState(EnableTableState.ENABLE_TABLE_MARK_REGIONS_ONLINE);
- break;
- case ENABLE_TABLE_MARK_REGIONS_ONLINE:
- Connection connection = env.getMasterServices().getConnection();
- // we will need to get the tableDescriptor here to see if there is a change in the replica
- // count
- TableDescriptor hTableDescriptor =
- env.getMasterServices().getTableDescriptors().get(tableName);
+ case ENABLE_TABLE_PREPARE:
+ if (prepareEnable(env)) {
+ setNextState(EnableTableState.ENABLE_TABLE_PRE_OPERATION);
+ } else {
+ assert isFailed() : "enable should have an exception here";
+ return Flow.NO_MORE_STATE;
+ }
+ break;
+ case ENABLE_TABLE_PRE_OPERATION:
+ preEnable(env, state);
+ setNextState(EnableTableState.ENABLE_TABLE_SET_ENABLING_TABLE_STATE);
+ break;
+ case ENABLE_TABLE_SET_ENABLING_TABLE_STATE:
+ setTableStateToEnabling(env, tableName);
+ setNextState(EnableTableState.ENABLE_TABLE_MARK_REGIONS_ONLINE);
+ break;
+ case ENABLE_TABLE_MARK_REGIONS_ONLINE:
+ Connection connection = env.getMasterServices().getConnection();
+ // we will need to get the tableDescriptor here to see if there is a change in the replica
+ // count
+ TableDescriptor hTableDescriptor =
+ env.getMasterServices().getTableDescriptors().get(tableName);
- // Get the replica count
- int regionReplicaCount = hTableDescriptor.getRegionReplication();
+ // Get the replica count
+ int regionReplicaCount = hTableDescriptor.getRegionReplication();
- // Get the regions for the table from memory; get both online and offline regions ('true').
- List<RegionInfo> regionsOfTable =
- env.getAssignmentManager().getRegionStates().getRegionsOfTable(tableName, true);
+ // Get the regions for the table from memory; get both online and offline regions
+ // ('true').
+ List<RegionInfo> regionsOfTable =
+ env.getAssignmentManager().getRegionStates().getRegionsOfTable(tableName, true);
- if (regionReplicaCount > 1) {
int currentMaxReplica = 0;
// Check if the regions in memory have replica regions as marked in META table
for (RegionInfo regionInfo : regionsOfTable) {
@@ -166,36 +166,33 @@ public class EnableTableProcedure
}
} else {
// the replicasFound is less than the regionReplication
- LOG.info(
- "The number of replicas has been changed(increased)."
- + " Lets assign the new region replicas. The previous replica count was "
- + (currentMaxReplica + 1) + ". The current replica count is "
- + regionReplicaCount);
+ LOG.info("The number of replicas has been changed(increased)."
+ + " Lets assign the new region replicas. The previous replica count was "
+ + (currentMaxReplica + 1) + ". The current replica count is " + regionReplicaCount);
regionsOfTable = RegionReplicaUtil.addReplicas(hTableDescriptor, regionsOfTable,
currentMaxReplica + 1, regionReplicaCount);
}
- }
- // Assign all the table regions. (including region replicas if added).
- // createAssignProcedure will try to retain old assignments if possible.
- addChildProcedure(env.getAssignmentManager().createAssignProcedures(regionsOfTable));
- setNextState(EnableTableState.ENABLE_TABLE_SET_ENABLED_TABLE_STATE);
- break;
- case ENABLE_TABLE_SET_ENABLED_TABLE_STATE:
- setTableStateToEnabled(env, tableName);
- setNextState(EnableTableState.ENABLE_TABLE_POST_OPERATION);
- break;
- case ENABLE_TABLE_POST_OPERATION:
- postEnable(env, state);
- return Flow.NO_MORE_STATE;
- default:
- throw new UnsupportedOperationException("unhandled state=" + state);
+ // Assign all the table regions. (including region replicas if added).
+ // createAssignProcedure will try to retain old assignments if possible.
+ addChildProcedure(env.getAssignmentManager().createAssignProcedures(regionsOfTable));
+ setNextState(EnableTableState.ENABLE_TABLE_SET_ENABLED_TABLE_STATE);
+ break;
+ case ENABLE_TABLE_SET_ENABLED_TABLE_STATE:
+ setTableStateToEnabled(env, tableName);
+ setNextState(EnableTableState.ENABLE_TABLE_POST_OPERATION);
+ break;
+ case ENABLE_TABLE_POST_OPERATION:
+ postEnable(env, state);
+ return Flow.NO_MORE_STATE;
+ default:
+ throw new UnsupportedOperationException("unhandled state=" + state);
}
} catch (IOException e) {
if (isRollbackSupported(state)) {
setFailure("master-enable-table", e);
} else {
- LOG.warn("Retriable error trying to enable table=" + tableName +
- " (in state=" + state + ")", e);
+ LOG.warn(
+ "Retriable error trying to enable table=" + tableName + " (in state=" + state + ")", e);
}
}
return Flow.HAS_MORE_STATE;
http://git-wip-us.apache.org/repos/asf/hbase/blob/b4eacdab/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java
index f7865ee..279b9ab 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/TestMetaTableAccessor.java
@@ -22,7 +22,7 @@ import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertNull;
import static org.junit.Assert.assertTrue;
-import static org.mockito.Matchers.anyObject;
+import static org.mockito.ArgumentMatchers.anyObject;
import static org.mockito.Mockito.doReturn;
import static org.mockito.Mockito.mock;
import static org.mockito.Mockito.reset;
@@ -68,6 +68,7 @@ import org.slf4j.Logger;
import org.slf4j.LoggerFactory;
import org.apache.hbase.thirdparty.com.google.common.collect.Lists;
+import org.apache.hbase.thirdparty.com.google.common.collect.Sets;
/**
* Test {@link org.apache.hadoop.hbase.MetaTableAccessor}.
@@ -439,6 +440,44 @@ public class TestMetaTableAccessor {
}
@Test
+ public void testMetaLocationForRegionReplicasIsRemovedAtTableDeletion() throws IOException {
+ long regionId = System.currentTimeMillis();
+ RegionInfo primary = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))
+ .setStartKey(HConstants.EMPTY_START_ROW).setEndKey(HConstants.EMPTY_END_ROW).setSplit(false)
+ .setRegionId(regionId).setReplicaId(0).build();
+
+ Table meta = MetaTableAccessor.getMetaHTable(connection);
+ try {
+ List<RegionInfo> regionInfos = Lists.newArrayList(primary);
+ MetaTableAccessor.addRegionsToMeta(connection, regionInfos, 3);
+ MetaTableAccessor.removeRegionReplicasFromMeta(Sets.newHashSet(primary.getRegionName()), 1, 2,
+ connection);
+ Get get = new Get(primary.getRegionName());
+ Result result = meta.get(get);
+ for (int replicaId = 0; replicaId < 3; replicaId++) {
+ Cell serverCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
+ MetaTableAccessor.getServerColumn(replicaId));
+ Cell startCodeCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
+ MetaTableAccessor.getStartCodeColumn(replicaId));
+ Cell stateCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
+ MetaTableAccessor.getRegionStateColumn(replicaId));
+ Cell snCell = result.getColumnLatestCell(HConstants.CATALOG_FAMILY,
+ MetaTableAccessor.getServerNameColumn(replicaId));
+ if (replicaId == 0) {
+ assertNotNull(stateCell);
+ } else {
+ assertNull(serverCell);
+ assertNull(startCodeCell);
+ assertNull(stateCell);
+ assertNull(snCell);
+ }
+ }
+ } finally {
+ meta.close();
+ }
+ }
+
+ @Test
public void testMetaLocationForRegionReplicasIsAddedAtTableCreation() throws IOException {
long regionId = System.currentTimeMillis();
RegionInfo primary = RegionInfoBuilder.newBuilder(TableName.valueOf(name.getMethodName()))