You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by zj...@apache.org on 2013/03/24 11:26:22 UTC
svn commit: r1460306 [1/5] - in /hbase/trunk:
hbase-client/src/main/java/org/apache/hadoop/hbase/
hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/
hbase-client/src/main/java/org/apache/hadoop/hbase/client/
hbase-client/src/main/java/org/apac...
Author: zjushch
Date: Sun Mar 24 10:26:21 2013
New Revision: 1460306
URL: http://svn.apache.org/r1460306
Log:
HBASE-7403 Online Merge (Chunhui shen)
Added:
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/DispatchMergingRegionHandler.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/handler/MergedRegionHandler.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeRequest.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionMergeTransaction.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransaction.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionMergeTransactionOnCluster.java
Modified:
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/AdminProtos.java
hbase/trunk/hbase-protocol/src/main/java/org/apache/hadoop/hbase/protobuf/generated/MasterAdminProtos.java
hbase/trunk/hbase-protocol/src/main/protobuf/Admin.proto
hbase/trunk/hbase-protocol/src/main/protobuf/MasterAdmin.proto
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/catalog/MetaEditor.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/AssignmentManager.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/CatalogJanitor.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/master/ServerManager.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/CompactSplitThread.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegion.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionFileSystem.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/HRegionServer.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/RegionServerServices.java
hbase/trunk/hbase-server/src/main/java/org/apache/hadoop/hbase/regionserver/SplitTransaction.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/HBaseTestingUtility.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockRegionServer.java
hbase/trunk/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/HRegionInfo.java Sun Mar 24 10:26:21 2013
@@ -1107,4 +1107,27 @@ public class HRegionInfo implements Comp
return hris;
}
+ /**
+ * Check whether two regions are adjacent
+ * @param regionA
+ * @param regionB
+ * @return true if two regions are adjacent
+ */
+ public static boolean areAdjacent(HRegionInfo regionA, HRegionInfo regionB) {
+ if (regionA == null || regionB == null) {
+ throw new IllegalArgumentException(
+ "Can't check whether adjacent for null region");
+ }
+ HRegionInfo a = regionA;
+ HRegionInfo b = regionB;
+ if (Bytes.compareTo(a.getStartKey(), b.getStartKey()) > 0) {
+ a = regionB;
+ b = regionA;
+ }
+ if (Bytes.compareTo(a.getEndKey(), b.getStartKey()) == 0) {
+ return true;
+ }
+ return false;
+ }
+
}
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/catalog/MetaReader.java Sun Mar 24 10:26:21 2013
@@ -234,6 +234,38 @@ public class MetaReader {
}
/**
+ * Gets the result in META for the specified region.
+ * @param catalogTracker
+ * @param regionName
+ * @return result of the specified region
+ * @throws IOException
+ */
+ public static Result getRegionResult(CatalogTracker catalogTracker,
+ byte[] regionName) throws IOException {
+ Get get = new Get(regionName);
+ get.addFamily(HConstants.CATALOG_FAMILY);
+ return get(getCatalogHTable(catalogTracker), get);
+ }
+
+ /**
+ * Get regions from the merge qualifier of the specified merged region
+ * @return null if it doesn't contain merge qualifier, else two merge regions
+ * @throws IOException
+ */
+ public static Pair<HRegionInfo, HRegionInfo> getRegionsFromMergeQualifier(
+ CatalogTracker catalogTracker, byte[] regionName) throws IOException {
+ Result result = getRegionResult(catalogTracker, regionName);
+ HRegionInfo mergeA = HRegionInfo.getHRegionInfo(result,
+ HConstants.MERGEA_QUALIFIER);
+ HRegionInfo mergeB = HRegionInfo.getHRegionInfo(result,
+ HConstants.MERGEB_QUALIFIER);
+ if (mergeA == null && mergeB == null) {
+ return null;
+ }
+ return new Pair<HRegionInfo, HRegionInfo>(mergeA, mergeB);
+ }
+
+ /**
* Checks if the specified table exists. Looks at the META table hosted on
* the specified server.
* @param catalogTracker
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java Sun Mar 24 10:26:21 2013
@@ -18,8 +18,18 @@
*/
package org.apache.hadoop.hbase.client;
-import com.google.protobuf.ByteString;
-import com.google.protobuf.ServiceException;
+import java.io.Closeable;
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.net.SocketTimeoutException;
+import java.util.Arrays;
+import java.util.LinkedList;
+import java.util.List;
+import java.util.concurrent.Callable;
+import java.util.concurrent.atomic.AtomicInteger;
+import java.util.concurrent.atomic.AtomicReference;
+import java.util.regex.Pattern;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience;
@@ -78,6 +88,7 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteSnapshotRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DisableTableRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DispatchMergingRegionsRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsRestoreSnapshotDoneRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsRestoreSnapshotDoneResponse;
@@ -109,17 +120,8 @@ import org.apache.hadoop.ipc.RemoteExcep
import org.apache.hadoop.util.StringUtils;
import org.apache.zookeeper.KeeperException;
-import java.io.Closeable;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.net.SocketTimeoutException;
-import java.util.Arrays;
-import java.util.LinkedList;
-import java.util.List;
-import java.util.concurrent.Callable;
-import java.util.concurrent.atomic.AtomicInteger;
-import java.util.concurrent.atomic.AtomicReference;
-import java.util.regex.Pattern;
+import com.google.protobuf.ByteString;
+import com.google.protobuf.ServiceException;
/**
* Provides an interface to manage HBase database table metadata + general
@@ -1690,6 +1692,38 @@ public class HBaseAdmin implements Abort
}
/**
+ * Merge two regions. Asynchronous operation.
+ * @param encodedNameOfRegionA encoded name of region a
+ * @param encodedNameOfRegionB encoded name of region b
+ * @param forcible true if do a compulsory merge, otherwise we will only merge
+ * two adjacent regions
+ * @throws IOException
+ */
+ public void mergeRegions(final byte[] encodedNameOfRegionA,
+ final byte[] encodedNameOfRegionB, final boolean forcible)
+ throws IOException {
+ MasterAdminKeepAliveConnection master = connection
+ .getKeepAliveMasterAdmin();
+ try {
+ DispatchMergingRegionsRequest request = RequestConverter
+ .buildDispatchMergingRegionsRequest(encodedNameOfRegionA,
+ encodedNameOfRegionB, forcible);
+ master.dispatchMergingRegions(null, request);
+ } catch (ServiceException se) {
+ IOException ioe = ProtobufUtil.getRemoteException(se);
+ if (ioe instanceof UnknownRegionException) {
+ throw (UnknownRegionException) ioe;
+ }
+ LOG.error("Unexpected exception: " + se
+ + " from calling HMaster.dispatchMergingRegions");
+ } catch (DeserializationException de) {
+ LOG.error("Could not parse destination server name: " + de);
+ } finally {
+ master.close();
+ }
+ }
+
+ /**
* Split a table or an individual region.
* Asynchronous operation.
*
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/executor/EventType.java Sun Mar 24 10:26:21 2013
@@ -43,6 +43,8 @@ public enum EventType {
RS_ZK_REGION_SPLITTING (5, null), // RS has started a region split
RS_ZK_REGION_SPLIT (6, ExecutorType.MASTER_SERVER_OPERATIONS), // RS split has completed.
RS_ZK_REGION_FAILED_OPEN (7, ExecutorType.MASTER_CLOSE_REGION), // RS failed to open a region
+ RS_ZK_REGION_MERGING (8, null), // RS has started merging regions
+ RS_ZK_REGION_MERGE (9, ExecutorType.MASTER_SERVER_OPERATIONS), // RS region merge has completed.
// Messages originating from Master to RS
M_RS_OPEN_REGION (20, ExecutorType.RS_OPEN_REGION), // Master asking RS to open a region
@@ -53,6 +55,7 @@ public enum EventType {
M_RS_CLOSE_META (25, ExecutorType.RS_CLOSE_META), // Master asking RS to close meta
// Messages originating from Client to Master
+ C_M_MERGE_REGION (30, ExecutorType.MASTER_TABLE_OPERATIONS), // Client asking Master to merge regions
C_M_DELETE_TABLE (40, ExecutorType.MASTER_TABLE_OPERATIONS), // Client asking Master to delete a table
C_M_DISABLE_TABLE (41, ExecutorType.MASTER_TABLE_OPERATIONS), // Client asking Master to disable a table
C_M_ENABLE_TABLE (42, ExecutorType.MASTER_TABLE_OPERATIONS), // Client asking Master to enable a table
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/ProtobufUtil.java Sun Mar 24 10:26:21 2013
@@ -85,6 +85,7 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoResponse;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileResponse;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.ServerInfo;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.SplitRegionRequest;
@@ -1426,6 +1427,28 @@ public final class ProtobufUtil {
}
}
+ /**
+ * A helper to merge regions using admin protocol. Send request to
+ * regionserver.
+ * @param admin
+ * @param region_a
+ * @param region_b
+ * @param forcible true if do a compulsory merge, otherwise we will only merge
+ * two adjacent regions
+ * @throws IOException
+ */
+ public static void mergeRegions(final AdminProtocol admin,
+ final HRegionInfo region_a, final HRegionInfo region_b,
+ final boolean forcible) throws IOException {
+ MergeRegionsRequest request = RequestConverter.buildMergeRegionsRequest(
+ region_a.getRegionName(), region_b.getRegionName(),forcible);
+ try {
+ admin.mergeRegions(null, request);
+ } catch (ServiceException se) {
+ throw ProtobufUtil.getRemoteException(se);
+ }
+ }
+
// End helpers for Admin
/*
Modified: hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java (original)
+++ hbase/trunk/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java Sun Mar 24 10:26:21 2013
@@ -47,6 +47,7 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetRegionInfoRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetServerInfoRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.GetStoreFileRequest;
+import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.MergeRegionsRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.OpenRegionRequest.RegionOpenInfo;
import org.apache.hadoop.hbase.protobuf.generated.AdminProtos.RollWALWriterRequest;
@@ -78,6 +79,7 @@ import org.apache.hadoop.hbase.protobuf.
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteColumnRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DeleteTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DisableTableRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.DispatchMergingRegionsRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableCatalogJanitorRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.EnableTableRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterAdminProtos.IsCatalogJanitorEnabledRequest;
@@ -777,6 +779,26 @@ public final class RequestConverter {
return builder.build();
}
+ /**
+ * Create a MergeRegionsRequest for the given regions
+ * @param regionA name of region a
+ * @param regionB name of region b
+ * @param forcible true if it is a compulsory merge
+ * @return a MergeRegionsRequest
+ */
+ public static MergeRegionsRequest buildMergeRegionsRequest(
+ final byte[] regionA, final byte[] regionB, final boolean forcible) {
+ MergeRegionsRequest.Builder builder = MergeRegionsRequest.newBuilder();
+ RegionSpecifier regionASpecifier = buildRegionSpecifier(
+ RegionSpecifierType.REGION_NAME, regionA);
+ RegionSpecifier regionBSpecifier = buildRegionSpecifier(
+ RegionSpecifierType.REGION_NAME, regionB);
+ builder.setRegionA(regionASpecifier);
+ builder.setRegionB(regionBSpecifier);
+ builder.setForcible(forcible);
+ return builder.build();
+ }
+
/**
* Create a CompactRegionRequest for a given region name
*
@@ -936,6 +958,18 @@ public final class RequestConverter {
return builder.build();
}
+ public static DispatchMergingRegionsRequest buildDispatchMergingRegionsRequest(
+ final byte[] encodedNameOfRegionA, final byte[] encodedNameOfRegionB,
+ final boolean forcible) throws DeserializationException {
+ DispatchMergingRegionsRequest.Builder builder = DispatchMergingRegionsRequest.newBuilder();
+ builder.setRegionA(buildRegionSpecifier(
+ RegionSpecifierType.ENCODED_REGION_NAME, encodedNameOfRegionA));
+ builder.setRegionB(buildRegionSpecifier(
+ RegionSpecifierType.ENCODED_REGION_NAME, encodedNameOfRegionB));
+ builder.setForcible(forcible);
+ return builder.build();
+ }
+
/**
* Create a protocol buffer AssignRegionRequest
*
Modified: hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java
URL: http://svn.apache.org/viewvc/hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java?rev=1460306&r1=1460305&r2=1460306&view=diff
==============================================================================
--- hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java (original)
+++ hbase/trunk/hbase-common/src/main/java/org/apache/hadoop/hbase/HConstants.java Sun Mar 24 10:26:21 2013
@@ -381,6 +381,12 @@ public final class HConstants {
/** The upper-half split region column qualifier */
public static final byte [] SPLITB_QUALIFIER = Bytes.toBytes("splitB");
+ /** The lower-half merge region column qualifier */
+ public static final byte[] MERGEA_QUALIFIER = Bytes.toBytes("mergeA");
+
+ /** The upper-half merge region column qualifier */
+ public static final byte[] MERGEB_QUALIFIER = Bytes.toBytes("mergeB");
+
/**
* The meta table version column qualifier.
* We keep current version of the meta table in this column in <code>-ROOT-</code>