You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by wc...@apache.org on 2023/10/31 10:43:40 UTC
(hbase) branch branch-2 updated: HBASE-28064:Implement truncate_region command (#5480)
This is an automated email from the ASF dual-hosted git repository.
wchevreuil pushed a commit to branch branch-2
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-2 by this push:
new a208ba2a141 HBASE-28064:Implement truncate_region command (#5480)
a208ba2a141 is described below
commit a208ba2a1412eec9bd3500df361c261d24c78b3f
Author: Vaibhav Joshi <vj...@cloudera.com>
AuthorDate: Tue Oct 31 16:13:35 2023 +0530
HBASE-28064:Implement truncate_region command (#5480)
Signed-off-by: Wellington Chevreuil <wc...@apache.org>
---
.../java/org/apache/hadoop/hbase/client/Admin.java | 14 ++
.../org/apache/hadoop/hbase/client/AsyncAdmin.java | 6 +
.../hadoop/hbase/client/AsyncHBaseAdmin.java | 5 +
.../hbase/client/ConnectionImplementation.java | 6 +
.../org/apache/hadoop/hbase/client/HBaseAdmin.java | 59 ++++++
.../hadoop/hbase/client/RawAsyncHBaseAdmin.java | 66 +++++++
.../hbase/client/ShortCircuitMasterConnection.java | 6 +
.../hbase/shaded/protobuf/RequestConverter.java | 11 ++
.../src/main/protobuf/Master.proto | 16 ++
.../src/main/protobuf/MasterProcedure.proto | 8 +
.../hadoop/hbase/coprocessor/MasterObserver.java | 40 ++++
.../org/apache/hadoop/hbase/master/HMaster.java | 31 +++
.../hadoop/hbase/master/MasterCoprocessorHost.java | 54 +++++
.../hadoop/hbase/master/MasterRpcServices.java | 12 ++
.../apache/hadoop/hbase/master/MasterServices.java | 9 +
.../hbase/master/assignment/AssignmentManager.java | 6 +
.../AbstractStateMachineRegionProcedure.java | 6 +
.../master/procedure/TableProcedureInterface.java | 5 +-
.../hadoop/hbase/master/procedure/TableQueue.java | 1 +
.../master/procedure/TruncateRegionProcedure.java | 219 +++++++++++++++++++++
.../org/apache/hadoop/hbase/client/TestAdmin1.java | 94 ++++++++-
.../hbase/client/TestAsyncRegionAdminApi2.java | 83 ++++++++
.../hbase/master/MockNoopMasterServices.java | 6 +
.../procedure/TestTruncateRegionProcedure.java | 202 +++++++++++++++++++
hbase-shell/src/main/ruby/hbase/admin.rb | 10 +
hbase-shell/src/main/ruby/shell.rb | 1 +
.../main/ruby/shell/commands/truncate_region.rb | 36 ++++
hbase-shell/src/test/ruby/hbase/admin_test.rb | 11 ++
.../hadoop/hbase/thrift2/client/ThriftAdmin.java | 10 +
29 files changed, 1030 insertions(+), 3 deletions(-)
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 3fc844b1aac..ab3df597bfa 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -3303,4 +3303,18 @@ public interface Admin extends Abortable, Closeable {
* Flush master local region
*/
void flushMasterStore() throws IOException;
+
+ /**
+ * Truncate an individual region.
+ * @param regionName region to truncate
+ * @throws IOException if a remote or network exception occurs
+ */
+ void truncateRegion(byte[] regionName) throws IOException;
+
+ /**
+ * Truncate an individual region. Asynchronous operation.
+ * @param regionName region to truncate
+ * @throws IOException if a remote or network exception occurs
+ */
+ Future<Void> truncateRegionAsync(byte[] regionName) throws IOException;
}
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
index 9509a888cc5..6b95e19a22a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncAdmin.java
@@ -625,6 +625,12 @@ public interface AsyncAdmin {
*/
CompletableFuture<Void> splitRegion(byte[] regionName, byte[] splitPoint);
+ /**
+ * Truncate an individual region.
+ * @param regionName region to truncate
+ */
+ CompletableFuture<Void> truncateRegion(byte[] regionName);
+
/**
* Assign an individual region.
* @param regionName Encoded or full name of region to assign.
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
index 442fb911556..4c9138e7306 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/AsyncHBaseAdmin.java
@@ -386,6 +386,11 @@ class AsyncHBaseAdmin implements AsyncAdmin {
return wrap(rawAdmin.splitRegion(regionName, splitPoint));
}
+ @Override
+ public CompletableFuture<Void> truncateRegion(byte[] regionName) {
+ return wrap(rawAdmin.truncateRegion(regionName));
+ }
+
@Override
public CompletableFuture<Void> assign(byte[] regionName) {
return wrap(rawAdmin.assign(regionName));
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index 70d5760df48..268a2495d69 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -1559,6 +1559,12 @@ public class ConnectionImplementation implements ClusterConnection, Closeable {
return stub.splitRegion(controller, request);
}
+ @Override
+ public MasterProtos.TruncateRegionResponse truncateRegion(RpcController controller,
+ MasterProtos.TruncateRegionRequest request) throws ServiceException {
+ return stub.truncateRegion(controller, request);
+ }
+
@Override
public MasterProtos.DeleteTableResponse deleteTable(RpcController controller,
MasterProtos.DeleteTableRequest request) throws ServiceException {
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index b4e9390c13e..c54e561cc21 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -2057,6 +2057,65 @@ public class HBaseAdmin implements Admin {
splitRegionAsync(regionServerPair.getFirst(), splitPoint);
}
+ @Override
+ public void truncateRegion(byte[] regionName) throws IOException {
+ get(truncateRegionAsync(regionName), syncWaitTimeout, TimeUnit.MILLISECONDS);
+ }
+
+ @Override
+ public Future<Void> truncateRegionAsync(byte[] regionName) throws IOException {
+ Pair<RegionInfo, ServerName> pair = getRegion(regionName);
+ RegionInfo hri;
+
+ if (pair != null) {
+ hri = pair.getFirst();
+ if (hri != null && hri.getReplicaId() != HRegionInfo.DEFAULT_REPLICA_ID) {
+ throw new IllegalArgumentException(
+ "Can't truncate replicas directly.Replicas are auto-truncated "
+ + "when their primary is truncated.");
+ }
+ } else {
+ throw new IllegalArgumentException("Invalid region: " + Bytes.toStringBinary(regionName));
+ }
+
+ TableName tableName = hri.getTable();
+
+ MasterProtos.TruncateRegionResponse response =
+ executeCallable(getTruncateRegionCallable(tableName, hri));
+
+ return new TruncateRegionFuture(this, tableName, response);
+ }
+
+ private MasterCallable<MasterProtos.TruncateRegionResponse>
+ getTruncateRegionCallable(TableName tableName, RegionInfo hri) {
+ return new MasterCallable<MasterProtos.TruncateRegionResponse>(getConnection(),
+ getRpcControllerFactory()) {
+ Long nonceGroup = ng.getNonceGroup();
+ Long nonce = ng.newNonce();
+
+ @Override
+ protected MasterProtos.TruncateRegionResponse rpcCall() throws Exception {
+ setPriority(tableName);
+ MasterProtos.TruncateRegionRequest request =
+ RequestConverter.buildTruncateRegionRequest(hri, nonceGroup, nonce);
+ return master.truncateRegion(getRpcController(), request);
+ }
+ };
+ }
+
+ private static class TruncateRegionFuture extends TableFuture<Void> {
+ public TruncateRegionFuture(final HBaseAdmin admin, final TableName tableName,
+ final MasterProtos.TruncateRegionResponse response) {
+ super(admin, tableName,
+ (response != null && response.hasProcId()) ? response.getProcId() : null);
+ }
+
+ @Override
+ public String getOperationType() {
+ return "TRUNCATE_REGION";
+ }
+ }
+
private static class ModifyTableFuture extends TableFuture<Void> {
public ModifyTableFuture(final HBaseAdmin admin, final TableName tableName,
final ModifyTableResponse response) {
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
index 9231c423ce6..7a560e51e0a 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/RawAsyncHBaseAdmin.java
@@ -1603,6 +1603,60 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
return future;
}
+ @Override
+ public CompletableFuture<Void> truncateRegion(byte[] regionName) {
+ CompletableFuture<Void> future = new CompletableFuture<>();
+ addListener(getRegionLocation(regionName), (location, err) -> {
+ if (err != null) {
+ future.completeExceptionally(err);
+ return;
+ }
+ RegionInfo regionInfo = location.getRegion();
+ if (regionInfo.getReplicaId() != RegionInfo.DEFAULT_REPLICA_ID) {
+ future.completeExceptionally(new IllegalArgumentException(
+ "Can't truncate replicas directly.Replicas are auto-truncated "
+ + "when their primary is truncated."));
+ return;
+ }
+ ServerName serverName = location.getServerName();
+ if (serverName == null) {
+ future
+ .completeExceptionally(new NoServerForRegionException(Bytes.toStringBinary(regionName)));
+ return;
+ }
+ addListener(truncateRegion(regionInfo), (ret, err2) -> {
+ if (err2 != null) {
+ future.completeExceptionally(err2);
+ } else {
+ future.complete(ret);
+ }
+ });
+ });
+ return future;
+ }
+
+ private CompletableFuture<Void> truncateRegion(final RegionInfo hri) {
+ CompletableFuture<Void> future = new CompletableFuture<>();
+ TableName tableName = hri.getTable();
+ final MasterProtos.TruncateRegionRequest request;
+ try {
+ request = RequestConverter.buildTruncateRegionRequest(hri, ng.getNonceGroup(), ng.newNonce());
+ } catch (DeserializationException e) {
+ future.completeExceptionally(e);
+ return future;
+ }
+ addListener(this.procedureCall(tableName, request, MasterService.Interface::truncateRegion,
+ MasterProtos.TruncateRegionResponse::getProcId,
+ new TruncateRegionProcedureBiConsumer(tableName)), (ret, err2) -> {
+ if (err2 != null) {
+ future.completeExceptionally(err2);
+ } else {
+ future.complete(ret);
+ }
+ });
+ return future;
+ }
+
@Override
public CompletableFuture<Void> assign(byte[] regionName) {
CompletableFuture<Void> future = new CompletableFuture<>();
@@ -2850,6 +2904,18 @@ class RawAsyncHBaseAdmin implements AsyncAdmin {
}
}
+ private static class TruncateRegionProcedureBiConsumer extends TableProcedureBiConsumer {
+
+ TruncateRegionProcedureBiConsumer(TableName tableName) {
+ super(tableName);
+ }
+
+ @Override
+ String getOperationType() {
+ return "TRUNCATE_REGION";
+ }
+ }
+
private static class SnapshotProcedureBiConsumer extends TableProcedureBiConsumer {
SnapshotProcedureBiConsumer(TableName tableName) {
super(tableName);
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java
index ea874742041..57f3cba8605 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ShortCircuitMasterConnection.java
@@ -730,6 +730,12 @@ public class ShortCircuitMasterConnection implements MasterKeepAliveConnection {
return stub.splitRegion(controller, request);
}
+ @Override
+ public MasterProtos.TruncateRegionResponse truncateRegion(RpcController controller,
+ MasterProtos.TruncateRegionRequest request) throws ServiceException {
+ return stub.truncateRegion(controller, request);
+ }
+
@Override
public SwitchRpcThrottleResponse switchRpcThrottle(RpcController controller,
SwitchRpcThrottleRequest request) throws ServiceException {
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
index d6956f4e9df..cd6fd2bb9c4 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/shaded/protobuf/RequestConverter.java
@@ -1187,6 +1187,17 @@ public final class RequestConverter {
return builder.build();
}
+ public static MasterProtos.TruncateRegionRequest
+ buildTruncateRegionRequest(final RegionInfo regionInfo, final long nonceGroup, final long nonce)
+ throws DeserializationException {
+ MasterProtos.TruncateRegionRequest.Builder builder =
+ MasterProtos.TruncateRegionRequest.newBuilder();
+ builder.setRegionInfo(ProtobufUtil.toRegionInfo(regionInfo));
+ builder.setNonceGroup(nonceGroup);
+ builder.setNonce(nonce);
+ return builder.build();
+ }
+
/**
* Create a protocol buffer AssignRegionRequest
* @return an AssignRegionRequest
diff --git a/hbase-protocol-shaded/src/main/protobuf/Master.proto b/hbase-protocol-shaded/src/main/protobuf/Master.proto
index 6ca065dfc94..ce736cd799e 100644
--- a/hbase-protocol-shaded/src/main/protobuf/Master.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/Master.proto
@@ -136,6 +136,16 @@ message SplitTableRegionResponse {
optional uint64 proc_id = 1;
}
+message TruncateRegionRequest {
+ required RegionInfo region_info = 1;
+ optional uint64 nonce_group = 2 [default = 0];
+ optional uint64 nonce = 3 [default = 0];
+}
+
+message TruncateRegionResponse {
+ optional uint64 proc_id = 1;
+}
+
message CreateTableRequest {
required TableSchema table_schema = 1;
repeated bytes split_keys = 2;
@@ -863,6 +873,12 @@ service MasterService {
rpc SplitRegion(SplitTableRegionRequest)
returns(SplitTableRegionResponse);
+ /**
+ * Truncate region
+ */
+ rpc TruncateRegion(TruncateRegionRequest)
+ returns(TruncateRegionResponse);
+
/** Deletes a table */
rpc DeleteTable(DeleteTableRequest)
returns(DeleteTableResponse);
diff --git a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
index 90d0341df64..b81507269ae 100644
--- a/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
+++ b/hbase-protocol-shaded/src/main/protobuf/MasterProcedure.proto
@@ -102,6 +102,14 @@ message TruncateTableStateData {
repeated RegionInfo region_info = 5;
}
+enum TruncateRegionState {
+ TRUNCATE_REGION_PRE_OPERATION = 1;
+ TRUNCATE_REGION_MAKE_OFFLINE = 2;
+ TRUNCATE_REGION_REMOVE = 3;
+ TRUNCATE_REGION_MAKE_ONLINE = 4;
+ TRUNCATE_REGION_POST_OPERATION = 5;
+}
+
enum DeleteTableState {
DELETE_TABLE_PRE_OPERATION = 1;
DELETE_TABLE_REMOVE_FROM_META = 2;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
index a49f5764843..5de99adf885 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/coprocessor/MasterObserver.java
@@ -665,6 +665,46 @@ public interface MasterObserver {
final TableName tableName, final byte[] splitRow) throws IOException {
}
+ /**
+ * Called before the region is truncated.
+ * @param c The environment to interact with the framework and master
+ * @param regionInfo The Region being truncated
+ */
+ @SuppressWarnings("unused")
+ default void preTruncateRegionAction(final ObserverContext<MasterCoprocessorEnvironment> c,
+ final RegionInfo regionInfo) {
+ }
+
+ /**
+ * Called before the truncate region procedure is called.
+ * @param c The environment to interact with the framework and master
+ * @param regionInfo The Region being truncated
+ */
+ @SuppressWarnings("unused")
+ default void preTruncateRegion(final ObserverContext<MasterCoprocessorEnvironment> c,
+ RegionInfo regionInfo) {
+ }
+
+ /**
+ * Called after the truncate region procedure is called.
+ * @param c The environment to interact with the framework and master
+ * @param regionInfo The Region being truncated
+ */
+ @SuppressWarnings("unused")
+ default void postTruncateRegion(final ObserverContext<MasterCoprocessorEnvironment> c,
+ RegionInfo regionInfo) {
+ }
+
+ /**
+ * Called post the region is truncated.
+ * @param c The environment to interact with the framework and master
+ * @param regionInfo The Region To be truncated
+ */
+ @SuppressWarnings("unused")
+ default void postTruncateRegionAction(final ObserverContext<MasterCoprocessorEnvironment> c,
+ final RegionInfo regionInfo) {
+ }
+
/**
* Called after the region is split.
* @param c the environment to interact with the framework and master
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
index 95ce05c6ef4..c92b3559a90 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/HMaster.java
@@ -163,6 +163,7 @@ import org.apache.hadoop.hbase.master.procedure.ProcedurePrepareLatch;
import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
import org.apache.hadoop.hbase.master.procedure.ReopenTableRegionsProcedure;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.procedure.TruncateRegionProcedure;
import org.apache.hadoop.hbase.master.procedure.TruncateTableProcedure;
import org.apache.hadoop.hbase.master.region.MasterRegion;
import org.apache.hadoop.hbase.master.region.MasterRegionFactory;
@@ -2493,6 +2494,36 @@ public class HMaster extends HRegionServer implements MasterServices {
});
}
+ @Override
+ public long truncateRegion(final RegionInfo regionInfo, final long nonceGroup, final long nonce)
+ throws IOException {
+ checkInitialized();
+
+ return MasterProcedureUtil
+ .submitProcedure(new MasterProcedureUtil.NonceProcedureRunnable(this, nonceGroup, nonce) {
+ @Override
+ protected void run() throws IOException {
+ getMaster().getMasterCoprocessorHost().preTruncateRegion(regionInfo);
+
+ LOG.info(
+ getClientIdAuditPrefix() + " truncate region " + regionInfo.getRegionNameAsString());
+
+ // Execute the operation asynchronously
+ ProcedurePrepareLatch latch = ProcedurePrepareLatch.createLatch(2, 0);
+ submitProcedure(
+ new TruncateRegionProcedure(procedureExecutor.getEnvironment(), regionInfo, latch));
+ latch.await();
+
+ getMaster().getMasterCoprocessorHost().postTruncateRegion(regionInfo);
+ }
+
+ @Override
+ protected String getDescription() {
+ return "TruncateRegionProcedure";
+ }
+ });
+ }
+
@Override
public long addColumn(final TableName tableName, final ColumnFamilyDescriptor column,
final long nonceGroup, final long nonce) throws IOException {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
index aa20c2afea2..95c183b4a39 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterCoprocessorHost.java
@@ -862,6 +862,60 @@ public class MasterCoprocessorHost
});
}
+ /**
+ * Invoked just before calling the truncate region procedure
+ * @param regionInfo region being truncated
+ */
+ public void preTruncateRegion(RegionInfo regionInfo) throws IOException {
+ execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
+ @Override
+ public void call(MasterObserver observer) {
+ observer.preTruncateRegion(this, regionInfo);
+ }
+ });
+ }
+
+ /**
+ * Invoked after calling the truncate region procedure
+ * @param regionInfo region being truncated
+ */
+ public void postTruncateRegion(RegionInfo regionInfo) throws IOException {
+ execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation() {
+ @Override
+ public void call(MasterObserver observer) {
+ observer.postTruncateRegion(this, regionInfo);
+ }
+ });
+ }
+
+ /**
+ * Invoked just before calling the truncate region procedure
+ * @param region Region to be truncated
+ * @param user The user
+ */
+ public void preTruncateRegionAction(final RegionInfo region, User user) throws IOException {
+ execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation(user) {
+ @Override
+ public void call(MasterObserver observer) throws IOException {
+ observer.preTruncateRegionAction(this, region);
+ }
+ });
+ }
+
+ /**
+ * Invoked after calling the truncate region procedure
+ * @param region Region which was truncated
+ * @param user The user
+ */
+ public void postTruncateRegionAction(final RegionInfo region, User user) throws IOException {
+ execOperation(coprocEnvironments.isEmpty() ? null : new MasterObserverOperation(user) {
+ @Override
+ public void call(MasterObserver observer) throws IOException {
+ observer.postTruncateRegionAction(this, region);
+ }
+ });
+ }
+
/**
* This will be called before update META step as part of split table region procedure.
* @param user the user
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
index 468a477288c..d4cfaaaf47a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterRpcServices.java
@@ -884,6 +884,18 @@ public class MasterRpcServices extends RSRpcServices
}
}
+ @Override
+ public MasterProtos.TruncateRegionResponse truncateRegion(RpcController controller,
+ final MasterProtos.TruncateRegionRequest request) throws ServiceException {
+ try {
+ long procId = master.truncateRegion(ProtobufUtil.toRegionInfo(request.getRegionInfo()),
+ request.getNonceGroup(), request.getNonce());
+ return MasterProtos.TruncateRegionResponse.newBuilder().setProcId(procId).build();
+ } catch (IOException ie) {
+ throw new ServiceException(ie);
+ }
+ }
+
@Override
public ClientProtos.CoprocessorServiceResponse execMasterService(final RpcController controller,
final ClientProtos.CoprocessorServiceRequest request) throws ServiceException {
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
index d465a4a6727..35044fb761a 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/MasterServices.java
@@ -448,4 +448,13 @@ public interface MasterServices extends Server {
*/
long flushTable(final TableName tableName, final List<byte[]> columnFamilies,
final long nonceGroup, final long nonce) throws IOException;
+
+ /**
+ * Truncate region
+ * @param regionInfo region to be truncated
+ * @param nonceGroup the nonce group
+ * @param nonce the nonce
+ * @return procedure Id
+ */
+ long truncateRegion(RegionInfo regionInfo, long nonceGroup, long nonce) throws IOException;
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
index 55edb5af5d5..b5892b42917 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/assignment/AssignmentManager.java
@@ -70,6 +70,7 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureScheduler;
import org.apache.hadoop.hbase.master.procedure.ProcedureSyncWait;
import org.apache.hadoop.hbase.master.procedure.ServerCrashProcedure;
+import org.apache.hadoop.hbase.master.procedure.TruncateRegionProcedure;
import org.apache.hadoop.hbase.master.region.MasterRegion;
import org.apache.hadoop.hbase.procedure2.Procedure;
import org.apache.hadoop.hbase.procedure2.ProcedureEvent;
@@ -1075,6 +1076,11 @@ public class AssignmentManager {
return new SplitTableRegionProcedure(getProcedureEnvironment(), regionToSplit, splitKey);
}
+ public TruncateRegionProcedure createTruncateRegionProcedure(final RegionInfo regionToTruncate)
+ throws IOException {
+ return new TruncateRegionProcedure(getProcedureEnvironment(), regionToTruncate);
+ }
+
public MergeTableRegionsProcedure createMergeProcedure(RegionInfo... ris) throws IOException {
return new MergeTableRegionsProcedure(getProcedureEnvironment(), ris, false);
}
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AbstractStateMachineRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AbstractStateMachineRegionProcedure.java
index fe98a78b4d7..cf886e9824b 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AbstractStateMachineRegionProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/AbstractStateMachineRegionProcedure.java
@@ -41,6 +41,12 @@ public abstract class AbstractStateMachineRegionProcedure<TState>
this.hri = hri;
}
+ protected AbstractStateMachineRegionProcedure(MasterProcedureEnv env, RegionInfo hri,
+ ProcedurePrepareLatch latch) {
+ super(env, latch);
+ this.hri = hri;
+ }
+
protected AbstractStateMachineRegionProcedure() {
// Required by the Procedure framework to create the procedure on replay
super();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java
index b5aaada9c45..1018535a173 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java
@@ -42,8 +42,9 @@ public interface TableProcedureInterface {
REGION_ASSIGN,
REGION_UNASSIGN,
REGION_GC,
- MERGED_REGIONS_GC/* region operations */
- };
+ MERGED_REGIONS_GC/* region operations */,
+ REGION_TRUNCATE
+ }
/** Returns the name of the table the procedure is operating on */
TableName getTableName();
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java
index d3eae050b56..910a684b51f 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableQueue.java
@@ -69,6 +69,7 @@ class TableQueue extends Queue<TableName> {
case REGION_GC:
case MERGED_REGIONS_GC:
case REGION_SNAPSHOT:
+ case REGION_TRUNCATE:
return false;
default:
break;
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java
new file mode 100644
index 00000000000..5e907c1681a
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TruncateRegionProcedure.java
@@ -0,0 +1,219 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import java.io.IOException;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
+import org.apache.hadoop.hbase.master.MasterFileSystem;
+import org.apache.hadoop.hbase.master.assignment.RegionStateNode;
+import org.apache.hadoop.hbase.master.assignment.TransitRegionStateProcedure;
+import org.apache.hadoop.hbase.regionserver.HRegionFileSystem;
+import org.apache.hadoop.hbase.util.CommonFSUtils;
+import org.apache.yetus.audience.InterfaceAudience;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.MasterProcedureProtos.TruncateRegionState;
+
+@InterfaceAudience.Private
+public class TruncateRegionProcedure
+ extends AbstractStateMachineRegionProcedure<TruncateRegionState> {
+ private static final Logger LOG = LoggerFactory.getLogger(TruncateRegionProcedure.class);
+
+ @SuppressWarnings("unused")
+ public TruncateRegionProcedure() {
+ // Required by the Procedure framework to create the procedure on replay
+ super();
+ }
+
+ public TruncateRegionProcedure(final MasterProcedureEnv env, final RegionInfo hri)
+ throws HBaseIOException {
+ super(env, hri);
+ checkOnline(env, getRegion());
+ }
+
+ public TruncateRegionProcedure(final MasterProcedureEnv env, final RegionInfo region,
+ ProcedurePrepareLatch latch) throws HBaseIOException {
+ super(env, region, latch);
+ preflightChecks(env, true);
+ }
+
+ @Override
+ protected Flow executeFromState(final MasterProcedureEnv env, TruncateRegionState state)
+ throws InterruptedException {
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(this + " execute state=" + state);
+ }
+ try {
+ switch (state) {
+ case TRUNCATE_REGION_PRE_OPERATION:
+ if (!prepareTruncate()) {
+ assert isFailed() : "the truncate should have an exception here";
+ return Flow.NO_MORE_STATE;
+ }
+ checkOnline(env, getRegion());
+ assert getRegion().getReplicaId() == RegionInfo.DEFAULT_REPLICA_ID || isFailed()
+ : "Can't truncate replicas directly. "
+ + "Replicas are auto-truncated when their primary is truncated.";
+ preTruncate(env);
+ setNextState(TruncateRegionState.TRUNCATE_REGION_MAKE_OFFLINE);
+ break;
+ case TRUNCATE_REGION_MAKE_OFFLINE:
+ addChildProcedure(createUnAssignProcedures(env));
+ setNextState(TruncateRegionState.TRUNCATE_REGION_REMOVE);
+ break;
+ case TRUNCATE_REGION_REMOVE:
+ deleteRegionFromFileSystem(env);
+ setNextState(TruncateRegionState.TRUNCATE_REGION_MAKE_ONLINE);
+ break;
+ case TRUNCATE_REGION_MAKE_ONLINE:
+ addChildProcedure(createAssignProcedures(env));
+ setNextState(TruncateRegionState.TRUNCATE_REGION_POST_OPERATION);
+ break;
+ case TRUNCATE_REGION_POST_OPERATION:
+ postTruncate(env);
+ LOG.debug("truncate '" + getTableName() + "' completed");
+ return Flow.NO_MORE_STATE;
+ default:
+ throw new UnsupportedOperationException("unhandled state=" + state);
+ }
+ } catch (IOException e) {
+ if (isRollbackSupported(state)) {
+ setFailure("master-truncate-region", e);
+ } else {
+ LOG.warn("Retriable error trying to truncate region=" + getRegion().getRegionNameAsString()
+ + " state=" + state, e);
+ }
+ }
+ return Flow.HAS_MORE_STATE;
+ }
+
+ private void deleteRegionFromFileSystem(final MasterProcedureEnv env) throws IOException {
+ RegionStateNode regionNode =
+ env.getAssignmentManager().getRegionStates().getRegionStateNode(getRegion());
+ try {
+ regionNode.lock();
+ final MasterFileSystem mfs = env.getMasterServices().getMasterFileSystem();
+ final Path tableDir = CommonFSUtils.getTableDir(mfs.getRootDir(), getTableName());
+ HRegionFileSystem.deleteRegionFromFileSystem(env.getMasterConfiguration(),
+ mfs.getFileSystem(), tableDir, getRegion());
+ } finally {
+ regionNode.unlock();
+ }
+ }
+
+ @Override
+ protected void rollbackState(final MasterProcedureEnv env, final TruncateRegionState state)
+ throws IOException {
+ if (state == TruncateRegionState.TRUNCATE_REGION_PRE_OPERATION) {
+ // Nothing to rollback, pre-truncate is just table-state checks.
+ return;
+ }
+ if (state == TruncateRegionState.TRUNCATE_REGION_MAKE_OFFLINE) {
+ RegionStateNode regionNode =
+ env.getAssignmentManager().getRegionStates().getRegionStateNode(getRegion());
+ if (regionNode == null) {
+ // Region was unassigned by state TRUNCATE_REGION_MAKE_OFFLINE.
+ // So Assign it back
+ addChildProcedure(createAssignProcedures(env));
+ }
+ return;
+ }
+ // The truncate doesn't have a rollback. The execution will succeed, at some point.
+ throw new UnsupportedOperationException("unhandled state=" + state);
+ }
+
+ @Override
+ protected void completionCleanup(final MasterProcedureEnv env) {
+ releaseSyncLatch();
+ }
+
+ @Override
+ protected boolean isRollbackSupported(final TruncateRegionState state) {
+ switch (state) {
+ case TRUNCATE_REGION_PRE_OPERATION:
+ return true;
+ case TRUNCATE_REGION_MAKE_OFFLINE:
+ return true;
+ default:
+ return false;
+ }
+ }
+
+ @Override
+ protected TruncateRegionState getState(final int stateId) {
+ return TruncateRegionState.forNumber(stateId);
+ }
+
+ @Override
+ protected int getStateId(final TruncateRegionState state) {
+ return state.getNumber();
+ }
+
+ @Override
+ protected TruncateRegionState getInitialState() {
+ return TruncateRegionState.TRUNCATE_REGION_PRE_OPERATION;
+ }
+
+ @Override
+ public void toStringClassDetails(StringBuilder sb) {
+ sb.append(getClass().getSimpleName());
+ sb.append(" (region=");
+ sb.append(getRegion().getRegionNameAsString());
+ sb.append(")");
+ }
+
+ private boolean prepareTruncate() throws IOException {
+ if (getTableName().equals(TableName.META_TABLE_NAME)) {
+ throw new IOException("Can't truncate region in catalog tables");
+ }
+ return true;
+ }
+
+ private void preTruncate(final MasterProcedureEnv env) throws IOException {
+ final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
+ if (cpHost != null) {
+ cpHost.preTruncateRegionAction(getRegion(), getUser());
+ }
+ }
+
+ private void postTruncate(final MasterProcedureEnv env) throws IOException {
+ final MasterCoprocessorHost cpHost = env.getMasterCoprocessorHost();
+ if (cpHost != null) {
+ cpHost.postTruncateRegionAction(getRegion(), getUser());
+ }
+ }
+
+ @Override
+ public TableOperationType getTableOperationType() {
+ return TableOperationType.REGION_TRUNCATE;
+ }
+
+ private TransitRegionStateProcedure createUnAssignProcedures(MasterProcedureEnv env)
+ throws IOException {
+ return env.getAssignmentManager().createOneUnassignProcedure(getRegion(), true);
+ }
+
+ private TransitRegionStateProcedure createAssignProcedures(MasterProcedureEnv env) {
+ return env.getAssignmentManager().createOneAssignProcedure(getRegion(), true);
+ }
+}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
index ea28a6aa0df..4b022d72e7d 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAdmin1.java
@@ -17,6 +17,7 @@
*/
package org.apache.hadoop.hbase.client;
+import static org.apache.hadoop.hbase.TableName.META_TABLE_NAME;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
import static org.junit.Assert.assertNotNull;
@@ -39,6 +40,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.exceptions.MergeRegionException;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
import org.apache.hadoop.hbase.master.janitor.CatalogJanitor;
import org.apache.hadoop.hbase.regionserver.DisabledRegionSplitPolicy;
import org.apache.hadoop.hbase.regionserver.HRegion;
@@ -83,7 +85,7 @@ public class TestAdmin1 extends TestAdminBase {
Region metaRegion = null;
for (int i = 0; i < NB_SERVERS; i++) {
HRegionServer rs = TEST_UTIL.getMiniHBaseCluster().getRegionServer(i);
- List<HRegion> onlineRegions = rs.getRegions(TableName.META_TABLE_NAME);
+ List<HRegion> onlineRegions = rs.getRegions(META_TABLE_NAME);
if (!onlineRegions.isEmpty()) {
metaRegion = onlineRegions.get(0);
break;
@@ -682,4 +684,94 @@ public class TestAdmin1 extends TestAdminBase {
MetaTableAccessor.getTableRegions(ADMIN.getConnection(), tableName, true);
assertEquals(1, allRegions.size());
}
+
+ @Test
+ public void testTruncateRegions() throws Exception {
+ // Arrange - Create table, insert data, identify region to truncate.
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") };
+ String family1 = "f1";
+ String family2 = "f2";
+ final byte[][] bFamilies = new byte[][] { Bytes.toBytes(family1), Bytes.toBytes(family2) };
+ final String[] sFamilies = new String[] { family1, family2 };
+ int replicaCount = 2;
+ try {
+ TEST_UTIL.createTable(tableName, bFamilies, splitKeys, replicaCount);
+ TEST_UTIL.waitTableAvailable(tableName);
+
+ AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 21, sFamilies);
+ AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 31, sFamilies);
+ AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 61, sFamilies);
+ AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 91, sFamilies);
+
+ List<RegionInfo> tableRegions = ADMIN.getRegions(tableName);
+
+ RegionInfo regionToBeTruncated = tableRegions.get(0);
+ int countBeforeTruncate = TEST_UTIL.countRows(tableName);
+
+ // Act - Truncate the first region
+ ADMIN.truncateRegion(regionToBeTruncated.getRegionName());
+
+ int countAfterTruncate = TEST_UTIL.countRows(tableName);
+
+ // Assert - Assert that before truncate count was 8 and after truncate its 6
+ assertEquals(8, countBeforeTruncate);
+ assertEquals(6, countAfterTruncate);
+ } finally {
+ ADMIN.disableTable(tableName);
+ ADMIN.deleteTable(tableName);
+ }
+ }
+
+ @Test
+ public void testTruncateReplicaRegionNotAllowed() throws Exception {
+ // Arrange - Create table, insert data, identify region to truncate.
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") };
+ String family1 = "f1";
+ String family2 = "f2";
+ final byte[][] bFamilies = new byte[][] { Bytes.toBytes(family1), Bytes.toBytes(family2) };
+ int replicaCount = 2;
+ try {
+ TEST_UTIL.createTable(tableName, bFamilies, splitKeys, replicaCount);
+ TEST_UTIL.waitTableAvailable(tableName);
+
+ List<RegionInfo> tableRegions = ADMIN.getRegions(tableName);
+
+ RegionInfo firstRegion = tableRegions.get(0);
+ RegionInfo regionToBeTruncated = RegionReplicaUtil.getRegionInfoForReplica(firstRegion, 1);
+
+ // Act - Truncate the first region replica
+ try {
+ ADMIN.truncateRegion(regionToBeTruncated.getRegionName());
+ } catch (Exception e) {
+ // Assert
+ assertEquals("Expected message is different",
+ "Can't truncate replicas directly.Replicas are auto-truncated "
+ + "when their primary is truncated.",
+ e.getMessage());
+ }
+ } finally {
+ ADMIN.disableTable(tableName);
+ ADMIN.deleteTable(tableName);
+ }
+ }
+
+ @Test
+ public void testTruncateRegionMetaTableRegionsNotAllowed() throws Exception {
+ // Arrange - Get the region of META table
+ List<RegionInfo> regions = ADMIN.getRegions(META_TABLE_NAME);
+ RegionInfo regionToBeTruncated = regions.get(0);
+
+ // Act
+ try {
+ ADMIN.truncateRegion(regionToBeTruncated.getRegionName());
+ } catch (Exception e) {
+ String expectedErrorMessage =
+ "Invalid region: " + Bytes.toStringBinary(regionToBeTruncated.getRegionName());
+ assertEquals(expectedErrorMessage, e.getMessage());
+ }
+ }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java
index 44106fb4ccf..b31369cbdd3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestAsyncRegionAdminApi2.java
@@ -22,10 +22,12 @@ import static org.hamcrest.CoreMatchers.instanceOf;
import static org.hamcrest.MatcherAssert.assertThat;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertNotEquals;
import static org.junit.Assert.assertNotNull;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
+import java.io.IOException;
import java.util.ArrayList;
import java.util.Collections;
import java.util.List;
@@ -37,6 +39,7 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.master.HMaster;
+import org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil;
import org.apache.hadoop.hbase.master.janitor.CatalogJanitor;
import org.apache.hadoop.hbase.testclassification.ClientTests;
import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -283,4 +286,84 @@ public class TestAsyncRegionAdminApi2 extends TestAsyncAdminBase {
}
assertEquals(2, count);
}
+
+ @Test
+ public void testTruncateRegion() throws Exception {
+ // Arrange - Create table, insert data, identify region to truncate.
+ final byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") };
+ String family1 = "f1";
+ String family2 = "f2";
+
+ final String[] sFamilies = new String[] { family1, family2 };
+ final byte[][] bFamilies = new byte[][] { Bytes.toBytes(family1), Bytes.toBytes(family2) };
+ createTableWithDefaultConf(tableName, splitKeys, bFamilies);
+
+ AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
+ List<HRegionLocation> regionLocations =
+ AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
+ RegionInfo regionToBeTruncated = regionLocations.get(0).getRegion();
+
+ assertEquals(4, regionLocations.size());
+
+ AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 21, sFamilies);
+ AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 31, sFamilies);
+ AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 61, sFamilies);
+ AssignmentTestingUtil.insertData(TEST_UTIL, tableName, 2, 91, sFamilies);
+ int rowCountBeforeTruncate = TEST_UTIL.countRows(tableName);
+
+ // Act - Truncate the first region
+ admin.truncateRegion(regionToBeTruncated.getRegionName()).get();
+
+ // Assert
+ int rowCountAfterTruncate = TEST_UTIL.countRows(tableName);
+ assertNotEquals(rowCountBeforeTruncate, rowCountAfterTruncate);
+ int expectedRowCount = rowCountBeforeTruncate - 2;// Since region with 2 rows was truncated.
+ assertEquals(expectedRowCount, rowCountAfterTruncate);
+ }
+
+ @Test
+ public void testTruncateReplicaRegionNotAllowed() throws Exception {
+ // Arrange - Create table, insert data, identify region to truncate.
+ final byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") };
+ String family1 = "f1";
+ String family2 = "f2";
+
+ final byte[][] bFamilies = new byte[][] { Bytes.toBytes(family1), Bytes.toBytes(family2) };
+ createTableWithDefaultConf(tableName, 2, splitKeys, bFamilies);
+
+ AsyncTable<AdvancedScanResultConsumer> metaTable = ASYNC_CONN.getTable(META_TABLE_NAME);
+ List<HRegionLocation> regionLocations =
+ AsyncMetaTableAccessor.getTableHRegionLocations(metaTable, tableName).get();
+ RegionInfo primaryRegion = regionLocations.get(0).getRegion();
+
+ RegionInfo firstReplica = RegionReplicaUtil.getRegionInfoForReplica(primaryRegion, 1);
+
+ // Act - Truncate the first region
+ try {
+ admin.truncateRegion(firstReplica.getRegionName()).get();
+ } catch (Exception e) {
+ // Assert
+ assertEquals("Expected message is different",
+ "Can't truncate replicas directly.Replicas are auto-truncated "
+ + "when their primary is truncated.",
+ e.getCause().getMessage());
+ }
+ }
+
+ @Test
+ public void testTruncateRegionsMetaTableRegionsNotAllowed() throws Exception {
+ AsyncTableRegionLocator locator = ASYNC_CONN.getRegionLocator(META_TABLE_NAME);
+ List<HRegionLocation> regionLocations = locator.getAllRegionLocations().get();
+ HRegionLocation regionToBeTruncated = regionLocations.get(0);
+ try {
+ admin.truncateRegion(regionToBeTruncated.getRegion().getRegionName()).get();
+ fail();
+ } catch (ExecutionException e) {
+ // expected
+ assertThat(e.getCause(), instanceOf(IOException.class));
+ assertEquals("Can't truncate region in catalog tables", e.getCause().getMessage());
+ }
+ }
}
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index d78caff9a39..a0951480124 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -74,6 +74,12 @@ public class MockNoopMasterServices implements MasterServices {
// no-op
}
+ @Override
+ public long truncateRegion(RegionInfo regionInfo, long nonceGroup, long nonce)
+ throws IOException {
+ return 0;
+ }
+
@Override
public long createTable(final TableDescriptor desc, final byte[][] splitKeys,
final long nonceGroup, final long nonce) throws IOException {
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateRegionProcedure.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateRegionProcedure.java
new file mode 100644
index 00000000000..8ce60ee5550
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/procedure/TestTruncateRegionProcedure.java
@@ -0,0 +1,202 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.master.procedure;
+
+import static org.apache.hadoop.hbase.master.assignment.AssignmentTestingUtil.insertData;
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.Arrays;
+import java.util.List;
+import java.util.stream.Collectors;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptor;
+import org.apache.hadoop.hbase.client.ColumnFamilyDescriptorBuilder;
+import org.apache.hadoop.hbase.client.RegionInfo;
+import org.apache.hadoop.hbase.client.RegionReplicaUtil;
+import org.apache.hadoop.hbase.client.TableDescriptor;
+import org.apache.hadoop.hbase.client.TableDescriptorBuilder;
+import org.apache.hadoop.hbase.master.assignment.TestSplitTableRegionProcedure;
+import org.apache.hadoop.hbase.procedure2.Procedure;
+import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
+import org.apache.hadoop.hbase.procedure2.ProcedureTestingUtility;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.testclassification.MasterTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.junit.After;
+import org.junit.AfterClass;
+import org.junit.Before;
+import org.junit.BeforeClass;
+import org.junit.ClassRule;
+import org.junit.Rule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.rules.TestName;
+import org.slf4j.Logger;
+import org.slf4j.LoggerFactory;
+
+import org.apache.hadoop.hbase.shaded.protobuf.generated.ProcedureProtos;
+
+@SuppressWarnings("OptionalGetWithoutIsPresent")
+@Category({ MasterTests.class, LargeTests.class })
+public class TestTruncateRegionProcedure extends TestTableDDLProcedureBase {
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestTruncateRegionProcedure.class);
+ private static final Logger LOG = LoggerFactory.getLogger(TestTruncateRegionProcedure.class);
+
+ @Rule
+ public TestName name = new TestName();
+
+ private static void setupConf(Configuration conf) {
+ conf.setInt(MasterProcedureConstants.MASTER_PROCEDURE_THREADS, 1);
+ conf.setLong(HConstants.MAJOR_COMPACTION_PERIOD, 0);
+ conf.set("hbase.coprocessor.region.classes",
+ TestSplitTableRegionProcedure.RegionServerHostingReplicaSlowOpenCopro.class.getName());
+ conf.setInt("hbase.client.sync.wait.timeout.msec", 60000);
+ }
+
+ @BeforeClass
+ public static void setupCluster() throws Exception {
+ setupConf(UTIL.getConfiguration());
+ UTIL.startMiniCluster(3);
+ }
+
+ @AfterClass
+ public static void cleanupTest() throws Exception {
+ try {
+ UTIL.shutdownMiniCluster();
+ } catch (Exception e) {
+ LOG.warn("failure shutting down cluster", e);
+ }
+ }
+
+ @Before
+ public void setup() throws Exception {
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
+
+ // Turn off balancer, so it doesn't cut in and mess up our placements.
+ UTIL.getAdmin().balancerSwitch(false, true);
+ // Turn off the meta scanner, so it doesn't remove, parent on us.
+ UTIL.getHBaseCluster().getMaster().setCatalogJanitorEnabled(false);
+ }
+
+ @After
+ public void tearDown() throws Exception {
+ ProcedureTestingUtility.setKillAndToggleBeforeStoreUpdate(getMasterProcedureExecutor(), false);
+ for (TableDescriptor htd : UTIL.getAdmin().listTableDescriptors()) {
+ UTIL.deleteTable(htd.getTableName());
+ }
+ }
+
+ @Test
+ public void testTruncateRegionProcedure() throws Exception {
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+ // Arrange - Load table and prepare arguments values.
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final String[] families = new String[] { "f1", "f2" };
+ final byte[][] splitKeys =
+ new byte[][] { Bytes.toBytes("30"), Bytes.toBytes("60"), Bytes.toBytes("90") };
+
+ MasterProcedureTestingUtility.createTable(procExec, tableName, splitKeys, families);
+
+ insertData(UTIL, tableName, 2, 20, families);
+ insertData(UTIL, tableName, 2, 31, families);
+ insertData(UTIL, tableName, 2, 61, families);
+ insertData(UTIL, tableName, 2, 91, families);
+
+ assertEquals(8, UTIL.countRows(tableName));
+
+ int rowsBeforeDropRegion = 8;
+
+ MasterProcedureEnv environment = procExec.getEnvironment();
+ RegionInfo regionToBeTruncated = environment.getAssignmentManager().getAssignedRegions()
+ .stream().filter(r -> tableName.getNameAsString().equals(r.getTable().getNameAsString()))
+ .min((o1, o2) -> Bytes.compareTo(o1.getStartKey(), o2.getStartKey())).get();
+
+ // Act - Execute Truncate region procedure
+ long procId =
+ procExec.submitProcedure(new TruncateRegionProcedure(environment, regionToBeTruncated));
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ assertEquals(8 - 2, UTIL.countRows(tableName));
+
+ int rowsAfterDropRegion = UTIL.countRows(tableName);
+ assertTrue("Row counts after truncate region should be less than row count before it",
+ rowsAfterDropRegion < rowsBeforeDropRegion);
+ assertEquals(rowsBeforeDropRegion, rowsAfterDropRegion + 2);
+
+ insertData(UTIL, tableName, 2, 20, families);
+ assertEquals(8, UTIL.countRows(tableName));
+ }
+
+ @Test
+ public void testTruncateRegionProcedureErrorWhenSpecifiedReplicaRegionID() throws Exception {
+ final ProcedureExecutor<MasterProcedureEnv> procExec = getMasterProcedureExecutor();
+ // Arrange - Load table and prepare arguments values.
+ final TableName tableName = TableName.valueOf(name.getMethodName());
+ final String[] families = new String[] { "f1", "f2" };
+ createTable(tableName, families, 2);
+ insertData(UTIL, tableName, 2, 20, families);
+ insertData(UTIL, tableName, 2, 30, families);
+ insertData(UTIL, tableName, 2, 60, families);
+
+ assertEquals(6, UTIL.countRows(tableName));
+
+ MasterProcedureEnv environment = procExec.getEnvironment();
+ RegionInfo regionToBeTruncated = environment.getAssignmentManager().getAssignedRegions()
+ .stream().filter(r -> tableName.getNameAsString().equals(r.getTable().getNameAsString()))
+ .min((o1, o2) -> Bytes.compareTo(o1.getStartKey(), o2.getStartKey())).get();
+
+ RegionInfo replicatedRegionId =
+ RegionReplicaUtil.getRegionInfoForReplica(regionToBeTruncated, 1);
+
+ // Act - Execute Truncate region procedure
+ long procId =
+ procExec.submitProcedure(new TruncateRegionProcedure(environment, replicatedRegionId));
+
+ ProcedureTestingUtility.waitProcedure(procExec, procId);
+ Procedure<MasterProcedureEnv> result = procExec.getResult(procId);
+ // Asserts
+
+ assertEquals(ProcedureProtos.ProcedureState.ROLLEDBACK, result.getState());
+ assertTrue(result.getException().getMessage()
+ .endsWith("Can't truncate replicas directly. Replicas are auto-truncated "
+ + "when their primary is truncated."));
+ }
+
+ private TableDescriptor tableDescriptor(final TableName tableName, String[] families,
+ final int replicaCount) {
+ return TableDescriptorBuilder.newBuilder(tableName).setRegionReplication(replicaCount)
+ .setColumnFamilies(columnFamilyDescriptor(families)).build();
+ }
+
+ private List<ColumnFamilyDescriptor> columnFamilyDescriptor(String[] families) {
+ return Arrays.stream(families).map(ColumnFamilyDescriptorBuilder::of)
+ .collect(Collectors.toList());
+ }
+
+ @SuppressWarnings("SameParameterValue")
+ private void createTable(final TableName tableName, String[] families, final int replicaCount)
+ throws IOException {
+ UTIL.getAdmin().createTable(tableDescriptor(tableName, families, replicaCount));
+ }
+}
diff --git a/hbase-shell/src/main/ruby/hbase/admin.rb b/hbase-shell/src/main/ruby/hbase/admin.rb
index 91e1e2b8472..2110e72d155 100644
--- a/hbase-shell/src/main/ruby/hbase/admin.rb
+++ b/hbase-shell/src/main/ruby/hbase/admin.rb
@@ -163,6 +163,16 @@ module Hbase
end
end
+ #----------------------------------------------------------------------------------------------
+ # Requests a region truncate
+ def truncate_region(region_name)
+ begin
+ org.apache.hadoop.hbase.util.FutureUtils.get(@admin.truncateRegionAsync(region_name.to_java_bytes))
+ rescue java.lang.IllegalArgumentException, org.apache.hadoop.hbase.UnknownRegionException
+ @admin.truncate_region(region_name.to_java_bytes)
+ end
+ end
+
#----------------------------------------------------------------------------------------------
# Enable/disable one split or merge switch
# Returns previous switch setting.
diff --git a/hbase-shell/src/main/ruby/shell.rb b/hbase-shell/src/main/ruby/shell.rb
index e0a29b32082..3d95aa1623b 100644
--- a/hbase-shell/src/main/ruby/shell.rb
+++ b/hbase-shell/src/main/ruby/shell.rb
@@ -487,6 +487,7 @@ Shell.load_command_group(
list_decommissioned_regionservers
decommission_regionservers
recommission_regionserver
+ truncate_region
],
# TODO: remove older hlog_roll command
aliases: {
diff --git a/hbase-shell/src/main/ruby/shell/commands/truncate_region.rb b/hbase-shell/src/main/ruby/shell/commands/truncate_region.rb
new file mode 100644
index 00000000000..b443d3210bc
--- /dev/null
+++ b/hbase-shell/src/main/ruby/shell/commands/truncate_region.rb
@@ -0,0 +1,36 @@
+#
+#
+# Licensed to the Apache Software Foundation (ASF) under one
+# or more contributor license agreements. See the NOTICE file
+# distributed with this work for additional information
+# regarding copyright ownership. The ASF licenses this file
+# to you under the Apache License, Version 2.0 (the
+# "License"); you may not use this file except in compliance
+# with the License. You may obtain a copy of the License at
+#
+# http://www.apache.org/licenses/LICENSE-2.0
+#
+# Unless required by applicable law or agreed to in writing, software
+# distributed under the License is distributed on an "AS IS" BASIS,
+# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+# See the License for the specific language governing permissions and
+# limitations under the License.
+#
+
+module Shell
+ module Commands
+ class TruncateRegion < Command
+ def help
+ <<-EOF
+Truncate individual region.
+Examples:
+ truncate_region 'REGIONNAME'
+ truncate_region 'ENCODED_REGIONNAME'
+EOF
+ end
+ def command(region_name)
+ admin.truncate_region(region_name)
+ end
+ end
+ end
+end
diff --git a/hbase-shell/src/test/ruby/hbase/admin_test.rb b/hbase-shell/src/test/ruby/hbase/admin_test.rb
index e84ffd6385a..dcd3d850343 100644
--- a/hbase-shell/src/test/ruby/hbase/admin_test.rb
+++ b/hbase-shell/src/test/ruby/hbase/admin_test.rb
@@ -158,6 +158,17 @@ module Hbase
#-------------------------------------------------------------------------------
+ define_test "truncate region should work" do
+ @t_name = 'hbase_shell_truncate_region'
+ drop_test_table(@t_name)
+ admin.create(@t_name, 'a', NUMREGIONS => 10, SPLITALGO => 'HexStringSplit')
+ r1 = command(:locate_region, @t_name, '1')
+ region1 = r1.getRegion.getRegionNameAsString
+ command(:truncate_region, region1)
+ end
+
+ #-------------------------------------------------------------------------------
+
define_test "drop should fail on non-existent tables" do
assert_raise(ArgumentError) do
command(:drop, 'NOT.EXISTS')
diff --git a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java
index 05fc3c932dc..183efd6288a 100644
--- a/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java
+++ b/hbase-thrift/src/main/java/org/apache/hadoop/hbase/thrift2/client/ThriftAdmin.java
@@ -888,6 +888,16 @@ public class ThriftAdmin implements Admin {
throw new NotImplementedException("split not supported in ThriftAdmin");
}
+ @Override
+ public void truncateRegion(byte[] regionName) throws IOException {
+ throw new NotImplementedException("Truncate Region not supported in ThriftAdmin");
+ }
+
+ @Override
+ public Future<Void> truncateRegionAsync(byte[] regionName) {
+ throw new NotImplementedException("Truncate Region Async not supported in ThriftAdmin");
+ }
+
@Override
public void splitRegion(byte[] regionName, byte[] splitPoint) {
throw new NotImplementedException("splitRegion not supported in ThriftAdmin");