You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2016/09/07 17:27:40 UTC
[3/3] hbase git commit: HBASE-15565 Rewrite restore with Procedure V2
HBASE-15565 Rewrite restore with Procedure V2
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/e01ac71f
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/e01ac71f
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/e01ac71f
Branch: refs/heads/HBASE-7912
Commit: e01ac71fba752d1eab00a62a2961c515c551a6b7
Parents: 28737d0
Author: tedyu <yu...@gmail.com>
Authored: Wed Sep 7 10:27:19 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Wed Sep 7 10:27:19 2016 -0700
----------------------------------------------------------------------
.../backup/BackupRestoreClientFactory.java | 55 -
.../hadoop/hbase/backup/RestoreClient.java | 48 -
.../hadoop/hbase/backup/RestoreRequest.java | 18 +-
.../org/apache/hadoop/hbase/client/Admin.java | 16 +-
.../apache/hadoop/hbase/client/BackupAdmin.java | 23 +-
.../hbase/client/ConnectionImplementation.java | 7 +
.../apache/hadoop/hbase/client/HBaseAdmin.java | 50 +
.../hadoop/hbase/client/HBaseBackupAdmin.java | 10 +-
.../hadoop/hbase/protobuf/RequestConverter.java | 24 +
.../hbase/protobuf/generated/MasterProtos.java | 3103 ++++++++++++++++--
hbase-protocol/src/main/protobuf/Master.proto | 24 +
.../hadoop/hbase/backup/RestoreDriver.java | 11 +-
.../hbase/backup/impl/RestoreClientImpl.java | 300 --
.../backup/impl/RestoreTablesProcedure.java | 398 +++
.../hbase/backup/util/RestoreServerUtil.java | 47 +
.../org/apache/hadoop/hbase/master/HMaster.java | 33 +
.../hadoop/hbase/master/MasterRpcServices.java | 21 +
.../hadoop/hbase/master/MasterServices.java | 7 +
.../procedure/TableProcedureInterface.java | 2 +-
.../hadoop/hbase/backup/TestBackupBase.java | 16 +-
.../hbase/backup/TestBackupDeleteRestore.java | 13 +-
.../hadoop/hbase/backup/TestFullRestore.java | 43 +-
.../hbase/backup/TestIncrementalBackup.java | 7 +-
.../TestIncrementalBackupDeleteTable.java | 7 +-
.../hadoop/hbase/backup/TestRemoteBackup.java | 5 +-
.../hadoop/hbase/backup/TestRemoteRestore.java | 5 +-
.../hbase/backup/TestRestoreBoundaryTests.java | 11 +-
.../hadoop/hbase/master/TestCatalogJanitor.java | 6 +
28 files changed, 3499 insertions(+), 811 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/e01ac71f/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreClientFactory.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreClientFactory.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreClientFactory.java
deleted file mode 100644
index b60ab21..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreClientFactory.java
+++ /dev/null
@@ -1,55 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.util.ReflectionUtils;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public final class BackupRestoreClientFactory {
- private static final Log LOG = LogFactory.getLog(BackupRestoreClientFactory.class);
-
- private BackupRestoreClientFactory(){
- throw new AssertionError("Instantiating utility class...");
- }
-
-
- /**
- * Gets restore client implementation
- * @param conf - configuration
- * @return backup client
- */
- public static RestoreClient getRestoreClient(Configuration conf) {
- try{
- Class<?> cls =
- conf.getClassByName("org.apache.hadoop.hbase.backup.impl.RestoreClientImpl");
-
- RestoreClient client = (RestoreClient) ReflectionUtils.newInstance(cls, conf);
- client.setConf(conf);
- return client;
- } catch(Exception e){
- LOG.error("Can not instantiate RestoreClient. Make sure you have hbase-server jar in CLASSPATH", e);
- }
- return null;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e01ac71f/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java
deleted file mode 100644
index 07f573e..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreClient.java
+++ /dev/null
@@ -1,48 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public interface RestoreClient {
-
- public void setConf(Configuration conf);
-
- /**
- * Restore operation.
- * @param backupRootDir The root dir for backup image
- * @param backupId The backup id for image to be restored
- * @param check True if only do dependency check
- * @param sTableArray The array of tables to be restored
- * @param tTableArray The array of mapping tables to restore to
- * @param isOverwrite True then do restore overwrite if target table exists, otherwise fail the
- * request if target table exists
- * @throws IOException if any failure during restore
- */
- public void restore(
- String backupRootDir,
- String backupId, boolean check, TableName[] sTableArray,
- TableName[] tTableArray, boolean isOverwrite) throws IOException;
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e01ac71f/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java
index c78f0bc..7490d20 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java
@@ -21,22 +21,21 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-
/**
* POJO class for restore request
*/
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
public class RestoreRequest {
-
- private String backupRootDir;
- private String backupId;
+
+ private String backupRootDir;
+ private String backupId;
private boolean check = false;
private TableName[] fromTables;
private TableName[] toTables;
private boolean overwrite = false;
-
- public RestoreRequest(){
+
+ public RestoreRequest() {
}
public String getBackupRootDir() {
@@ -73,7 +72,6 @@ public class RestoreRequest {
public RestoreRequest setFromTables(TableName[] fromTables) {
this.fromTables = fromTables;
return this;
-
}
public TableName[] getToTables() {
@@ -93,6 +91,4 @@ public class RestoreRequest {
this.overwrite = overwrite;
return this;
}
-
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e01ac71f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index f46c855..37b38a5 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -38,7 +38,7 @@ import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.backup.BackupRequest;
-import org.apache.hadoop.hbase.backup.BackupType;
+import org.apache.hadoop.hbase.backup.RestoreRequest;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
import org.apache.hadoop.hbase.client.security.SecurityCapability;
@@ -916,6 +916,20 @@ public interface Admin extends Abortable, Closeable {
/**
+ * Restore operation. Asynchronous version.
+ * @param request RestoreRequest instance
+ * @throws IOException
+ */
+ public Future<Void> restoreTablesAsync(final RestoreRequest request) throws IOException;
+
+ /**
+ * Restore operation. Synchronous version.
+ * @param request RestoreRequest instance
+ * @throws IOException
+ */
+ public void restoreTables(final RestoreRequest userRequest) throws IOException;
+
+ /**
* Modify an existing table, more IRB friendly version. Asynchronous operation. This means that
* it may be a while before your schema change is updated across all of the table.
* You can use Future.get(long, TimeUnit) to wait on the operation to complete.
http://git-wip-us.apache.org/repos/asf/hbase/blob/e01ac71f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BackupAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BackupAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BackupAdmin.java
index 4134cc8..b19dea1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BackupAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/BackupAdmin.java
@@ -73,6 +73,21 @@ public interface BackupAdmin extends Closeable{
public Future<String> backupTablesAsync(final BackupRequest userRequest) throws IOException;
/**
+ * Restore backup
+ * @param request - restore request
+ * @throws IOException exception
+ */
+ public void restore(RestoreRequest request) throws IOException;
+
+ /**
+ * Restore backup
+ * @param request - restore request
+ * @return Future which client can wait on
+ * @throws IOException exception
+ */
+ public Future<Void> restoreAsync(RestoreRequest request) throws IOException;
+
+ /**
* Describe backup image command
* @param backupId - backup id
* @return backup info
@@ -156,12 +171,4 @@ public interface BackupAdmin extends Closeable{
* @throws IOException exception
*/
public void removeFromBackupSet(String name, String[] tables) throws IOException;
-
- /**
- * Restore backup
- * @param request - restore request
- * @throws IOException exception
- */
- public void restore(RestoreRequest request) throws IOException;
-
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/e01ac71f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
index c245ef4..1ac43f9 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/ConnectionImplementation.java
@@ -1408,6 +1408,13 @@ class ConnectionImplementation implements ClusterConnection, Closeable {
}
@Override
+ public MasterProtos.RestoreTablesResponse restoreTables(
+ RpcController controller,
+ MasterProtos.RestoreTablesRequest request) throws ServiceException {
+ return stub.restoreTables(controller, request);
+ }
+
+ @Override
public MasterProtos.AddColumnResponse addColumn(
RpcController controller,
MasterProtos.AddColumnRequest request) throws ServiceException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/e01ac71f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 8ba26ba..2b1dcd1 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -62,6 +62,7 @@ import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.UnknownRegionException;
import org.apache.hadoop.hbase.ZooKeeperConnectionException;
import org.apache.hadoop.hbase.backup.BackupRequest;
+import org.apache.hadoop.hbase.backup.RestoreRequest;
import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
@@ -148,6 +149,8 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableRespon
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreSnapshotResponse;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreTablesRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreTablesResponse;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SecurityCapabilitiesRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
@@ -218,6 +221,7 @@ public class HBaseAdmin implements Admin {
private final int retryLongerMultiplier;
private final int syncWaitTimeout;
private final long backupWaitTimeout;
+ private final long restoreWaitTimeout;
private boolean aborted;
private int operationTimeout;
@@ -246,6 +250,8 @@ public class HBaseAdmin implements Admin {
"hbase.client.sync.wait.timeout.msec", 10 * 60000); // 10min
this.backupWaitTimeout = this.conf.getInt(
"hbase.client.backup.wait.timeout.sec", 24 * 3600); // 24 h
+ this.restoreWaitTimeout = this.conf.getInt(
+ "hbase.client.restore.wait.timeout.sec", 24 * 3600); // 24 h
this.rpcCallerFactory = RpcRetryingCallerFactory.instantiate(this.conf);
this.ng = this.connection.getNonceGenerator();
@@ -1630,6 +1636,50 @@ public class HBaseAdmin implements Admin {
}
}
+ /**
+ * Restore operation.
+ * @param request RestoreRequest instance
+ * @throws IOException
+ */
+ @Override
+ public Future<Void> restoreTablesAsync(final RestoreRequest userRequest) throws IOException {
+ RestoreTablesResponse response = executeCallable(
+ new MasterCallable<RestoreTablesResponse>(getConnection()) {
+ @Override
+ public RestoreTablesResponse call(int callTimeout) throws ServiceException {
+ try {
+ RestoreTablesRequest request = RequestConverter.buildRestoreTablesRequest(
+ userRequest.getBackupRootDir(), userRequest.getBackupId(),
+ userRequest.isCheck(), userRequest.getFromTables(), userRequest.getToTables(),
+ userRequest.isOverwrite(), ng.getNonceGroup(), ng.newNonce());
+ return master.restoreTables(null, request);
+ } catch (IOException ioe) {
+ throw new ServiceException(ioe);
+ }
+ }
+ });
+ return new TableRestoreFuture(this, TableName.BACKUP_TABLE_NAME, response);
+ }
+
+ @Override
+ public void restoreTables(final RestoreRequest userRequest) throws IOException {
+ get(restoreTablesAsync(userRequest),
+ restoreWaitTimeout, TimeUnit.SECONDS);
+ }
+
+ private static class TableRestoreFuture extends TableFuture<Void> {
+ public TableRestoreFuture(final HBaseAdmin admin, final TableName tableName,
+ final RestoreTablesResponse response) {
+ super(admin, tableName,
+ (response != null) ? response.getProcId() : null);
+ }
+
+ @Override
+ public String getOperationType() {
+ return "RESTORE";
+ }
+ }
+
@Override
public Future<Void> modifyTable(final TableName tableName, final HTableDescriptor htd)
throws IOException {
http://git-wip-us.apache.org/repos/asf/hbase/blob/e01ac71f/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java
index a67d3d2..8e08fd0 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseBackupAdmin.java
@@ -36,9 +36,7 @@ import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.BackupInfo;
import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
import org.apache.hadoop.hbase.backup.BackupRequest;
-import org.apache.hadoop.hbase.backup.BackupRestoreClientFactory;
import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.RestoreClient;
import org.apache.hadoop.hbase.backup.RestoreRequest;
import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
@@ -407,10 +405,12 @@ public class HBaseBackupAdmin implements BackupAdmin {
@Override
public void restore(RestoreRequest request) throws IOException {
- RestoreClient client = BackupRestoreClientFactory.getRestoreClient(admin.getConfiguration());
- client.restore(request.getBackupRootDir(), request.getBackupId(), request.isCheck(),
- request.getFromTables(), request.getToTables(), request.isOverwrite());
+ admin.restoreTables(request);
+ }
+ @Override
+ public Future<Void> restoreAsync(RestoreRequest request) throws IOException {
+ return admin.restoreTablesAsync(request);
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/e01ac71f/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
index 5babfb8..eba01ce 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/protobuf/RequestConverter.java
@@ -105,6 +105,7 @@ import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.ModifyTableReques
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.MoveRegionRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.NormalizeRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.OfflineRegionRequest;
+import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreTablesRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RunCatalogScanRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetBalancerRunningRequest;
import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.SetNormalizerRunningRequest;
@@ -1286,6 +1287,29 @@ public final class RequestConverter {
return builder.build();
}
+ public static RestoreTablesRequest buildRestoreTablesRequest(String backupRootDir,
+ String backupId, boolean check, TableName[] sTableList,
+ TableName[] tTableList, boolean isOverwrite, final long nonceGroup, final long nonce)
+ throws IOException {
+ RestoreTablesRequest.Builder builder = RestoreTablesRequest.newBuilder();
+ builder.setBackupId(backupId).setBackupRootDir(backupRootDir);
+ builder.setDependencyCheckOnly(check).setOverwrite(isOverwrite);
+ if (sTableList != null) {
+ for (TableName table : sTableList) {
+ builder.addTables(ProtobufUtil.toProtoTableName(table));
+ }
+ } else {
+ throw new IOException("Source table list shouldn't be empty");
+ }
+ if (tTableList != null) {
+ for (TableName table : tTableList) {
+ builder.addTargetTables(ProtobufUtil.toProtoTableName(table));
+ }
+ }
+ builder.setNonceGroup(nonceGroup).setNonce(nonce);
+ return builder.build();
+ }
+
/**
* Creates a protocol buffer GetSchemaAlterStatusRequest
*