You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2016/09/22 21:24:00 UTC
hbase git commit: HBASE-16683 Address review comments for mega patch
of backup / restore feature
Repository: hbase
Updated Branches:
refs/heads/HBASE-7912 7e6f19581 -> 9952b134f
HBASE-16683 Address review comments for mega patch of backup / restore feature
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/9952b134
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/9952b134
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/9952b134
Branch: refs/heads/HBASE-7912
Commit: 9952b134fc26b3e4745b2a93afaddb12565975a7
Parents: 7e6f195
Author: tedyu <yu...@gmail.com>
Authored: Thu Sep 22 14:23:29 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Thu Sep 22 14:23:29 2016 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/client/Admin.java | 15 ---
.../apache/hadoop/hbase/client/HBaseAdmin.java | 3 -
.../backup/impl/IncrementalBackupManager.java | 5 +-
.../backup/master/FullTableBackupProcedure.java | 7 +-
.../master/IncrementalTableBackupProcedure.java | 2 +-
.../backup/master/RestoreTablesProcedure.java | 2 +-
.../master/procedure/MasterProcedureUtil.java | 68 -------------
.../procedure/TableProcedureInterface.java | 2 +-
.../hadoop/hbase/procedure/ProcedureUtil.java | 100 +++++++++++++++++++
.../org/apache/hadoop/hbase/util/LogUtils.java | 2 +
10 files changed, 112 insertions(+), 94 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/9952b134/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
index 37b38a5..343dad4 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/Admin.java
@@ -914,21 +914,6 @@ public interface Admin extends Abortable, Closeable {
void splitRegion(final byte[] regionName, final byte[] splitPoint)
throws IOException;
-
- /**
- * Restore operation. Asynchronous version.
- * @param request RestoreRequest instance
- * @throws IOException
- */
- public Future<Void> restoreTablesAsync(final RestoreRequest request) throws IOException;
-
- /**
- * Restore operation. Synchronous version.
- * @param request RestoreRequest instance
- * @throws IOException
- */
- public void restoreTables(final RestoreRequest userRequest) throws IOException;
-
/**
* Modify an existing table, more IRB friendly version. Asynchronous operation. This means that
* it may be a while before your schema change is updated across all of the table.
http://git-wip-us.apache.org/repos/asf/hbase/blob/9952b134/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
index 9245cdb..6702bbc 100644
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
+++ b/hbase-client/src/main/java/org/apache/hadoop/hbase/client/HBaseAdmin.java
@@ -1571,7 +1571,6 @@ public class HBaseAdmin implements Admin {
ProtobufUtil.split(admin, hri, splitPoint);
}
-
Future<String> backupTablesAsync(final BackupRequest userRequest) throws IOException {
BackupClientUtil.checkTargetDir(userRequest.getTargetRootDir(), conf);
if (userRequest.getTableList() != null) {
@@ -1641,7 +1640,6 @@ public class HBaseAdmin implements Admin {
* @param request RestoreRequest instance
* @throws IOException
*/
- @Override
public Future<Void> restoreTablesAsync(final RestoreRequest userRequest) throws IOException {
RestoreTablesResponse response = executeCallable(
new MasterCallable<RestoreTablesResponse>(getConnection()) {
@@ -1661,7 +1659,6 @@ public class HBaseAdmin implements Admin {
return new TableRestoreFuture(this, TableName.BACKUP_TABLE_NAME, response);
}
- @Override
public void restoreTables(final RestoreRequest userRequest) throws IOException {
get(restoreTablesAsync(userRequest),
restoreWaitTimeout, TimeUnit.SECONDS);
http://git-wip-us.apache.org/repos/asf/hbase/blob/9952b134/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
index cde6c04..8f6aeb8 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/IncrementalBackupManager.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.master.MasterServices;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
+import org.apache.hadoop.hbase.procedure.ProcedureUtil;
import org.apache.hadoop.hbase.util.FSUtils;
import org.apache.hadoop.hbase.wal.DefaultWALProvider;
import org.apache.hadoop.hbase.backup.impl.BackupSystemTable.WALItem;
@@ -110,10 +111,10 @@ public class IncrementalBackupManager {
props.put("backupRoot", backupContext.getTargetRootDir());
MasterProcedureManager mpm = svc.getMasterProcedureManagerHost()
.getProcedureManager(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE);
- long waitTime = MasterProcedureUtil.execProcedure(mpm,
+ long waitTime = ProcedureUtil.execProcedure(mpm,
LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE,
LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props);
- MasterProcedureUtil.waitForProcedure(mpm,
+ ProcedureUtil.waitForProcedure(mpm,
LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE,
LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props, waitTime,
conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
http://git-wip-us.apache.org/repos/asf/hbase/blob/9952b134/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java
index 94e991f..2d41423 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java
@@ -56,6 +56,7 @@ import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface;
import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
+import org.apache.hadoop.hbase.procedure.ProcedureUtil;
import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
import org.apache.hadoop.hbase.protobuf.generated.BackupProtos;
import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.FullTableBackupState;
@@ -525,10 +526,10 @@ public class FullTableBackupProcedure
.getProcedureManager(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE);
Map<String, String> props= new HashMap<String, String>();
props.put("backupRoot", backupContext.getTargetRootDir());
- long waitTime = MasterProcedureUtil.execProcedure(mpm,
+ long waitTime = ProcedureUtil.execProcedure(mpm,
LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE,
LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props);
- MasterProcedureUtil.waitForProcedure(mpm,
+ ProcedureUtil.waitForProcedure(mpm,
LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE,
LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props, waitTime,
conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
@@ -758,7 +759,7 @@ public class FullTableBackupProcedure
@Override
public TableOperationType getTableOperationType() {
- return TableOperationType.BACKUP;
+ return TableOperationType.EDIT;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/9952b134/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java
index 64c1fb4..e877ebd 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java
@@ -382,7 +382,7 @@ public class IncrementalTableBackupProcedure
@Override
public TableOperationType getTableOperationType() {
- return TableOperationType.BACKUP;
+ return TableOperationType.EDIT;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/9952b134/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/RestoreTablesProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/RestoreTablesProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/RestoreTablesProcedure.java
index 2678278..8fd7621 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/RestoreTablesProcedure.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/RestoreTablesProcedure.java
@@ -369,7 +369,7 @@ public class RestoreTablesProcedure
@Override
public TableOperationType getTableOperationType() {
- return TableOperationType.RESTORE;
+ return TableOperationType.EDIT;
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/9952b134/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureUtil.java
index e2409d4..6339db2 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/MasterProcedureUtil.java
@@ -18,22 +18,11 @@
package org.apache.hadoop.hbase.master.procedure;
-import java.io.IOException;
-import java.io.InterruptedIOException;
-import java.util.Map;
-import java.util.Map.Entry;
-
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.HBaseAdmin;
-import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.UserInformation;
-import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
import org.apache.hadoop.security.UserGroupInformation;
@InterfaceAudience.Private
@@ -65,61 +54,4 @@ public final class MasterProcedureUtil {
return null;
}
- public static ProcedureDescription buildProcedure(String signature, String instance,
- Map<String, String> props) {
- ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
- builder.setSignature(signature).setInstance(instance);
- for (Entry<String, String> entry : props.entrySet()) {
- NameStringPair pair = NameStringPair.newBuilder().setName(entry.getKey())
- .setValue(entry.getValue()).build();
- builder.addConfiguration(pair);
- }
- ProcedureDescription desc = builder.build();
- return desc;
- }
-
- public static long execProcedure(MasterProcedureManager mpm, String signature, String instance,
- Map<String, String> props) throws IOException {
- if (mpm == null) {
- throw new IOException("The procedure is not registered: " + signature);
- }
- ProcedureDescription desc = buildProcedure(signature, instance, props);
- mpm.execProcedure(desc);
-
- // send back the max amount of time the client should wait for the procedure
- // to complete
- long waitTime = SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME;
- return waitTime;
- }
-
- public static void waitForProcedure(MasterProcedureManager mpm, String signature, String instance,
- Map<String, String> props, long max, int numRetries, long pause) throws IOException {
- ProcedureDescription desc = buildProcedure(signature, instance, props);
- long start = EnvironmentEdgeManager.currentTime();
- long maxPauseTime = max / numRetries;
- int tries = 0;
- LOG.debug("Waiting a max of " + max + " ms for procedure '" +
- signature + " : " + instance + "'' to complete. (max " + maxPauseTime + " ms per retry)");
- boolean done = false;
- while (tries == 0
- || ((EnvironmentEdgeManager.currentTime() - start) < max && !done)) {
- try {
- // sleep a backoff <= pauseTime amount
- long sleep = HBaseAdmin.getPauseTime(tries++, pause);
- sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
- LOG.debug("(#" + tries + ") Sleeping: " + sleep +
- "ms while waiting for procedure completion.");
- Thread.sleep(sleep);
- } catch (InterruptedException e) {
- throw (InterruptedIOException) new InterruptedIOException("Interrupted").initCause(e);
- }
- LOG.debug("Getting current status of procedure from master...");
- done = mpm.isProcedureDone(desc);
- }
- if (!done) {
- throw new IOException("Procedure '" + signature + " : " + instance
- + "' wasn't completed in expectedTime:" + max + " ms");
- }
-
- }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/9952b134/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java
index 5356d2d..cc088f3 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/master/procedure/TableProcedureInterface.java
@@ -30,7 +30,7 @@ import org.apache.hadoop.hbase.classification.InterfaceStability;
@InterfaceStability.Evolving
public interface TableProcedureInterface {
public enum TableOperationType {
- CREATE, DELETE, DISABLE, EDIT, ENABLE, READ, BACKUP, RESTORE,
+ CREATE, DELETE, DISABLE, EDIT, ENABLE, READ,
};
/**
http://git-wip-us.apache.org/repos/asf/hbase/blob/9952b134/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureUtil.java
new file mode 100644
index 0000000..0ce8e70
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/procedure/ProcedureUtil.java
@@ -0,0 +1,100 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.procedure;
+
+import java.io.IOException;
+import java.io.InterruptedIOException;
+import java.util.Map;
+import java.util.Map.Entry;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.NameStringPair;
+import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.ProcedureDescription;
+import org.apache.hadoop.hbase.protobuf.generated.RPCProtos.UserInformation;
+import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.apache.hadoop.security.UserGroupInformation;
+
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class ProcedureUtil {
+ private static final Log LOG = LogFactory.getLog(ProcedureUtil.class);
+
+ public static ProcedureDescription buildProcedure(String signature, String instance,
+ Map<String, String> props) {
+ ProcedureDescription.Builder builder = ProcedureDescription.newBuilder();
+ builder.setSignature(signature).setInstance(instance);
+ for (Entry<String, String> entry : props.entrySet()) {
+ NameStringPair pair = NameStringPair.newBuilder().setName(entry.getKey())
+ .setValue(entry.getValue()).build();
+ builder.addConfiguration(pair);
+ }
+ ProcedureDescription desc = builder.build();
+ return desc;
+ }
+
+ public static long execProcedure(MasterProcedureManager mpm, String signature, String instance,
+ Map<String, String> props) throws IOException {
+ if (mpm == null) {
+ throw new IOException("The procedure is not registered: " + signature);
+ }
+ ProcedureDescription desc = buildProcedure(signature, instance, props);
+ mpm.execProcedure(desc);
+
+ // send back the max amount of time the client should wait for the procedure
+ // to complete
+ long waitTime = SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME;
+ return waitTime;
+ }
+
+ public static void waitForProcedure(MasterProcedureManager mpm, String signature, String instance,
+ Map<String, String> props, long max, int numRetries, long pause) throws IOException {
+ ProcedureDescription desc = buildProcedure(signature, instance, props);
+ long start = EnvironmentEdgeManager.currentTime();
+ long maxPauseTime = max / numRetries;
+ int tries = 0;
+ LOG.debug("Waiting a max of " + max + " ms for procedure '" +
+ signature + " : " + instance + "'' to complete. (max " + maxPauseTime + " ms per retry)");
+ boolean done = false;
+ while (tries == 0
+ || ((EnvironmentEdgeManager.currentTime() - start) < max && !done)) {
+ try {
+ // sleep a backoff <= pauseTime amount
+ long sleep = HBaseAdmin.getPauseTime(tries++, pause);
+ sleep = sleep > maxPauseTime ? maxPauseTime : sleep;
+ LOG.debug("(#" + tries + ") Sleeping: " + sleep +
+ "ms while waiting for procedure completion.");
+ Thread.sleep(sleep);
+ } catch (InterruptedException e) {
+ throw (InterruptedIOException) new InterruptedIOException("Interrupted").initCause(e);
+ }
+ LOG.debug("Getting current status of procedure from master...");
+ done = mpm.isProcedureDone(desc);
+ }
+ if (!done) {
+ throw new IOException("Procedure '" + signature + " : " + instance
+ + "' wasn't completed in expectedTime:" + max + " ms");
+ }
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/9952b134/hbase-server/src/main/java/org/apache/hadoop/hbase/util/LogUtils.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/LogUtils.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/LogUtils.java
index 26f261c..2b50d49 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/util/LogUtils.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/util/LogUtils.java
@@ -18,9 +18,11 @@
package org.apache.hadoop.hbase.util;
import org.apache.commons.logging.Log;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
import org.apache.log4j.Level;
import org.apache.log4j.Logger;
+@InterfaceAudience.Private
public final class LogUtils {
private LogUtils() {