You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2016/06/01 04:05:10 UTC
hbase git commit: HBASE-15861 Add support for table sets in restore
operation (Vlad)
Repository: hbase
Updated Branches:
refs/heads/HBASE-7912 a402006de -> 486fc54c6
HBASE-15861 Add support for table sets in restore operation (Vlad)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/486fc54c
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/486fc54c
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/486fc54c
Branch: refs/heads/HBASE-7912
Commit: 486fc54c6c36eb488248c7e41bd5235f825c83dd
Parents: a402006
Author: tedyu <yu...@gmail.com>
Authored: Tue May 31 21:04:33 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Tue May 31 21:04:33 2016 -0700
----------------------------------------------------------------------
.../hadoop/hbase/backup/RestoreDriver.java | 62 +++++++--
.../hadoop/hbase/backup/TestFullBackupSet.java | 22 +++-
.../backup/TestFullBackupSetRestoreSet.java | 127 +++++++++++++++++++
3 files changed, 198 insertions(+), 13 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/486fc54c/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
index 83c8297..0dba079 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
@@ -18,14 +18,20 @@
package org.apache.hadoop.hbase.backup;
import java.io.IOException;
+import java.util.List;
import org.apache.commons.cli.CommandLine;
+import org.apache.commons.lang.StringUtils;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants;
+import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
import org.apache.hadoop.hbase.backup.util.BackupServerUtil;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.util.AbstractHBaseTool;
import org.apache.hadoop.hbase.util.LogUtils;
import org.apache.hadoop.util.ToolRunner;
@@ -40,9 +46,12 @@ public class RestoreDriver extends AbstractHBaseTool {
private static final String OPTION_OVERWRITE = "overwrite";
private static final String OPTION_CHECK = "check";
private static final String OPTION_AUTOMATIC = "automatic";
+ private static final String OPTION_SET = "set";
+ private static final String OPTION_DEBUG = "debug";
+
private static final String USAGE =
- "Usage: hbase restore <backup_root_path> <backup_id> <tables> [tableMapping] \n"
+ "Usage: hbase restore [-set set_name] <backup_root_path> <backup_id> <tables> [tableMapping] \n"
+ " [-overwrite] [-check] [-automatic]\n"
+ " backup_root_path The parent location where the backup images are stored\n"
+ " backup_id The id identifying the backup image\n"
@@ -61,7 +70,8 @@ public class RestoreDriver extends AbstractHBaseTool {
+ " The restore dependencies can be checked by using \"-check\" "
+ "option,\n"
+ " or using \"hbase backup describe\" command. Without this option, "
- + "only\n" + " this backup image is restored\n";
+ + "only\n" + " this backup image is restored\n"
+ + " -set set_name Backup set to restore, mutually exclusive with table list <tables>.";
protected RestoreDriver() throws IOException
@@ -75,7 +85,8 @@ public class RestoreDriver extends AbstractHBaseTool {
"Overwrite the data if any of the restore target tables exists");
addOptNoArg(OPTION_CHECK, "Check restore sequence and dependencies");
addOptNoArg(OPTION_AUTOMATIC, "Restore all dependencies");
- addOptNoArg("debug", "Enable debug logging");
+ addOptNoArg(OPTION_DEBUG, "Enable debug logging");
+ addOptWithArg(OPTION_SET, "Backup set name");
// disable irrelevant loggers to avoid it mess up command output
LogUtils.disableUselessLoggers(LOG);
@@ -85,7 +96,7 @@ public class RestoreDriver extends AbstractHBaseTool {
// enable debug logging
Logger backupClientLogger = Logger.getLogger("org.apache.hadoop.hbase.backup");
- if (cmd.hasOption("debug")) {
+ if (cmd.hasOption(OPTION_DEBUG)) {
backupClientLogger.setLevel(Level.DEBUG);
}
@@ -112,16 +123,36 @@ public class RestoreDriver extends AbstractHBaseTool {
// parse main restore command options
String[] remainArgs = cmd.getArgs();
- if (remainArgs.length < 3) {
+ if (remainArgs.length < 3 && !cmd.hasOption(OPTION_SET) ||
+ (cmd.hasOption(OPTION_SET) && remainArgs.length < 2)) {
+ System.out.println("ERROR: remain args length="+ remainArgs.length);
System.out.println(USAGE);
return -1;
- }
+ }
String backupRootDir = remainArgs[0];
String backupId = remainArgs[1];
- String tables = remainArgs[2];
-
- String tableMapping = (remainArgs.length > 3) ? remainArgs[3] : null;
+ String tables = null;
+ String tableMapping = null;
+ // Check backup set
+ if (cmd.hasOption(OPTION_SET)) {
+ String setName = cmd.getOptionValue(OPTION_SET);
+ try{
+ tables = getTablesForSet(setName, conf);
+ } catch(IOException e){
+ System.out.println("ERROR: "+ e.getMessage()+" for setName="+setName);
+ return -2;
+ }
+ if (tables == null) {
+ System.out.println("ERROR: Backup set '" + setName
+ + "' is either empty or does not exist");
+ return -3;
+ }
+ tableMapping = (remainArgs.length > 2) ? remainArgs[2] : null;
+ } else {
+ tables = remainArgs[2];
+ tableMapping = (remainArgs.length > 3) ? remainArgs[3] : null;
+ }
TableName[] sTableArray = BackupServerUtil.parseTableNames(tables);
TableName[] tTableArray = BackupServerUtil.parseTableNames(tableMapping);
@@ -129,7 +160,7 @@ public class RestoreDriver extends AbstractHBaseTool {
if (sTableArray != null && tTableArray != null && (sTableArray.length != tTableArray.length)) {
System.out.println("ERROR: table mapping mismatch: " + tables + " : " + tableMapping);
System.out.println(USAGE);
- return -2;
+ return -4;
}
@@ -139,11 +170,20 @@ public class RestoreDriver extends AbstractHBaseTool {
tTableArray, isOverwrite);
} catch (Exception e){
e.printStackTrace();
- return -3;
+ return -5;
}
return 0;
}
+ private String getTablesForSet(String name, Configuration conf) throws IOException {
+ try (final Connection conn = ConnectionFactory.createConnection(conf);
+ final BackupSystemTable table = new BackupSystemTable(conn)) {
+ List<TableName> tables = table.describeBackupSet(name);
+ if (tables == null) return null;
+ return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
+ }
+ }
+
@Override
protected void addOptions() {
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/486fc54c/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java
index a4c0fa1..a8fa7de 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSet.java
@@ -27,6 +27,7 @@ import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.util.ToolRunner;
import org.junit.Test;
@@ -45,7 +46,7 @@ public class TestFullBackupSet extends TestBackupBase {
@Test
public void testFullBackupSetExist() throws Exception {
- LOG.info("TFBSE test full backup, backup set exists");
+ LOG.info("Test full backup, backup set exists");
//Create set
try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) {
@@ -65,7 +66,24 @@ public class TestFullBackupSet extends TestBackupBase {
assertTrue(backups.size() == 1);
String backupId = backups.get(0).getBackupId();
assertTrue(checkSucceeded(backupId));
- LOG.info("TFBSE backup complete");
+
+ LOG.info("backup complete");
+
+ // Restore from set into other table
+ args = new String[]{BACKUP_ROOT_DIR, backupId,
+ "-set", name, table1_restore.getNameAsString(), "-overwrite" };
+ // Run backup
+ ret = ToolRunner.run(conf1, new RestoreDriver(), args);
+ assertTrue(ret == 0);
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table1_restore));
+ // Verify number of rows in both tables
+ assertEquals(TEST_UTIL.countRows(table1), TEST_UTIL.countRows(table1_restore));
+ TEST_UTIL.deleteTable(table1_restore);
+ LOG.info("restore into other table is complete");
+ hba.close();
+
+
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/486fc54c/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java
new file mode 100644
index 0000000..35e84b1
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java
@@ -0,0 +1,127 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.*;
+
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestFullBackupSetRestoreSet extends TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestFullBackupSetRestoreSet.class);
+
+ @Test
+ public void testFullRestoreSetToOtherTable() throws Exception {
+
+ LOG.info("Test full restore set");
+
+ // Create set
+ try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) {
+ String name = "name";
+ table.addToBackupSet(name, new String[] { table1.getNameAsString() });
+ List<TableName> names = table.describeBackupSet(name);
+
+ assertNotNull(names);
+ assertTrue(names.size() == 1);
+ assertTrue(names.get(0).equals(table1));
+
+ String[] args = new String[] { "create", "full", BACKUP_ROOT_DIR, "-set", name };
+ // Run backup
+ int ret = ToolRunner.run(conf1, new BackupDriver(), args);
+ assertTrue(ret == 0);
+ ArrayList<BackupInfo> backups = table.getBackupHistory();
+ assertTrue(backups.size() == 1);
+ String backupId = backups.get(0).getBackupId();
+ assertTrue(checkSucceeded(backupId));
+
+ LOG.info("backup complete");
+
+ // Restore from set into other table
+ args =
+ new String[] { BACKUP_ROOT_DIR, backupId, "-set", name, table1_restore.getNameAsString(),
+ "-overwrite" };
+ // Run backup
+ ret = ToolRunner.run(conf1, new RestoreDriver(), args);
+ assertTrue(ret == 0);
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table1_restore));
+ // Verify number of rows in both tables
+ assertEquals(TEST_UTIL.countRows(table1), TEST_UTIL.countRows(table1_restore));
+ TEST_UTIL.deleteTable(table1_restore);
+ LOG.info("restore into other table is complete");
+ hba.close();
+ }
+ }
+
+ @Test
+ public void testFullRestoreSetToSameTable() throws Exception {
+
+ LOG.info("Test full restore set to same table");
+
+ // Create set
+ try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) {
+ String name = "name1";
+ table.addToBackupSet(name, new String[] { table1.getNameAsString() });
+ List<TableName> names = table.describeBackupSet(name);
+
+ assertNotNull(names);
+ assertTrue(names.size() == 1);
+ assertTrue(names.get(0).equals(table1));
+
+ String[] args = new String[] { "create", "full", BACKUP_ROOT_DIR, "-set", name };
+ // Run backup
+ int ret = ToolRunner.run(conf1, new BackupDriver(), args);
+ assertTrue(ret == 0);
+ ArrayList<BackupInfo> backups = table.getBackupHistory();
+ String backupId = backups.get(0).getBackupId();
+ assertTrue(checkSucceeded(backupId));
+
+ LOG.info("backup complete");
+ int count = TEST_UTIL.countRows(table1);
+ TEST_UTIL.deleteTable(table1);
+
+ // Restore from set into other table
+ args = new String[] { BACKUP_ROOT_DIR, backupId, "-set", name, "-overwrite" };
+ // Run backup
+ ret = ToolRunner.run(conf1, new RestoreDriver(), args);
+ assertTrue(ret == 0);
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table1));
+ // Verify number of rows in both tables
+ assertEquals(count, TEST_UTIL.countRows(table1));
+ LOG.info("restore into same table is complete");
+ hba.close();
+
+ }
+
+ }
+
+}
\ No newline at end of file