You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2016/11/14 17:22:48 UTC
[02/11] hbase git commit: HBASE-14123 HBase Backup/Restore Phase 2
(Vladimir Rodionov)
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java
new file mode 100644
index 0000000..77ade03
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullBackupSetRestoreSet.java
@@ -0,0 +1,129 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertEquals;
+import static org.junit.Assert.assertNotNull;
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestFullBackupSetRestoreSet extends TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestFullBackupSetRestoreSet.class);
+
+ @Test
+ public void testFullRestoreSetToOtherTable() throws Exception {
+
+ LOG.info("Test full restore set");
+
+ // Create set
+ try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) {
+ String name = "name";
+ table.addToBackupSet(name, new String[] { table1.getNameAsString() });
+ List<TableName> names = table.describeBackupSet(name);
+
+ assertNotNull(names);
+ assertTrue(names.size() == 1);
+ assertTrue(names.get(0).equals(table1));
+
+ String[] args = new String[] { "create", "full", BACKUP_ROOT_DIR, "-s", name };
+ // Run backup
+ int ret = ToolRunner.run(conf1, new BackupDriver(), args);
+ assertTrue(ret == 0);
+ List<BackupInfo> backups = table.getBackupHistory();
+ assertTrue(backups.size() == 1);
+ String backupId = backups.get(0).getBackupId();
+ assertTrue(checkSucceeded(backupId));
+ assertTrue(backupId.startsWith(name));
+
+ LOG.info("backup complete");
+
+ // Restore from set into other table
+ args =
+ new String[] { BACKUP_ROOT_DIR, backupId, "-s", name, "-m", table1_restore.getNameAsString(),
+ "-o" };
+ // Run backup
+ ret = ToolRunner.run(conf1, new RestoreDriver(), args);
+ assertTrue(ret == 0);
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table1_restore));
+ // Verify number of rows in both tables
+ assertEquals(TEST_UTIL.countRows(table1), TEST_UTIL.countRows(table1_restore));
+ TEST_UTIL.deleteTable(table1_restore);
+ LOG.info("restore into other table is complete");
+ hba.close();
+ }
+ }
+
+ @Test
+ public void testFullRestoreSetToSameTable() throws Exception {
+
+ LOG.info("Test full restore set to same table");
+
+ // Create set
+ try (BackupSystemTable table = new BackupSystemTable(TEST_UTIL.getConnection())) {
+ String name = "name1";
+ table.addToBackupSet(name, new String[] { table1.getNameAsString() });
+ List<TableName> names = table.describeBackupSet(name);
+
+ assertNotNull(names);
+ assertTrue(names.size() == 1);
+ assertTrue(names.get(0).equals(table1));
+
+ String[] args = new String[] { "create", "full", BACKUP_ROOT_DIR, "-s", name };
+ // Run backup
+ int ret = ToolRunner.run(conf1, new BackupDriver(), args);
+ assertTrue(ret == 0);
+ List<BackupInfo> backups = table.getBackupHistory();
+ String backupId = backups.get(0).getBackupId();
+ assertTrue(checkSucceeded(backupId));
+
+ LOG.info("backup complete");
+ int count = TEST_UTIL.countRows(table1);
+ TEST_UTIL.deleteTable(table1);
+
+ // Restore from set into other table
+ args = new String[] { BACKUP_ROOT_DIR, backupId, "-s", name, "-o" };
+ // Run backup
+ ret = ToolRunner.run(conf1, new RestoreDriver(), args);
+ assertTrue(ret == 0);
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table1));
+ // Verify number of rows in both tables
+ assertEquals(count, TEST_UTIL.countRows(table1));
+ LOG.info("restore into same table is complete");
+ hba.close();
+
+ }
+
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java
new file mode 100644
index 0000000..aa01e9a
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestFullRestore.java
@@ -0,0 +1,322 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.List;
+
+import org.apache.commons.lang.StringUtils;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.util.RestoreServerUtil;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.util.ToolRunner;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.collect.Lists;
+
+@Category(LargeTests.class)
+public class TestFullRestore extends TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestFullRestore.class);
+
+ /**
+ * Verify that a single table is restored to a new table
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreSingle() throws Exception {
+
+ LOG.info("test full restore on a single table empty table");
+
+ List<TableName> tables = Lists.newArrayList(table1);
+ String backupId = fullTableBackup(tables);
+ assertTrue(checkSucceeded(backupId));
+
+ LOG.info("backup complete");
+
+ TableName[] tableset = new TableName[] { table1 };
+ TableName[] tablemap = new TableName[] { table1_restore };
+ BackupAdmin client = getBackupAdmin();
+ client.restore(RestoreServerUtil.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false,
+ tableset, tablemap, false));
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table1_restore));
+ TEST_UTIL.deleteTable(table1_restore);
+ hba.close();
+ }
+
+
+ @Test
+ public void testFullRestoreSingleCommand() throws Exception {
+
+ LOG.info("test full restore on a single table empty table: command-line");
+
+ List<TableName> tables = Lists.newArrayList(table1);
+ String backupId = fullTableBackup(tables);
+ LOG.info("backup complete");
+ assertTrue(checkSucceeded(backupId));
+ //restore <backup_root_path> <backup_id> <tables> [tableMapping]
+ String[] args = new String[]{BACKUP_ROOT_DIR, backupId,
+ table1.getNameAsString(), "-m", table1_restore.getNameAsString() };
+ // Run backup
+ int ret = ToolRunner.run(conf1, new RestoreDriver(), args);
+
+ assertTrue(ret==0);
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table1_restore));
+ TEST_UTIL.deleteTable(table1_restore);
+ hba.close();
+ }
+
+ /**
+ * Verify that multiple tables are restored to new tables.
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreMultiple() throws Exception {
+ LOG.info("create full backup image on multiple tables");
+ List<TableName> tables = Lists.newArrayList(table2, table3);
+ String backupId = fullTableBackup(tables);
+ assertTrue(checkSucceeded(backupId));
+
+ TableName[] restore_tableset = new TableName[] { table2, table3 };
+ TableName[] tablemap = new TableName[] { table2_restore, table3_restore };
+ BackupAdmin client = getBackupAdmin();
+ client.restore(RestoreServerUtil.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false,
+ restore_tableset, tablemap, false));
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table2_restore));
+ assertTrue(hba.tableExists(table3_restore));
+ TEST_UTIL.deleteTable(table2_restore);
+ TEST_UTIL.deleteTable(table3_restore);
+ hba.close();
+ }
+
+ /**
+ * Verify that multiple tables are restored to new tables.
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreMultipleCommand() throws Exception {
+ LOG.info("create full backup image on multiple tables: command-line");
+ List<TableName> tables = Lists.newArrayList(table2, table3);
+ String backupId = fullTableBackup(tables);
+ assertTrue(checkSucceeded(backupId));
+
+ TableName[] restore_tableset = new TableName[] { table2, table3 };
+ TableName[] tablemap = new TableName[] { table2_restore, table3_restore };
+
+
+ //restore <backup_root_path> <backup_id> <tables> [tableMapping]
+ String[] args = new String[]{BACKUP_ROOT_DIR, backupId,
+ StringUtils.join(restore_tableset, ","),
+ "-m", StringUtils.join(tablemap, ",") };
+ // Run backup
+ int ret = ToolRunner.run(conf1, new RestoreDriver(), args);
+
+ assertTrue(ret==0);
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table2_restore));
+ assertTrue(hba.tableExists(table3_restore));
+ TEST_UTIL.deleteTable(table2_restore);
+ TEST_UTIL.deleteTable(table3_restore);
+ hba.close();
+ }
+
+
+ /**
+ * Verify that a single table is restored using overwrite
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreSingleOverwrite() throws Exception {
+
+ LOG.info("test full restore on a single table empty table");
+ List<TableName> tables = Lists.newArrayList(table1);
+ String backupId = fullTableBackup(tables);
+ assertTrue(checkSucceeded(backupId));
+
+ LOG.info("backup complete");
+
+ TableName[] tableset = new TableName[] { table1 };
+ BackupAdmin client = getBackupAdmin();
+ client.restore(RestoreServerUtil.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false,
+ tableset, null, true));
+ }
+
+ /**
+ * Verify that a single table is restored using overwrite
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreSingleOverwriteCommand() throws Exception {
+
+ LOG.info("test full restore on a single table empty table: command-line");
+ List<TableName> tables = Lists.newArrayList(table1);
+ String backupId = fullTableBackup(tables);
+ assertTrue(checkSucceeded(backupId));
+ LOG.info("backup complete");
+ TableName[] tableset = new TableName[] { table1 };
+ //restore <backup_root_path> <backup_id> <tables> [tableMapping]
+ String[] args = new String[]{BACKUP_ROOT_DIR, backupId,
+ StringUtils.join(tableset, ","), "-o" };
+ // Run restore
+ int ret = ToolRunner.run(conf1, new RestoreDriver(), args);
+ assertTrue(ret==0);
+
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table1));
+ hba.close();
+
+ }
+
+ /**
+ * Verify that multiple tables are restored to new tables using overwrite.
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreMultipleOverwrite() throws Exception {
+ LOG.info("create full backup image on multiple tables");
+
+ List<TableName> tables = Lists.newArrayList(table2, table3);
+ String backupId = fullTableBackup(tables);
+ assertTrue(checkSucceeded(backupId));
+
+ TableName[] restore_tableset = new TableName[] { table2, table3 };
+ BackupAdmin client = getBackupAdmin();
+ client.restore(RestoreServerUtil.createRestoreRequest(BACKUP_ROOT_DIR, backupId,
+ false, restore_tableset, null, true));
+ }
+
+ /**
+ * Verify that multiple tables are restored to new tables using overwrite.
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreMultipleOverwriteCommand() throws Exception {
+ LOG.info("create full backup image on multiple tables: command-line");
+
+ List<TableName> tables = Lists.newArrayList(table2, table3);
+ String backupId = fullTableBackup(tables);
+ assertTrue(checkSucceeded(backupId));
+
+ TableName[] restore_tableset = new TableName[] { table2, table3 };
+ //restore <backup_root_path> <backup_id> <tables> [tableMapping]
+ String[] args = new String[]{BACKUP_ROOT_DIR, backupId,
+ StringUtils.join(restore_tableset, ","), "-o" };
+ // Run backup
+ int ret = ToolRunner.run(conf1, new RestoreDriver(), args);
+
+ assertTrue(ret==0);
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table2));
+ assertTrue(hba.tableExists(table3));
+ hba.close();
+ }
+
+ /**
+ * Verify that restore fails on a single table that does not exist.
+ * @throws Exception
+ */
+ @Test(expected = IOException.class)
+ public void testFullRestoreSingleDNE() throws Exception {
+
+ LOG.info("test restore fails on a single table that does not exist");
+ List<TableName> tables = Lists.newArrayList(table1);
+ String backupId = fullTableBackup(tables);
+ assertTrue(checkSucceeded(backupId));
+
+ LOG.info("backup complete");
+
+ TableName[] tableset = new TableName[] { TableName.valueOf("faketable") };
+ TableName[] tablemap = new TableName[] { table1_restore };
+ BackupAdmin client = getBackupAdmin();
+ client.restore(RestoreServerUtil.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false,
+ tableset, tablemap, false));
+ }
+
+
+ /**
+ * Verify that restore fails on a single table that does not exist.
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreSingleDNECommand() throws Exception {
+
+ LOG.info("test restore fails on a single table that does not exist: command-line");
+ List<TableName> tables = Lists.newArrayList(table1);
+ String backupId = fullTableBackup(tables);
+ assertTrue(checkSucceeded(backupId));
+
+ LOG.info("backup complete");
+
+ TableName[] tableset = new TableName[] { TableName.valueOf("faketable") };
+ TableName[] tablemap = new TableName[] { table1_restore };
+ String[] args = new String[]{BACKUP_ROOT_DIR, backupId,
+ StringUtils.join(tableset, ","),
+ "-m", StringUtils.join(tablemap, ",") };
+ // Run restore
+ int ret = ToolRunner.run(conf1, new RestoreDriver(), args);
+ assertTrue(ret != 0);
+
+ }
+ /**
+ * Verify that restore fails on multiple tables that do not exist.
+ * @throws Exception
+ */
+ @Test(expected = IOException.class)
+ public void testFullRestoreMultipleDNE() throws Exception {
+
+ LOG.info("test restore fails on multiple tables that do not exist");
+
+ List<TableName> tables = Lists.newArrayList(table2, table3);
+ String backupId = fullTableBackup(tables);
+ assertTrue(checkSucceeded(backupId));
+
+ TableName[] restore_tableset
+ = new TableName[] { TableName.valueOf("faketable1"), TableName.valueOf("faketable2") };
+ TableName[] tablemap = new TableName[] { table2_restore, table3_restore };
+ BackupAdmin client = getBackupAdmin();
+ client.restore(RestoreServerUtil.createRestoreRequest(BACKUP_ROOT_DIR, backupId, false,
+ restore_tableset, tablemap, false));
+ }
+
+ /**
+ * Verify that restore fails on multiple tables that do not exist.
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreMultipleDNECommand() throws Exception {
+
+ LOG.info("test restore fails on multiple tables that do not exist: command-line");
+
+ List<TableName> tables = Lists.newArrayList(table2, table3);
+ String backupId = fullTableBackup(tables);
+ assertTrue(checkSucceeded(backupId));
+
+ TableName[] restore_tableset
+ = new TableName[] { TableName.valueOf("faketable1"), TableName.valueOf("faketable2") };
+ TableName[] tablemap = new TableName[] { table2_restore, table3_restore };
+ String[] args = new String[]{BACKUP_ROOT_DIR, backupId,
+ StringUtils.join(restore_tableset, ","),
+ "-m", StringUtils.join(tablemap, ",") };
+ // Run restore
+ int ret = ToolRunner.run(conf1, new RestoreDriver(), args);
+ assertTrue(ret != 0);
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java
new file mode 100644
index 0000000..9a845ba
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackup.java
@@ -0,0 +1,208 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.ArrayList;
+import java.util.Collection;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.MiniHBaseCluster;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.impl.HBaseBackupAdmin;
+import org.apache.hadoop.hbase.backup.util.RestoreServerUtil;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.regionserver.HRegion;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
+import org.hamcrest.CoreMatchers;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+import org.junit.runner.RunWith;
+import org.junit.runners.Parameterized;
+
+import com.google.common.collect.Lists;
+
+@Category(LargeTests.class)
+@RunWith(Parameterized.class)
+public class TestIncrementalBackup extends TestBackupBase {
+ private static final Log LOG = LogFactory.getLog(TestIncrementalBackup.class);
+
+ @Parameterized.Parameters
+ public static Collection<Object[]> data() {
+ provider = "multiwal";
+ List<Object[]> params = new ArrayList<Object[]>();
+ params.add(new Object[] {Boolean.TRUE});
+ return params;
+ }
+ public TestIncrementalBackup(Boolean b) {
+ }
+
+ //implement all test cases in 1 test since incremental backup/restore has dependencies
+ @Test
+ public void TestIncBackupRestore() throws Exception {
+
+ int ADD_ROWS = 99;
+ // #1 - create full backup for all tables
+ LOG.info("create full backup image for all tables");
+
+ List<TableName> tables = Lists.newArrayList(table1, table2);
+ final byte[] fam3Name = Bytes.toBytes("f3");
+ table1Desc.addFamily(new HColumnDescriptor(fam3Name));
+ HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc);
+
+ Connection conn = ConnectionFactory.createConnection(conf1);
+ int NB_ROWS_FAM3 = 6;
+ insertIntoTable(conn, table1, fam3Name, 3, NB_ROWS_FAM3).close();
+
+ HBaseAdmin admin = null;
+ admin = (HBaseAdmin) conn.getAdmin();
+ HBaseBackupAdmin client = new HBaseBackupAdmin(conn);
+
+ BackupRequest request = new BackupRequest();
+ request.setBackupType(BackupType.FULL).setTableList(tables).setTargetRootDir(BACKUP_ROOT_DIR);
+ String backupIdFull = client.backupTables(request);
+
+ assertTrue(checkSucceeded(backupIdFull));
+
+ // #2 - insert some data to table
+ HTable t1 = insertIntoTable(conn, table1, famName, 1, ADD_ROWS);
+ LOG.debug("writing " + ADD_ROWS + " rows to " + table1);
+
+ Assert.assertThat(TEST_UTIL.countRows(t1),
+ CoreMatchers.equalTo(NB_ROWS_IN_BATCH + ADD_ROWS + NB_ROWS_FAM3));
+ t1.close();
+ LOG.debug("written " + ADD_ROWS + " rows to " + table1);
+
+ HTable t2 = (HTable) conn.getTable(table2);
+ Put p2;
+ for (int i = 0; i < 5; i++) {
+ p2 = new Put(Bytes.toBytes("row-t2" + i));
+ p2.addColumn(famName, qualName, Bytes.toBytes("val" + i));
+ t2.put(p2);
+ }
+
+ Assert.assertThat(TEST_UTIL.countRows(t2), CoreMatchers.equalTo(NB_ROWS_IN_BATCH + 5));
+ t2.close();
+ LOG.debug("written " + 5 + " rows to " + table2);
+ // split table1
+ MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
+ List<HRegion> regions = cluster.getRegions(table1);
+
+ byte[] name = regions.get(0).getRegionInfo().getRegionName();
+ long startSplitTime = EnvironmentEdgeManager.currentTime();
+ admin.splitRegion(name);
+
+ while (!admin.isTableAvailable(table1)) {
+ Thread.sleep(100);
+ }
+
+ long endSplitTime = EnvironmentEdgeManager.currentTime();
+
+ // split finished
+ LOG.debug("split finished in =" + (endSplitTime - startSplitTime));
+
+ // #3 - incremental backup for multiple tables
+ tables = Lists.newArrayList(table1, table2);
+ request = new BackupRequest();
+ request.setBackupType(BackupType.INCREMENTAL).setTableList(tables)
+ .setTargetRootDir(BACKUP_ROOT_DIR);
+ String backupIdIncMultiple = client.backupTables(request);
+ assertTrue(checkSucceeded(backupIdIncMultiple));
+
+ // add column family f2 to table1
+ final byte[] fam2Name = Bytes.toBytes("f2");
+ table1Desc.addFamily(new HColumnDescriptor(fam2Name));
+ // drop column family f3
+ table1Desc.removeFamily(fam3Name);
+ HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc);
+
+ int NB_ROWS_FAM2 = 7;
+ HTable t3 = insertIntoTable(conn, table1, fam2Name, 2, NB_ROWS_FAM2);
+ t3.close();
+
+ // #3 - incremental backup for multiple tables
+ request = new BackupRequest();
+ request.setBackupType(BackupType.INCREMENTAL).setTableList(tables)
+ .setTargetRootDir(BACKUP_ROOT_DIR);
+ String backupIdIncMultiple2 = client.backupTables(request);
+ assertTrue(checkSucceeded(backupIdIncMultiple2));
+
+ // #4 - restore full backup for all tables, without overwrite
+ TableName[] tablesRestoreFull = new TableName[] { table1, table2 };
+
+ TableName[] tablesMapFull = new TableName[] { table1_restore, table2_restore };
+
+ LOG.debug("Restoring full " + backupIdFull);
+ client.restore(RestoreServerUtil.createRestoreRequest(BACKUP_ROOT_DIR, backupIdFull, false,
+ tablesRestoreFull, tablesMapFull, false));
+
+ // #5.1 - check tables for full restore
+ HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hAdmin.tableExists(table1_restore));
+ assertTrue(hAdmin.tableExists(table2_restore));
+
+ hAdmin.close();
+
+ // #5.2 - checking row count of tables for full restore
+ HTable hTable = (HTable) conn.getTable(table1_restore);
+ Assert.assertThat(TEST_UTIL.countRows(hTable),
+ CoreMatchers.equalTo(NB_ROWS_IN_BATCH + NB_ROWS_FAM3));
+ hTable.close();
+
+ hTable = (HTable) conn.getTable(table2_restore);
+ Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH));
+ hTable.close();
+
+ // #6 - restore incremental backup for multiple tables, with overwrite
+ TableName[] tablesRestoreIncMultiple = new TableName[] { table1, table2 };
+ TableName[] tablesMapIncMultiple = new TableName[] { table1_restore, table2_restore };
+ client.restore(RestoreServerUtil.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple2,
+ false, tablesRestoreIncMultiple, tablesMapIncMultiple, true));
+
+ hTable = (HTable) conn.getTable(table1_restore);
+ LOG.debug("After incremental restore: " + hTable.getTableDescriptor());
+ LOG.debug("f1 has " + TEST_UTIL.countRows(hTable, famName) + " rows");
+ Assert.assertThat(TEST_UTIL.countRows(hTable, famName),
+ CoreMatchers.equalTo(NB_ROWS_IN_BATCH + ADD_ROWS));
+ LOG.debug("f2 has " + TEST_UTIL.countRows(hTable, fam2Name) + " rows");
+ Assert.assertThat(TEST_UTIL.countRows(hTable, fam2Name), CoreMatchers.equalTo(NB_ROWS_FAM2));
+ hTable.close();
+
+ hTable = (HTable) conn.getTable(table2_restore);
+ Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH + 5));
+ hTable.close();
+
+ admin.close();
+ conn.close();
+
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java
new file mode 100644
index 0000000..52e247c
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestIncrementalBackupDeleteTable.java
@@ -0,0 +1,140 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.impl.HBaseBackupAdmin;
+import org.apache.hadoop.hbase.backup.util.RestoreServerUtil;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.hamcrest.CoreMatchers;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.collect.Lists;
+
+/**
+ *
+ * 1. Create table t1, t2
+ * 2. Load data to t1, t2
+ * 3 Full backup t1, t2
+ * 4 Delete t2
+ * 5 Load data to t1
+ * 6 Incremental backup t1
+ */
+@Category(LargeTests.class)
+public class TestIncrementalBackupDeleteTable extends TestBackupBase {
+ private static final Log LOG = LogFactory.getLog(TestIncrementalBackupDeleteTable.class);
+ //implement all test cases in 1 test since incremental backup/restore has dependencies
+ @Test
+ public void TestIncBackupDeleteTable() throws Exception {
+ // #1 - create full backup for all tables
+ LOG.info("create full backup image for all tables");
+
+ List<TableName> tables = Lists.newArrayList(table1, table2);
+ HBaseAdmin admin = null;
+ Connection conn = ConnectionFactory.createConnection(conf1);
+ admin = (HBaseAdmin) conn.getAdmin();
+ HBaseBackupAdmin client = new HBaseBackupAdmin(conn);
+
+ BackupRequest request = new BackupRequest();
+ request.setBackupType(BackupType.FULL).setTableList(tables).setTargetRootDir(BACKUP_ROOT_DIR);
+ String backupIdFull = client.backupTables(request);
+
+ assertTrue(checkSucceeded(backupIdFull));
+
+ // #2 - insert some data to table table1
+ HTable t1 = (HTable) conn.getTable(table1);
+ Put p1;
+ for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
+ p1 = new Put(Bytes.toBytes("row-t1" + i));
+ p1.addColumn(famName, qualName, Bytes.toBytes("val" + i));
+ t1.put(p1);
+ }
+
+ Assert.assertThat(TEST_UTIL.countRows(t1), CoreMatchers.equalTo(NB_ROWS_IN_BATCH * 2));
+ t1.close();
+
+ // Delete table table2
+ admin.disableTable(table2);
+ admin.deleteTable(table2);
+
+ // #3 - incremental backup for table1
+ tables = Lists.newArrayList(table1);
+ request = new BackupRequest();
+ request.setBackupType(BackupType.INCREMENTAL).setTableList(tables)
+ .setTargetRootDir(BACKUP_ROOT_DIR);
+ String backupIdIncMultiple = client.backupTables(request);
+ assertTrue(checkSucceeded(backupIdIncMultiple));
+
+ // #4 - restore full backup for all tables, without overwrite
+ TableName[] tablesRestoreFull =
+ new TableName[] { table1, table2};
+
+ TableName[] tablesMapFull =
+ new TableName[] { table1_restore, table2_restore };
+
+ client.restore(RestoreServerUtil.createRestoreRequest(BACKUP_ROOT_DIR, backupIdFull, false,
+ tablesRestoreFull,
+ tablesMapFull, false));
+
+ // #5.1 - check tables for full restore
+ HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hAdmin.tableExists(table1_restore));
+ assertTrue(hAdmin.tableExists(table2_restore));
+
+
+ // #5.2 - checking row count of tables for full restore
+ HTable hTable = (HTable) conn.getTable(table1_restore);
+ Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH));
+ hTable.close();
+
+ hTable = (HTable) conn.getTable(table2_restore);
+ Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH));
+ hTable.close();
+
+
+ // #6 - restore incremental backup for table1
+ TableName[] tablesRestoreIncMultiple =
+ new TableName[] { table1 };
+ TableName[] tablesMapIncMultiple =
+ new TableName[] { table1_restore };
+ client.restore(RestoreServerUtil.createRestoreRequest(BACKUP_ROOT_DIR, backupIdIncMultiple,
+ false, tablesRestoreIncMultiple, tablesMapIncMultiple, true));
+
+ hTable = (HTable) conn.getTable(table1_restore);
+ Assert.assertThat(TEST_UTIL.countRows(hTable), CoreMatchers.equalTo(NB_ROWS_IN_BATCH * 2));
+ hTable.close();
+ admin.close();
+ conn.close();
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java
new file mode 100644
index 0000000..5111dd2
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteBackup.java
@@ -0,0 +1,132 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.concurrent.CountDownLatch;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.HBaseTestingUtility;
+import org.apache.hadoop.hbase.HColumnDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.util.RestoreServerUtil;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.snapshot.MobSnapshotTestingUtils;
+import org.apache.hadoop.hbase.snapshot.SnapshotTestingUtils;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.hamcrest.CoreMatchers;
+import org.junit.Assert;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.collect.Lists;
+
+@Category(LargeTests.class)
+public class TestRemoteBackup extends TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestRemoteBackup.class);
+
+ /**
+ * Verify that a remote full backup is created on a single table with data correctly.
+ * @throws Exception
+ */
+ @Test
+ public void testFullBackupRemote() throws Exception {
+ LOG.info("test remote full backup on a single table");
+ final CountDownLatch latch = new CountDownLatch(1);
+ final int NB_ROWS_IN_FAM3 = 6;
+ final byte[] fam3Name = Bytes.toBytes("f3");
+ final byte[] fam2Name = Bytes.toBytes("f2");
+ final Connection conn = ConnectionFactory.createConnection(conf1);
+ Thread t = new Thread() {
+ @Override
+ public void run() {
+ try {
+ latch.await();
+ } catch (InterruptedException ie) {
+ }
+ try {
+ HTable t1 = (HTable) conn.getTable(table1);
+ Put p1;
+ for (int i = 0; i < NB_ROWS_IN_FAM3; i++) {
+ p1 = new Put(Bytes.toBytes("row-t1" + i));
+ p1.addColumn(fam3Name, qualName, Bytes.toBytes("val" + i));
+ t1.put(p1);
+ }
+ LOG.debug("Wrote " + NB_ROWS_IN_FAM3 + " rows into family3");
+ t1.close();
+ } catch (IOException ioe) {
+ throw new RuntimeException(ioe);
+ }
+ }
+ };
+ t.start();
+
+ table1Desc.addFamily(new HColumnDescriptor(fam3Name));
+ // family 2 is MOB enabled
+ HColumnDescriptor hcd = new HColumnDescriptor(fam2Name);
+ hcd.setMobEnabled(true);
+ hcd.setMobThreshold(0L);
+ table1Desc.addFamily(hcd);
+ HBaseTestingUtility.modifyTableSync(TEST_UTIL.getAdmin(), table1Desc);
+
+ SnapshotTestingUtils.loadData(TEST_UTIL, table1, 50, fam2Name);
+ HTable t1 = (HTable) conn.getTable(table1);
+ int rows0 = MobSnapshotTestingUtils.countMobRows(t1, fam2Name);
+
+ latch.countDown();
+ String backupId = backupTables(BackupType.FULL,
+ Lists.newArrayList(table1), BACKUP_REMOTE_ROOT_DIR);
+ assertTrue(checkSucceeded(backupId));
+
+ LOG.info("backup complete " + backupId);
+ Assert.assertThat(TEST_UTIL.countRows(t1, famName), CoreMatchers.equalTo(NB_ROWS_IN_BATCH));
+
+ t.join();
+ Assert.assertThat(TEST_UTIL.countRows(t1, fam3Name), CoreMatchers.equalTo(NB_ROWS_IN_FAM3));
+ t1.close();
+
+ TableName[] tablesRestoreFull =
+ new TableName[] { table1 };
+
+ TableName[] tablesMapFull =
+ new TableName[] { table1_restore };
+
+ BackupAdmin client = getBackupAdmin();
+ client.restore(RestoreServerUtil.createRestoreRequest(BACKUP_REMOTE_ROOT_DIR, backupId, false,
+ tablesRestoreFull, tablesMapFull, false));
+
+ // check tables for full restore
+ HBaseAdmin hAdmin = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hAdmin.tableExists(table1_restore));
+
+ // #5.2 - checking row count of tables for full restore
+ HTable hTable = (HTable) conn.getTable(table1_restore);
+ Assert.assertThat(TEST_UTIL.countRows(hTable, famName), CoreMatchers.equalTo(NB_ROWS_IN_BATCH));
+ int cnt3 = TEST_UTIL.countRows(hTable, fam3Name);
+ Assert.assertTrue(cnt3 >= 0 && cnt3 <= NB_ROWS_IN_FAM3);
+
+ int rows1 = MobSnapshotTestingUtils.countMobRows(t1, fam2Name);
+ Assert.assertEquals(rows0, rows1);
+ hTable.close();
+
+ hAdmin.close();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java
new file mode 100644
index 0000000..988d07c
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRemoteRestore.java
@@ -0,0 +1,51 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one or more contributor license
+ * agreements. See the NOTICE file distributed with this work for additional information regarding
+ * copyright ownership. The ASF licenses this file to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance with the License. You may obtain a
+ * copy of the License at http://www.apache.org/licenses/LICENSE-2.0 Unless required by applicable
+ * law or agreed to in writing, software distributed under the License is distributed on an "AS IS"
+ * BASIS, WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. See the License
+ * for the specific language governing permissions and limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertTrue;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.util.RestoreServerUtil;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestRemoteRestore extends TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestRemoteRestore.class);
+
+ /**
+ * Verify that a remote restore on a single table is successful.
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreRemote() throws Exception {
+
+ LOG.info("test remote full backup on a single table");
+ String backupId = backupTables(BackupType.FULL, toList(table1.getNameAsString()),
+ BACKUP_REMOTE_ROOT_DIR);
+ LOG.info("backup complete");
+ TableName[] tableset = new TableName[] { table1 };
+ TableName[] tablemap = new TableName[] { table1_restore };
+ getBackupAdmin().restore(RestoreServerUtil.createRestoreRequest(BACKUP_REMOTE_ROOT_DIR,
+ backupId, false, tableset, tablemap, false));
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table1_restore));
+ TEST_UTIL.deleteTable(table1_restore);
+ hba.close();
+ }
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java
new file mode 100644
index 0000000..e20f2cc
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestRestoreBoundaryTests.java
@@ -0,0 +1,78 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import static org.junit.Assert.assertTrue;
+
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.util.RestoreServerUtil;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestRestoreBoundaryTests extends TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestRestoreBoundaryTests.class);
+
+ /**
+ * Verify that a single empty table is restored to a new table
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreSingleEmpty() throws Exception {
+ LOG.info("test full restore on a single table empty table");
+ String backupId = fullTableBackup(toList(table1.getNameAsString()));
+ LOG.info("backup complete");
+ TableName[] tableset = new TableName[] { table1 };
+ TableName[] tablemap = new TableName[] { table1_restore };
+ getBackupAdmin().restore(RestoreServerUtil.createRestoreRequest(BACKUP_ROOT_DIR, backupId,
+ false, tableset, tablemap, false));
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table1_restore));
+ TEST_UTIL.deleteTable(table1_restore);
+ }
+
+ /**
+ * Verify that multiple tables are restored to new tables.
+ * @throws Exception
+ */
+ @Test
+ public void testFullRestoreMultipleEmpty() throws Exception {
+ LOG.info("create full backup image on multiple tables");
+
+ List<TableName> tables = toList(table2.getNameAsString(), table3.getNameAsString());
+ String backupId = fullTableBackup(tables);
+ TableName[] restore_tableset = new TableName[] { table2, table3};
+ TableName[] tablemap = new TableName[] { table2_restore, table3_restore };
+ getBackupAdmin().restore(RestoreServerUtil.createRestoreRequest(BACKUP_ROOT_DIR, backupId,
+ false, restore_tableset, tablemap, false));
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ assertTrue(hba.tableExists(table2_restore));
+ assertTrue(hba.tableExists(table3_restore));
+ TEST_UTIL.deleteTable(table2_restore);
+ TEST_UTIL.deleteTable(table3_restore);
+ hba.close();
+ }
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java
new file mode 100644
index 0000000..2d94fd3
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/TestSystemTableSnapshot.java
@@ -0,0 +1,59 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
+import org.apache.hadoop.hbase.client.HBaseAdmin;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+@Category(LargeTests.class)
+public class TestSystemTableSnapshot extends TestBackupBase {
+
+ private static final Log LOG = LogFactory.getLog(TestSystemTableSnapshot.class);
+
+ /**
+ * Verify that a single table is restored to a new table
+ * @throws Exception
+ */
+ @Test
+ public void testBackupRestoreSystemTable() throws Exception {
+
+ LOG.info("test snapshot system table");
+
+
+ TableName backupSystem = BackupSystemTable.getTableName();
+
+
+ HBaseAdmin hba = TEST_UTIL.getHBaseAdmin();
+ String snapshotName = "sysTable";
+ hba.snapshot(snapshotName, backupSystem);
+
+ hba.disableTable(backupSystem);
+ hba.restoreSnapshot(snapshotName);
+ hba.enableTable(backupSystem);
+ hba.close();
+ }
+
+
+}
\ No newline at end of file
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java
new file mode 100644
index 0000000..dffed12
--- /dev/null
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/backup/master/TestBackupLogCleaner.java
@@ -0,0 +1,162 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup.master;
+
+import static org.junit.Assert.assertFalse;
+import static org.junit.Assert.assertTrue;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.List;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupType;
+import org.apache.hadoop.hbase.backup.TestBackupBase;
+import org.apache.hadoop.hbase.backup.impl.BackupSystemTable;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+import org.apache.hadoop.hbase.client.HTable;
+import org.apache.hadoop.hbase.client.Put;
+import org.apache.hadoop.hbase.testclassification.LargeTests;
+import org.apache.hadoop.hbase.util.Bytes;
+import org.apache.hadoop.hbase.util.FSUtils;
+import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import com.google.common.collect.Iterables;
+import com.google.common.collect.Lists;
+
+@Category(LargeTests.class)
+public class TestBackupLogCleaner extends TestBackupBase {
+ private static final Log LOG = LogFactory.getLog(TestBackupLogCleaner.class);
+
+ // implements all test cases in 1 test since incremental full backup/
+ // incremental backup has dependencies
+ @Test
+ public void testBackupLogCleaner() throws Exception {
+
+ // #1 - create full backup for all tables
+ LOG.info("create full backup image for all tables");
+
+ List<TableName> tableSetFullList = Lists.newArrayList(table1, table2, table3, table4);
+
+ try (BackupSystemTable systemTable = new BackupSystemTable(TEST_UTIL.getConnection())) {
+ // Verify that we have no backup sessions yet
+ assertFalse(systemTable.hasBackupSessions());
+
+ List<FileStatus> walFiles = getListOfWALFiles(TEST_UTIL.getConfiguration());
+ List<String> swalFiles = convert(walFiles);
+ BackupLogCleaner cleaner = new BackupLogCleaner();
+ cleaner.setConf(TEST_UTIL.getConfiguration());
+ cleaner.init(null);
+ cleaner.setConf(TEST_UTIL.getConfiguration());
+
+ Iterable<FileStatus> deletable = cleaner.getDeletableFiles(walFiles);
+ int size = Iterables.size(deletable);
+
+ // We can delete all files because we do not have yet recorded backup sessions
+ assertTrue(size == walFiles.size());
+
+ systemTable.addWALFiles(swalFiles, "backup", "root");
+ String backupIdFull = fullTableBackup(tableSetFullList);
+ assertTrue(checkSucceeded(backupIdFull));
+ // Check one more time
+ deletable = cleaner.getDeletableFiles(walFiles);
+ // We can delete wal files because they were saved into hbase:backup table
+ size = Iterables.size(deletable);
+ assertTrue(size == walFiles.size());
+
+ List<FileStatus> newWalFiles = getListOfWALFiles(TEST_UTIL.getConfiguration());
+ LOG.debug("WAL list after full backup");
+ convert(newWalFiles);
+
+ // New list of wal files is greater than the previous one,
+ // because new wal per RS have been opened after full backup
+ assertTrue(walFiles.size() < newWalFiles.size());
+ Connection conn = ConnectionFactory.createConnection(conf1);
+ // #2 - insert some data to table
+ HTable t1 = (HTable) conn.getTable(table1);
+ Put p1;
+ for (int i = 0; i < NB_ROWS_IN_BATCH; i++) {
+ p1 = new Put(Bytes.toBytes("row-t1" + i));
+ p1.addColumn(famName, qualName, Bytes.toBytes("val" + i));
+ t1.put(p1);
+ }
+
+ t1.close();
+
+ HTable t2 = (HTable) conn.getTable(table2);
+ Put p2;
+ for (int i = 0; i < 5; i++) {
+ p2 = new Put(Bytes.toBytes("row-t2" + i));
+ p2.addColumn(famName, qualName, Bytes.toBytes("val" + i));
+ t2.put(p2);
+ }
+
+ t2.close();
+
+ // #3 - incremental backup for multiple tables
+
+ List<TableName> tableSetIncList = Lists.newArrayList(table1, table2, table3);
+ String backupIdIncMultiple = backupTables(BackupType.INCREMENTAL, tableSetIncList,
+ BACKUP_ROOT_DIR);
+ assertTrue(checkSucceeded(backupIdIncMultiple));
+ deletable = cleaner.getDeletableFiles(newWalFiles);
+
+ assertTrue(Iterables.size(deletable) == newWalFiles.size());
+
+ conn.close();
+ }
+ }
+
+ private List<String> convert(List<FileStatus> walFiles) {
+ List<String> result = new ArrayList<String>();
+ for (FileStatus fs : walFiles) {
+ LOG.debug("+++WAL: " + fs.getPath().toString());
+ result.add(fs.getPath().toString());
+ }
+ return result;
+ }
+
+ private List<FileStatus> getListOfWALFiles(Configuration c) throws IOException {
+ Path logRoot = new Path(FSUtils.getRootDir(c), HConstants.HREGION_LOGDIR_NAME);
+ FileSystem fs = FileSystem.get(c);
+ RemoteIterator<LocatedFileStatus> it = fs.listFiles(logRoot, true);
+ List<FileStatus> logFiles = new ArrayList<FileStatus>();
+ while (it.hasNext()) {
+ LocatedFileStatus lfs = it.next();
+ if (lfs.isFile() && !AbstractFSWALProvider.isMetaFile(lfs.getPath())) {
+ logFiles.add(lfs);
+ LOG.info(lfs);
+ }
+ }
+ return logFiles;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
index 721391a..29fd1c5 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/client/TestMetaWithReplicas.java
@@ -29,7 +29,6 @@ import java.util.Collection;
import java.util.List;
import java.util.concurrent.ExecutorService;
-import edu.umd.cs.findbugs.annotations.Nullable;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.conf.Configuration;
@@ -45,6 +44,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.TableNotFoundException;
import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.regionserver.StorefileRefresherChore;
import org.apache.hadoop.hbase.testclassification.LargeTests;
@@ -64,6 +64,8 @@ import org.junit.Test;
import org.junit.experimental.categories.Category;
import org.junit.rules.TestRule;
+import edu.umd.cs.findbugs.annotations.Nullable;
+
/**
* Tests the scenarios where replicas are enabled for the meta table
*/
@@ -82,7 +84,11 @@ public class TestMetaWithReplicas {
TEST_UTIL.getConfiguration().setInt(HConstants.META_REPLICAS_NUM, 3);
TEST_UTIL.getConfiguration().setInt(
StorefileRefresherChore.REGIONSERVER_STOREFILE_REFRESH_PERIOD, 1000);
+ TEST_UTIL.getConfiguration().setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
TEST_UTIL.startMiniCluster(3);
+
+ TEST_UTIL.waitUntilAllSystemRegionsAssigned();
+
// disable the balancer
LoadBalancerTracker l = new LoadBalancerTracker(TEST_UTIL.getZooKeeperWatcher(),
new Abortable() {
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java
index 061068c..67c20f3 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithAbort.java
@@ -36,6 +36,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
@@ -107,6 +108,7 @@ public class TestMasterCoprocessorExceptionWithAbort {
HTableDescriptor desc, HRegionInfo[] regions) throws IOException {
// cause a NullPointerException and don't catch it: this will cause the
// master to abort().
+ if (desc.getTableName().isSystemTable()) return;
Integer i;
i = null;
i = i++;
@@ -143,6 +145,7 @@ public class TestMasterCoprocessorExceptionWithAbort {
@BeforeClass
public static void setupBeforeClass() throws Exception {
Configuration conf = UTIL.getConfiguration();
+ conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
BuggyMasterObserver.class.getName());
conf.setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, true);
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java
index bbb855c..d438ebc 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/coprocessor/TestMasterCoprocessorExceptionWithRemove.java
@@ -34,6 +34,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.master.HMaster;
import org.apache.hadoop.hbase.master.MasterCoprocessorHost;
@@ -85,6 +86,7 @@ public class TestMasterCoprocessorExceptionWithRemove {
// Cause a NullPointerException and don't catch it: this should cause the
// master to throw an o.apache.hadoop.hbase.DoNotRetryIOException to the
// client.
+ if (desc.getTableName().isSystemTable()) return;
Integer i;
i = null;
i = i++;
@@ -125,6 +127,7 @@ public class TestMasterCoprocessorExceptionWithRemove {
@BeforeClass
public static void setupBeforeClass() throws Exception {
Configuration conf = UTIL.getConfiguration();
+ conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
conf.set(CoprocessorHost.MASTER_COPROCESSOR_CONF_KEY,
BuggyMasterObserver.class.getName());
UTIL.getConfiguration().setBoolean(CoprocessorHost.ABORT_ON_ERROR_KEY, false);
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableOutputFormatConnectionExhaust.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableOutputFormatConnectionExhaust.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableOutputFormatConnectionExhaust.java
index 835117c..50008a1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableOutputFormatConnectionExhaust.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/mapred/TestTableOutputFormatConnectionExhaust.java
@@ -17,6 +17,10 @@
*/
package org.apache.hadoop.hbase.mapred;
+import static org.junit.Assert.fail;
+
+import java.io.IOException;
+
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.hbase.HBaseTestingUtility;
@@ -31,10 +35,6 @@ import org.junit.BeforeClass;
import org.junit.Test;
import org.junit.experimental.categories.Category;
-import java.io.IOException;
-
-import static org.junit.Assert.fail;
-
/**
* Spark creates many instances of TableOutputFormat within a single process. We need to make
* sure we can have many instances and not leak connections.
@@ -55,7 +55,7 @@ public class TestTableOutputFormatConnectionExhaust {
public static void beforeClass() throws Exception {
// Default in ZookeeperMiniCluster is 1000, setting artificially low to trigger exhaustion.
// need min of 7 to properly start the default mini HBase cluster
- UTIL.getConfiguration().setInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, 10);
+ UTIL.getConfiguration().setInt(HConstants.ZOOKEEPER_MAX_CLIENT_CNXNS, 11);
UTIL.startMiniCluster();
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
index 2630068..8609f7f 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/MockNoopMasterServices.java
@@ -39,10 +39,8 @@ import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
import org.apache.hadoop.hbase.procedure2.ProcedureExecutor;
import org.apache.hadoop.hbase.quotas.MasterQuotaManager;
-import org.apache.hadoop.hbase.security.User;
import org.apache.hadoop.hbase.zookeeper.MetaTableLocator;
import org.apache.hadoop.hbase.zookeeper.ZooKeeperWatcher;
-import org.mockito.Mockito;
import com.google.protobuf.Service;
@@ -215,6 +213,17 @@ public class MockNoopMasterServices implements MasterServices, Server {
return null; //To change body of implemented methods use File | Settings | File Templates.
}
+
+ /*
+ * Restore table set
+ */
+ public long restoreTables(String backupRootDir,
+ String backupId, boolean check, List<TableName> sTableList,
+ List<TableName> tTableList, boolean isOverwrite, long nonceGroup, long nonce)
+ throws IOException {
+ return -1;
+ }
+
@Override
public List<HTableDescriptor> listTableDescriptorsByNamespace(String name) throws IOException {
return null; //To change body of implemented methods use File | Settings | File Templates.
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
index ebba4c8..5d4d014 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestCatalogJanitor.java
@@ -54,6 +54,11 @@ import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination;
import org.apache.hadoop.hbase.coordination.SplitLogManagerCoordination.SplitLogManagerDetails;
import org.apache.hadoop.hbase.io.Reference;
import org.apache.hadoop.hbase.master.CatalogJanitor.SplitParentFirstComparator;
+import org.apache.hadoop.hbase.master.snapshot.SnapshotManager;
+import org.apache.hadoop.hbase.procedure.MasterProcedureManagerHost;
+import org.apache.hadoop.hbase.regionserver.HStore;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController;
+import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
import org.apache.hadoop.hbase.shaded.protobuf.generated.AdminProtos;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos;
@@ -64,9 +69,6 @@ import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.MutateResp
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionAction;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.RegionActionResult;
import org.apache.hadoop.hbase.shaded.protobuf.generated.ClientProtos.ResultOrException;
-import org.apache.hadoop.hbase.regionserver.HStore;
-import org.apache.hadoop.hbase.shaded.com.google.protobuf.RpcController;
-import org.apache.hadoop.hbase.shaded.com.google.protobuf.ServiceException;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.testclassification.SmallTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -138,6 +140,16 @@ public class TestCatalogJanitor {
}
@Override
+ public SnapshotManager getSnapshotManager() {
+ return null;
+ }
+
+ @Override
+ public MasterProcedureManagerHost getMasterProcedureManagerHost() {
+ return null;
+ }
+
+ @Override
public AssignmentManager getAssignmentManager() {
return this.asm;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
index e34c9cd..9adfaee 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestDistributedLogSplitting.java
@@ -67,6 +67,7 @@ import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.SplitLogCounters;
import org.apache.hadoop.hbase.TableName;
import org.apache.hadoop.hbase.Waiter;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
import org.apache.hadoop.hbase.client.ClusterConnection;
import org.apache.hadoop.hbase.client.CompactionState;
import org.apache.hadoop.hbase.client.ConnectionUtils;
@@ -80,7 +81,6 @@ import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Result;
import org.apache.hadoop.hbase.client.RetriesExhaustedWithDetailsException;
import org.apache.hadoop.hbase.client.Table;
-import org.apache.hadoop.hbase.coordination.BaseCoordinatedStateManager;
import org.apache.hadoop.hbase.coordination.ZKSplitLogManagerCoordination;
import org.apache.hadoop.hbase.exceptions.OperationConflictException;
import org.apache.hadoop.hbase.exceptions.RegionInRecoveryException;
@@ -101,7 +101,6 @@ import org.apache.hadoop.hbase.util.JVMClusterUtil.MasterThread;
import org.apache.hadoop.hbase.util.JVMClusterUtil.RegionServerThread;
import org.apache.hadoop.hbase.util.Threads;
import org.apache.hadoop.hbase.wal.AbstractFSWALProvider;
-import org.apache.hadoop.hbase.wal.FSHLogProvider;
import org.apache.hadoop.hbase.wal.WAL;
import org.apache.hadoop.hbase.wal.WALFactory;
import org.apache.hadoop.hbase.wal.WALSplitter;
@@ -163,6 +162,7 @@ public class TestDistributedLogSplitting {
private void startCluster(int num_rs) throws Exception {
SplitLogCounters.resetCounters();
LOG.info("Starting cluster");
+ conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
conf.getLong("hbase.splitlog.max.resubmit", 0);
// Make the failure test faster
conf.setInt("zookeeper.recovery.retry", 0);
@@ -1143,8 +1143,8 @@ public class TestDistributedLogSplitting {
out.write(Bytes.toBytes("corrupted bytes"));
out.close();
ZKSplitLogManagerCoordination coordination =
- (ZKSplitLogManagerCoordination) ((BaseCoordinatedStateManager) master
- .getCoordinatedStateManager()).getSplitLogManagerCoordination();
+ (ZKSplitLogManagerCoordination) master
+ .getCoordinatedStateManager().getSplitLogManagerCoordination();
coordination.setIgnoreDeleteForTesting(true);
executor = Executors.newSingleThreadExecutor();
Runnable runnable = new Runnable() {
@@ -1507,14 +1507,14 @@ public class TestDistributedLogSplitting {
for (String oregion : regions)
LOG.debug("Region still online: " + oregion);
}
- assertEquals(2 + existingRegions, regions.size());
+ assertTrue(2 + existingRegions <= regions.size());
LOG.debug("Enabling table\n");
TEST_UTIL.getHBaseAdmin().enableTable(table);
LOG.debug("Waiting for no more RIT\n");
blockUntilNoRIT(zkw, master);
LOG.debug("Verifying there are " + numRegions + " assigned on cluster\n");
regions = HBaseTestingUtility.getAllOnlineRegions(cluster);
- assertEquals(numRegions + 2 + existingRegions, regions.size());
+ assertTrue(numRegions + 2 + existingRegions <= regions.size());
return ht;
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
index f57d6b9..fe9302c 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterFailover.java
@@ -41,6 +41,7 @@ import org.apache.hadoop.hbase.MetaTableAccessor;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.master.RegionState.State;
@@ -98,7 +99,7 @@ public class TestMasterFailover {
// Start the cluster
HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
-
+ TEST_UTIL.getConfiguration().setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
@@ -205,6 +206,7 @@ public class TestMasterFailover {
// Start the cluster
HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
+ TEST_UTIL.getConfiguration().setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
log("Cluster started");
@@ -238,9 +240,9 @@ public class TestMasterFailover {
log("Regions in hbase:meta and namespace have been created");
- // at this point we only expect 3 regions to be assigned out
- // (catalogs and namespace, + 1 online region)
- assertEquals(3, cluster.countServedRegions());
+ // at this point we expect at least 3 regions to be assigned out
+ // (meta and namespace, + 1 online region)
+ assertTrue(3 <= cluster.countServedRegions());
HRegionInfo hriOnline = null;
try (RegionLocator locator =
TEST_UTIL.getConnection().getRegionLocator(TableName.valueOf("onlineTable"))) {
@@ -333,6 +335,7 @@ public class TestMasterFailover {
// Start the cluster
HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility();
+ TEST_UTIL.getConfiguration().setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
log("Cluster started");
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
index 7a4baf3..6aefe6e 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterOperationsForRegionReplicas.java
@@ -38,11 +38,12 @@ import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.HRegionLocation;
import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.MetaTableAccessor;
+import org.apache.hadoop.hbase.MetaTableAccessor.Visitor;
import org.apache.hadoop.hbase.RegionLocations;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.MetaTableAccessor;
-import org.apache.hadoop.hbase.MetaTableAccessor.Visitor;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
@@ -71,12 +72,14 @@ public class TestMasterOperationsForRegionReplicas {
public static void setupBeforeClass() throws Exception {
conf = TEST_UTIL.getConfiguration();
conf.setBoolean("hbase.tests.use.shortcircuit.reads", false);
+ conf.setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
TEST_UTIL.startMiniCluster(numSlaves);
CONNECTION = ConnectionFactory.createConnection(TEST_UTIL.getConfiguration());
ADMIN = CONNECTION.getAdmin();
while(ADMIN.getClusterStatus().getServers().size() < numSlaves) {
Thread.sleep(100);
}
+ TEST_UTIL.waitUntilAllSystemRegionsAssigned();
}
@AfterClass
@@ -305,7 +308,7 @@ public class TestMasterOperationsForRegionReplicas {
connection);
snapshot.initialize();
Map<HRegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
- assert(regionToServerMap.size() == numRegions * numReplica + 1); //'1' for the namespace
+ assert(regionToServerMap.size() == numRegions * numReplica + 2); //'1' for the namespace
Map<ServerName, List<HRegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
for (Map.Entry<ServerName, List<HRegionInfo>> entry : serverToRegionMap.entrySet()) {
if (entry.getKey().equals(util.getHBaseCluster().getMaster().getServerName())) {
@@ -332,14 +335,14 @@ public class TestMasterOperationsForRegionReplicas {
connection);
snapshot.initialize();
Map<HRegionInfo, ServerName> regionToServerMap = snapshot.getRegionToRegionServerMap();
- assertEquals(regionToServerMap.size(), numRegions * numReplica + 1); //'1' for the namespace
+ assertEquals(regionToServerMap.size(), numRegions * numReplica + 2); //'2' for the ns, backup
Map<ServerName, List<HRegionInfo>> serverToRegionMap = snapshot.getRegionServerToRegionMap();
assertEquals(serverToRegionMap.keySet().size(), 2); // 1 rs + 1 master
for (Map.Entry<ServerName, List<HRegionInfo>> entry : serverToRegionMap.entrySet()) {
if (entry.getKey().equals(TEST_UTIL.getHBaseCluster().getMaster().getServerName())) {
continue;
}
- assertEquals(entry.getValue().size(), numRegions * numReplica);
+ assertEquals(entry.getValue().size(), numRegions * numReplica +1);
}
}
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
index b62279e..4f0fa73 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestMasterRestartAfterDisablingTable.java
@@ -31,6 +31,7 @@ import org.apache.hadoop.hbase.HBaseConfiguration;
import org.apache.hadoop.hbase.HBaseTestingUtility;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
import org.apache.hadoop.hbase.client.Admin;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
@@ -58,6 +59,7 @@ public class TestMasterRestartAfterDisablingTable {
log("Starting cluster");
Configuration conf = HBaseConfiguration.create();
HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
+ TEST_UTIL.getConfiguration().setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
log("Waiting for active/ready master");
@@ -79,9 +81,12 @@ public class TestMasterRestartAfterDisablingTable {
TEST_UTIL.getHBaseAdmin().disableTable(table);
NavigableSet<String> regions = HBaseTestingUtility.getAllOnlineRegions(cluster);
- assertEquals(
- "The number of regions for the table tableRestart should be 0 and only"
- + "the catalog and namespace tables should be present.", 2, regions.size());
+ for (String region : regions) {
+ assertTrue(
+ "The number of regions for the table tableRestart should be 0 and only"
+ + "the catalog and namespace tables should be present.",
+ !region.startsWith(table.getNameAsString()));
+ }
List<MasterThread> masterThreads = cluster.getMasterThreads();
MasterThread activeMaster = null;
@@ -110,7 +115,7 @@ public class TestMasterRestartAfterDisablingTable {
regions = HBaseTestingUtility.getAllOnlineRegions(cluster);
assertEquals("The assigned regions were not onlined after master"
+ " switch except for the catalog and namespace tables.",
- 6, regions.size());
+ 7, regions.size());
assertTrue("The table should be in enabled state",
cluster.getMaster().getTableStateManager()
.isTableState(TableName.valueOf("tableRestart"), TableState.State.ENABLED));
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
index 7c41c0f..8e8fcce 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRestartCluster.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableExistsException;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
import org.apache.hadoop.hbase.testclassification.LargeTests;
import org.apache.hadoop.hbase.testclassification.MasterTests;
import org.apache.hadoop.hbase.util.Bytes;
@@ -62,6 +63,7 @@ public class TestRestartCluster {
@Test (timeout=300000)
public void testClusterRestart() throws Exception {
+ UTIL.getConfiguration().setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
UTIL.startMiniCluster(3);
while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
Threads.sleep(1);
@@ -75,7 +77,7 @@ public class TestRestartCluster {
}
List<HRegionInfo> allRegions = MetaTableAccessor.getAllRegions(UTIL.getConnection(), false);
- assertEquals(4, allRegions.size());
+ assertTrue(4 <= allRegions.size());
LOG.info("\n\nShutting down cluster");
UTIL.shutdownMiniHBaseCluster();
@@ -90,7 +92,7 @@ public class TestRestartCluster {
// Otherwise we're reusing an Connection that has gone stale because
// the shutdown of the cluster also called shut of the connection.
allRegions = MetaTableAccessor.getAllRegions(UTIL.getConnection(), false);
- assertEquals(4, allRegions.size());
+ assertTrue(4 <= allRegions.size());
LOG.info("\n\nWaiting for tables to be available");
for(TableName TABLE: TABLES) {
try {
@@ -108,6 +110,7 @@ public class TestRestartCluster {
*/
@Test (timeout=300000)
public void testRetainAssignmentOnRestart() throws Exception {
+ UTIL.getConfiguration().setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
UTIL.startMiniCluster(2);
while (!UTIL.getMiniHBaseCluster().getMaster().isInitialized()) {
Threads.sleep(1);
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java
index 9d79c6c..1a20957 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/master/TestRollingRestart.java
@@ -35,6 +35,7 @@ import org.apache.hadoop.hbase.HRegionInfo;
import org.apache.hadoop.hbase.MiniHBaseCluster;
import org.apache.hadoop.hbase.ServerName;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
import org.apache.hadoop.hbase.client.RegionLocator;
import org.apache.hadoop.hbase.client.Table;
import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
@@ -67,6 +68,7 @@ public class TestRollingRestart {
log("Starting cluster");
Configuration conf = HBaseConfiguration.create();
HBaseTestingUtility TEST_UTIL = new HBaseTestingUtility(conf);
+ TEST_UTIL.getConfiguration().setBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY, true);
TEST_UTIL.startMiniCluster(NUM_MASTERS, NUM_RS);
MiniHBaseCluster cluster = TEST_UTIL.getHBaseCluster();
log("Waiting for active/ready master");
@@ -93,7 +95,7 @@ public class TestRollingRestart {
if (regions.size() != 2) {
for (String oregion : regions) log("Region still online: " + oregion);
}
- assertEquals(2, regions.size());
+ assertEquals(3, regions.size());
log("Enabling table\n");
TEST_UTIL.getHBaseAdmin().enableTable(table);
log("Waiting for no more RIT\n");
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
index f49bdb1..25d2ae1 100644
--- a/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
+++ b/hbase-server/src/test/java/org/apache/hadoop/hbase/regionserver/TestRegionOpen.java
@@ -29,12 +29,12 @@ import org.apache.hadoop.hbase.HColumnDescriptor;
import org.apache.hadoop.hbase.HConstants;
import org.apache.hadoop.hbase.HTableDescriptor;
import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
import org.apache.hadoop.hbase.client.ConnectionFactory;
import org.apache.hadoop.hbase.executor.ExecutorType;
import org.apache.hadoop.hbase.testclassification.MediumTests;
import org.apache.hadoop.hbase.testclassification.RegionServerTests;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
import org.junit.AfterClass;
import org.junit.BeforeClass;
import org.junit.Test;
@@ -52,6 +52,7 @@ public class TestRegionOpen {
@BeforeClass
public static void before() throws Exception {
HTU.startMiniCluster(NB_SERVERS);
+ HTU.waitUntilAllSystemRegionsAssigned();
}
@AfterClass
@@ -68,7 +69,7 @@ public class TestRegionOpen {
ThreadPoolExecutor exec = getRS().getExecutorService()
.getExecutorThreadPool(ExecutorType.RS_OPEN_PRIORITY_REGION);
- assertEquals(0, exec.getCompletedTaskCount());
+ long taskCount = exec.getCompletedTaskCount(); // System table regions
HTableDescriptor htd = new HTableDescriptor(tableName);
htd.setPriority(HConstants.HIGH_QOS);
@@ -78,6 +79,6 @@ public class TestRegionOpen {
admin.createTable(htd);
}
- assertEquals(1, exec.getCompletedTaskCount());
+ assertEquals(taskCount+1, exec.getCompletedTaskCount());
}
}