You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2016/11/14 17:22:54 UTC
[08/11] hbase git commit: HBASE-14123 HBase Backup/Restore Phase 2
(Vladimir Rodionov)
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
new file mode 100644
index 0000000..c4227f5
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
@@ -0,0 +1,754 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup.impl;
+
+import java.io.IOException;
+import java.net.URI;
+import java.util.List;
+
+import org.apache.commons.cli.CommandLine;
+import org.apache.commons.cli.HelpFormatter;
+import org.apache.commons.cli.Options;
+import org.apache.commons.lang.StringUtils;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.conf.Configured;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HBaseConfiguration;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupInfo;
+import org.apache.hadoop.hbase.backup.BackupRequest;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
+import org.apache.hadoop.hbase.backup.BackupType;
+import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
+import org.apache.hadoop.hbase.backup.util.BackupSet;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.Connection;
+import org.apache.hadoop.hbase.client.ConnectionFactory;
+
+import com.google.common.collect.Lists;
+
+/**
+ * General backup commands, options and usage messages
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class BackupCommands implements BackupRestoreConstants {
+
+ public final static String INCORRECT_USAGE = "Incorrect usage";
+
+ public static final String USAGE =
+ "Usage: bin/hbase backup COMMAND [command-specific arguments]\n"
+ + "where COMMAND is one of:\n"
+ + " create create a new backup image\n"
+ + " delete delete an existing backup image\n"
+ + " describe show the detailed information of a backup image\n"
+ + " history show history of all successful backups\n"
+ + " progress show the progress of the latest backup request\n"
+ + " set backup set management\n"
+ + "Run \'bin/hbase backup COMMAND -h\' to see help message for each command\n";
+
+ public static final String CREATE_CMD_USAGE =
+ "Usage: bin/hbase backup create <type> <backup_root> [tables] [options]\n"
+ + " type \"full\" to create a full backup image\n"
+ + " \"incremental\" to create an incremental backup image\n"
+ + " backup_root Full path to store the backup image\n"
+ + " tables If no tables (\"\") are specified, all tables are backed up.\n"
+ + " otherwise it is a comma separated list of tables.";
+
+
+ public static final String PROGRESS_CMD_USAGE = "Usage: bin/hbase backup progress <backupId>\n"
+ + " backupId Backup image id\n";
+ public static final String NO_INFO_FOUND = "No info was found for backup id: ";
+
+ public static final String DESCRIBE_CMD_USAGE = "Usage: bin/hbase backup describe <backupId>\n"
+ + " backupId Backup image id\n";
+
+ public static final String HISTORY_CMD_USAGE =
+ "Usage: bin/hbase backup history [options]";
+
+
+
+ public static final String DELETE_CMD_USAGE = "Usage: bin/hbase backup delete <backupId>\n"
+ + " backupId Backup image id\n";
+
+ public static final String CANCEL_CMD_USAGE = "Usage: bin/hbase backup cancel <backupId>\n"
+ + " backupId Backup image id\n";
+
+ public static final String SET_CMD_USAGE = "Usage: bin/hbase backup set COMMAND [name] [tables]\n"
+ + " name Backup set name\n"
+ + " tables If no tables (\"\") are specified, all tables will belong to the set.\n"
+ + " Otherwise it is a comma separated list of tables.\n"
+ + "COMMAND is one of:\n"
+ + " add add tables to a set, create a set if needed\n"
+ + " remove remove tables from a set\n"
+ + " list list all backup sets in the system\n"
+ + " describe describe set\n"
+ + " delete delete backup set\n";
+
+ public static final String USAGE_FOOTER = "";
+
+ public static abstract class Command extends Configured {
+ CommandLine cmdline;
+
+ Command(Configuration conf) {
+ super(conf);
+ }
+
+ public void execute() throws IOException
+ {
+ if (cmdline.hasOption("h") || cmdline.hasOption("help")) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+ }
+
+ protected abstract void printUsage();
+ }
+
+ private BackupCommands() {
+ throw new AssertionError("Instantiating utility class...");
+ }
+
+ public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) {
+ Command cmd = null;
+ switch (type) {
+ case CREATE:
+ cmd = new CreateCommand(conf, cmdline);
+ break;
+ case DESCRIBE:
+ cmd = new DescribeCommand(conf, cmdline);
+ break;
+ case PROGRESS:
+ cmd = new ProgressCommand(conf, cmdline);
+ break;
+ case DELETE:
+ cmd = new DeleteCommand(conf, cmdline);
+ break;
+ case CANCEL:
+ cmd = new CancelCommand(conf, cmdline);
+ break;
+ case HISTORY:
+ cmd = new HistoryCommand(conf, cmdline);
+ break;
+ case SET:
+ cmd = new BackupSetCommand(conf, cmdline);
+ break;
+ case HELP:
+ default:
+ cmd = new HelpCommand(conf, cmdline);
+ break;
+ }
+ return cmd;
+ }
+
+ static int numOfArgs(String[] args) {
+ if (args == null) return 0;
+ return args.length;
+ }
+
+ public static class CreateCommand extends Command {
+
+ CreateCommand(Configuration conf, CommandLine cmdline) {
+ super(conf);
+ this.cmdline = cmdline;
+ }
+
+ @Override
+ public void execute() throws IOException {
+ super.execute();
+ if (cmdline == null || cmdline.getArgs() == null) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+ String[] args = cmdline.getArgs();
+ if (args.length < 3 || args.length > 4) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+
+ if (!BackupType.FULL.toString().equalsIgnoreCase(args[1])
+ && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])) {
+ System.out.println("ERROR: invalid backup type: "+ args[1]);
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+ if(!verifyPath(args[2])) {
+ System.out.println("ERROR: invalid backup destination: "+ args[2]);
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+
+ String tables = null;
+ Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create();
+
+ // Check if we have both: backup set and list of tables
+ if(args.length == 4 && cmdline.hasOption(OPTION_SET)) {
+ System.out.println("ERROR: You can specify either backup set or list"+
+ " of tables, but not both");
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+
+ // Check backup set
+ String setName = null;
+ if (cmdline.hasOption(OPTION_SET)) {
+ setName = cmdline.getOptionValue(OPTION_SET);
+ tables = getTablesForSet(setName, conf);
+
+ if (tables == null) {
+ System.out.println("ERROR: Backup set '" + setName+ "' is either empty or does not exist");
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+ } else {
+ tables = (args.length == 4) ? args[3] : null;
+ }
+ int bandwidth = cmdline.hasOption(OPTION_BANDWIDTH) ?
+ Integer.parseInt(cmdline.getOptionValue(OPTION_BANDWIDTH)) : -1;
+ int workers = cmdline.hasOption(OPTION_WORKERS) ?
+ Integer.parseInt(cmdline.getOptionValue(OPTION_WORKERS)) : -1;
+
+ try (Connection conn = ConnectionFactory.createConnection(getConf());
+ HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);) {
+ BackupRequest request = new BackupRequest();
+ request.setBackupType(BackupType.valueOf(args[1].toUpperCase()))
+ .setTableList(tables != null?Lists.newArrayList(BackupClientUtil.parseTableNames(tables)): null)
+ .setTargetRootDir(args[2]).setWorkers(workers).setBandwidth(bandwidth)
+ .setBackupSetName(setName);
+
+ String backupId = admin.backupTables(request);
+ System.out.println("Backup session "+ backupId+" finished. Status: SUCCESS");
+ } catch (IOException e) {
+ System.out.println("Backup session finished. Status: FAILURE");
+ throw e;
+ }
+ }
+
+ private boolean verifyPath(String path) {
+ try{
+ Path p = new Path(path);
+ Configuration conf = getConf() != null? getConf():
+ HBaseConfiguration.create();
+ URI uri = p.toUri();
+ if(uri.getScheme() == null) return false;
+ FileSystem.get(uri, conf);
+ return true;
+ } catch(Exception e){
+ return false;
+ }
+ }
+
+ private String getTablesForSet(String name, Configuration conf)
+ throws IOException {
+ try (final Connection conn = ConnectionFactory.createConnection(conf);
+ final BackupSystemTable table = new BackupSystemTable(conn)) {
+ List<TableName> tables = table.describeBackupSet(name);
+ if (tables == null) return null;
+ return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
+ }
+ }
+
+ @Override
+ protected void printUsage() {
+ System.out.println(CREATE_CMD_USAGE);
+ Options options = new Options();
+ options.addOption(OPTION_WORKERS, true, OPTION_WORKERS_DESC);
+ options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC);
+ options.addOption(OPTION_SET, true, OPTION_SET_RESTORE_DESC);
+
+ HelpFormatter helpFormatter = new HelpFormatter();
+ helpFormatter.setLeftPadding(2);
+ helpFormatter.setDescPadding(8);
+ helpFormatter.setWidth(100);
+ helpFormatter.setSyntaxPrefix("Options:");
+ helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
+
+ }
+ }
+
+ private static class HelpCommand extends Command {
+
+ HelpCommand(Configuration conf, CommandLine cmdline) {
+ super(conf);
+ this.cmdline = cmdline;
+ }
+
+ @Override
+ public void execute() throws IOException {
+ super.execute();
+ if (cmdline == null) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+
+ String[] args = cmdline.getArgs();
+ if (args == null || args.length == 0) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+
+ if (args.length != 2) {
+ System.out.println("ERROR: Only supports help message of a single command type");
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+
+ String type = args[1];
+
+ if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) {
+ System.out.println(CREATE_CMD_USAGE);
+ } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) {
+ System.out.println(DESCRIBE_CMD_USAGE);
+ } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) {
+ System.out.println(HISTORY_CMD_USAGE);
+ } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) {
+ System.out.println(PROGRESS_CMD_USAGE);
+ } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) {
+ System.out.println(DELETE_CMD_USAGE);
+ } else if (BackupCommand.CANCEL.name().equalsIgnoreCase(type)) {
+ System.out.println(CANCEL_CMD_USAGE);
+ } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) {
+ System.out.println(SET_CMD_USAGE);
+ } else {
+ System.out.println("Unknown command : " + type);
+ printUsage();
+ }
+ }
+
+ @Override
+ protected void printUsage() {
+ System.out.println(USAGE);
+ }
+ }
+
+ private static class DescribeCommand extends Command {
+
+ DescribeCommand(Configuration conf, CommandLine cmdline) {
+ super(conf);
+ this.cmdline = cmdline;
+ }
+
+ @Override
+ public void execute() throws IOException {
+ super.execute();
+ if (cmdline == null || cmdline.getArgs() == null) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+ String[] args = cmdline.getArgs();
+ if (args.length != 2) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+
+ String backupId = args[1];
+ Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
+ try (final Connection conn = ConnectionFactory.createConnection(conf);
+ final BackupSystemTable sysTable = new BackupSystemTable(conn);) {
+ BackupInfo info = sysTable.readBackupInfo(backupId);
+ if (info == null) {
+ System.out.println("ERROR: " + backupId + " does not exist");
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+ System.out.println(info.getShortDescription());
+ }
+ }
+
+ @Override
+ protected void printUsage() {
+ System.out.println(DESCRIBE_CMD_USAGE);
+ }
+ }
+
+ private static class ProgressCommand extends Command {
+
+ ProgressCommand(Configuration conf, CommandLine cmdline) {
+ super(conf);
+ this.cmdline = cmdline;
+ }
+
+ @Override
+ public void execute() throws IOException {
+ super.execute();
+
+ if (cmdline == null || cmdline.getArgs() == null ||
+ cmdline.getArgs().length == 1) {
+ System.err.println("No backup id was specified, "
+ + "will retrieve the most recent (ongoing) sessions");
+ }
+ String[] args = cmdline == null ? null : cmdline.getArgs();
+ if (args != null && args.length > 2) {
+ System.err.println("ERROR: wrong number of arguments: " + args.length);
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+
+ String backupId = (args == null || args.length <= 1) ? null : args[1];
+ Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create();
+ try(final Connection conn = ConnectionFactory.createConnection(conf);
+ final BackupSystemTable sysTable = new BackupSystemTable(conn);){
+ BackupInfo info = sysTable.readBackupInfo(backupId);
+ int progress = info == null? -1: info.getProgress();
+ if(progress < 0){
+ System.out.println(NO_INFO_FOUND + backupId);
+ } else{
+ System.out.println(backupId+" progress=" + progress+"%");
+ }
+ }
+ }
+
+ @Override
+ protected void printUsage() {
+ System.out.println(PROGRESS_CMD_USAGE);
+ }
+ }
+
+ private static class DeleteCommand extends Command {
+
+ DeleteCommand(Configuration conf, CommandLine cmdline) {
+ super(conf);
+ this.cmdline = cmdline;
+ }
+
+ @Override
+ public void execute() throws IOException {
+ super.execute();
+ if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+
+ String[] args = cmdline.getArgs();
+
+ String[] backupIds = new String[args.length - 1];
+ System.arraycopy(args, 1, backupIds, 0, backupIds.length);
+ Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
+ try (final Connection conn = ConnectionFactory.createConnection(conf);
+ HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);) {
+ int deleted = admin.deleteBackups(args);
+ System.out.println("Deleted " + deleted + " backups. Total requested: " + args.length);
+ }
+
+ }
+
+ @Override
+ protected void printUsage() {
+ System.out.println(DELETE_CMD_USAGE);
+ }
+ }
+
+// TODO Cancel command
+
+ private static class CancelCommand extends Command {
+
+ CancelCommand(Configuration conf, CommandLine cmdline) {
+ super(conf);
+ this.cmdline = cmdline;
+ }
+
+ @Override
+ public void execute() throws IOException {
+ super.execute();
+ if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
+ System.out.println("No backup id(s) was specified, will use the most recent one");
+ }
+ Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
+ try (final Connection conn = ConnectionFactory.createConnection(conf);
+ HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);) {
+ // TODO cancel backup
+ }
+ }
+
+ @Override
+ protected void printUsage() {
+ }
+ }
+
+ private static class HistoryCommand extends Command {
+
+ private final static int DEFAULT_HISTORY_LENGTH = 10;
+
+ HistoryCommand(Configuration conf, CommandLine cmdline) {
+ super(conf);
+ this.cmdline = cmdline;
+ }
+
+ @Override
+ public void execute() throws IOException {
+
+ super.execute();
+
+ int n = parseHistoryLength();
+ final TableName tableName = getTableName();
+ final String setName = getTableSetName();
+ BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() {
+ @Override
+ public boolean apply(BackupInfo info) {
+ if (tableName == null) return true;
+ List<TableName> names = info.getTableNames();
+ return names.contains(tableName);
+ }
+ };
+ BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() {
+ @Override
+ public boolean apply(BackupInfo info) {
+ if (setName == null) return true;
+ String backupId = info.getBackupId();
+ return backupId.startsWith(setName);
+ }
+ };
+ Path backupRootPath = getBackupRootPath();
+ List<BackupInfo> history = null;
+ Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
+ if (backupRootPath == null) {
+ // Load from hbase:backup
+ try (final Connection conn = ConnectionFactory.createConnection(conf);
+ final BackupSystemTable sysTable = new BackupSystemTable(conn);) {
+
+ history = sysTable.getBackupHistory(n, tableNameFilter, tableSetFilter);
+ }
+ } else {
+ // load from backup FS
+ history = BackupClientUtil.getHistory(conf, n, backupRootPath,
+ tableNameFilter, tableSetFilter);
+ }
+ for (BackupInfo info : history) {
+ System.out.println(info.getShortDescription());
+ }
+ }
+
+ private Path getBackupRootPath() throws IOException {
+ String value = null;
+ try{
+ value = cmdline.getOptionValue(OPTION_PATH);
+ if (value == null) return null;
+ return new Path(value);
+ } catch (IllegalArgumentException e) {
+ System.out.println("ERROR: Illegal argument for backup root path: "+ value);
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+ }
+
+ private TableName getTableName() throws IOException {
+ String value = cmdline.getOptionValue(OPTION_TABLE);
+ if (value == null) return null;
+ try{
+ return TableName.valueOf(value);
+ } catch (IllegalArgumentException e){
+ System.out.println("Illegal argument for table name: "+ value);
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+ }
+
+ private String getTableSetName() throws IOException {
+ String value = cmdline.getOptionValue(OPTION_SET);
+ return value;
+ }
+
+ private int parseHistoryLength() throws IOException {
+ String value = cmdline.getOptionValue(OPTION_RECORD_NUMBER);
+ try{
+ if (value == null) return DEFAULT_HISTORY_LENGTH;
+ return Integer.parseInt(value);
+ } catch(NumberFormatException e) {
+ System.out.println("Illegal argument for history length: "+ value);
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+ }
+
+ @Override
+ protected void printUsage() {
+ System.out.println(HISTORY_CMD_USAGE);
+ Options options = new Options();
+ options.addOption(OPTION_RECORD_NUMBER, true, OPTION_RECORD_NUMBER_DESC);
+ options.addOption(OPTION_PATH, true, OPTION_PATH_DESC);
+ options.addOption(OPTION_TABLE, true, OPTION_TABLE_DESC);
+ options.addOption(OPTION_SET, true, OPTION_SET_DESC);
+
+ HelpFormatter helpFormatter = new HelpFormatter();
+ helpFormatter.setLeftPadding(2);
+ helpFormatter.setDescPadding(8);
+ helpFormatter.setWidth(100);
+ helpFormatter.setSyntaxPrefix("Options:");
+ helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
+ }
+ }
+
+ private static class BackupSetCommand extends Command {
+ private final static String SET_ADD_CMD = "add";
+ private final static String SET_REMOVE_CMD = "remove";
+ private final static String SET_DELETE_CMD = "delete";
+ private final static String SET_DESCRIBE_CMD = "describe";
+ private final static String SET_LIST_CMD = "list";
+
+ BackupSetCommand(Configuration conf, CommandLine cmdline) {
+ super(conf);
+ this.cmdline = cmdline;
+ }
+
+ @Override
+ public void execute() throws IOException {
+ super.execute();
+ // Command-line must have at least one element
+ if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+
+ String[] args = cmdline.getArgs();
+ String cmdStr = args[1];
+ BackupCommand cmd = getCommand(cmdStr);
+
+ switch (cmd) {
+ case SET_ADD:
+ processSetAdd(args);
+ break;
+ case SET_REMOVE:
+ processSetRemove(args);
+ break;
+ case SET_DELETE:
+ processSetDelete(args);
+ break;
+ case SET_DESCRIBE:
+ processSetDescribe(args);
+ break;
+ case SET_LIST:
+ processSetList(args);
+ break;
+ default:
+ break;
+
+ }
+ }
+
+ private void processSetList(String[] args) throws IOException {
+ // List all backup set names
+ // does not expect any args
+ Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create();
+ try(final Connection conn = ConnectionFactory.createConnection(conf);
+ HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);){
+ List<BackupSet> list = admin.listBackupSets();
+ for(BackupSet bs: list){
+ System.out.println(bs);
+ }
+ }
+ }
+
+ private void processSetDescribe(String[] args) throws IOException {
+ if (args == null || args.length != 3) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+ String setName = args[2];
+ Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create();
+ try(final Connection conn = ConnectionFactory.createConnection(conf);
+ final BackupSystemTable sysTable = new BackupSystemTable(conn);){
+ List<TableName> tables = sysTable.describeBackupSet(setName);
+ BackupSet set = tables == null? null : new BackupSet(setName, tables);
+ if(set == null) {
+ System.out.println("Set '"+setName+"' does not exist.");
+ } else{
+ System.out.println(set);
+ }
+ }
+ }
+
+ private void processSetDelete(String[] args) throws IOException {
+ if (args == null || args.length != 3) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+ String setName = args[2];
+ Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create();
+ try(final Connection conn = ConnectionFactory.createConnection(conf);
+ final HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);){
+ boolean result = admin.deleteBackupSet(setName);
+ if(result){
+ System.out.println("Delete set "+setName+" OK.");
+ } else{
+ System.out.println("Set "+setName+" does not exist");
+ }
+ }
+ }
+
+ private void processSetRemove(String[] args) throws IOException {
+ if (args == null || args.length != 4) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+
+ String setName = args[2];
+ String[] tables = args[3].split(",");
+ Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create();
+ try(final Connection conn = ConnectionFactory.createConnection(conf);
+ final HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);){
+ admin.removeFromBackupSet(setName, tables);
+ }
+ }
+
+ private void processSetAdd(String[] args) throws IOException {
+ if (args == null || args.length != 4) {
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+ String setName = args[2];
+ String[] tables = args[3].split(",");
+ TableName[] tableNames = new TableName[tables.length];
+ for(int i=0; i < tables.length; i++){
+ tableNames[i] = TableName.valueOf(tables[i]);
+ }
+ Configuration conf = getConf() != null? getConf():HBaseConfiguration.create();
+ try(final Connection conn = ConnectionFactory.createConnection(conf);
+ final HBaseBackupAdmin admin = new HBaseBackupAdmin(conn);){
+ admin.addToBackupSet(setName, tableNames);
+ }
+
+ }
+
+ private BackupCommand getCommand(String cmdStr) throws IOException {
+ if (cmdStr.equals(SET_ADD_CMD)) {
+ return BackupCommand.SET_ADD;
+ } else if (cmdStr.equals(SET_REMOVE_CMD)) {
+ return BackupCommand.SET_REMOVE;
+ } else if (cmdStr.equals(SET_DELETE_CMD)) {
+ return BackupCommand.SET_DELETE;
+ } else if (cmdStr.equals(SET_DESCRIBE_CMD)) {
+ return BackupCommand.SET_DESCRIBE;
+ } else if (cmdStr.equals(SET_LIST_CMD)) {
+ return BackupCommand.SET_LIST;
+ } else {
+ System.out.println("ERROR: Unknown command for 'set' :" + cmdStr);
+ printUsage();
+ throw new IOException(INCORRECT_USAGE);
+ }
+ }
+
+ @Override
+ protected void printUsage() {
+ System.out.println(SET_CMD_USAGE);
+ }
+
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java
new file mode 100644
index 0000000..ca204b4
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java
@@ -0,0 +1,86 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup.impl;
+
+import org.apache.hadoop.hbase.HBaseIOException;
+import org.apache.hadoop.hbase.backup.BackupInfo;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * Backup exception
+ */
+@SuppressWarnings("serial")
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BackupException extends HBaseIOException {
+ private BackupInfo description;
+
+ /**
+ * Some exception happened for a backup and don't even know the backup that it was about
+ * @param msg Full description of the failure
+ */
+ public BackupException(String msg) {
+ super(msg);
+ }
+
+ /**
+ * Some exception happened for a backup with a cause
+ * @param cause the cause
+ */
+ public BackupException(Throwable cause) {
+ super(cause);
+ }
+
+ /**
+ * Exception for the given backup that has no previous root cause
+ * @param msg reason why the backup failed
+ * @param desc description of the backup that is being failed
+ */
+ public BackupException(String msg, BackupInfo desc) {
+ super(msg);
+ this.description = desc;
+ }
+
+ /**
+ * Exception for the given backup due to another exception
+ * @param msg reason why the backup failed
+ * @param cause root cause of the failure
+ * @param desc description of the backup that is being failed
+ */
+ public BackupException(String msg, Throwable cause, BackupInfo desc) {
+ super(msg, cause);
+ this.description = desc;
+ }
+
+ /**
+ * Exception when the description of the backup cannot be determined, due to some other root
+ * cause
+ * @param message description of what caused the failure
+ * @param e root cause
+ */
+ public BackupException(String message, Exception e) {
+ super(message, e);
+ }
+
+ public BackupInfo getBackupContext() {
+ return this.description;
+ }
+
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java
new file mode 100644
index 0000000..6fb7cfd
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java
@@ -0,0 +1,502 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup.impl;
+
+import java.io.Closeable;
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.Set;
+import java.util.concurrent.ExecutorService;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.ThreadPoolExecutor;
+import java.util.concurrent.TimeUnit;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.HTableDescriptor;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupInfo;
+import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
+import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
+import org.apache.hadoop.hbase.backup.BackupType;
+import org.apache.hadoop.hbase.backup.HBackupFileSystem;
+import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
+import org.apache.hadoop.hbase.backup.master.BackupController;
+import org.apache.hadoop.hbase.backup.master.BackupLogCleaner;
+import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
+import org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.client.Admin;
+import org.apache.hadoop.hbase.client.Connection;
+
+import com.google.common.util.concurrent.ThreadFactoryBuilder;
+
+/**
+ * Handles backup requests on server-side, creates backup context records in hbase:backup
+ * to keep track backup. The timestamps kept in hbase:backup table will be used for future
+ * incremental backup. Creates BackupContext and DispatchRequest.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BackupManager implements Closeable {
+ private static final Log LOG = LogFactory.getLog(BackupManager.class);
+
+ private Configuration conf = null;
+ private BackupInfo backupContext = null;
+ private ExecutorService pool = null;
+ private BackupSystemTable systemTable;
+ private final Connection conn;
+
+ /**
+ * Backup manager constructor.
+ * @param conf configuration
+ * @throws IOException exception
+ */
+ public BackupManager(Connection conn, Configuration conf) throws IOException {
+ if (!conf.getBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY,
+ BackupRestoreConstants.BACKUP_ENABLE_DEFAULT)) {
+ throw new BackupException("HBase backup is not enabled. Check your " +
+ BackupRestoreConstants.BACKUP_ENABLE_KEY + " setting.");
+ }
+ this.conf = conf;
+ this.conn = conn;
+ this.systemTable = new BackupSystemTable(conn);
+
+ }
+
+ /**
+ * Return backup context
+ */
+ protected BackupInfo getBackupContext()
+ {
+ return backupContext;
+ }
+ /**
+ * This method modifies the master's configuration in order to inject backup-related features
+ * @param conf configuration
+ */
+ public static void decorateMasterConfiguration(Configuration conf) {
+ if (!isBackupEnabled(conf)) {
+ return;
+ }
+ // Add WAL archive cleaner plug-in
+ String plugins = conf.get(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
+ String cleanerClass = BackupLogCleaner.class.getCanonicalName();
+ if (!plugins.contains(cleanerClass)) {
+ conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, plugins + "," + cleanerClass);
+ }
+
+ String classes = conf.get("hbase.procedure.master.classes");
+ String masterProcedureClass = LogRollMasterProcedureManager.class.getName();
+ if(classes == null){
+ conf.set("hbase.procedure.master.classes", masterProcedureClass);
+ } else if(!classes.contains(masterProcedureClass)){
+ conf.set("hbase.procedure.master.classes", classes +","+masterProcedureClass);
+ }
+
+ // Set Master Observer - Backup Controller
+ classes = conf.get("hbase.coprocessor.master.classes");
+ String observerClass = BackupController.class.getName();
+ if(classes == null){
+ conf.set("hbase.coprocessor.master.classes", observerClass);
+ } else if(!classes.contains(observerClass)){
+ conf.set("hbase.coprocessor.master.classes", classes +","+observerClass);
+ }
+
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Added log cleaner: " + cleanerClass);
+ LOG.debug("Added master procedure manager: "+masterProcedureClass);
+ LOG.debug("Added master observer: "+observerClass);
+ }
+
+ }
+
+ /**
+ * This method modifies the RS configuration in order to inject backup-related features
+ * @param conf configuration
+ */
+ public static void decorateRSConfiguration(Configuration conf) {
+ if (!isBackupEnabled(conf)) {
+ return;
+ }
+
+ String classes = conf.get("hbase.procedure.regionserver.classes");
+ String regionProcedureClass = LogRollRegionServerProcedureManager.class.getName();
+ if(classes == null){
+ conf.set("hbase.procedure.regionserver.classes", regionProcedureClass);
+ } else if(!classes.contains(regionProcedureClass)){
+ conf.set("hbase.procedure.regionserver.classes", classes +","+regionProcedureClass);
+ }
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Added region procedure manager: "+regionProcedureClass);
+ }
+
+ }
+
+
+ public static boolean isBackupEnabled(Configuration conf) {
+ return conf.getBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY,
+ BackupRestoreConstants.BACKUP_ENABLE_DEFAULT);
+ }
+
+ /**
+ * Get configuration
+ * @return configuration
+ */
+ Configuration getConf() {
+ return conf;
+ }
+
+ /**
+ * Stop all the work of backup.
+ */
+ @Override
+ public void close() {
+ // currently, we shutdown now for all ongoing back handlers, we may need to do something like
+ // record the failed list somewhere later
+ if (this.pool != null) {
+ this.pool.shutdownNow();
+ }
+ if (systemTable != null) {
+ try {
+ systemTable.close();
+ } catch (Exception e) {
+ LOG.error(e);
+ }
+ }
+ if (conn != null) {
+ try {
+ conn.close();
+ } catch (IOException e) {
+ LOG.error(e);
+ }
+ }
+ }
+
+ /**
+ * Create a BackupContext based on input backup request.
+ * @param backupId backup id
+ * @param type type
+ * @param tableList table list
+ * @param targetRootDir root dir
+ * @param workers number of parallel workers
+ * @param bandwidth bandwidth per worker in MB per sec
+ * @return BackupInfo
+ * @throws BackupException exception
+ */
+ public BackupInfo createBackupContext(String backupId, BackupType type,
+ List<TableName> tableList, String targetRootDir, int workers, long bandwidth)
+ throws BackupException {
+ if (targetRootDir == null) {
+ throw new BackupException("Wrong backup request parameter: target backup root directory");
+ }
+
+ if (type == BackupType.FULL && (tableList == null || tableList.isEmpty())) {
+ // If table list is null for full backup, which means backup all tables. Then fill the table
+ // list with all user tables from meta. It no table available, throw the request exception.
+
+ HTableDescriptor[] htds = null;
+ try (Admin admin = conn.getAdmin()) {
+ htds = admin.listTables();
+ } catch (Exception e) {
+ throw new BackupException(e);
+ }
+
+ if (htds == null) {
+ throw new BackupException("No table exists for full backup of all tables.");
+ } else {
+ tableList = new ArrayList<>();
+ for (HTableDescriptor hTableDescriptor : htds) {
+ tableList.add(hTableDescriptor.getTableName());
+ }
+
+ LOG.info("Full backup all the tables available in the cluster: " + tableList);
+ }
+ }
+
+ // there are one or more tables in the table list
+ backupContext = new BackupInfo(backupId, type,
+ tableList.toArray(new TableName[tableList.size()]),
+ targetRootDir);
+ backupContext.setBandwidth(bandwidth);
+ backupContext.setWorkers(workers);
+ return backupContext;
+ }
+
+ /**
+ * Check if any ongoing backup. Currently, we only reply on checking status in hbase:backup. We
+ * need to consider to handle the case of orphan records in the future. Otherwise, all the coming
+ * request will fail.
+ * @return the ongoing backup id if on going backup exists, otherwise null
+ * @throws IOException exception
+ */
+ private String getOngoingBackupId() throws IOException {
+
+ ArrayList<BackupInfo> sessions = systemTable.getBackupContexts(BackupState.RUNNING);
+ if (sessions.size() == 0) {
+ return null;
+ }
+ return sessions.get(0).getBackupId();
+ }
+
+ /**
+ * Start the backup manager service.
+ * @throws IOException exception
+ */
+ public void initialize() throws IOException {
+ String ongoingBackupId = this.getOngoingBackupId();
+ if (ongoingBackupId != null) {
+ LOG.info("There is a ongoing backup " + ongoingBackupId
+ + ". Can not launch new backup until no ongoing backup remains.");
+ throw new BackupException("There is ongoing backup.");
+ }
+
+ // Initialize thread pools
+ int nrThreads = this.conf.getInt("hbase.backup.threads.max", 1);
+ ThreadFactoryBuilder builder = new ThreadFactoryBuilder();
+ builder.setNameFormat("BackupHandler-%1$d");
+ this.pool =
+ new ThreadPoolExecutor(nrThreads, nrThreads, 60, TimeUnit.SECONDS,
+ new LinkedBlockingQueue<Runnable>(), builder.build());
+ ((ThreadPoolExecutor) pool).allowCoreThreadTimeOut(true);
+ }
+
+ public void setBackupContext(BackupInfo backupContext) {
+ this.backupContext = backupContext;
+ }
+
+ /**
+ * Get direct ancestors of the current backup.
+ * @param backupCtx The backup context for the current backup
+ * @return The ancestors for the current backup
+ * @throws IOException exception
+ * @throws BackupException exception
+ */
+ public ArrayList<BackupImage> getAncestors(BackupInfo backupCtx) throws IOException,
+ BackupException {
+ LOG.debug("Getting the direct ancestors of the current backup "+
+ backupCtx.getBackupId());
+
+ ArrayList<BackupImage> ancestors = new ArrayList<BackupImage>();
+
+ // full backup does not have ancestor
+ if (backupCtx.getType() == BackupType.FULL) {
+ LOG.debug("Current backup is a full backup, no direct ancestor for it.");
+ return ancestors;
+ }
+
+ // get all backup history list in descending order
+
+ ArrayList<BackupInfo> allHistoryList = getBackupHistory(true);
+ for (BackupInfo backup : allHistoryList) {
+ BackupImage image =
+ new BackupImage(backup.getBackupId(), backup.getType(),
+ backup.getTargetRootDir(),
+ backup.getTableNames(), backup.getStartTs(), backup
+ .getEndTs());
+ // add the full backup image as an ancestor until the last incremental backup
+ if (backup.getType().equals(BackupType.FULL)) {
+ // check the backup image coverage, if previous image could be covered by the newer ones,
+ // then no need to add
+ if (!BackupManifest.canCoverImage(ancestors, image)) {
+ ancestors.add(image);
+ }
+ } else {
+ // found last incremental backup, if previously added full backup ancestor images can cover
+ // it, then this incremental ancestor is not the dependent of the current incremental
+ // backup, that is to say, this is the backup scope boundary of current table set.
+ // Otherwise, this incremental backup ancestor is the dependent ancestor of the ongoing
+ // incremental backup
+ if (BackupManifest.canCoverImage(ancestors, image)) {
+ LOG.debug("Met the backup boundary of the current table set. "
+ + "The root full backup images for the current backup scope:");
+ for (BackupImage image1 : ancestors) {
+ LOG.debug(" BackupId: " + image1.getBackupId() + ", Backup directory: "
+ + image1.getRootDir());
+ }
+ } else {
+ Path logBackupPath =
+ HBackupFileSystem.getLogBackupPath(backup.getTargetRootDir(),
+ backup.getBackupId());
+ LOG.debug("Current backup has an incremental backup ancestor, "
+ + "touching its image manifest in " + logBackupPath.toString()
+ + " to construct the dependency.");
+ BackupManifest lastIncrImgManifest = new BackupManifest(conf, logBackupPath);
+ BackupImage lastIncrImage = lastIncrImgManifest.getBackupImage();
+ ancestors.add(lastIncrImage);
+
+ LOG.debug("Last dependent incremental backup image information:");
+ LOG.debug(" Token: " + lastIncrImage.getBackupId());
+ LOG.debug(" Backup directory: " + lastIncrImage.getRootDir());
+ }
+ }
+ }
+ LOG.debug("Got " + ancestors.size() + " ancestors for the current backup.");
+ return ancestors;
+ }
+
+ /**
+ * Get the direct ancestors of this backup for one table involved.
+ * @param backupContext backup context
+ * @param table table
+ * @return backupImages on the dependency list
+ * @throws BackupException exception
+ * @throws IOException exception
+ */
+ public ArrayList<BackupImage> getAncestors(BackupInfo backupContext, TableName table)
+ throws BackupException, IOException {
+ ArrayList<BackupImage> ancestors = getAncestors(backupContext);
+ ArrayList<BackupImage> tableAncestors = new ArrayList<BackupImage>();
+ for (BackupImage image : ancestors) {
+ if (image.hasTable(table)) {
+ tableAncestors.add(image);
+ if (image.getType() == BackupType.FULL) {
+ break;
+ }
+ }
+ }
+ return tableAncestors;
+ }
+
+ /*
+ * hbase:backup operations
+ */
+
+ /**
+ * Updates status (state) of a backup session in a persistent store
+ * @param context context
+ * @throws IOException exception
+ */
+ public void updateBackupInfo(BackupInfo context) throws IOException {
+ systemTable.updateBackupInfo(context);
+ }
+
+ /**
+ * Read the last backup start code (timestamp) of last successful backup. Will return null
+ * if there is no startcode stored in hbase:backup or the value is of length 0. These two
+ * cases indicate there is no successful backup completed so far.
+ * @return the timestamp of a last successful backup
+ * @throws IOException exception
+ */
+ public String readBackupStartCode() throws IOException {
+ return systemTable.readBackupStartCode(backupContext.getTargetRootDir());
+ }
+
+ /**
+ * Write the start code (timestamp) to hbase:backup. If passed in null, then write 0 byte.
+ * @param startCode start code
+ * @throws IOException exception
+ */
+ public void writeBackupStartCode(Long startCode) throws IOException {
+ systemTable.writeBackupStartCode(startCode, backupContext.getTargetRootDir());
+ }
+
+ /**
+ * Get the RS log information after the last log roll from hbase:backup.
+ * @return RS log info
+ * @throws IOException exception
+ */
+ public HashMap<String, Long> readRegionServerLastLogRollResult() throws IOException {
+ return systemTable.readRegionServerLastLogRollResult(backupContext.getTargetRootDir());
+ }
+
+ /**
+ * Get all completed backup information (in desc order by time)
+ * @return history info of BackupCompleteData
+ * @throws IOException exception
+ */
+ public List<BackupInfo> getBackupHistory() throws IOException {
+ return systemTable.getBackupHistory();
+ }
+
+ public ArrayList<BackupInfo> getBackupHistory(boolean completed) throws IOException {
+ return systemTable.getBackupHistory(completed);
+ }
+ /**
+ * Write the current timestamps for each regionserver to hbase:backup after a successful full or
+ * incremental backup. Each table may have a different set of log timestamps. The saved timestamp
+ * is of the last log file that was backed up already.
+ * @param tables tables
+ * @throws IOException exception
+ */
+ public void writeRegionServerLogTimestamp(Set<TableName> tables,
+ HashMap<String, Long> newTimestamps) throws IOException {
+ systemTable.writeRegionServerLogTimestamp(tables, newTimestamps,
+ backupContext.getTargetRootDir());
+ }
+
+ /**
+ * Read the timestamp for each region server log after the last successful backup. Each table has
+ * its own set of the timestamps.
+ * @return the timestamp for each region server. key: tableName value:
+ * RegionServer,PreviousTimeStamp
+ * @throws IOException exception
+ */
+ public HashMap<TableName, HashMap<String, Long>> readLogTimestampMap() throws IOException {
+ return systemTable.readLogTimestampMap(backupContext.getTargetRootDir());
+ }
+
+ /**
+ * Return the current tables covered by incremental backup.
+ * @return set of tableNames
+ * @throws IOException exception
+ */
+ public Set<TableName> getIncrementalBackupTableSet() throws IOException {
+ return systemTable.getIncrementalBackupTableSet(backupContext.getTargetRootDir());
+ }
+
+ /**
+ * Adds set of tables to overall incremental backup table set
+ * @param tables tables
+ * @throws IOException exception
+ */
+ public void addIncrementalBackupTableSet(Set<TableName> tables) throws IOException {
+ systemTable.addIncrementalBackupTableSet(tables, backupContext.getTargetRootDir());
+ }
+
+ /**
+ * Saves list of WAL files after incremental backup operation. These files will be stored until
+ * TTL expiration and are used by Backup Log Cleaner plugin to determine which WAL files can be
+ * safely purged.
+ */
+ public void recordWALFiles(List<String> files) throws IOException {
+ systemTable.addWALFiles(files,
+ backupContext.getBackupId(), backupContext.getTargetRootDir());
+ }
+
+ /**
+ * Get WAL files iterator
+ * @return WAL files iterator from hbase:backup
+ * @throws IOException
+ */
+ public Iterator<BackupSystemTable.WALItem> getWALFilesFromBackupSystem() throws IOException {
+ return systemTable.getWALFilesIterator(backupContext.getTargetRootDir());
+ }
+
+ public Connection getConnection() {
+ return conn;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java
new file mode 100644
index 0000000..c3dc539
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java
@@ -0,0 +1,711 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup.impl;
+
+import java.io.IOException;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.HashMap;
+import java.util.List;
+import java.util.Map;
+import java.util.Map.Entry;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FSDataInputStream;
+import org.apache.hadoop.fs.FSDataOutputStream;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupInfo;
+import org.apache.hadoop.hbase.backup.BackupType;
+import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos;
+import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
+
+
+/**
+ * Backup manifest Contains all the meta data of a backup image. The manifest info will be bundled
+ * as manifest file together with data. So that each backup image will contain all the info needed
+ * for restore.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public class BackupManifest {
+
+ private static final Log LOG = LogFactory.getLog(BackupManifest.class);
+
+ // manifest file name
+ public static final String MANIFEST_FILE_NAME = ".backup.manifest";
+
+ // backup image, the dependency graph is made up by series of backup images
+
+ public static class BackupImage implements Comparable<BackupImage> {
+
+ private String backupId;
+ private BackupType type;
+ private String rootDir;
+ private List<TableName> tableList;
+ private long startTs;
+ private long completeTs;
+ private ArrayList<BackupImage> ancestors;
+ private HashMap<TableName, HashMap<String, Long>> incrTimeRanges;
+
+ public BackupImage() {
+ super();
+ }
+
+ public BackupImage(String backupId, BackupType type, String rootDir,
+ List<TableName> tableList, long startTs, long completeTs) {
+ this.backupId = backupId;
+ this.type = type;
+ this.rootDir = rootDir;
+ this.tableList = tableList;
+ this.startTs = startTs;
+ this.completeTs = completeTs;
+ }
+
+ static BackupImage fromProto(BackupProtos.BackupImage im) {
+ String backupId = im.getBackupId();
+ String rootDir = im.getRootDir();
+ long startTs = im.getStartTs();
+ long completeTs = im.getCompleteTs();
+ List<HBaseProtos.TableName> tableListList = im.getTableListList();
+ List<TableName> tableList = new ArrayList<TableName>();
+ for(HBaseProtos.TableName tn : tableListList) {
+ tableList.add(ProtobufUtil.toTableName(tn));
+ }
+
+ List<BackupProtos.BackupImage> ancestorList = im.getAncestorsList();
+
+ BackupType type =
+ im.getBackupType() == BackupProtos.BackupType.FULL ? BackupType.FULL:
+ BackupType.INCREMENTAL;
+
+ BackupImage image = new BackupImage(backupId, type, rootDir, tableList, startTs, completeTs);
+ for(BackupProtos.BackupImage img: ancestorList) {
+ image.addAncestor(fromProto(img));
+ }
+ image.setIncrTimeRanges(loadIncrementalTimestampMap(im));
+ return image;
+ }
+
+ BackupProtos.BackupImage toProto() {
+ BackupProtos.BackupImage.Builder builder = BackupProtos.BackupImage.newBuilder();
+ builder.setBackupId(backupId);
+ builder.setCompleteTs(completeTs);
+ builder.setStartTs(startTs);
+ builder.setRootDir(rootDir);
+ if (type == BackupType.FULL) {
+ builder.setBackupType(BackupProtos.BackupType.FULL);
+ } else{
+ builder.setBackupType(BackupProtos.BackupType.INCREMENTAL);
+ }
+
+ for (TableName name: tableList) {
+ builder.addTableList(ProtobufUtil.toProtoTableNameShaded(name));
+ }
+
+ if (ancestors != null){
+ for (BackupImage im: ancestors){
+ builder.addAncestors(im.toProto());
+ }
+ }
+
+ setIncrementalTimestampMap(builder);
+ return builder.build();
+ }
+
+
+ private static HashMap<TableName, HashMap<String, Long>>
+ loadIncrementalTimestampMap(BackupProtos.BackupImage proto) {
+ List<BackupProtos.TableServerTimestamp> list = proto.getTstMapList();
+
+ HashMap<TableName, HashMap<String, Long>> incrTimeRanges =
+ new HashMap<TableName, HashMap<String, Long>>();
+ if(list == null || list.size() == 0) return incrTimeRanges;
+ for(BackupProtos.TableServerTimestamp tst: list){
+ TableName tn = ProtobufUtil.toTableName(tst.getTable());
+ HashMap<String, Long> map = incrTimeRanges.get(tn);
+ if(map == null){
+ map = new HashMap<String, Long>();
+ incrTimeRanges.put(tn, map);
+ }
+ List<BackupProtos.ServerTimestamp> listSt = tst.getServerTimestampList();
+ for(BackupProtos.ServerTimestamp stm: listSt) {
+ ServerName sn = ProtobufUtil.toServerNameShaded(stm.getServer());
+ map.put(sn.getHostname() +":" + sn.getPort(), stm.getTimestamp());
+ }
+ }
+ return incrTimeRanges;
+ }
+
+
+ private void setIncrementalTimestampMap(BackupProtos.BackupImage.Builder builder) {
+ if (this.incrTimeRanges == null) {
+ return;
+ }
+ for (Entry<TableName, HashMap<String,Long>> entry: this.incrTimeRanges.entrySet()) {
+ TableName key = entry.getKey();
+ HashMap<String, Long> value = entry.getValue();
+ BackupProtos.TableServerTimestamp.Builder tstBuilder =
+ BackupProtos.TableServerTimestamp.newBuilder();
+ tstBuilder.setTable(ProtobufUtil.toProtoTableNameShaded(key));
+
+ for (Map.Entry<String, Long> entry2 : value.entrySet()) {
+ String s = entry2.getKey();
+ BackupProtos.ServerTimestamp.Builder stBuilder = BackupProtos.ServerTimestamp.newBuilder();
+ HBaseProtos.ServerName.Builder snBuilder = HBaseProtos.ServerName.newBuilder();
+ ServerName sn = ServerName.parseServerName(s);
+ snBuilder.setHostName(sn.getHostname());
+ snBuilder.setPort(sn.getPort());
+ stBuilder.setServer(snBuilder.build());
+ stBuilder.setTimestamp(entry2.getValue());
+ tstBuilder.addServerTimestamp(stBuilder.build());
+ }
+ builder.addTstMap(tstBuilder.build());
+ }
+ }
+
+ public String getBackupId() {
+ return backupId;
+ }
+
+ public void setBackupId(String backupId) {
+ this.backupId = backupId;
+ }
+
+ public BackupType getType() {
+ return type;
+ }
+
+ public void setType(BackupType type) {
+ this.type = type;
+ }
+
+ public String getRootDir() {
+ return rootDir;
+ }
+
+ public void setRootDir(String rootDir) {
+ this.rootDir = rootDir;
+ }
+
+ public List<TableName> getTableNames() {
+ return tableList;
+ }
+
+ public void setTableList(List<TableName> tableList) {
+ this.tableList = tableList;
+ }
+
+ public long getStartTs() {
+ return startTs;
+ }
+
+ public void setStartTs(long startTs) {
+ this.startTs = startTs;
+ }
+
+ public long getCompleteTs() {
+ return completeTs;
+ }
+
+ public void setCompleteTs(long completeTs) {
+ this.completeTs = completeTs;
+ }
+
+ public ArrayList<BackupImage> getAncestors() {
+ if (this.ancestors == null) {
+ this.ancestors = new ArrayList<BackupImage>();
+ }
+ return this.ancestors;
+ }
+
+ public void addAncestor(BackupImage backupImage) {
+ this.getAncestors().add(backupImage);
+ }
+
+ public boolean hasAncestor(String token) {
+ for (BackupImage image : this.getAncestors()) {
+ if (image.getBackupId().equals(token)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ public boolean hasTable(TableName table) {
+ for (TableName t : tableList) {
+ if (t.equals(table)) {
+ return true;
+ }
+ }
+ return false;
+ }
+
+ @Override
+ public int compareTo(BackupImage other) {
+ String thisBackupId = this.getBackupId();
+ String otherBackupId = other.getBackupId();
+ int index1 = thisBackupId.lastIndexOf("_");
+ int index2 = otherBackupId.lastIndexOf("_");
+ String name1 = thisBackupId.substring(0, index1);
+ String name2 = otherBackupId.substring(0, index2);
+ if(name1.equals(name2)) {
+ Long thisTS = Long.valueOf(thisBackupId.substring(index1 + 1));
+ Long otherTS = Long.valueOf(otherBackupId.substring(index2 + 1));
+ return thisTS.compareTo(otherTS);
+ } else {
+ return name1.compareTo(name2);
+ }
+ }
+ @Override
+ public boolean equals(Object obj) {
+ if (obj instanceof BackupImage) {
+ return this.compareTo((BackupImage)obj) == 0;
+ }
+ return false;
+ }
+ @Override
+ public int hashCode() {
+ int hash = 33 * this.getBackupId().hashCode() + type.hashCode();
+ hash = 33 * hash + rootDir.hashCode();
+ hash = 33 * hash + Long.valueOf(startTs).hashCode();
+ hash = 33 * hash + Long.valueOf(completeTs).hashCode();
+ for (TableName table : tableList) {
+ hash = 33 * hash + table.hashCode();
+ }
+ return hash;
+ }
+
+ public HashMap<TableName, HashMap<String, Long>> getIncrTimeRanges() {
+ return incrTimeRanges;
+ }
+
+ public void setIncrTimeRanges(HashMap<TableName, HashMap<String, Long>> incrTimeRanges) {
+ this.incrTimeRanges = incrTimeRanges;
+ }
+ }
+
+ // hadoop hbase configuration
+ protected Configuration config = null;
+
+ // backup root directory
+ private String rootDir = null;
+
+ // backup image directory
+ private String tableBackupDir = null;
+
+ // backup log directory if this is an incremental backup
+ private String logBackupDir = null;
+
+ // backup token
+ private String backupId;
+
+ // backup type, full or incremental
+ private BackupType type;
+
+ // the table list for the backup
+ private ArrayList<TableName> tableList;
+
+ // actual start timestamp of the backup process
+ private long startTs;
+
+ // actual complete timestamp of the backup process
+ private long completeTs;
+
+ // the region server timestamp for tables:
+ // <table, <rs, timestamp>>
+ private Map<TableName, HashMap<String, Long>> incrTimeRanges;
+
+ // dependency of this backup, including all the dependent images to do PIT recovery
+ //private Map<String, BackupImage> dependency;
+ private BackupImage backupImage;
+
+ /**
+ * Construct manifest for a ongoing backup.
+ * @param backupCtx The ongoing backup context
+ */
+ public BackupManifest(BackupInfo backupCtx) {
+ this.backupId = backupCtx.getBackupId();
+ this.type = backupCtx.getType();
+ this.rootDir = backupCtx.getTargetRootDir();
+ if (this.type == BackupType.INCREMENTAL) {
+ this.logBackupDir = backupCtx.getHLogTargetDir();
+ }
+ this.startTs = backupCtx.getStartTs();
+ this.completeTs = backupCtx.getEndTs();
+ this.loadTableList(backupCtx.getTableNames());
+ this.backupImage = new BackupImage(this.backupId, this.type, this.rootDir, tableList, this.startTs,
+ this.completeTs);
+ }
+
+
+ /**
+ * Construct a table level manifest for a backup of the named table.
+ * @param backupCtx The ongoing backup context
+ */
+ public BackupManifest(BackupInfo backupCtx, TableName table) {
+ this.backupId = backupCtx.getBackupId();
+ this.type = backupCtx.getType();
+ this.rootDir = backupCtx.getTargetRootDir();
+ this.tableBackupDir = backupCtx.getBackupStatus(table).getTargetDir();
+ if (this.type == BackupType.INCREMENTAL) {
+ this.logBackupDir = backupCtx.getHLogTargetDir();
+ }
+ this.startTs = backupCtx.getStartTs();
+ this.completeTs = backupCtx.getEndTs();
+ List<TableName> tables = new ArrayList<TableName>();
+ tables.add(table);
+ this.loadTableList(tables);
+ this.backupImage = new BackupImage(this.backupId, this.type, this.rootDir, tableList, this.startTs,
+ this.completeTs);
+ }
+
+ /**
+ * Construct manifest from a backup directory.
+ * @param conf configuration
+ * @param backupPath backup path
+ * @throws IOException
+ */
+
+ public BackupManifest(Configuration conf, Path backupPath) throws IOException {
+ this(backupPath.getFileSystem(conf), backupPath);
+ }
+
+ /**
+ * Construct manifest from a backup directory.
+ * @param fs the FileSystem
+ * @param backupPath backup path
+ * @throws BackupException exception
+ */
+
+ public BackupManifest(FileSystem fs, Path backupPath) throws BackupException {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Loading manifest from: " + backupPath.toString());
+ }
+ // The input backupDir may not exactly be the backup table dir.
+ // It could be the backup log dir where there is also a manifest file stored.
+ // This variable's purpose is to keep the correct and original location so
+ // that we can store/persist it.
+ this.tableBackupDir = backupPath.toString();
+ this.config = fs.getConf();
+ try {
+
+ FileStatus[] subFiles = BackupClientUtil.listStatus(fs, backupPath, null);
+ if (subFiles == null) {
+ String errorMsg = backupPath.toString() + " does not exist";
+ LOG.error(errorMsg);
+ throw new IOException(errorMsg);
+ }
+ for (FileStatus subFile : subFiles) {
+ if (subFile.getPath().getName().equals(MANIFEST_FILE_NAME)) {
+
+ // load and set manifest field from file content
+ FSDataInputStream in = fs.open(subFile.getPath());
+ long len = subFile.getLen();
+ byte[] pbBytes = new byte[(int) len];
+ in.readFully(pbBytes);
+ BackupProtos.BackupImage proto = null;
+ try{
+ proto = BackupProtos.BackupImage.parseFrom(pbBytes);
+ } catch(Exception e){
+ throw new BackupException(e);
+ }
+ this.backupImage = BackupImage.fromProto(proto);
+ // Here the parameter backupDir is where the manifest file is.
+ // There should always be a manifest file under:
+ // backupRootDir/namespace/table/backupId/.backup.manifest
+ this.rootDir = backupPath.getParent().getParent().getParent().toString();
+
+ Path p = backupPath.getParent();
+ if (p.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
+ this.rootDir = p.getParent().toString();
+ } else {
+ this.rootDir = p.getParent().getParent().toString();
+ }
+ this.backupId = this.backupImage.getBackupId();
+ this.startTs = this.backupImage.getStartTs();
+ this.completeTs = this.backupImage.getCompleteTs();
+ this.type = this.backupImage.getType();
+ this.tableList = (ArrayList<TableName>)this.backupImage.getTableNames();
+ this.incrTimeRanges = this.backupImage.getIncrTimeRanges();
+ LOG.debug("Loaded manifest instance from manifest file: "
+ + BackupClientUtil.getPath(subFile.getPath()));
+ return;
+ }
+ }
+ String errorMsg = "No manifest file found in: " + backupPath.toString();
+ throw new IOException(errorMsg);
+
+ } catch (IOException e) {
+ throw new BackupException(e.getMessage());
+ }
+ }
+
+ public BackupType getType() {
+ return type;
+ }
+
+ public void setType(BackupType type) {
+ this.type = type;
+ }
+
+ /**
+ * Loads table list.
+ * @param tableList Table list
+ */
+ private void loadTableList(List<TableName> tableList) {
+
+ this.tableList = this.getTableList();
+ if (this.tableList.size() > 0) {
+ this.tableList.clear();
+ }
+ for (int i = 0; i < tableList.size(); i++) {
+ this.tableList.add(tableList.get(i));
+ }
+
+ LOG.debug(tableList.size() + " tables exist in table set.");
+ }
+
+ /**
+ * Get the table set of this image.
+ * @return The table set list
+ */
+ public ArrayList<TableName> getTableList() {
+ if (this.tableList == null) {
+ this.tableList = new ArrayList<TableName>();
+ }
+ return this.tableList;
+ }
+
+ /**
+ * Persist the manifest file.
+ * @throws IOException IOException when storing the manifest file.
+ */
+
+ public void store(Configuration conf) throws BackupException {
+ byte[] data = backupImage.toProto().toByteArray();
+ // write the file, overwrite if already exist
+ Path manifestFilePath =
+ new Path(new Path((this.tableBackupDir != null ? this.tableBackupDir : this.logBackupDir))
+ ,MANIFEST_FILE_NAME);
+ try {
+ FSDataOutputStream out =
+ manifestFilePath.getFileSystem(conf).create(manifestFilePath, true);
+ out.write(data);
+ out.close();
+ } catch (IOException e) {
+ throw new BackupException(e.getMessage());
+ }
+
+ LOG.info("Manifest file stored to " + manifestFilePath);
+ }
+
+
+ /**
+ * Get this backup image.
+ * @return the backup image.
+ */
+ public BackupImage getBackupImage() {
+ return backupImage;
+ }
+
+ /**
+ * Add dependent backup image for this backup.
+ * @param image The direct dependent backup image
+ */
+ public void addDependentImage(BackupImage image) {
+ this.backupImage.addAncestor(image);
+ }
+
+ /**
+ * Set the incremental timestamp map directly.
+ * @param incrTimestampMap timestamp map
+ */
+ public void setIncrTimestampMap(HashMap<TableName, HashMap<String, Long>> incrTimestampMap) {
+ this.incrTimeRanges = incrTimestampMap;
+ this.backupImage.setIncrTimeRanges(incrTimestampMap);
+ }
+
+ public Map<TableName, HashMap<String, Long>> getIncrTimestampMap() {
+ if (this.incrTimeRanges == null) {
+ this.incrTimeRanges = new HashMap<TableName, HashMap<String, Long>>();
+ }
+ return this.incrTimeRanges;
+ }
+
+ /**
+ * Get the image list of this backup for restore in time order.
+ * @param reverse If true, then output in reverse order, otherwise in time order from old to new
+ * @return the backup image list for restore in time order
+ */
+ public ArrayList<BackupImage> getRestoreDependentList(boolean reverse) {
+ TreeMap<Long, BackupImage> restoreImages = new TreeMap<Long, BackupImage>();
+ restoreImages.put(backupImage.startTs, backupImage);
+ for (BackupImage image : backupImage.getAncestors()) {
+ restoreImages.put(Long.valueOf(image.startTs), image);
+ }
+ return new ArrayList<BackupImage>(reverse ? (restoreImages.descendingMap().values())
+ : (restoreImages.values()));
+ }
+
+ /**
+ * Get the dependent image list for a specific table of this backup in time order from old to new
+ * if want to restore to this backup image level.
+ * @param table table
+ * @return the backup image list for a table in time order
+ */
+ public ArrayList<BackupImage> getDependentListByTable(TableName table) {
+ ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>();
+ ArrayList<BackupImage> imageList = getRestoreDependentList(true);
+ for (BackupImage image : imageList) {
+ if (image.hasTable(table)) {
+ tableImageList.add(image);
+ if (image.getType() == BackupType.FULL) {
+ break;
+ }
+ }
+ }
+ Collections.reverse(tableImageList);
+ return tableImageList;
+ }
+
+ /**
+ * Get the full dependent image list in the whole dependency scope for a specific table of this
+ * backup in time order from old to new.
+ * @param table table
+ * @return the full backup image list for a table in time order in the whole scope of the
+ * dependency of this image
+ */
+ public ArrayList<BackupImage> getAllDependentListByTable(TableName table) {
+ ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>();
+ ArrayList<BackupImage> imageList = getRestoreDependentList(false);
+ for (BackupImage image : imageList) {
+ if (image.hasTable(table)) {
+ tableImageList.add(image);
+ }
+ }
+ return tableImageList;
+ }
+
+ /**
+ * Check whether backup image1 could cover backup image2 or not.
+ * @param image1 backup image 1
+ * @param image2 backup image 2
+ * @return true if image1 can cover image2, otherwise false
+ */
+ public static boolean canCoverImage(BackupImage image1, BackupImage image2) {
+ // image1 can cover image2 only when the following conditions are satisfied:
+ // - image1 must not be an incremental image;
+ // - image1 must be taken after image2 has been taken;
+ // - table set of image1 must cover the table set of image2.
+ if (image1.getType() == BackupType.INCREMENTAL) {
+ return false;
+ }
+ if (image1.getStartTs() < image2.getStartTs()) {
+ return false;
+ }
+ List<TableName> image1TableList = image1.getTableNames();
+ List<TableName> image2TableList = image2.getTableNames();
+ boolean found = false;
+ for (int i = 0; i < image2TableList.size(); i++) {
+ found = false;
+ for (int j = 0; j < image1TableList.size(); j++) {
+ if (image2TableList.get(i).equals(image1TableList.get(j))) {
+ found = true;
+ break;
+ }
+ }
+ if (!found) {
+ return false;
+ }
+ }
+
+ LOG.debug("Backup image " + image1.getBackupId() + " can cover " + image2.getBackupId());
+ return true;
+ }
+
+ /**
+ * Check whether backup image set could cover a backup image or not.
+ * @param fullImages The backup image set
+ * @param image The target backup image
+ * @return true if fullImages can cover image, otherwise false
+ */
+ public static boolean canCoverImage(ArrayList<BackupImage> fullImages, BackupImage image) {
+ // fullImages can cover image only when the following conditions are satisfied:
+ // - each image of fullImages must not be an incremental image;
+ // - each image of fullImages must be taken after image has been taken;
+ // - sum table set of fullImages must cover the table set of image.
+ for (BackupImage image1 : fullImages) {
+ if (image1.getType() == BackupType.INCREMENTAL) {
+ return false;
+ }
+ if (image1.getStartTs() < image.getStartTs()) {
+ return false;
+ }
+ }
+
+ ArrayList<String> image1TableList = new ArrayList<String>();
+ for (BackupImage image1 : fullImages) {
+ List<TableName> tableList = image1.getTableNames();
+ for (TableName table : tableList) {
+ image1TableList.add(table.getNameAsString());
+ }
+ }
+ ArrayList<String> image2TableList = new ArrayList<String>();
+ List<TableName> tableList = image.getTableNames();
+ for (TableName table : tableList) {
+ image2TableList.add(table.getNameAsString());
+ }
+
+ for (int i = 0; i < image2TableList.size(); i++) {
+ if (image1TableList.contains(image2TableList.get(i)) == false) {
+ return false;
+ }
+ }
+
+ LOG.debug("Full image set can cover image " + image.getBackupId());
+ return true;
+ }
+
+ public BackupInfo toBackupInfo()
+ {
+ BackupInfo info = new BackupInfo();
+ info.setType(type);
+ TableName[] tables = new TableName[tableList.size()];
+ info.addTables(getTableList().toArray(tables));
+ info.setBackupId(backupId);
+ info.setStartTs(startTs);
+ info.setTargetRootDir(rootDir);
+ if(type == BackupType.INCREMENTAL) {
+ info.setHlogTargetDir(logBackupDir);
+ }
+ return info;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/2725fb25/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSnapshotCopy.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSnapshotCopy.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSnapshotCopy.java
new file mode 100644
index 0000000..731ccd3
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupSnapshotCopy.java
@@ -0,0 +1,36 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup.impl;
+
+import org.apache.hadoop.hbase.snapshot.ExportSnapshot;
+
+/* this class will be extended in future jira to support progress report */
+public class BackupSnapshotCopy extends ExportSnapshot {
+ private String table;
+
+ public BackupSnapshotCopy(String table) {
+ super();
+ this.table = table;
+ }
+
+ public String getTable() {
+ return this.table;
+ }
+
+}