You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by st...@apache.org on 2017/12/02 01:22:51 UTC
[08/10] hbase git commit: HBASE-19407 [branch-2] Remove backup/restore
http://git-wip-us.apache.org/repos/asf/hbase/blob/79ac70ac/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
deleted file mode 100644
index 102d45d..0000000
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
+++ /dev/null
@@ -1,1019 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.impl;
-
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_BACKUP_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_DEBUG_DESC;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.List;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Options;
-import org.apache.commons.lang3.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupAdmin;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
-import org.apache.hadoop.hbase.backup.BackupRequest;
-import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
-import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.util.BackupSet;
-import org.apache.hadoop.hbase.backup.util.BackupUtils;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.yetus.audience.InterfaceAudience;
-
-/**
- * General backup commands, options and usage messages
- */
-
-@InterfaceAudience.Private
-public final class BackupCommands {
-
- public final static String INCORRECT_USAGE = "Incorrect usage";
-
- public final static String TOP_LEVEL_NOT_ALLOWED =
- "Top level (root) folder is not allowed to be a backup destination";
-
- public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n"
- + "where COMMAND is one of:\n" + " create create a new backup image\n"
- + " delete delete an existing backup image\n"
- + " describe show the detailed information of a backup image\n"
- + " history show history of all successful backups\n"
- + " progress show the progress of the latest backup request\n"
- + " set backup set management\n"
- + " repair repair backup system table\n"
- + " merge merge backup images\n"
- + "Run \'hbase backup COMMAND -h\' to see help message for each command\n";
-
- public static final String CREATE_CMD_USAGE =
- "Usage: hbase backup create <type> <backup_path> [options]\n"
- + " type \"full\" to create a full backup image\n"
- + " \"incremental\" to create an incremental backup image\n"
- + " backup_path Full path to store the backup image\n";
-
- public static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress <backup_id>\n"
- + " backup_id Backup image id (optional). If no id specified, the command will show\n"
- + " progress for currently running backup session.";
- public static final String NO_INFO_FOUND = "No info was found for backup id: ";
- public static final String NO_ACTIVE_SESSION_FOUND = "No active backup sessions found.";
-
- public static final String DESCRIBE_CMD_USAGE = "Usage: hbase backup describe <backup_id>\n"
- + " backup_id Backup image id\n";
-
- public static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [options]";
-
- public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete <backup_id>\n"
- + " backup_id Backup image id\n";
-
- public static final String REPAIR_CMD_USAGE = "Usage: hbase backup repair\n";
-
- public static final String SET_CMD_USAGE = "Usage: hbase backup set COMMAND [name] [tables]\n"
- + " name Backup set name\n"
- + " tables Comma separated list of tables.\n" + "COMMAND is one of:\n"
- + " add add tables to a set, create a set if needed\n"
- + " remove remove tables from a set\n"
- + " list list all backup sets in the system\n"
- + " describe describe set\n" + " delete delete backup set\n";
- public static final String MERGE_CMD_USAGE = "Usage: hbase backup merge [backup_ids]\n"
- + " backup_ids Comma separated list of backup image ids.\n";
-
- public static final String USAGE_FOOTER = "";
-
- public static abstract class Command extends Configured {
- CommandLine cmdline;
- Connection conn;
-
- Command(Configuration conf) {
- if (conf == null) {
- conf = HBaseConfiguration.create();
- }
- setConf(conf);
- }
-
- public void execute() throws IOException {
- if (cmdline.hasOption("h") || cmdline.hasOption("help")) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- // Create connection
- conn = ConnectionFactory.createConnection(getConf());
- if (requiresNoActiveSession()) {
- // Check active session
- try (BackupSystemTable table = new BackupSystemTable(conn);) {
- List<BackupInfo> sessions = table.getBackupInfos(BackupState.RUNNING);
-
- if (sessions.size() > 0) {
- System.err.println("Found backup session in a RUNNING state: ");
- System.err.println(sessions.get(0));
- System.err.println("This may indicate that a previous session has failed abnormally.");
- System.err.println("In this case, backup recovery is recommended.");
- throw new IOException("Active session found, aborted command execution");
- }
- }
- }
- if (requiresConsistentState()) {
- // Check failed delete
- try (BackupSystemTable table = new BackupSystemTable(conn);) {
- String[] ids = table.getListOfBackupIdsFromDeleteOperation();
-
- if (ids != null && ids.length > 0) {
- System.err.println("Found failed backup DELETE coommand. ");
- System.err.println("Backup system recovery is required.");
- throw new IOException("Failed backup DELETE found, aborted command execution");
- }
-
- ids = table.getListOfBackupIdsFromMergeOperation();
- if (ids != null && ids.length > 0) {
- System.err.println("Found failed backup MERGE coommand. ");
- System.err.println("Backup system recovery is required.");
- throw new IOException("Failed backup MERGE found, aborted command execution");
- }
-
- }
- }
- }
-
- public void finish() throws IOException {
- if (conn != null) {
- conn.close();
- }
- }
-
- protected abstract void printUsage();
-
- /**
- * The command can't be run if active backup session is in progress
- * @return true if no active sessions are in progress
- */
- protected boolean requiresNoActiveSession() {
- return false;
- }
-
- /**
- * Command requires consistent state of a backup system Backup system may become inconsistent
- * because of an abnormal termination of a backup session or delete command
- * @return true, if yes
- */
- protected boolean requiresConsistentState() {
- return false;
- }
- }
-
- private BackupCommands() {
- throw new AssertionError("Instantiating utility class...");
- }
-
- public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) {
- Command cmd = null;
- switch (type) {
- case CREATE:
- cmd = new CreateCommand(conf, cmdline);
- break;
- case DESCRIBE:
- cmd = new DescribeCommand(conf, cmdline);
- break;
- case PROGRESS:
- cmd = new ProgressCommand(conf, cmdline);
- break;
- case DELETE:
- cmd = new DeleteCommand(conf, cmdline);
- break;
- case HISTORY:
- cmd = new HistoryCommand(conf, cmdline);
- break;
- case SET:
- cmd = new BackupSetCommand(conf, cmdline);
- break;
- case REPAIR:
- cmd = new RepairCommand(conf, cmdline);
- break;
- case MERGE:
- cmd = new MergeCommand(conf, cmdline);
- break;
- case HELP:
- default:
- cmd = new HelpCommand(conf, cmdline);
- break;
- }
- return cmd;
- }
-
- static int numOfArgs(String[] args) {
- if (args == null) return 0;
- return args.length;
- }
-
- public static class CreateCommand extends Command {
-
- CreateCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- protected boolean requiresNoActiveSession() {
- return true;
- }
-
- @Override
- protected boolean requiresConsistentState() {
- return true;
- }
-
- @Override
- public void execute() throws IOException {
- if (cmdline == null || cmdline.getArgs() == null) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- String[] args = cmdline.getArgs();
- if (args.length != 3) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- if (!BackupType.FULL.toString().equalsIgnoreCase(args[1])
- && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])) {
- System.out.println("ERROR: invalid backup type: " + args[1]);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- if (!verifyPath(args[2])) {
- System.out.println("ERROR: invalid backup destination: " + args[2]);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- String targetBackupDir = args[2];
- // Check if backup destination is top level (root) folder - not allowed
- if (isRootFolder(targetBackupDir)) {
- throw new IOException(TOP_LEVEL_NOT_ALLOWED);
- }
- String tables = null;
-
- // Check if we have both: backup set and list of tables
- if (cmdline.hasOption(OPTION_TABLE) && cmdline.hasOption(OPTION_SET)) {
- System.out.println("ERROR: You can specify either backup set or list"
- + " of tables, but not both");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- // Creates connection
- super.execute();
- // Check backup set
- String setName = null;
- if (cmdline.hasOption(OPTION_SET)) {
- setName = cmdline.getOptionValue(OPTION_SET);
- tables = getTablesForSet(setName, getConf());
-
- if (tables == null) {
- System.out.println("ERROR: Backup set '" + setName
- + "' is either empty or does not exist");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- } else {
- tables = cmdline.getOptionValue(OPTION_TABLE);
- }
- int bandwidth =
- cmdline.hasOption(OPTION_BANDWIDTH) ? Integer.parseInt(cmdline
- .getOptionValue(OPTION_BANDWIDTH)) : -1;
- int workers =
- cmdline.hasOption(OPTION_WORKERS) ? Integer.parseInt(cmdline
- .getOptionValue(OPTION_WORKERS)) : -1;
-
- if (cmdline.hasOption(OPTION_YARN_QUEUE_NAME)) {
- String queueName = cmdline.getOptionValue(OPTION_YARN_QUEUE_NAME);
- // Set system property value for MR job
- System.setProperty("mapreduce.job.queuename", queueName);
- }
-
- try (BackupAdminImpl admin = new BackupAdminImpl(conn);) {
-
- BackupRequest.Builder builder = new BackupRequest.Builder();
- BackupRequest request =
- builder
- .withBackupType(BackupType.valueOf(args[1].toUpperCase()))
- .withTableList(
- tables != null ? Lists.newArrayList(BackupUtils.parseTableNames(tables)) : null)
- .withTargetRootDir(targetBackupDir).withTotalTasks(workers)
- .withBandwidthPerTasks(bandwidth).withBackupSetName(setName).build();
- String backupId = admin.backupTables(request);
- System.out.println("Backup session " + backupId + " finished. Status: SUCCESS");
- } catch (IOException e) {
- System.out.println("Backup session finished. Status: FAILURE");
- throw e;
- }
- }
-
- private boolean isRootFolder(String targetBackupDir) {
- Path p = new Path(targetBackupDir);
- return p.isRoot();
- }
-
- private boolean verifyPath(String path) {
- try {
- Path p = new Path(path);
- Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
- URI uri = p.toUri();
- if (uri.getScheme() == null) return false;
- FileSystem.get(uri, conf);
- return true;
- } catch (Exception e) {
- return false;
- }
- }
-
- private String getTablesForSet(String name, Configuration conf) throws IOException {
- try (final BackupSystemTable table = new BackupSystemTable(conn)) {
- List<TableName> tables = table.describeBackupSet(name);
- if (tables == null) return null;
- return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(CREATE_CMD_USAGE);
- Options options = new Options();
- options.addOption(OPTION_WORKERS, true, OPTION_WORKERS_DESC);
- options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC);
- options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC);
- options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC);
- options.addOption(OPTION_YARN_QUEUE_NAME, true, OPTION_YARN_QUEUE_NAME_DESC);
- options.addOption(OPTION_DEBUG, false, OPTION_DEBUG_DESC);
-
- HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.setLeftPadding(2);
- helpFormatter.setDescPadding(8);
- helpFormatter.setWidth(100);
- helpFormatter.setSyntaxPrefix("Options:");
- helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
-
- }
- }
-
- private static class HelpCommand extends Command {
-
- HelpCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- if (cmdline == null) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String[] args = cmdline.getArgs();
- if (args == null || args.length == 0) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- if (args.length != 2) {
- System.out.println("ERROR: Only supports help message of a single command type");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String type = args[1];
-
- if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) {
- System.out.println(CREATE_CMD_USAGE);
- } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) {
- System.out.println(DESCRIBE_CMD_USAGE);
- } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) {
- System.out.println(HISTORY_CMD_USAGE);
- } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) {
- System.out.println(PROGRESS_CMD_USAGE);
- } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) {
- System.out.println(DELETE_CMD_USAGE);
- } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) {
- System.out.println(SET_CMD_USAGE);
- } else {
- System.out.println("Unknown command : " + type);
- printUsage();
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(USAGE);
- }
- }
-
- private static class DescribeCommand extends Command {
-
- DescribeCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- if (cmdline == null || cmdline.getArgs() == null) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- String[] args = cmdline.getArgs();
- if (args.length != 2) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- super.execute();
-
- String backupId = args[1];
- try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) {
- BackupInfo info = sysTable.readBackupInfo(backupId);
- if (info == null) {
- System.out.println("ERROR: " + backupId + " does not exist");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- System.out.println(info.getShortDescription());
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(DESCRIBE_CMD_USAGE);
- }
- }
-
- private static class ProgressCommand extends Command {
-
- ProgressCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
-
- if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length == 1) {
- System.out.println("No backup id was specified, "
- + "will retrieve the most recent (ongoing) session");
- }
- String[] args = cmdline == null ? null : cmdline.getArgs();
- if (args != null && args.length > 2) {
- System.err.println("ERROR: wrong number of arguments: " + args.length);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- super.execute();
-
- String backupId = (args == null || args.length <= 1) ? null : args[1];
- try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) {
- BackupInfo info = null;
-
- if (backupId != null) {
- info = sysTable.readBackupInfo(backupId);
- } else {
- List<BackupInfo> infos = sysTable.getBackupInfos(BackupState.RUNNING);
- if (infos != null && infos.size() > 0) {
- info = infos.get(0);
- backupId = info.getBackupId();
- System.out.println("Found ongoing session with backupId=" + backupId);
- } else {
- }
- }
- int progress = info == null ? -1 : info.getProgress();
- if (progress < 0) {
- if (backupId != null) {
- System.out.println(NO_INFO_FOUND + backupId);
- } else {
- System.err.println(NO_ACTIVE_SESSION_FOUND);
- }
- } else {
- System.out.println(backupId + " progress=" + progress + "%");
- }
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(PROGRESS_CMD_USAGE);
- }
- }
-
- private static class DeleteCommand extends Command {
-
- DeleteCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- protected boolean requiresNoActiveSession() {
- return true;
- }
-
- @Override
- public void execute() throws IOException {
- if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- super.execute();
-
- String[] args = cmdline.getArgs();
- String[] backupIds = new String[args.length - 1];
- System.arraycopy(args, 1, backupIds, 0, backupIds.length);
- try (BackupAdminImpl admin = new BackupAdminImpl(conn);) {
- int deleted = admin.deleteBackups(backupIds);
- System.out.println("Deleted " + deleted + " backups. Total requested: " + (args.length -1));
- } catch (IOException e) {
- System.err
- .println("Delete command FAILED. Please run backup repair tool to restore backup system integrity");
- throw e;
- }
-
- }
-
- @Override
- protected void printUsage() {
- System.out.println(DELETE_CMD_USAGE);
- }
- }
-
- private static class RepairCommand extends Command {
-
- RepairCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- super.execute();
-
- String[] args = cmdline == null ? null : cmdline.getArgs();
- if (args != null && args.length > 1) {
- System.err.println("ERROR: wrong number of arguments: " + args.length);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
- try (final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupSystemTable sysTable = new BackupSystemTable(conn);) {
-
- // Failed backup
- BackupInfo backupInfo;
- List<BackupInfo> list = sysTable.getBackupInfos(BackupState.RUNNING);
- if (list.size() == 0) {
- // No failed sessions found
- System.out.println("REPAIR status: no failed sessions found."
- + " Checking failed delete backup operation ...");
- repairFailedBackupDeletionIfAny(conn, sysTable);
- repairFailedBackupMergeIfAny(conn, sysTable);
- return;
- }
- backupInfo = list.get(0);
- // If this is a cancel exception, then we've already cleaned.
- // set the failure timestamp of the overall backup
- backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime());
- // set failure message
- backupInfo.setFailedMsg("REPAIR status: repaired after failure:\n" + backupInfo);
- // set overall backup status: failed
- backupInfo.setState(BackupState.FAILED);
- // compose the backup failed data
- String backupFailedData =
- "BackupId=" + backupInfo.getBackupId() + ",startts=" + backupInfo.getStartTs()
- + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase="
- + backupInfo.getPhase() + ",failedmessage=" + backupInfo.getFailedMsg();
- System.out.println(backupFailedData);
- TableBackupClient.cleanupAndRestoreBackupSystem(conn, backupInfo, conf);
- // If backup session is updated to FAILED state - means we
- // processed recovery already.
- sysTable.updateBackupInfo(backupInfo);
- sysTable.finishBackupExclusiveOperation();
- System.out.println("REPAIR status: finished repair failed session:\n " + backupInfo);
-
- }
- }
-
- private void repairFailedBackupDeletionIfAny(Connection conn, BackupSystemTable sysTable)
- throws IOException {
- String[] backupIds = sysTable.getListOfBackupIdsFromDeleteOperation();
- if (backupIds == null || backupIds.length == 0) {
- System.out.println("No failed backup DELETE operation found");
- // Delete backup table snapshot if exists
- BackupSystemTable.deleteSnapshot(conn);
- return;
- }
- System.out.println("Found failed DELETE operation for: " + StringUtils.join(backupIds));
- System.out.println("Running DELETE again ...");
- // Restore table from snapshot
- BackupSystemTable.restoreFromSnapshot(conn);
- // Finish previous failed session
- sysTable.finishBackupExclusiveOperation();
- try (BackupAdmin admin = new BackupAdminImpl(conn);) {
- admin.deleteBackups(backupIds);
- }
- System.out.println("DELETE operation finished OK: " + StringUtils.join(backupIds));
-
- }
-
- private void repairFailedBackupMergeIfAny(Connection conn, BackupSystemTable sysTable)
- throws IOException {
- String[] backupIds = sysTable.getListOfBackupIdsFromMergeOperation();
- if (backupIds == null || backupIds.length == 0) {
- System.out.println("No failed backup MERGE operation found");
- // Delete backup table snapshot if exists
- BackupSystemTable.deleteSnapshot(conn);
- return;
- }
- System.out.println("Found failed MERGE operation for: " + StringUtils.join(backupIds));
- System.out.println("Running MERGE again ...");
- // Restore table from snapshot
- BackupSystemTable.restoreFromSnapshot(conn);
- // Unlock backupo system
- sysTable.finishBackupExclusiveOperation();
- // Finish previous failed session
- sysTable.finishMergeOperation();
- try (BackupAdmin admin = new BackupAdminImpl(conn);) {
- admin.mergeBackups(backupIds);
- }
- System.out.println("MERGE operation finished OK: " + StringUtils.join(backupIds));
-
- }
-
- @Override
- protected void printUsage() {
- System.out.println(REPAIR_CMD_USAGE);
- }
- }
-
- private static class MergeCommand extends Command {
-
- MergeCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- protected boolean requiresNoActiveSession() {
- return true;
- }
-
- @Override
- protected boolean requiresConsistentState() {
- return true;
- }
-
- @Override
- public void execute() throws IOException {
- super.execute();
-
- String[] args = cmdline == null ? null : cmdline.getArgs();
- if (args == null || (args.length != 2)) {
- System.err.println("ERROR: wrong number of arguments: "
- + (args == null ? null : args.length));
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String[] backupIds = args[1].split(",");
- if (backupIds.length < 2) {
- String msg = "ERROR: can not merge a single backup image. "+
- "Number of images must be greater than 1.";
- System.err.println(msg);
- throw new IOException(msg);
-
- }
- Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
- try (final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupAdminImpl admin = new BackupAdminImpl(conn);) {
- admin.mergeBackups(backupIds);
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(MERGE_CMD_USAGE);
- }
- }
-
- private static class HistoryCommand extends Command {
-
- private final static int DEFAULT_HISTORY_LENGTH = 10;
-
- HistoryCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
-
- int n = parseHistoryLength();
- final TableName tableName = getTableName();
- final String setName = getTableSetName();
- BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() {
- @Override
- public boolean apply(BackupInfo info) {
- if (tableName == null) return true;
- List<TableName> names = info.getTableNames();
- return names.contains(tableName);
- }
- };
- BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() {
- @Override
- public boolean apply(BackupInfo info) {
- if (setName == null) return true;
- String backupId = info.getBackupId();
- return backupId.startsWith(setName);
- }
- };
- Path backupRootPath = getBackupRootPath();
- List<BackupInfo> history = null;
- if (backupRootPath == null) {
- // Load from backup system table
- super.execute();
- try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) {
- history = sysTable.getBackupHistory(n, tableNameFilter, tableSetFilter);
- }
- } else {
- // load from backup FS
- history =
- BackupUtils.getHistory(getConf(), n, backupRootPath, tableNameFilter, tableSetFilter);
- }
- for (BackupInfo info : history) {
- System.out.println(info.getShortDescription());
- }
- }
-
- private Path getBackupRootPath() throws IOException {
- String value = null;
- try {
- value = cmdline.getOptionValue(OPTION_PATH);
- if (value == null) return null;
- return new Path(value);
- } catch (IllegalArgumentException e) {
- System.out.println("ERROR: Illegal argument for backup root path: " + value);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- }
-
- private TableName getTableName() throws IOException {
- String value = cmdline.getOptionValue(OPTION_TABLE);
- if (value == null) return null;
- try {
- return TableName.valueOf(value);
- } catch (IllegalArgumentException e) {
- System.out.println("Illegal argument for table name: " + value);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- }
-
- private String getTableSetName() throws IOException {
- String value = cmdline.getOptionValue(OPTION_SET);
- return value;
- }
-
- private int parseHistoryLength() throws IOException {
- String value = cmdline.getOptionValue(OPTION_RECORD_NUMBER);
- try {
- if (value == null) return DEFAULT_HISTORY_LENGTH;
- return Integer.parseInt(value);
- } catch (NumberFormatException e) {
- System.out.println("Illegal argument for history length: " + value);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(HISTORY_CMD_USAGE);
- Options options = new Options();
- options.addOption(OPTION_RECORD_NUMBER, true, OPTION_RECORD_NUMBER_DESC);
- options.addOption(OPTION_PATH, true, OPTION_PATH_DESC);
- options.addOption(OPTION_TABLE, true, OPTION_TABLE_DESC);
- options.addOption(OPTION_SET, true, OPTION_SET_DESC);
-
- HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.setLeftPadding(2);
- helpFormatter.setDescPadding(8);
- helpFormatter.setWidth(100);
- helpFormatter.setSyntaxPrefix("Options:");
- helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
- }
- }
-
- private static class BackupSetCommand extends Command {
- private final static String SET_ADD_CMD = "add";
- private final static String SET_REMOVE_CMD = "remove";
- private final static String SET_DELETE_CMD = "delete";
- private final static String SET_DESCRIBE_CMD = "describe";
- private final static String SET_LIST_CMD = "list";
-
- BackupSetCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- // Command-line must have at least one element
- if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String[] args = cmdline.getArgs();
- String cmdStr = args[1];
- BackupCommand cmd = getCommand(cmdStr);
-
- switch (cmd) {
- case SET_ADD:
- processSetAdd(args);
- break;
- case SET_REMOVE:
- processSetRemove(args);
- break;
- case SET_DELETE:
- processSetDelete(args);
- break;
- case SET_DESCRIBE:
- processSetDescribe(args);
- break;
- case SET_LIST:
- processSetList(args);
- break;
- default:
- break;
-
- }
- }
-
- private void processSetList(String[] args) throws IOException {
- super.execute();
-
- // List all backup set names
- // does not expect any args
- try (BackupAdminImpl admin = new BackupAdminImpl(conn);) {
- List<BackupSet> list = admin.listBackupSets();
- for (BackupSet bs : list) {
- System.out.println(bs);
- }
- }
- }
-
- private void processSetDescribe(String[] args) throws IOException {
- if (args == null || args.length != 3) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- super.execute();
-
- String setName = args[2];
- try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) {
- List<TableName> tables = sysTable.describeBackupSet(setName);
- BackupSet set = tables == null ? null : new BackupSet(setName, tables);
- if (set == null) {
- System.out.println("Set '" + setName + "' does not exist.");
- } else {
- System.out.println(set);
- }
- }
- }
-
- private void processSetDelete(String[] args) throws IOException {
- if (args == null || args.length != 3) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- super.execute();
-
- String setName = args[2];
- try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) {
- boolean result = admin.deleteBackupSet(setName);
- if (result) {
- System.out.println("Delete set " + setName + " OK.");
- } else {
- System.out.println("Set " + setName + " does not exist");
- }
- }
- }
-
- private void processSetRemove(String[] args) throws IOException {
- if (args == null || args.length != 4) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- super.execute();
-
- String setName = args[2];
- String[] tables = args[3].split(",");
- TableName[] tableNames = toTableNames(tables);
- try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) {
- admin.removeFromBackupSet(setName, tableNames);
- }
- }
-
- private TableName[] toTableNames(String[] tables) {
- TableName[] arr = new TableName[tables.length];
- for (int i = 0; i < tables.length; i++) {
- arr[i] = TableName.valueOf(tables[i]);
- }
- return arr;
- }
-
- private void processSetAdd(String[] args) throws IOException {
- if (args == null || args.length != 4) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- super.execute();
-
- String setName = args[2];
- String[] tables = args[3].split(",");
- TableName[] tableNames = new TableName[tables.length];
- for (int i = 0; i < tables.length; i++) {
- tableNames[i] = TableName.valueOf(tables[i]);
- }
- try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) {
- admin.addToBackupSet(setName, tableNames);
- }
-
- }
-
- private BackupCommand getCommand(String cmdStr) throws IOException {
- if (cmdStr.equals(SET_ADD_CMD)) {
- return BackupCommand.SET_ADD;
- } else if (cmdStr.equals(SET_REMOVE_CMD)) {
- return BackupCommand.SET_REMOVE;
- } else if (cmdStr.equals(SET_DELETE_CMD)) {
- return BackupCommand.SET_DELETE;
- } else if (cmdStr.equals(SET_DESCRIBE_CMD)) {
- return BackupCommand.SET_DESCRIBE;
- } else if (cmdStr.equals(SET_LIST_CMD)) {
- return BackupCommand.SET_LIST;
- } else {
- System.out.println("ERROR: Unknown command for 'set' :" + cmdStr);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(SET_CMD_USAGE);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/79ac70ac/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java
deleted file mode 100644
index 0147c29..0000000
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.impl;
-
-import org.apache.hadoop.hbase.HBaseIOException;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.yetus.audience.InterfaceAudience;
-
-/**
- * Backup exception
- */
-@SuppressWarnings("serial")
-@InterfaceAudience.Private
-public class BackupException extends HBaseIOException {
- private BackupInfo info;
-
- /**
- * Some exception happened for a backup and don't even know the backup that it was about
- * @param msg Full description of the failure
- */
- public BackupException(String msg) {
- super(msg);
- }
-
- /**
- * Some exception happened for a backup with a cause
- * @param cause the cause
- */
- public BackupException(Throwable cause) {
- super(cause);
- }
-
- /**
- * Exception for the given backup that has no previous root cause
- * @param msg reason why the backup failed
- * @param desc description of the backup that is being failed
- */
- public BackupException(String msg, BackupInfo desc) {
- super(msg);
- this.info = desc;
- }
-
- /**
- * Exception for the given backup due to another exception
- * @param msg reason why the backup failed
- * @param cause root cause of the failure
- * @param desc description of the backup that is being failed
- */
- public BackupException(String msg, Throwable cause, BackupInfo desc) {
- super(msg, cause);
- this.info = desc;
- }
-
- /**
- * Exception when the description of the backup cannot be determined, due to some other root
- * cause
- * @param message description of what caused the failure
- * @param e root cause
- */
- public BackupException(String message, Exception e) {
- super(message, e);
- }
-
- public BackupInfo getBackupInfo() {
- return this.info;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/79ac70ac/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java
deleted file mode 100644
index 7ac94d8..0000000
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java
+++ /dev/null
@@ -1,502 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.impl;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
-import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.HBackupFileSystem;
-import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
-import org.apache.hadoop.hbase.backup.master.BackupLogCleaner;
-import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
-import org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.procedure.ProcedureManagerHost;
-import org.apache.hadoop.hbase.util.Pair;
-
-import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
-
-/**
- * Handles backup requests, creates backup info records in backup system table to
- * keep track of backup sessions, dispatches backup request.
- */
-@InterfaceAudience.Private
-public class BackupManager implements Closeable {
- private static final Log LOG = LogFactory.getLog(BackupManager.class);
-
- protected Configuration conf = null;
- protected BackupInfo backupInfo = null;
- protected BackupSystemTable systemTable;
- protected final Connection conn;
-
- /**
- * Backup manager constructor.
- * @param conn connection
- * @param conf configuration
- * @throws IOException exception
- */
- public BackupManager(Connection conn, Configuration conf) throws IOException {
- if (!conf.getBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY,
- BackupRestoreConstants.BACKUP_ENABLE_DEFAULT)) {
- throw new BackupException("HBase backup is not enabled. Check your "
- + BackupRestoreConstants.BACKUP_ENABLE_KEY + " setting.");
- }
- this.conf = conf;
- this.conn = conn;
- this.systemTable = new BackupSystemTable(conn);
-
- }
-
- /**
- * Returns backup info
- */
- protected BackupInfo getBackupInfo() {
- return backupInfo;
- }
-
- /**
- * This method modifies the master's configuration in order to inject backup-related features
- * (TESTs only)
- * @param conf configuration
- */
- @VisibleForTesting
- public static void decorateMasterConfiguration(Configuration conf) {
- if (!isBackupEnabled(conf)) {
- return;
- }
- // Add WAL archive cleaner plug-in
- String plugins = conf.get(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
- String cleanerClass = BackupLogCleaner.class.getCanonicalName();
- if (!plugins.contains(cleanerClass)) {
- conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, plugins + "," + cleanerClass);
- }
-
- String classes = conf.get(ProcedureManagerHost.MASTER_PROCEDURE_CONF_KEY);
- String masterProcedureClass = LogRollMasterProcedureManager.class.getName();
- if (classes == null) {
- conf.set(ProcedureManagerHost.MASTER_PROCEDURE_CONF_KEY, masterProcedureClass);
- } else if (!classes.contains(masterProcedureClass)) {
- conf.set(ProcedureManagerHost.MASTER_PROCEDURE_CONF_KEY, classes + "," + masterProcedureClass);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Added log cleaner: " + cleanerClass + "\n" + "Added master procedure manager: "
- + masterProcedureClass);
- }
-
- }
-
- /**
- * This method modifies the Region Server configuration in order to inject backup-related features
- * TESTs only.
- * @param conf configuration
- */
- @VisibleForTesting
- public static void decorateRegionServerConfiguration(Configuration conf) {
- if (!isBackupEnabled(conf)) {
- return;
- }
-
- String classes = conf.get(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY);
- String regionProcedureClass = LogRollRegionServerProcedureManager.class.getName();
- if (classes == null) {
- conf.set(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY, regionProcedureClass);
- } else if (!classes.contains(regionProcedureClass)) {
- conf.set(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY, classes + ","
- + regionProcedureClass);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Added region procedure manager: " + regionProcedureClass);
- }
-
- }
-
- public static boolean isBackupEnabled(Configuration conf) {
- return conf.getBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY,
- BackupRestoreConstants.BACKUP_ENABLE_DEFAULT);
- }
-
- /**
- * Get configuration
- * @return configuration
- */
- Configuration getConf() {
- return conf;
- }
-
- /**
- * Stop all the work of backup.
- */
- @Override
- public void close() {
-
- if (systemTable != null) {
- try {
- systemTable.close();
- } catch (Exception e) {
- LOG.error(e);
- }
- }
- }
-
- /**
- * Creates a backup info based on input backup request.
- * @param backupId backup id
- * @param type type
- * @param tableList table list
- * @param targetRootDir root dir
- * @param workers number of parallel workers
- * @param bandwidth bandwidth per worker in MB per sec
- * @return BackupInfo
- * @throws BackupException exception
- */
- public BackupInfo createBackupInfo(String backupId, BackupType type, List<TableName> tableList,
- String targetRootDir, int workers, long bandwidth) throws BackupException {
- if (targetRootDir == null) {
- throw new BackupException("Wrong backup request parameter: target backup root directory");
- }
-
- if (type == BackupType.FULL && (tableList == null || tableList.isEmpty())) {
- // If table list is null for full backup, which means backup all tables. Then fill the table
- // list with all user tables from meta. It no table available, throw the request exception.
-
- HTableDescriptor[] htds = null;
- try (Admin admin = conn.getAdmin()) {
- htds = admin.listTables();
- } catch (Exception e) {
- throw new BackupException(e);
- }
-
- if (htds == null) {
- throw new BackupException("No table exists for full backup of all tables.");
- } else {
- tableList = new ArrayList<>();
- for (HTableDescriptor hTableDescriptor : htds) {
- TableName tn = hTableDescriptor.getTableName();
- if (tn.equals(BackupSystemTable.getTableName(conf))) {
- // skip backup system table
- continue;
- }
- tableList.add(hTableDescriptor.getTableName());
- }
-
- LOG.info("Full backup all the tables available in the cluster: " + tableList);
- }
- }
-
- // there are one or more tables in the table list
- backupInfo =
- new BackupInfo(backupId, type, tableList.toArray(new TableName[tableList.size()]),
- targetRootDir);
- backupInfo.setBandwidth(bandwidth);
- backupInfo.setWorkers(workers);
- return backupInfo;
- }
-
- /**
- * Check if any ongoing backup. Currently, we only reply on checking status in backup system
- * table. We need to consider to handle the case of orphan records in the future. Otherwise, all
- * the coming request will fail.
- * @return the ongoing backup id if on going backup exists, otherwise null
- * @throws IOException exception
- */
- private String getOngoingBackupId() throws IOException {
-
- ArrayList<BackupInfo> sessions = systemTable.getBackupInfos(BackupState.RUNNING);
- if (sessions.size() == 0) {
- return null;
- }
- return sessions.get(0).getBackupId();
- }
-
- /**
- * Start the backup manager service.
- * @throws IOException exception
- */
- public void initialize() throws IOException {
- String ongoingBackupId = this.getOngoingBackupId();
- if (ongoingBackupId != null) {
- LOG.info("There is a ongoing backup " + ongoingBackupId
- + ". Can not launch new backup until no ongoing backup remains.");
- throw new BackupException("There is ongoing backup.");
- }
- }
-
- public void setBackupInfo(BackupInfo backupInfo) {
- this.backupInfo = backupInfo;
- }
-
- /**
- * Get direct ancestors of the current backup.
- * @param backupInfo The backup info for the current backup
- * @return The ancestors for the current backup
- * @throws IOException exception
- * @throws BackupException exception
- */
- public ArrayList<BackupImage> getAncestors(BackupInfo backupInfo) throws IOException,
- BackupException {
- LOG.debug("Getting the direct ancestors of the current backup " + backupInfo.getBackupId());
-
- ArrayList<BackupImage> ancestors = new ArrayList<BackupImage>();
-
- // full backup does not have ancestor
- if (backupInfo.getType() == BackupType.FULL) {
- LOG.debug("Current backup is a full backup, no direct ancestor for it.");
- return ancestors;
- }
-
- // get all backup history list in descending order
-
- ArrayList<BackupInfo> allHistoryList = getBackupHistory(true);
- for (BackupInfo backup : allHistoryList) {
-
- BackupImage.Builder builder = BackupImage.newBuilder();
-
- BackupImage image =
- builder.withBackupId(backup.getBackupId()).withType(backup.getType())
- .withRootDir(backup.getBackupRootDir()).withTableList(backup.getTableNames())
- .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build();
-
- // add the full backup image as an ancestor until the last incremental backup
- if (backup.getType().equals(BackupType.FULL)) {
- // check the backup image coverage, if previous image could be covered by the newer ones,
- // then no need to add
- if (!BackupManifest.canCoverImage(ancestors, image)) {
- ancestors.add(image);
- }
- } else {
- // found last incremental backup, if previously added full backup ancestor images can cover
- // it, then this incremental ancestor is not the dependent of the current incremental
- // backup, that is to say, this is the backup scope boundary of current table set.
- // Otherwise, this incremental backup ancestor is the dependent ancestor of the ongoing
- // incremental backup
- if (BackupManifest.canCoverImage(ancestors, image)) {
- LOG.debug("Met the backup boundary of the current table set:");
- for (BackupImage image1 : ancestors) {
- LOG.debug(" BackupID=" + image1.getBackupId() + ", BackupDir=" + image1.getRootDir());
- }
- } else {
- Path logBackupPath =
- HBackupFileSystem.getBackupPath(backup.getBackupRootDir(), backup.getBackupId());
- LOG.debug("Current backup has an incremental backup ancestor, "
- + "touching its image manifest in " + logBackupPath.toString()
- + " to construct the dependency.");
- BackupManifest lastIncrImgManifest = new BackupManifest(conf, logBackupPath);
- BackupImage lastIncrImage = lastIncrImgManifest.getBackupImage();
- ancestors.add(lastIncrImage);
-
- LOG.debug("Last dependent incremental backup image: " + "{BackupID="
- + lastIncrImage.getBackupId() + "," + "BackupDir=" + lastIncrImage.getRootDir() + "}");
- }
- }
- }
- LOG.debug("Got " + ancestors.size() + " ancestors for the current backup.");
- return ancestors;
- }
-
- /**
- * Get the direct ancestors of this backup for one table involved.
- * @param backupInfo backup info
- * @param table table
- * @return backupImages on the dependency list
- * @throws BackupException exception
- * @throws IOException exception
- */
- public ArrayList<BackupImage> getAncestors(BackupInfo backupInfo, TableName table)
- throws BackupException, IOException {
- ArrayList<BackupImage> ancestors = getAncestors(backupInfo);
- ArrayList<BackupImage> tableAncestors = new ArrayList<BackupImage>();
- for (BackupImage image : ancestors) {
- if (image.hasTable(table)) {
- tableAncestors.add(image);
- if (image.getType() == BackupType.FULL) {
- break;
- }
- }
- }
- return tableAncestors;
- }
-
- /*
- * backup system table operations
- */
-
- /**
- * Updates status (state) of a backup session in a persistent store
- * @param context context
- * @throws IOException exception
- */
- public void updateBackupInfo(BackupInfo context) throws IOException {
- systemTable.updateBackupInfo(context);
- }
-
- /**
- * Starts new backup session
- * @throws IOException if active session already exists
- */
- public void startBackupSession() throws IOException {
- systemTable.startBackupExclusiveOperation();
- }
-
- /**
- * Finishes active backup session
- * @throws IOException if no active session
- */
- public void finishBackupSession() throws IOException {
- systemTable.finishBackupExclusiveOperation();
- }
-
- /**
- * Read the last backup start code (timestamp) of last successful backup. Will return null if
- * there is no startcode stored in backup system table or the value is of length 0. These two
- * cases indicate there is no successful backup completed so far.
- * @return the timestamp of a last successful backup
- * @throws IOException exception
- */
- public String readBackupStartCode() throws IOException {
- return systemTable.readBackupStartCode(backupInfo.getBackupRootDir());
- }
-
- /**
- * Write the start code (timestamp) to backup system table. If passed in null, then write 0 byte.
- * @param startCode start code
- * @throws IOException exception
- */
- public void writeBackupStartCode(Long startCode) throws IOException {
- systemTable.writeBackupStartCode(startCode, backupInfo.getBackupRootDir());
- }
-
- /**
- * Get the RS log information after the last log roll from backup system table.
- * @return RS log info
- * @throws IOException exception
- */
- public HashMap<String, Long> readRegionServerLastLogRollResult() throws IOException {
- return systemTable.readRegionServerLastLogRollResult(backupInfo.getBackupRootDir());
- }
-
- public Pair<Map<TableName, Map<String, Map<String, List<Pair<String, Boolean>>>>>, List<byte[]>>
- readBulkloadRows(List<TableName> tableList) throws IOException {
- return systemTable.readBulkloadRows(tableList);
- }
-
- public void removeBulkLoadedRows(List<TableName> lst, List<byte[]> rows) throws IOException {
- systemTable.removeBulkLoadedRows(lst, rows);
- }
-
- public void writeBulkLoadedFiles(List<TableName> sTableList, Map<byte[], List<Path>>[] maps)
- throws IOException {
- systemTable.writeBulkLoadedFiles(sTableList, maps, backupInfo.getBackupId());
- }
-
- /**
- * Get all completed backup information (in desc order by time)
- * @return history info of BackupCompleteData
- * @throws IOException exception
- */
- public List<BackupInfo> getBackupHistory() throws IOException {
- return systemTable.getBackupHistory();
- }
-
- public ArrayList<BackupInfo> getBackupHistory(boolean completed) throws IOException {
- return systemTable.getBackupHistory(completed);
- }
-
- /**
- * Write the current timestamps for each regionserver to backup system table after a successful
- * full or incremental backup. Each table may have a different set of log timestamps. The saved
- * timestamp is of the last log file that was backed up already.
- * @param tables tables
- * @throws IOException exception
- */
- public void writeRegionServerLogTimestamp(Set<TableName> tables,
- HashMap<String, Long> newTimestamps) throws IOException {
- systemTable.writeRegionServerLogTimestamp(tables, newTimestamps, backupInfo.getBackupRootDir());
- }
-
- /**
- * Read the timestamp for each region server log after the last successful backup. Each table has
- * its own set of the timestamps.
- * @return the timestamp for each region server. key: tableName value:
- * RegionServer,PreviousTimeStamp
- * @throws IOException exception
- */
- public HashMap<TableName, HashMap<String, Long>> readLogTimestampMap() throws IOException {
- return systemTable.readLogTimestampMap(backupInfo.getBackupRootDir());
- }
-
- /**
- * Return the current tables covered by incremental backup.
- * @return set of tableNames
- * @throws IOException exception
- */
- public Set<TableName> getIncrementalBackupTableSet() throws IOException {
- return systemTable.getIncrementalBackupTableSet(backupInfo.getBackupRootDir());
- }
-
- /**
- * Adds set of tables to overall incremental backup table set
- * @param tables tables
- * @throws IOException exception
- */
- public void addIncrementalBackupTableSet(Set<TableName> tables) throws IOException {
- systemTable.addIncrementalBackupTableSet(tables, backupInfo.getBackupRootDir());
- }
-
- /**
- * Saves list of WAL files after incremental backup operation. These files will be stored until
- * TTL expiration and are used by Backup Log Cleaner plug-in to determine which WAL files can be
- * safely purged.
- */
- public void recordWALFiles(List<String> files) throws IOException {
- systemTable.addWALFiles(files, backupInfo.getBackupId(), backupInfo.getBackupRootDir());
- }
-
- /**
- * Get WAL files iterator
- * @return WAL files iterator from backup system table
- * @throws IOException
- */
- public Iterator<BackupSystemTable.WALItem> getWALFilesFromBackupSystem() throws IOException {
- return systemTable.getWALFilesIterator(backupInfo.getBackupRootDir());
- }
-
- public Connection getConnection() {
- return conn;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/79ac70ac/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java
deleted file mode 100644
index 4e6f6ff..0000000
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java
+++ /dev/null
@@ -1,674 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.impl;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TreeMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.ServerName;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.HBackupFileSystem;
-import org.apache.hadoop.hbase.backup.util.BackupUtils;
-import org.apache.yetus.audience.InterfaceAudience;
-import org.apache.hadoop.hbase.shaded.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.BackupProtos;
-import org.apache.hadoop.hbase.shaded.protobuf.generated.HBaseProtos;
-
-/**
- * Backup manifest contains all the meta data of a backup image. The manifest info will be bundled
- * as manifest file together with data. So that each backup image will contain all the info needed
- * for restore. BackupManifest is a storage container for BackupImage.
- * It is responsible for storing/reading backup image data and has some additional utility methods.
- *
- */
-@InterfaceAudience.Private
-public class BackupManifest {
-
- private static final Log LOG = LogFactory.getLog(BackupManifest.class);
-
- // manifest file name
- public static final String MANIFEST_FILE_NAME = ".backup.manifest";
-
- /**
- * Backup image, the dependency graph is made up by series of backup images BackupImage contains
- * all the relevant information to restore the backup and is used during restore operation
- */
-
- public static class BackupImage implements Comparable<BackupImage> {
-
- static class Builder {
- BackupImage image;
-
- Builder() {
- image = new BackupImage();
- }
-
- Builder withBackupId(String backupId) {
- image.setBackupId(backupId);
- return this;
- }
-
- Builder withType(BackupType type) {
- image.setType(type);
- return this;
- }
-
- Builder withRootDir(String rootDir) {
- image.setRootDir(rootDir);
- return this;
- }
-
- Builder withTableList(List<TableName> tableList) {
- image.setTableList(tableList);
- return this;
- }
-
- Builder withStartTime(long startTime) {
- image.setStartTs(startTime);
- return this;
- }
-
- Builder withCompleteTime(long completeTime) {
- image.setCompleteTs(completeTime);
- return this;
- }
-
- BackupImage build() {
- return image;
- }
-
- }
-
- private String backupId;
- private BackupType type;
- private String rootDir;
- private List<TableName> tableList;
- private long startTs;
- private long completeTs;
- private ArrayList<BackupImage> ancestors;
- private HashMap<TableName, HashMap<String, Long>> incrTimeRanges;
-
- static Builder newBuilder() {
- return new Builder();
- }
-
- public BackupImage() {
- super();
- }
-
- private BackupImage(String backupId, BackupType type, String rootDir,
- List<TableName> tableList, long startTs, long completeTs) {
- this.backupId = backupId;
- this.type = type;
- this.rootDir = rootDir;
- this.tableList = tableList;
- this.startTs = startTs;
- this.completeTs = completeTs;
- }
-
- static BackupImage fromProto(BackupProtos.BackupImage im) {
- String backupId = im.getBackupId();
- String rootDir = im.getBackupRootDir();
- long startTs = im.getStartTs();
- long completeTs = im.getCompleteTs();
- List<HBaseProtos.TableName> tableListList = im.getTableListList();
- List<TableName> tableList = new ArrayList<TableName>();
- for (HBaseProtos.TableName tn : tableListList) {
- tableList.add(ProtobufUtil.toTableName(tn));
- }
-
- List<BackupProtos.BackupImage> ancestorList = im.getAncestorsList();
-
- BackupType type =
- im.getBackupType() == BackupProtos.BackupType.FULL ? BackupType.FULL
- : BackupType.INCREMENTAL;
-
- BackupImage image = new BackupImage(backupId, type, rootDir, tableList, startTs, completeTs);
- for (BackupProtos.BackupImage img : ancestorList) {
- image.addAncestor(fromProto(img));
- }
- image.setIncrTimeRanges(loadIncrementalTimestampMap(im));
- return image;
- }
-
- BackupProtos.BackupImage toProto() {
- BackupProtos.BackupImage.Builder builder = BackupProtos.BackupImage.newBuilder();
- builder.setBackupId(backupId);
- builder.setCompleteTs(completeTs);
- builder.setStartTs(startTs);
- builder.setBackupRootDir(rootDir);
- if (type == BackupType.FULL) {
- builder.setBackupType(BackupProtos.BackupType.FULL);
- } else {
- builder.setBackupType(BackupProtos.BackupType.INCREMENTAL);
- }
-
- for (TableName name : tableList) {
- builder.addTableList(ProtobufUtil.toProtoTableName(name));
- }
-
- if (ancestors != null) {
- for (BackupImage im : ancestors) {
- builder.addAncestors(im.toProto());
- }
- }
-
- setIncrementalTimestampMap(builder);
- return builder.build();
- }
-
- private static HashMap<TableName, HashMap<String, Long>> loadIncrementalTimestampMap(
- BackupProtos.BackupImage proto) {
- List<BackupProtos.TableServerTimestamp> list = proto.getTstMapList();
-
- HashMap<TableName, HashMap<String, Long>> incrTimeRanges =
- new HashMap<TableName, HashMap<String, Long>>();
- if (list == null || list.size() == 0) return incrTimeRanges;
- for (BackupProtos.TableServerTimestamp tst : list) {
- TableName tn = ProtobufUtil.toTableName(tst.getTableName());
- HashMap<String, Long> map = incrTimeRanges.get(tn);
- if (map == null) {
- map = new HashMap<String, Long>();
- incrTimeRanges.put(tn, map);
- }
- List<BackupProtos.ServerTimestamp> listSt = tst.getServerTimestampList();
- for (BackupProtos.ServerTimestamp stm : listSt) {
- ServerName sn = ProtobufUtil.toServerName(stm.getServerName());
- map.put(sn.getHostname() + ":" + sn.getPort(), stm.getTimestamp());
- }
- }
- return incrTimeRanges;
- }
-
- private void setIncrementalTimestampMap(BackupProtos.BackupImage.Builder builder) {
- if (this.incrTimeRanges == null) {
- return;
- }
- for (Entry<TableName, HashMap<String, Long>> entry : this.incrTimeRanges.entrySet()) {
- TableName key = entry.getKey();
- HashMap<String, Long> value = entry.getValue();
- BackupProtos.TableServerTimestamp.Builder tstBuilder =
- BackupProtos.TableServerTimestamp.newBuilder();
- tstBuilder.setTableName(ProtobufUtil.toProtoTableName(key));
-
- for (Map.Entry<String, Long> entry2 : value.entrySet()) {
- String s = entry2.getKey();
- BackupProtos.ServerTimestamp.Builder stBuilder =
- BackupProtos.ServerTimestamp.newBuilder();
- HBaseProtos.ServerName.Builder snBuilder = HBaseProtos.ServerName.newBuilder();
- ServerName sn = ServerName.parseServerName(s);
- snBuilder.setHostName(sn.getHostname());
- snBuilder.setPort(sn.getPort());
- stBuilder.setServerName(snBuilder.build());
- stBuilder.setTimestamp(entry2.getValue());
- tstBuilder.addServerTimestamp(stBuilder.build());
- }
- builder.addTstMap(tstBuilder.build());
- }
- }
-
- public String getBackupId() {
- return backupId;
- }
-
- private void setBackupId(String backupId) {
- this.backupId = backupId;
- }
-
- public BackupType getType() {
- return type;
- }
-
- private void setType(BackupType type) {
- this.type = type;
- }
-
- public String getRootDir() {
- return rootDir;
- }
-
- private void setRootDir(String rootDir) {
- this.rootDir = rootDir;
- }
-
- public List<TableName> getTableNames() {
- return tableList;
- }
-
- private void setTableList(List<TableName> tableList) {
- this.tableList = tableList;
- }
-
- public long getStartTs() {
- return startTs;
- }
-
- private void setStartTs(long startTs) {
- this.startTs = startTs;
- }
-
- public long getCompleteTs() {
- return completeTs;
- }
-
- private void setCompleteTs(long completeTs) {
- this.completeTs = completeTs;
- }
-
- public ArrayList<BackupImage> getAncestors() {
- if (this.ancestors == null) {
- this.ancestors = new ArrayList<BackupImage>();
- }
- return this.ancestors;
- }
-
- public void removeAncestors(List<String> backupIds) {
- List<BackupImage> toRemove = new ArrayList<BackupImage>();
- for (BackupImage im : this.ancestors) {
- if (backupIds.contains(im.getBackupId())) {
- toRemove.add(im);
- }
- }
- this.ancestors.removeAll(toRemove);
- }
-
- private void addAncestor(BackupImage backupImage) {
- this.getAncestors().add(backupImage);
- }
-
- public boolean hasAncestor(String token) {
- for (BackupImage image : this.getAncestors()) {
- if (image.getBackupId().equals(token)) {
- return true;
- }
- }
- return false;
- }
-
- public boolean hasTable(TableName table) {
- return tableList.contains(table);
- }
-
- @Override
- public int compareTo(BackupImage other) {
- String thisBackupId = this.getBackupId();
- String otherBackupId = other.getBackupId();
- int index1 = thisBackupId.lastIndexOf("_");
- int index2 = otherBackupId.lastIndexOf("_");
- String name1 = thisBackupId.substring(0, index1);
- String name2 = otherBackupId.substring(0, index2);
- if (name1.equals(name2)) {
- Long thisTS = Long.valueOf(thisBackupId.substring(index1 + 1));
- Long otherTS = Long.valueOf(otherBackupId.substring(index2 + 1));
- return thisTS.compareTo(otherTS);
- } else {
- return name1.compareTo(name2);
- }
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof BackupImage) {
- return this.compareTo((BackupImage) obj) == 0;
- }
- return false;
- }
-
- @Override
- public int hashCode() {
- int hash = 33 * this.getBackupId().hashCode() + type.hashCode();
- hash = 33 * hash + rootDir.hashCode();
- hash = 33 * hash + Long.valueOf(startTs).hashCode();
- hash = 33 * hash + Long.valueOf(completeTs).hashCode();
- for (TableName table : tableList) {
- hash = 33 * hash + table.hashCode();
- }
- return hash;
- }
-
- public HashMap<TableName, HashMap<String, Long>> getIncrTimeRanges() {
- return incrTimeRanges;
- }
-
- private void setIncrTimeRanges(HashMap<TableName, HashMap<String, Long>> incrTimeRanges) {
- this.incrTimeRanges = incrTimeRanges;
- }
- }
-
- // backup image directory
- private String tableBackupDir = null;
- private BackupImage backupImage;
-
- /**
- * Construct manifest for a ongoing backup.
- * @param backup The ongoing backup info
- */
- public BackupManifest(BackupInfo backup) {
-
- BackupImage.Builder builder = BackupImage.newBuilder();
- this.backupImage =
- builder.withBackupId(backup.getBackupId()).withType(backup.getType())
- .withRootDir(backup.getBackupRootDir()).withTableList(backup.getTableNames())
- .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build();
- }
-
- /**
- * Construct a table level manifest for a backup of the named table.
- * @param backup The ongoing backup session info
- */
- public BackupManifest(BackupInfo backup, TableName table) {
- this.tableBackupDir = backup.getTableBackupDir(table);
- List<TableName> tables = new ArrayList<TableName>();
- tables.add(table);
- BackupImage.Builder builder = BackupImage.newBuilder();
- this.backupImage =
- builder.withBackupId(backup.getBackupId()).withType(backup.getType())
- .withRootDir(backup.getBackupRootDir()).withTableList(tables)
- .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build();
- }
-
- /**
- * Construct manifest from a backup directory.
- * @param conf configuration
- * @param backupPath backup path
- * @throws IOException
- */
-
- public BackupManifest(Configuration conf, Path backupPath) throws IOException {
- this(backupPath.getFileSystem(conf), backupPath);
- }
-
- /**
- * Construct manifest from a backup directory.
- * @param fs the FileSystem
- * @param backupPath backup path
- * @throws BackupException exception
- */
-
- public BackupManifest(FileSystem fs, Path backupPath) throws BackupException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Loading manifest from: " + backupPath.toString());
- }
- // The input backupDir may not exactly be the backup table dir.
- // It could be the backup log dir where there is also a manifest file stored.
- // This variable's purpose is to keep the correct and original location so
- // that we can store/persist it.
- try {
-
- FileStatus[] subFiles = BackupUtils.listStatus(fs, backupPath, null);
- if (subFiles == null) {
- String errorMsg = backupPath.toString() + " does not exist";
- LOG.error(errorMsg);
- throw new IOException(errorMsg);
- }
- for (FileStatus subFile : subFiles) {
- if (subFile.getPath().getName().equals(MANIFEST_FILE_NAME)) {
-
- // load and set manifest field from file content
- FSDataInputStream in = fs.open(subFile.getPath());
- long len = subFile.getLen();
- byte[] pbBytes = new byte[(int) len];
- in.readFully(pbBytes);
- BackupProtos.BackupImage proto = null;
- try {
- proto = BackupProtos.BackupImage.parseFrom(pbBytes);
- } catch (Exception e) {
- throw new BackupException(e);
- }
- this.backupImage = BackupImage.fromProto(proto);
- LOG.debug("Loaded manifest instance from manifest file: "
- + BackupUtils.getPath(subFile.getPath()));
- return;
- }
- }
- String errorMsg = "No manifest file found in: " + backupPath.toString();
- throw new IOException(errorMsg);
-
- } catch (IOException e) {
- throw new BackupException(e.getMessage());
- }
- }
-
- public BackupType getType() {
- return backupImage.getType();
- }
-
- /**
- * Get the table set of this image.
- * @return The table set list
- */
- public List<TableName> getTableList() {
- return backupImage.getTableNames();
- }
-
- /**
- * TODO: fix it. Persist the manifest file.
- * @throws IOException IOException when storing the manifest file.
- */
-
- public void store(Configuration conf) throws BackupException {
- byte[] data = backupImage.toProto().toByteArray();
- // write the file, overwrite if already exist
- Path manifestFilePath =
- new Path(HBackupFileSystem.getBackupPath(backupImage.getRootDir(),
- backupImage.getBackupId()), MANIFEST_FILE_NAME);
- try (FSDataOutputStream out =
- manifestFilePath.getFileSystem(conf).create(manifestFilePath, true);) {
- out.write(data);
- } catch (IOException e) {
- throw new BackupException(e.getMessage());
- }
-
- LOG.info("Manifest file stored to " + manifestFilePath);
- }
-
- /**
- * Get this backup image.
- * @return the backup image.
- */
- public BackupImage getBackupImage() {
- return backupImage;
- }
-
- /**
- * Add dependent backup image for this backup.
- * @param image The direct dependent backup image
- */
- public void addDependentImage(BackupImage image) {
- this.backupImage.addAncestor(image);
- }
-
- /**
- * Set the incremental timestamp map directly.
- * @param incrTimestampMap timestamp map
- */
- public void setIncrTimestampMap(HashMap<TableName, HashMap<String, Long>> incrTimestampMap) {
- this.backupImage.setIncrTimeRanges(incrTimestampMap);
- }
-
- public Map<TableName, HashMap<String, Long>> getIncrTimestampMap() {
- return backupImage.getIncrTimeRanges();
- }
-
- /**
- * Get the image list of this backup for restore in time order.
- * @param reverse If true, then output in reverse order, otherwise in time order from old to new
- * @return the backup image list for restore in time order
- */
- public ArrayList<BackupImage> getRestoreDependentList(boolean reverse) {
- TreeMap<Long, BackupImage> restoreImages = new TreeMap<Long, BackupImage>();
- restoreImages.put(backupImage.startTs, backupImage);
- for (BackupImage image : backupImage.getAncestors()) {
- restoreImages.put(Long.valueOf(image.startTs), image);
- }
- return new ArrayList<BackupImage>(reverse ? (restoreImages.descendingMap().values())
- : (restoreImages.values()));
- }
-
- /**
- * Get the dependent image list for a specific table of this backup in time order from old to new
- * if want to restore to this backup image level.
- * @param table table
- * @return the backup image list for a table in time order
- */
- public ArrayList<BackupImage> getDependentListByTable(TableName table) {
- ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>();
- ArrayList<BackupImage> imageList = getRestoreDependentList(true);
- for (BackupImage image : imageList) {
- if (image.hasTable(table)) {
- tableImageList.add(image);
- if (image.getType() == BackupType.FULL) {
- break;
- }
- }
- }
- Collections.reverse(tableImageList);
- return tableImageList;
- }
-
- /**
- * Get the full dependent image list in the whole dependency scope for a specific table of this
- * backup in time order from old to new.
- * @param table table
- * @return the full backup image list for a table in time order in the whole scope of the
- * dependency of this image
- */
- public ArrayList<BackupImage> getAllDependentListByTable(TableName table) {
- ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>();
- ArrayList<BackupImage> imageList = getRestoreDependentList(false);
- for (BackupImage image : imageList) {
- if (image.hasTable(table)) {
- tableImageList.add(image);
- }
- }
- return tableImageList;
- }
-
- /**
- * Check whether backup image1 could cover backup image2 or not.
- * @param image1 backup image 1
- * @param image2 backup image 2
- * @return true if image1 can cover image2, otherwise false
- */
- public static boolean canCoverImage(BackupImage image1, BackupImage image2) {
- // image1 can cover image2 only when the following conditions are satisfied:
- // - image1 must not be an incremental image;
- // - image1 must be taken after image2 has been taken;
- // - table set of image1 must cover the table set of image2.
- if (image1.getType() == BackupType.INCREMENTAL) {
- return false;
- }
- if (image1.getStartTs() < image2.getStartTs()) {
- return false;
- }
- List<TableName> image1TableList = image1.getTableNames();
- List<TableName> image2TableList = image2.getTableNames();
- boolean found = false;
- for (int i = 0; i < image2TableList.size(); i++) {
- found = false;
- for (int j = 0; j < image1TableList.size(); j++) {
- if (image2TableList.get(i).equals(image1TableList.get(j))) {
- found = true;
- break;
- }
- }
- if (!found) {
- return false;
- }
- }
-
- LOG.debug("Backup image " + image1.getBackupId() + " can cover " + image2.getBackupId());
- return true;
- }
-
- /**
- * Check whether backup image set could cover a backup image or not.
- * @param fullImages The backup image set
- * @param image The target backup image
- * @return true if fullImages can cover image, otherwise false
- */
- public static boolean canCoverImage(ArrayList<BackupImage> fullImages, BackupImage image) {
- // fullImages can cover image only when the following conditions are satisfied:
- // - each image of fullImages must not be an incremental image;
- // - each image of fullImages must be taken after image has been taken;
- // - sum table set of fullImages must cover the table set of image.
- for (BackupImage image1 : fullImages) {
- if (image1.getType() == BackupType.INCREMENTAL) {
- return false;
- }
- if (image1.getStartTs() < image.getStartTs()) {
- return false;
- }
- }
-
- ArrayList<String> image1TableList = new ArrayList<String>();
- for (BackupImage image1 : fullImages) {
- List<TableName> tableList = image1.getTableNames();
- for (TableName table : tableList) {
- image1TableList.add(table.getNameAsString());
- }
- }
- ArrayList<String> image2TableList = new ArrayList<String>();
- List<TableName> tableList = image.getTableNames();
- for (TableName table : tableList) {
- image2TableList.add(table.getNameAsString());
- }
-
- for (int i = 0; i < image2TableList.size(); i++) {
- if (image1TableList.contains(image2TableList.get(i)) == false) {
- return false;
- }
- }
-
- LOG.debug("Full image set can cover image " + image.getBackupId());
- return true;
- }
-
- public BackupInfo toBackupInfo() {
- BackupInfo info = new BackupInfo();
- info.setType(backupImage.getType());
- List<TableName> list = backupImage.getTableNames();
- TableName[] tables = new TableName[list.size()];
- info.addTables(list.toArray(tables));
- info.setBackupId(backupImage.getBackupId());
- info.setStartTs(backupImage.getStartTs());
- info.setBackupRootDir(backupImage.getRootDir());
- if (backupImage.getType() == BackupType.INCREMENTAL) {
- info.setHLogTargetDir(BackupUtils.getLogBackupDir(backupImage.getRootDir(),
- backupImage.getBackupId()));
- }
- return info;
- }
-}