You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by el...@apache.org on 2017/08/23 16:47:23 UTC
[26/36] hbase git commit: HBASE-17614: Move Backup/Restore into
separate module (Vladimir Rodionov)
http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java
deleted file mode 100644
index 99fb06c..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupAdminImpl.java
+++ /dev/null
@@ -1,743 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup.impl;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.HashSet;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupAdmin;
-import org.apache.hadoop.hbase.backup.BackupClientFactory;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
-import org.apache.hadoop.hbase.backup.BackupMergeJob;
-import org.apache.hadoop.hbase.backup.BackupRequest;
-import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
-import org.apache.hadoop.hbase.backup.BackupRestoreFactory;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.HBackupFileSystem;
-import org.apache.hadoop.hbase.backup.RestoreRequest;
-import org.apache.hadoop.hbase.backup.util.BackupSet;
-import org.apache.hadoop.hbase.backup.util.BackupUtils;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-
-@InterfaceAudience.Private
-public class BackupAdminImpl implements BackupAdmin {
- public final static String CHECK_OK = "Checking backup images: OK";
- public final static String CHECK_FAILED =
- "Checking backup images: Failed. Some dependencies are missing for restore";
- private static final Log LOG = LogFactory.getLog(BackupAdminImpl.class);
-
- private final Connection conn;
-
- public BackupAdminImpl(Connection conn) {
- this.conn = conn;
- }
-
- @Override
- public void close() throws IOException {
- }
-
- @Override
- public BackupInfo getBackupInfo(String backupId) throws IOException {
- BackupInfo backupInfo = null;
- try (final BackupSystemTable table = new BackupSystemTable(conn)) {
- if (backupId == null) {
- ArrayList<BackupInfo> recentSessions = table.getBackupInfos(BackupState.RUNNING);
- if (recentSessions.isEmpty()) {
- LOG.warn("No ongoing sessions found.");
- return null;
- }
- // else show status for ongoing session
- // must be one maximum
- return recentSessions.get(0);
- } else {
- backupInfo = table.readBackupInfo(backupId);
- return backupInfo;
- }
- }
- }
-
- @Override
- public int deleteBackups(String[] backupIds) throws IOException {
-
- int totalDeleted = 0;
- Map<String, HashSet<TableName>> allTablesMap = new HashMap<String, HashSet<TableName>>();
-
- boolean deleteSessionStarted = false;
- boolean snapshotDone = false;
- try (final BackupSystemTable sysTable = new BackupSystemTable(conn)) {
-
- // Step 1: Make sure there is no active session
- // is running by using startBackupSession API
- // If there is an active session in progress, exception will be thrown
- try {
- sysTable.startBackupExclusiveOperation();
- deleteSessionStarted = true;
- } catch (IOException e) {
- LOG.warn("You can not run delete command while active backup session is in progress. \n"
- + "If there is no active backup session running, run backup repair utility to restore \n"
- + "backup system integrity.");
- return -1;
- }
-
- // Step 2: Make sure there is no failed session
- List<BackupInfo> list = sysTable.getBackupInfos(BackupState.RUNNING);
- if (list.size() != 0) {
- // ailed sessions found
- LOG.warn("Failed backup session found. Run backup repair tool first.");
- return -1;
- }
-
- // Step 3: Record delete session
- sysTable.startDeleteOperation(backupIds);
- // Step 4: Snapshot backup system table
- if (!BackupSystemTable.snapshotExists(conn)) {
- BackupSystemTable.snapshot(conn);
- } else {
- LOG.warn("Backup system table snapshot exists");
- }
- snapshotDone = true;
- try {
- for (int i = 0; i < backupIds.length; i++) {
- BackupInfo info = sysTable.readBackupInfo(backupIds[i]);
- if (info != null) {
- String rootDir = info.getBackupRootDir();
- HashSet<TableName> allTables = allTablesMap.get(rootDir);
- if (allTables == null) {
- allTables = new HashSet<TableName>();
- allTablesMap.put(rootDir, allTables);
- }
- allTables.addAll(info.getTableNames());
- totalDeleted += deleteBackup(backupIds[i], sysTable);
- }
- }
- finalizeDelete(allTablesMap, sysTable);
- // Finish
- sysTable.finishDeleteOperation();
- // delete snapshot
- BackupSystemTable.deleteSnapshot(conn);
- } catch (IOException e) {
- // Fail delete operation
- // Step 1
- if (snapshotDone) {
- if (BackupSystemTable.snapshotExists(conn)) {
- BackupSystemTable.restoreFromSnapshot(conn);
- // delete snapshot
- BackupSystemTable.deleteSnapshot(conn);
- // We still have record with unfinished delete operation
- LOG.error("Delete operation failed, please run backup repair utility to restore "
- + "backup system integrity", e);
- throw e;
- } else {
- LOG.warn("Delete operation succeeded, there were some errors: ", e);
- }
- }
-
- } finally {
- if (deleteSessionStarted) {
- sysTable.finishBackupExclusiveOperation();
- }
- }
- }
- return totalDeleted;
- }
-
- /**
- * Updates incremental backup set for every backupRoot
- * @param tablesMap map [backupRoot: Set<TableName>]
- * @param table backup system table
- * @throws IOException
- */
-
- private void finalizeDelete(Map<String, HashSet<TableName>> tablesMap, BackupSystemTable table)
- throws IOException {
- for (String backupRoot : tablesMap.keySet()) {
- Set<TableName> incrTableSet = table.getIncrementalBackupTableSet(backupRoot);
- Map<TableName, ArrayList<BackupInfo>> tableMap =
- table.getBackupHistoryForTableSet(incrTableSet, backupRoot);
- for (Map.Entry<TableName, ArrayList<BackupInfo>> entry : tableMap.entrySet()) {
- if (entry.getValue() == null) {
- // No more backups for a table
- incrTableSet.remove(entry.getKey());
- }
- }
- if (!incrTableSet.isEmpty()) {
- table.addIncrementalBackupTableSet(incrTableSet, backupRoot);
- } else { // empty
- table.deleteIncrementalBackupTableSet(backupRoot);
- }
- }
- }
-
- /**
- * Delete single backup and all related backups <br>
- * Algorithm:<br>
- * Backup type: FULL or INCREMENTAL <br>
- * Is this last backup session for table T: YES or NO <br>
- * For every table T from table list 'tables':<br>
- * if(FULL, YES) deletes only physical data (PD) <br>
- * if(FULL, NO), deletes PD, scans all newer backups and removes T from backupInfo,<br>
- * until we either reach the most recent backup for T in the system or FULL backup<br>
- * which includes T<br>
- * if(INCREMENTAL, YES) deletes only physical data (PD) if(INCREMENTAL, NO) deletes physical data
- * and for table T scans all backup images between last<br>
- * FULL backup, which is older than the backup being deleted and the next FULL backup (if exists) <br>
- * or last one for a particular table T and removes T from list of backup tables.
- * @param backupId backup id
- * @param sysTable backup system table
- * @return total number of deleted backup images
- * @throws IOException
- */
- private int deleteBackup(String backupId, BackupSystemTable sysTable) throws IOException {
-
- BackupInfo backupInfo = sysTable.readBackupInfo(backupId);
-
- int totalDeleted = 0;
- if (backupInfo != null) {
- LOG.info("Deleting backup " + backupInfo.getBackupId() + " ...");
- // Step 1: clean up data for backup session (idempotent)
- BackupUtils.cleanupBackupData(backupInfo, conn.getConfiguration());
- // List of tables in this backup;
- List<TableName> tables = backupInfo.getTableNames();
- long startTime = backupInfo.getStartTs();
- for (TableName tn : tables) {
- boolean isLastBackupSession = isLastBackupSession(sysTable, tn, startTime);
- if (isLastBackupSession) {
- continue;
- }
- // else
- List<BackupInfo> affectedBackups = getAffectedBackupSessions(backupInfo, tn, sysTable);
- for (BackupInfo info : affectedBackups) {
- if (info.equals(backupInfo)) {
- continue;
- }
- removeTableFromBackupImage(info, tn, sysTable);
- }
- }
- Map<byte[], String> map = sysTable.readBulkLoadedFiles(backupId);
- FileSystem fs = FileSystem.get(conn.getConfiguration());
- boolean success = true;
- int numDeleted = 0;
- for (String f : map.values()) {
- Path p = new Path(f);
- try {
- LOG.debug("Delete backup info " + p + " for " + backupInfo.getBackupId());
- if (!fs.delete(p)) {
- if (fs.exists(p)) {
- LOG.warn(f + " was not deleted");
- success = false;
- }
- } else {
- numDeleted++;
- }
- } catch (IOException ioe) {
- LOG.warn(f + " was not deleted", ioe);
- success = false;
- }
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug(numDeleted + " bulk loaded files out of " + map.size() + " were deleted");
- }
- if (success) {
- sysTable.deleteBulkLoadedFiles(map);
- }
-
- sysTable.deleteBackupInfo(backupInfo.getBackupId());
- LOG.info("Delete backup " + backupInfo.getBackupId() + " completed.");
- totalDeleted++;
- } else {
- LOG.warn("Delete backup failed: no information found for backupID=" + backupId);
- }
- return totalDeleted;
- }
-
- private void
- removeTableFromBackupImage(BackupInfo info, TableName tn, BackupSystemTable sysTable)
- throws IOException {
- List<TableName> tables = info.getTableNames();
- LOG.debug("Remove " + tn + " from " + info.getBackupId() + " tables="
- + info.getTableListAsString());
- if (tables.contains(tn)) {
- tables.remove(tn);
-
- if (tables.isEmpty()) {
- LOG.debug("Delete backup info " + info.getBackupId());
-
- sysTable.deleteBackupInfo(info.getBackupId());
- // Idempotent operation
- BackupUtils.cleanupBackupData(info, conn.getConfiguration());
- } else {
- info.setTables(tables);
- sysTable.updateBackupInfo(info);
- // Now, clean up directory for table (idempotent)
- cleanupBackupDir(info, tn, conn.getConfiguration());
- }
- }
- }
-
- private List<BackupInfo> getAffectedBackupSessions(BackupInfo backupInfo, TableName tn,
- BackupSystemTable table) throws IOException {
- LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn);
- long ts = backupInfo.getStartTs();
- List<BackupInfo> list = new ArrayList<BackupInfo>();
- List<BackupInfo> history = table.getBackupHistory(backupInfo.getBackupRootDir());
- // Scan from most recent to backupInfo
- // break when backupInfo reached
- for (BackupInfo info : history) {
- if (info.getStartTs() == ts) {
- break;
- }
- List<TableName> tables = info.getTableNames();
- if (tables.contains(tn)) {
- BackupType bt = info.getType();
- if (bt == BackupType.FULL) {
- // Clear list if we encounter FULL backup
- list.clear();
- } else {
- LOG.debug("GetAffectedBackupInfos for: " + backupInfo.getBackupId() + " table=" + tn
- + " added " + info.getBackupId() + " tables=" + info.getTableListAsString());
- list.add(info);
- }
- }
- }
- return list;
- }
-
- /**
- * Clean up the data at target directory
- * @throws IOException
- */
- private void cleanupBackupDir(BackupInfo backupInfo, TableName table, Configuration conf)
- throws IOException {
- try {
- // clean up the data at target directory
- String targetDir = backupInfo.getBackupRootDir();
- if (targetDir == null) {
- LOG.warn("No target directory specified for " + backupInfo.getBackupId());
- return;
- }
-
- FileSystem outputFs = FileSystem.get(new Path(backupInfo.getBackupRootDir()).toUri(), conf);
-
- Path targetDirPath =
- new Path(BackupUtils.getTableBackupDir(backupInfo.getBackupRootDir(),
- backupInfo.getBackupId(), table));
- if (outputFs.delete(targetDirPath, true)) {
- LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done.");
- } else {
- LOG.info("No data has been found in " + targetDirPath.toString() + ".");
- }
-
- } catch (IOException e1) {
- LOG.error("Cleaning up backup data of " + backupInfo.getBackupId() + " for table " + table
- + "at " + backupInfo.getBackupRootDir() + " failed due to " + e1.getMessage() + ".");
- throw e1;
- }
- }
-
- private boolean isLastBackupSession(BackupSystemTable table, TableName tn, long startTime)
- throws IOException {
- List<BackupInfo> history = table.getBackupHistory();
- for (BackupInfo info : history) {
- List<TableName> tables = info.getTableNames();
- if (!tables.contains(tn)) {
- continue;
- }
- if (info.getStartTs() <= startTime) {
- return true;
- } else {
- return false;
- }
- }
- return false;
- }
-
- @Override
- public List<BackupInfo> getHistory(int n) throws IOException {
- try (final BackupSystemTable table = new BackupSystemTable(conn)) {
- List<BackupInfo> history = table.getBackupHistory();
- if (history.size() <= n) return history;
- List<BackupInfo> list = new ArrayList<BackupInfo>();
- for (int i = 0; i < n; i++) {
- list.add(history.get(i));
- }
- return list;
- }
- }
-
- @Override
- public List<BackupInfo> getHistory(int n, BackupInfo.Filter... filters) throws IOException {
- if (filters.length == 0) return getHistory(n);
- try (final BackupSystemTable table = new BackupSystemTable(conn)) {
- List<BackupInfo> history = table.getBackupHistory();
- List<BackupInfo> result = new ArrayList<BackupInfo>();
- for (BackupInfo bi : history) {
- if (result.size() == n) break;
- boolean passed = true;
- for (int i = 0; i < filters.length; i++) {
- if (!filters[i].apply(bi)) {
- passed = false;
- break;
- }
- }
- if (passed) {
- result.add(bi);
- }
- }
- return result;
- }
- }
-
- @Override
- public List<BackupSet> listBackupSets() throws IOException {
- try (final BackupSystemTable table = new BackupSystemTable(conn)) {
- List<String> list = table.listBackupSets();
- List<BackupSet> bslist = new ArrayList<BackupSet>();
- for (String s : list) {
- List<TableName> tables = table.describeBackupSet(s);
- if (tables != null) {
- bslist.add(new BackupSet(s, tables));
- }
- }
- return bslist;
- }
- }
-
- @Override
- public BackupSet getBackupSet(String name) throws IOException {
- try (final BackupSystemTable table = new BackupSystemTable(conn)) {
- List<TableName> list = table.describeBackupSet(name);
- if (list == null) return null;
- return new BackupSet(name, list);
- }
- }
-
- @Override
- public boolean deleteBackupSet(String name) throws IOException {
- try (final BackupSystemTable table = new BackupSystemTable(conn)) {
- if (table.describeBackupSet(name) == null) {
- return false;
- }
- table.deleteBackupSet(name);
- return true;
- }
- }
-
- @Override
- public void addToBackupSet(String name, TableName[] tables) throws IOException {
- String[] tableNames = new String[tables.length];
- try (final BackupSystemTable table = new BackupSystemTable(conn);
- final Admin admin = conn.getAdmin();) {
- for (int i = 0; i < tables.length; i++) {
- tableNames[i] = tables[i].getNameAsString();
- if (!admin.tableExists(TableName.valueOf(tableNames[i]))) {
- throw new IOException("Cannot add " + tableNames[i] + " because it doesn't exist");
- }
- }
- table.addToBackupSet(name, tableNames);
- LOG.info("Added tables [" + StringUtils.join(tableNames, " ") + "] to '" + name
- + "' backup set");
- }
- }
-
- @Override
- public void removeFromBackupSet(String name, TableName[] tables) throws IOException {
- LOG.info("Removing tables [" + StringUtils.join(tables, " ") + "] from '" + name + "'");
- try (final BackupSystemTable table = new BackupSystemTable(conn)) {
- table.removeFromBackupSet(name, toStringArray(tables));
- LOG.info("Removing tables [" + StringUtils.join(tables, " ") + "] from '" + name
- + "' completed.");
- }
- }
-
- private String[] toStringArray(TableName[] list) {
- String[] arr = new String[list.length];
- for (int i = 0; i < list.length; i++) {
- arr[i] = list[i].toString();
- }
- return arr;
- }
-
- @Override
- public void restore(RestoreRequest request) throws IOException {
- if (request.isCheck()) {
- HashMap<TableName, BackupManifest> backupManifestMap = new HashMap<>();
- // check and load backup image manifest for the tables
- Path rootPath = new Path(request.getBackupRootDir());
- String backupId = request.getBackupId();
- TableName[] sTableArray = request.getFromTables();
- HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray,
- conn.getConfiguration(), rootPath, backupId);
-
- // Check and validate the backup image and its dependencies
-
- if (BackupUtils.validate(backupManifestMap, conn.getConfiguration())) {
- LOG.info(CHECK_OK);
- } else {
- LOG.error(CHECK_FAILED);
- }
- return;
- }
- // Execute restore request
- new RestoreTablesClient(conn, request).execute();
- }
-
- @Override
- public String backupTables(BackupRequest request) throws IOException {
- BackupType type = request.getBackupType();
- String targetRootDir = request.getTargetRootDir();
- List<TableName> tableList = request.getTableList();
-
- String backupId = BackupRestoreConstants.BACKUPID_PREFIX + EnvironmentEdgeManager.currentTime();
- if (type == BackupType.INCREMENTAL) {
- Set<TableName> incrTableSet = null;
- try (BackupSystemTable table = new BackupSystemTable(conn)) {
- incrTableSet = table.getIncrementalBackupTableSet(targetRootDir);
- }
-
- if (incrTableSet.isEmpty()) {
- String msg =
- "Incremental backup table set contains no tables. "
- + "You need to run full backup first "
- + (tableList != null ? "on " + StringUtils.join(tableList, ",") : "");
-
- throw new IOException(msg);
- }
- if (tableList != null) {
- tableList.removeAll(incrTableSet);
- if (!tableList.isEmpty()) {
- String extraTables = StringUtils.join(tableList, ",");
- String msg =
- "Some tables (" + extraTables + ") haven't gone through full backup. "
- + "Perform full backup on " + extraTables + " first, " + "then retry the command";
- throw new IOException(msg);
- }
- }
- tableList = Lists.newArrayList(incrTableSet);
- }
- if (tableList != null && !tableList.isEmpty()) {
- for (TableName table : tableList) {
- String targetTableBackupDir =
- HBackupFileSystem.getTableBackupDir(targetRootDir, backupId, table);
- Path targetTableBackupDirPath = new Path(targetTableBackupDir);
- FileSystem outputFs =
- FileSystem.get(targetTableBackupDirPath.toUri(), conn.getConfiguration());
- if (outputFs.exists(targetTableBackupDirPath)) {
- throw new IOException("Target backup directory " + targetTableBackupDir
- + " exists already.");
- }
- }
- ArrayList<TableName> nonExistingTableList = null;
- try (Admin admin = conn.getAdmin();) {
- for (TableName tableName : tableList) {
- if (!admin.tableExists(tableName)) {
- if (nonExistingTableList == null) {
- nonExistingTableList = new ArrayList<>();
- }
- nonExistingTableList.add(tableName);
- }
- }
- }
- if (nonExistingTableList != null) {
- if (type == BackupType.INCREMENTAL) {
- // Update incremental backup set
- tableList = excludeNonExistingTables(tableList, nonExistingTableList);
- } else {
- // Throw exception only in full mode - we try to backup non-existing table
- throw new IOException("Non-existing tables found in the table list: "
- + nonExistingTableList);
- }
- }
- }
-
- // update table list
- BackupRequest.Builder builder = new BackupRequest.Builder();
- request =
- builder.withBackupType(request.getBackupType()).withTableList(tableList)
- .withTargetRootDir(request.getTargetRootDir())
- .withBackupSetName(request.getBackupSetName()).withTotalTasks(request.getTotalTasks())
- .withBandwidthPerTasks((int) request.getBandwidth()).build();
-
- TableBackupClient client = null;
- try {
- client = BackupClientFactory.create(conn, backupId, request);
- } catch (IOException e) {
- LOG.error("There is an active session already running");
- throw e;
- }
-
- client.execute();
-
- return backupId;
- }
-
- private List<TableName> excludeNonExistingTables(List<TableName> tableList,
- List<TableName> nonExistingTableList) {
-
- for (TableName table : nonExistingTableList) {
- tableList.remove(table);
- }
- return tableList;
- }
-
- @Override
- public void mergeBackups(String[] backupIds) throws IOException {
- try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) {
- checkIfValidForMerge(backupIds, sysTable);
- BackupMergeJob job = BackupRestoreFactory.getBackupMergeJob(conn.getConfiguration());
- job.run(backupIds);
- }
- }
-
- /**
- * Verifies that backup images are valid for merge.
- *
- * <ul>
- * <li>All backups MUST be in the same destination
- * <li>No FULL backups are allowed - only INCREMENTAL
- * <li>All backups must be in COMPLETE state
- * <li>No holes in backup list are allowed
- * </ul>
- * <p>
- * @param backupIds list of backup ids
- * @param table backup system table
- * @throws IOException
- */
- private void checkIfValidForMerge(String[] backupIds, BackupSystemTable table) throws IOException {
- String backupRoot = null;
-
- final Set<TableName> allTables = new HashSet<TableName>();
- final Set<String> allBackups = new HashSet<String>();
- long minTime = Long.MAX_VALUE, maxTime = Long.MIN_VALUE;
- for (String backupId : backupIds) {
- BackupInfo bInfo = table.readBackupInfo(backupId);
- if (bInfo == null) {
- String msg = "Backup session " + backupId + " not found";
- throw new IOException(msg);
- }
- if (backupRoot == null) {
- backupRoot = bInfo.getBackupRootDir();
- } else if (!bInfo.getBackupRootDir().equals(backupRoot)) {
- throw new IOException("Found different backup destinations in a list of a backup sessions \n"
- + "1. " + backupRoot + "\n" + "2. " + bInfo.getBackupRootDir());
- }
- if (bInfo.getType() == BackupType.FULL) {
- throw new IOException("FULL backup image can not be merged for: \n" + bInfo);
- }
-
- if (bInfo.getState() != BackupState.COMPLETE) {
- throw new IOException("Backup image " + backupId
- + " can not be merged becuase of its state: " + bInfo.getState());
- }
- allBackups.add(backupId);
- allTables.addAll(bInfo.getTableNames());
- long time = bInfo.getStartTs();
- if (time < minTime) {
- minTime = time;
- }
- if (time > maxTime) {
- maxTime = time;
- }
- }
-
-
- final long startRangeTime = minTime;
- final long endRangeTime = maxTime;
- final String backupDest = backupRoot;
- // Check we have no 'holes' in backup id list
- // Filter 1 : backupRoot
- // Filter 2 : time range filter
- // Filter 3 : table filter
-
- BackupInfo.Filter destinationFilter = new BackupInfo.Filter() {
-
- @Override
- public boolean apply(BackupInfo info) {
- return info.getBackupRootDir().equals(backupDest);
- }
- };
-
- BackupInfo.Filter timeRangeFilter = new BackupInfo.Filter() {
-
- @Override
- public boolean apply(BackupInfo info) {
- long time = info.getStartTs();
- return time >= startRangeTime && time <= endRangeTime ;
- }
- };
-
- BackupInfo.Filter tableFilter = new BackupInfo.Filter() {
-
- @Override
- public boolean apply(BackupInfo info) {
- List<TableName> tables = info.getTableNames();
- return !Collections.disjoint(allTables, tables);
- }
- };
-
- BackupInfo.Filter typeFilter = new BackupInfo.Filter() {
-
- @Override
- public boolean apply(BackupInfo info) {
- return info.getType() == BackupType.INCREMENTAL;
- }
- };
-
- BackupInfo.Filter stateFilter = new BackupInfo.Filter() {
- @Override
- public boolean apply(BackupInfo info) {
- return info.getState() == BackupState.COMPLETE;
- }
- };
-
- List<BackupInfo> allInfos =
- table.getBackupHistory( -1, destinationFilter,
- timeRangeFilter, tableFilter, typeFilter, stateFilter);
- if (allInfos.size() != allBackups.size()) {
- // Yes we have at least one hole in backup image sequence
- List<String> missingIds = new ArrayList<String>();
- for(BackupInfo info: allInfos) {
- if(allBackups.contains(info.getBackupId())) {
- continue;
- }
- missingIds.add(info.getBackupId());
- }
- String errMsg =
- "Sequence of backup ids has 'holes'. The following backup images must be added:" +
- org.apache.hadoop.util.StringUtils.join(",", missingIds);
- throw new IOException(errMsg);
- }
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
deleted file mode 100644
index 650ba2e..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
+++ /dev/null
@@ -1,1022 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.impl;
-
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_BANDWIDTH_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_PATH_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_RECORD_NUMBER_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_BACKUP_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_SET_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS;
-import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC;
-
-import java.io.IOException;
-import java.net.URI;
-import java.util.List;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.cli.HelpFormatter;
-import org.apache.commons.cli.Options;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupAdmin;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
-import org.apache.hadoop.hbase.backup.BackupRequest;
-import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
-import org.apache.hadoop.hbase.backup.BackupRestoreConstants.BackupCommand;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.util.BackupSet;
-import org.apache.hadoop.hbase.backup.util.BackupUtils;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-import org.apache.hadoop.hbase.shaded.com.google.common.collect.Lists;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-
-/**
- * General backup commands, options and usage messages
- */
-
-@InterfaceAudience.Private
-public final class BackupCommands {
-
- public final static String INCORRECT_USAGE = "Incorrect usage";
-
- public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n"
- + "where COMMAND is one of:\n" + " create create a new backup image\n"
- + " delete delete an existing backup image\n"
- + " describe show the detailed information of a backup image\n"
- + " history show history of all successful backups\n"
- + " progress show the progress of the latest backup request\n"
- + " set backup set management\n"
- + " repair repair backup system table\n"
- + " merge merge backup images\n"
- + "Run \'hbase backup COMMAND -h\' to see help message for each command\n";
-
- public static final String CREATE_CMD_USAGE =
- "Usage: hbase backup create <type> <backup_path> [options]\n"
- + " type \"full\" to create a full backup image\n"
- + " \"incremental\" to create an incremental backup image\n"
- + " backup_path Full path to store the backup image\n";
-
- public static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress <backup_id>\n"
- + " backup_id Backup image id (optional). If no id specified, the command will show\n"
- + " progress for currently running backup session.";
- public static final String NO_INFO_FOUND = "No info was found for backup id: ";
- public static final String NO_ACTIVE_SESSION_FOUND = "No active backup sessions found.";
-
- public static final String DESCRIBE_CMD_USAGE = "Usage: hbase backup describe <backup_id>\n"
- + " backup_id Backup image id\n";
-
- public static final String HISTORY_CMD_USAGE = "Usage: hbase backup history [options]";
-
- public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete <backup_id>\n"
- + " backup_id Backup image id\n";
-
- public static final String REPAIR_CMD_USAGE = "Usage: hbase backup repair\n";
-
- public static final String CANCEL_CMD_USAGE = "Usage: hbase backup cancel <backup_id>\n"
- + " backup_id Backup image id\n";
-
- public static final String SET_CMD_USAGE = "Usage: hbase backup set COMMAND [name] [tables]\n"
- + " name Backup set name\n"
- + " tables Comma separated list of tables.\n" + "COMMAND is one of:\n"
- + " add add tables to a set, create a set if needed\n"
- + " remove remove tables from a set\n"
- + " list list all backup sets in the system\n"
- + " describe describe set\n" + " delete delete backup set\n";
- public static final String MERGE_CMD_USAGE = "Usage: hbase backup merge [backup_ids]\n"
- + " backup_ids Comma separated list of backup image ids.\n";
-
- public static final String USAGE_FOOTER = "";
-
- public static abstract class Command extends Configured {
- CommandLine cmdline;
- Connection conn;
-
- Command(Configuration conf) {
- if (conf == null) {
- conf = HBaseConfiguration.create();
- }
- setConf(conf);
- }
-
- public void execute() throws IOException {
- if (cmdline.hasOption("h") || cmdline.hasOption("help")) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- // Create connection
- conn = ConnectionFactory.createConnection(getConf());
- if (requiresNoActiveSession()) {
- // Check active session
- try (BackupSystemTable table = new BackupSystemTable(conn);) {
- List<BackupInfo> sessions = table.getBackupInfos(BackupState.RUNNING);
-
- if (sessions.size() > 0) {
- System.err.println("Found backup session in a RUNNING state: ");
- System.err.println(sessions.get(0));
- System.err.println("This may indicate that a previous session has failed abnormally.");
- System.err.println("In this case, backup recovery is recommended.");
- throw new IOException("Active session found, aborted command execution");
- }
- }
- }
- if (requiresConsistentState()) {
- // Check failed delete
- try (BackupSystemTable table = new BackupSystemTable(conn);) {
- String[] ids = table.getListOfBackupIdsFromDeleteOperation();
-
- if (ids != null && ids.length > 0) {
- System.err.println("Found failed backup DELETE coommand. ");
- System.err.println("Backup system recovery is required.");
- throw new IOException("Failed backup DELETE found, aborted command execution");
- }
-
- ids = table.getListOfBackupIdsFromMergeOperation();
- if (ids != null && ids.length > 0) {
- System.err.println("Found failed backup MERGE coommand. ");
- System.err.println("Backup system recovery is required.");
- throw new IOException("Failed backup MERGE found, aborted command execution");
- }
-
- }
- }
- }
-
- public void finish() throws IOException {
- if (conn != null) {
- conn.close();
- }
- }
-
- protected abstract void printUsage();
-
- /**
- * The command can't be run if active backup session is in progress
- * @return true if no active sessions are in progress
- */
- protected boolean requiresNoActiveSession() {
- return false;
- }
-
- /**
- * Command requires consistent state of a backup system Backup system may become inconsistent
- * because of an abnormal termination of a backup session or delete command
- * @return true, if yes
- */
- protected boolean requiresConsistentState() {
- return false;
- }
- }
-
- private BackupCommands() {
- throw new AssertionError("Instantiating utility class...");
- }
-
- public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) {
- Command cmd = null;
- switch (type) {
- case CREATE:
- cmd = new CreateCommand(conf, cmdline);
- break;
- case DESCRIBE:
- cmd = new DescribeCommand(conf, cmdline);
- break;
- case PROGRESS:
- cmd = new ProgressCommand(conf, cmdline);
- break;
- case DELETE:
- cmd = new DeleteCommand(conf, cmdline);
- break;
- case CANCEL:
- cmd = new CancelCommand(conf, cmdline);
- break;
- case HISTORY:
- cmd = new HistoryCommand(conf, cmdline);
- break;
- case SET:
- cmd = new BackupSetCommand(conf, cmdline);
- break;
- case REPAIR:
- cmd = new RepairCommand(conf, cmdline);
- break;
- case MERGE:
- cmd = new MergeCommand(conf, cmdline);
- break;
- case HELP:
- default:
- cmd = new HelpCommand(conf, cmdline);
- break;
- }
- return cmd;
- }
-
- static int numOfArgs(String[] args) {
- if (args == null) return 0;
- return args.length;
- }
-
- public static class CreateCommand extends Command {
-
- CreateCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- protected boolean requiresNoActiveSession() {
- return true;
- }
-
- @Override
- protected boolean requiresConsistentState() {
- return true;
- }
-
- @Override
- public void execute() throws IOException {
- if (cmdline == null || cmdline.getArgs() == null) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- String[] args = cmdline.getArgs();
- if (args.length != 3) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- if (!BackupType.FULL.toString().equalsIgnoreCase(args[1])
- && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])) {
- System.out.println("ERROR: invalid backup type: " + args[1]);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- if (!verifyPath(args[2])) {
- System.out.println("ERROR: invalid backup destination: " + args[2]);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String tables = null;
-
- // Check if we have both: backup set and list of tables
- if (cmdline.hasOption(OPTION_TABLE) && cmdline.hasOption(OPTION_SET)) {
- System.out.println("ERROR: You can specify either backup set or list"
- + " of tables, but not both");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- // Creates connection
- super.execute();
- // Check backup set
- String setName = null;
- if (cmdline.hasOption(OPTION_SET)) {
- setName = cmdline.getOptionValue(OPTION_SET);
- tables = getTablesForSet(setName, getConf());
-
- if (tables == null) {
- System.out.println("ERROR: Backup set '" + setName
- + "' is either empty or does not exist");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- } else {
- tables = cmdline.getOptionValue(OPTION_TABLE);
- }
- int bandwidth =
- cmdline.hasOption(OPTION_BANDWIDTH) ? Integer.parseInt(cmdline
- .getOptionValue(OPTION_BANDWIDTH)) : -1;
- int workers =
- cmdline.hasOption(OPTION_WORKERS) ? Integer.parseInt(cmdline
- .getOptionValue(OPTION_WORKERS)) : -1;
-
- try (BackupAdminImpl admin = new BackupAdminImpl(conn);) {
-
- BackupRequest.Builder builder = new BackupRequest.Builder();
- BackupRequest request =
- builder
- .withBackupType(BackupType.valueOf(args[1].toUpperCase()))
- .withTableList(
- tables != null ? Lists.newArrayList(BackupUtils.parseTableNames(tables)) : null)
- .withTargetRootDir(args[2]).withTotalTasks(workers)
- .withBandwidthPerTasks(bandwidth).withBackupSetName(setName).build();
- String backupId = admin.backupTables(request);
- System.out.println("Backup session " + backupId + " finished. Status: SUCCESS");
- } catch (IOException e) {
- System.out.println("Backup session finished. Status: FAILURE");
- throw e;
- }
- }
-
- private boolean verifyPath(String path) {
- try {
- Path p = new Path(path);
- Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
- URI uri = p.toUri();
- if (uri.getScheme() == null) return false;
- FileSystem.get(uri, conf);
- return true;
- } catch (Exception e) {
- return false;
- }
- }
-
- private String getTablesForSet(String name, Configuration conf) throws IOException {
- try (final BackupSystemTable table = new BackupSystemTable(conn)) {
- List<TableName> tables = table.describeBackupSet(name);
- if (tables == null) return null;
- return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(CREATE_CMD_USAGE);
- Options options = new Options();
- options.addOption(OPTION_WORKERS, true, OPTION_WORKERS_DESC);
- options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC);
- options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC);
- options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC);
-
- HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.setLeftPadding(2);
- helpFormatter.setDescPadding(8);
- helpFormatter.setWidth(100);
- helpFormatter.setSyntaxPrefix("Options:");
- helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
-
- }
- }
-
- private static class HelpCommand extends Command {
-
- HelpCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- if (cmdline == null) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String[] args = cmdline.getArgs();
- if (args == null || args.length == 0) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- if (args.length != 2) {
- System.out.println("ERROR: Only supports help message of a single command type");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String type = args[1];
-
- if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) {
- System.out.println(CREATE_CMD_USAGE);
- } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) {
- System.out.println(DESCRIBE_CMD_USAGE);
- } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) {
- System.out.println(HISTORY_CMD_USAGE);
- } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) {
- System.out.println(PROGRESS_CMD_USAGE);
- } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) {
- System.out.println(DELETE_CMD_USAGE);
- } else if (BackupCommand.CANCEL.name().equalsIgnoreCase(type)) {
- System.out.println(CANCEL_CMD_USAGE);
- } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) {
- System.out.println(SET_CMD_USAGE);
- } else {
- System.out.println("Unknown command : " + type);
- printUsage();
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(USAGE);
- }
- }
-
- private static class DescribeCommand extends Command {
-
- DescribeCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- if (cmdline == null || cmdline.getArgs() == null) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- String[] args = cmdline.getArgs();
- if (args.length != 2) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- super.execute();
-
- String backupId = args[1];
- try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) {
- BackupInfo info = sysTable.readBackupInfo(backupId);
- if (info == null) {
- System.out.println("ERROR: " + backupId + " does not exist");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- System.out.println(info.getShortDescription());
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(DESCRIBE_CMD_USAGE);
- }
- }
-
- private static class ProgressCommand extends Command {
-
- ProgressCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
-
- if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length == 1) {
- System.out.println("No backup id was specified, "
- + "will retrieve the most recent (ongoing) session");
- }
- String[] args = cmdline == null ? null : cmdline.getArgs();
- if (args != null && args.length > 2) {
- System.err.println("ERROR: wrong number of arguments: " + args.length);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- super.execute();
-
- String backupId = (args == null || args.length <= 1) ? null : args[1];
- try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) {
- BackupInfo info = null;
-
- if (backupId != null) {
- info = sysTable.readBackupInfo(backupId);
- } else {
- List<BackupInfo> infos = sysTable.getBackupInfos(BackupState.RUNNING);
- if (infos != null && infos.size() > 0) {
- info = infos.get(0);
- backupId = info.getBackupId();
- System.out.println("Found ongoing session with backupId=" + backupId);
- } else {
- }
- }
- int progress = info == null ? -1 : info.getProgress();
- if (progress < 0) {
- if (backupId != null) {
- System.out.println(NO_INFO_FOUND + backupId);
- } else {
- System.err.println(NO_ACTIVE_SESSION_FOUND);
- }
- } else {
- System.out.println(backupId + " progress=" + progress + "%");
- }
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(PROGRESS_CMD_USAGE);
- }
- }
-
- private static class DeleteCommand extends Command {
-
- DeleteCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- protected boolean requiresNoActiveSession() {
- return true;
- }
-
- @Override
- public void execute() throws IOException {
- if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- super.execute();
-
- String[] args = cmdline.getArgs();
- String[] backupIds = new String[args.length - 1];
- System.arraycopy(args, 1, backupIds, 0, backupIds.length);
- try (BackupAdminImpl admin = new BackupAdminImpl(conn);) {
- int deleted = admin.deleteBackups(backupIds);
- System.out.println("Deleted " + deleted + " backups. Total requested: " + args.length);
- } catch (IOException e) {
- System.err
- .println("Delete command FAILED. Please run backup repair tool to restore backup system integrity");
- throw e;
- }
-
- }
-
- @Override
- protected void printUsage() {
- System.out.println(DELETE_CMD_USAGE);
- }
- }
-
- private static class RepairCommand extends Command {
-
- RepairCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- super.execute();
-
- String[] args = cmdline == null ? null : cmdline.getArgs();
- if (args != null && args.length > 1) {
- System.err.println("ERROR: wrong number of arguments: " + args.length);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
- try (final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupSystemTable sysTable = new BackupSystemTable(conn);) {
-
- // Failed backup
- BackupInfo backupInfo;
- List<BackupInfo> list = sysTable.getBackupInfos(BackupState.RUNNING);
- if (list.size() == 0) {
- // No failed sessions found
- System.out.println("REPAIR status: no failed sessions found."
- + " Checking failed delete backup operation ...");
- repairFailedBackupDeletionIfAny(conn, sysTable);
- repairFailedBackupMergeIfAny(conn, sysTable);
- return;
- }
- backupInfo = list.get(0);
- // If this is a cancel exception, then we've already cleaned.
- // set the failure timestamp of the overall backup
- backupInfo.setCompleteTs(EnvironmentEdgeManager.currentTime());
- // set failure message
- backupInfo.setFailedMsg("REPAIR status: repaired after failure:\n" + backupInfo);
- // set overall backup status: failed
- backupInfo.setState(BackupState.FAILED);
- // compose the backup failed data
- String backupFailedData =
- "BackupId=" + backupInfo.getBackupId() + ",startts=" + backupInfo.getStartTs()
- + ",failedts=" + backupInfo.getCompleteTs() + ",failedphase="
- + backupInfo.getPhase() + ",failedmessage=" + backupInfo.getFailedMsg();
- System.out.println(backupFailedData);
- TableBackupClient.cleanupAndRestoreBackupSystem(conn, backupInfo, conf);
- // If backup session is updated to FAILED state - means we
- // processed recovery already.
- sysTable.updateBackupInfo(backupInfo);
- sysTable.finishBackupExclusiveOperation();
- System.out.println("REPAIR status: finished repair failed session:\n " + backupInfo);
-
- }
- }
-
- private void repairFailedBackupDeletionIfAny(Connection conn, BackupSystemTable sysTable)
- throws IOException {
- String[] backupIds = sysTable.getListOfBackupIdsFromDeleteOperation();
- if (backupIds == null || backupIds.length == 0) {
- System.out.println("No failed backup DELETE operation found");
- // Delete backup table snapshot if exists
- BackupSystemTable.deleteSnapshot(conn);
- return;
- }
- System.out.println("Found failed DELETE operation for: " + StringUtils.join(backupIds));
- System.out.println("Running DELETE again ...");
- // Restore table from snapshot
- BackupSystemTable.restoreFromSnapshot(conn);
- // Finish previous failed session
- sysTable.finishBackupExclusiveOperation();
- try (BackupAdmin admin = new BackupAdminImpl(conn);) {
- admin.deleteBackups(backupIds);
- }
- System.out.println("DELETE operation finished OK: " + StringUtils.join(backupIds));
-
- }
-
- private void repairFailedBackupMergeIfAny(Connection conn, BackupSystemTable sysTable)
- throws IOException {
- String[] backupIds = sysTable.getListOfBackupIdsFromMergeOperation();
- if (backupIds == null || backupIds.length == 0) {
- System.out.println("No failed backup MERGE operation found");
- // Delete backup table snapshot if exists
- BackupSystemTable.deleteSnapshot(conn);
- return;
- }
- System.out.println("Found failed MERGE operation for: " + StringUtils.join(backupIds));
- System.out.println("Running MERGE again ...");
- // Restore table from snapshot
- BackupSystemTable.restoreFromSnapshot(conn);
- // Unlock backupo system
- sysTable.finishBackupExclusiveOperation();
- // Finish previous failed session
- sysTable.finishMergeOperation();
- try (BackupAdmin admin = new BackupAdminImpl(conn);) {
- admin.mergeBackups(backupIds);
- }
- System.out.println("MERGE operation finished OK: " + StringUtils.join(backupIds));
-
- }
-
- @Override
- protected void printUsage() {
- System.out.println(REPAIR_CMD_USAGE);
- }
- }
-
- private static class MergeCommand extends Command {
-
- MergeCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- protected boolean requiresNoActiveSession() {
- return true;
- }
-
- @Override
- protected boolean requiresConsistentState() {
- return true;
- }
-
- @Override
- public void execute() throws IOException {
- super.execute();
-
- String[] args = cmdline == null ? null : cmdline.getArgs();
- if (args == null || (args.length != 2)) {
- System.err.println("ERROR: wrong number of arguments: "
- + (args == null ? null : args.length));
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String[] backupIds = args[1].split(",");
- if (backupIds.length < 2) {
- String msg = "ERROR: can not merge a single backup image. "+
- "Number of images must be greater than 1.";
- System.err.println(msg);
- throw new IOException(msg);
-
- }
- Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
- try (final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupAdminImpl admin = new BackupAdminImpl(conn);) {
- admin.mergeBackups(backupIds);
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(MERGE_CMD_USAGE);
- }
- }
-
- // TODO Cancel command
-
- private static class CancelCommand extends Command {
-
- CancelCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- throw new UnsupportedOperationException("Cancel command is not supported yet.");
- }
-
- @Override
- protected void printUsage() {
- }
- }
-
- private static class HistoryCommand extends Command {
-
- private final static int DEFAULT_HISTORY_LENGTH = 10;
-
- HistoryCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
-
- int n = parseHistoryLength();
- final TableName tableName = getTableName();
- final String setName = getTableSetName();
- BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() {
- @Override
- public boolean apply(BackupInfo info) {
- if (tableName == null) return true;
- List<TableName> names = info.getTableNames();
- return names.contains(tableName);
- }
- };
- BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() {
- @Override
- public boolean apply(BackupInfo info) {
- if (setName == null) return true;
- String backupId = info.getBackupId();
- return backupId.startsWith(setName);
- }
- };
- Path backupRootPath = getBackupRootPath();
- List<BackupInfo> history = null;
- if (backupRootPath == null) {
- // Load from backup system table
- super.execute();
- try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) {
- history = sysTable.getBackupHistory(n, tableNameFilter, tableSetFilter);
- }
- } else {
- // load from backup FS
- history =
- BackupUtils.getHistory(getConf(), n, backupRootPath, tableNameFilter, tableSetFilter);
- }
- for (BackupInfo info : history) {
- System.out.println(info.getShortDescription());
- }
- }
-
- private Path getBackupRootPath() throws IOException {
- String value = null;
- try {
- value = cmdline.getOptionValue(OPTION_PATH);
- if (value == null) return null;
- return new Path(value);
- } catch (IllegalArgumentException e) {
- System.out.println("ERROR: Illegal argument for backup root path: " + value);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- }
-
- private TableName getTableName() throws IOException {
- String value = cmdline.getOptionValue(OPTION_TABLE);
- if (value == null) return null;
- try {
- return TableName.valueOf(value);
- } catch (IllegalArgumentException e) {
- System.out.println("Illegal argument for table name: " + value);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- }
-
- private String getTableSetName() throws IOException {
- String value = cmdline.getOptionValue(OPTION_SET);
- return value;
- }
-
- private int parseHistoryLength() throws IOException {
- String value = cmdline.getOptionValue(OPTION_RECORD_NUMBER);
- try {
- if (value == null) return DEFAULT_HISTORY_LENGTH;
- return Integer.parseInt(value);
- } catch (NumberFormatException e) {
- System.out.println("Illegal argument for history length: " + value);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(HISTORY_CMD_USAGE);
- Options options = new Options();
- options.addOption(OPTION_RECORD_NUMBER, true, OPTION_RECORD_NUMBER_DESC);
- options.addOption(OPTION_PATH, true, OPTION_PATH_DESC);
- options.addOption(OPTION_TABLE, true, OPTION_TABLE_DESC);
- options.addOption(OPTION_SET, true, OPTION_SET_DESC);
-
- HelpFormatter helpFormatter = new HelpFormatter();
- helpFormatter.setLeftPadding(2);
- helpFormatter.setDescPadding(8);
- helpFormatter.setWidth(100);
- helpFormatter.setSyntaxPrefix("Options:");
- helpFormatter.printHelp(" ", null, options, USAGE_FOOTER);
- }
- }
-
- private static class BackupSetCommand extends Command {
- private final static String SET_ADD_CMD = "add";
- private final static String SET_REMOVE_CMD = "remove";
- private final static String SET_DELETE_CMD = "delete";
- private final static String SET_DESCRIBE_CMD = "describe";
- private final static String SET_LIST_CMD = "list";
-
- BackupSetCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- // Command-line must have at least one element
- if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String[] args = cmdline.getArgs();
- String cmdStr = args[1];
- BackupCommand cmd = getCommand(cmdStr);
-
- switch (cmd) {
- case SET_ADD:
- processSetAdd(args);
- break;
- case SET_REMOVE:
- processSetRemove(args);
- break;
- case SET_DELETE:
- processSetDelete(args);
- break;
- case SET_DESCRIBE:
- processSetDescribe(args);
- break;
- case SET_LIST:
- processSetList(args);
- break;
- default:
- break;
-
- }
- }
-
- private void processSetList(String[] args) throws IOException {
- super.execute();
-
- // List all backup set names
- // does not expect any args
- try (BackupAdminImpl admin = new BackupAdminImpl(conn);) {
- List<BackupSet> list = admin.listBackupSets();
- for (BackupSet bs : list) {
- System.out.println(bs);
- }
- }
- }
-
- private void processSetDescribe(String[] args) throws IOException {
- if (args == null || args.length != 3) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- super.execute();
-
- String setName = args[2];
- try (final BackupSystemTable sysTable = new BackupSystemTable(conn);) {
- List<TableName> tables = sysTable.describeBackupSet(setName);
- BackupSet set = tables == null ? null : new BackupSet(setName, tables);
- if (set == null) {
- System.out.println("Set '" + setName + "' does not exist.");
- } else {
- System.out.println(set);
- }
- }
- }
-
- private void processSetDelete(String[] args) throws IOException {
- if (args == null || args.length != 3) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- super.execute();
-
- String setName = args[2];
- try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) {
- boolean result = admin.deleteBackupSet(setName);
- if (result) {
- System.out.println("Delete set " + setName + " OK.");
- } else {
- System.out.println("Set " + setName + " does not exist");
- }
- }
- }
-
- private void processSetRemove(String[] args) throws IOException {
- if (args == null || args.length != 4) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- super.execute();
-
- String setName = args[2];
- String[] tables = args[3].split(",");
- TableName[] tableNames = toTableNames(tables);
- try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) {
- admin.removeFromBackupSet(setName, tableNames);
- }
- }
-
- private TableName[] toTableNames(String[] tables) {
- TableName[] arr = new TableName[tables.length];
- for (int i = 0; i < tables.length; i++) {
- arr[i] = TableName.valueOf(tables[i]);
- }
- return arr;
- }
-
- private void processSetAdd(String[] args) throws IOException {
- if (args == null || args.length != 4) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- super.execute();
-
- String setName = args[2];
- String[] tables = args[3].split(",");
- TableName[] tableNames = new TableName[tables.length];
- for (int i = 0; i < tables.length; i++) {
- tableNames[i] = TableName.valueOf(tables[i]);
- }
- try (final BackupAdminImpl admin = new BackupAdminImpl(conn);) {
- admin.addToBackupSet(setName, tableNames);
- }
-
- }
-
- private BackupCommand getCommand(String cmdStr) throws IOException {
- if (cmdStr.equals(SET_ADD_CMD)) {
- return BackupCommand.SET_ADD;
- } else if (cmdStr.equals(SET_REMOVE_CMD)) {
- return BackupCommand.SET_REMOVE;
- } else if (cmdStr.equals(SET_DELETE_CMD)) {
- return BackupCommand.SET_DELETE;
- } else if (cmdStr.equals(SET_DESCRIBE_CMD)) {
- return BackupCommand.SET_DESCRIBE;
- } else if (cmdStr.equals(SET_LIST_CMD)) {
- return BackupCommand.SET_LIST;
- } else {
- System.out.println("ERROR: Unknown command for 'set' :" + cmdStr);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- }
-
- @Override
- protected void printUsage() {
- System.out.println(SET_CMD_USAGE);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java
deleted file mode 100644
index 2c7d35f..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java
+++ /dev/null
@@ -1,84 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.impl;
-
-import org.apache.hadoop.hbase.HBaseIOException;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-
-/**
- * Backup exception
- */
-@SuppressWarnings("serial")
-@InterfaceAudience.Private
-public class BackupException extends HBaseIOException {
- private BackupInfo info;
-
- /**
- * Some exception happened for a backup and don't even know the backup that it was about
- * @param msg Full description of the failure
- */
- public BackupException(String msg) {
- super(msg);
- }
-
- /**
- * Some exception happened for a backup with a cause
- * @param cause the cause
- */
- public BackupException(Throwable cause) {
- super(cause);
- }
-
- /**
- * Exception for the given backup that has no previous root cause
- * @param msg reason why the backup failed
- * @param desc description of the backup that is being failed
- */
- public BackupException(String msg, BackupInfo desc) {
- super(msg);
- this.info = desc;
- }
-
- /**
- * Exception for the given backup due to another exception
- * @param msg reason why the backup failed
- * @param cause root cause of the failure
- * @param desc description of the backup that is being failed
- */
- public BackupException(String msg, Throwable cause, BackupInfo desc) {
- super(msg, cause);
- this.info = desc;
- }
-
- /**
- * Exception when the description of the backup cannot be determined, due to some other root
- * cause
- * @param message description of what caused the failure
- * @param e root cause
- */
- public BackupException(String message, Exception e) {
- super(message, e);
- }
-
- public BackupInfo getBackupInfo() {
- return this.info;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/37c65946/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java
deleted file mode 100644
index 8fe5eaf..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManager.java
+++ /dev/null
@@ -1,502 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.impl;
-
-import java.io.Closeable;
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.Iterator;
-import java.util.List;
-import java.util.Map;
-import java.util.Set;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.HTableDescriptor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
-import org.apache.hadoop.hbase.backup.BackupRestoreConstants;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.HBackupFileSystem;
-import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
-import org.apache.hadoop.hbase.backup.master.BackupLogCleaner;
-import org.apache.hadoop.hbase.backup.master.LogRollMasterProcedureManager;
-import org.apache.hadoop.hbase.backup.regionserver.LogRollRegionServerProcedureManager;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.procedure.ProcedureManagerHost;
-import org.apache.hadoop.hbase.util.Pair;
-
-import org.apache.hadoop.hbase.shaded.com.google.common.annotations.VisibleForTesting;
-
-/**
- * Handles backup requests, creates backup info records in backup system table to
- * keep track of backup sessions, dispatches backup request.
- */
-@InterfaceAudience.Private
-public class BackupManager implements Closeable {
- private static final Log LOG = LogFactory.getLog(BackupManager.class);
-
- protected Configuration conf = null;
- protected BackupInfo backupInfo = null;
- protected BackupSystemTable systemTable;
- protected final Connection conn;
-
- /**
- * Backup manager constructor.
- * @param conn connection
- * @param conf configuration
- * @throws IOException exception
- */
- public BackupManager(Connection conn, Configuration conf) throws IOException {
- if (!conf.getBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY,
- BackupRestoreConstants.BACKUP_ENABLE_DEFAULT)) {
- throw new BackupException("HBase backup is not enabled. Check your "
- + BackupRestoreConstants.BACKUP_ENABLE_KEY + " setting.");
- }
- this.conf = conf;
- this.conn = conn;
- this.systemTable = new BackupSystemTable(conn);
-
- }
-
- /**
- * Returns backup info
- */
- protected BackupInfo getBackupInfo() {
- return backupInfo;
- }
-
- /**
- * This method modifies the master's configuration in order to inject backup-related features
- * (TESTs only)
- * @param conf configuration
- */
- @VisibleForTesting
- public static void decorateMasterConfiguration(Configuration conf) {
- if (!isBackupEnabled(conf)) {
- return;
- }
- // Add WAL archive cleaner plug-in
- String plugins = conf.get(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS);
- String cleanerClass = BackupLogCleaner.class.getCanonicalName();
- if (!plugins.contains(cleanerClass)) {
- conf.set(HConstants.HBASE_MASTER_LOGCLEANER_PLUGINS, plugins + "," + cleanerClass);
- }
-
- String classes = conf.get(ProcedureManagerHost.MASTER_PROCEDURE_CONF_KEY);
- String masterProcedureClass = LogRollMasterProcedureManager.class.getName();
- if (classes == null) {
- conf.set(ProcedureManagerHost.MASTER_PROCEDURE_CONF_KEY, masterProcedureClass);
- } else if (!classes.contains(masterProcedureClass)) {
- conf.set(ProcedureManagerHost.MASTER_PROCEDURE_CONF_KEY, classes + "," + masterProcedureClass);
- }
-
- if (LOG.isDebugEnabled()) {
- LOG.debug("Added log cleaner: " + cleanerClass + "\n" + "Added master procedure manager: "
- + masterProcedureClass);
- }
-
- }
-
- /**
- * This method modifies the Region Server configuration in order to inject backup-related features
- * TESTs only.
- * @param conf configuration
- */
- @VisibleForTesting
- public static void decorateRegionServerConfiguration(Configuration conf) {
- if (!isBackupEnabled(conf)) {
- return;
- }
-
- String classes = conf.get(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY);
- String regionProcedureClass = LogRollRegionServerProcedureManager.class.getName();
- if (classes == null) {
- conf.set(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY, regionProcedureClass);
- } else if (!classes.contains(regionProcedureClass)) {
- conf.set(ProcedureManagerHost.REGIONSERVER_PROCEDURE_CONF_KEY, classes + ","
- + regionProcedureClass);
- }
- if (LOG.isDebugEnabled()) {
- LOG.debug("Added region procedure manager: " + regionProcedureClass);
- }
-
- }
-
- public static boolean isBackupEnabled(Configuration conf) {
- return conf.getBoolean(BackupRestoreConstants.BACKUP_ENABLE_KEY,
- BackupRestoreConstants.BACKUP_ENABLE_DEFAULT);
- }
-
- /**
- * Get configuration
- * @return configuration
- */
- Configuration getConf() {
- return conf;
- }
-
- /**
- * Stop all the work of backup.
- */
- @Override
- public void close() {
-
- if (systemTable != null) {
- try {
- systemTable.close();
- } catch (Exception e) {
- LOG.error(e);
- }
- }
- }
-
- /**
- * Creates a backup info based on input backup request.
- * @param backupId backup id
- * @param type type
- * @param tableList table list
- * @param targetRootDir root dir
- * @param workers number of parallel workers
- * @param bandwidth bandwidth per worker in MB per sec
- * @return BackupInfo
- * @throws BackupException exception
- */
- public BackupInfo createBackupInfo(String backupId, BackupType type, List<TableName> tableList,
- String targetRootDir, int workers, long bandwidth) throws BackupException {
- if (targetRootDir == null) {
- throw new BackupException("Wrong backup request parameter: target backup root directory");
- }
-
- if (type == BackupType.FULL && (tableList == null || tableList.isEmpty())) {
- // If table list is null for full backup, which means backup all tables. Then fill the table
- // list with all user tables from meta. It no table available, throw the request exception.
-
- HTableDescriptor[] htds = null;
- try (Admin admin = conn.getAdmin()) {
- htds = admin.listTables();
- } catch (Exception e) {
- throw new BackupException(e);
- }
-
- if (htds == null) {
- throw new BackupException("No table exists for full backup of all tables.");
- } else {
- tableList = new ArrayList<>();
- for (HTableDescriptor hTableDescriptor : htds) {
- TableName tn = hTableDescriptor.getTableName();
- if (tn.equals(BackupSystemTable.getTableName(conf))) {
- // skip backup system table
- continue;
- }
- tableList.add(hTableDescriptor.getTableName());
- }
-
- LOG.info("Full backup all the tables available in the cluster: " + tableList);
- }
- }
-
- // there are one or more tables in the table list
- backupInfo =
- new BackupInfo(backupId, type, tableList.toArray(new TableName[tableList.size()]),
- targetRootDir);
- backupInfo.setBandwidth(bandwidth);
- backupInfo.setWorkers(workers);
- return backupInfo;
- }
-
- /**
- * Check if any ongoing backup. Currently, we only reply on checking status in backup system
- * table. We need to consider to handle the case of orphan records in the future. Otherwise, all
- * the coming request will fail.
- * @return the ongoing backup id if on going backup exists, otherwise null
- * @throws IOException exception
- */
- private String getOngoingBackupId() throws IOException {
-
- ArrayList<BackupInfo> sessions = systemTable.getBackupInfos(BackupState.RUNNING);
- if (sessions.size() == 0) {
- return null;
- }
- return sessions.get(0).getBackupId();
- }
-
- /**
- * Start the backup manager service.
- * @throws IOException exception
- */
- public void initialize() throws IOException {
- String ongoingBackupId = this.getOngoingBackupId();
- if (ongoingBackupId != null) {
- LOG.info("There is a ongoing backup " + ongoingBackupId
- + ". Can not launch new backup until no ongoing backup remains.");
- throw new BackupException("There is ongoing backup.");
- }
- }
-
- public void setBackupInfo(BackupInfo backupInfo) {
- this.backupInfo = backupInfo;
- }
-
- /**
- * Get direct ancestors of the current backup.
- * @param backupInfo The backup info for the current backup
- * @return The ancestors for the current backup
- * @throws IOException exception
- * @throws BackupException exception
- */
- public ArrayList<BackupImage> getAncestors(BackupInfo backupInfo) throws IOException,
- BackupException {
- LOG.debug("Getting the direct ancestors of the current backup " + backupInfo.getBackupId());
-
- ArrayList<BackupImage> ancestors = new ArrayList<BackupImage>();
-
- // full backup does not have ancestor
- if (backupInfo.getType() == BackupType.FULL) {
- LOG.debug("Current backup is a full backup, no direct ancestor for it.");
- return ancestors;
- }
-
- // get all backup history list in descending order
-
- ArrayList<BackupInfo> allHistoryList = getBackupHistory(true);
- for (BackupInfo backup : allHistoryList) {
-
- BackupImage.Builder builder = BackupImage.newBuilder();
-
- BackupImage image =
- builder.withBackupId(backup.getBackupId()).withType(backup.getType())
- .withRootDir(backup.getBackupRootDir()).withTableList(backup.getTableNames())
- .withStartTime(backup.getStartTs()).withCompleteTime(backup.getCompleteTs()).build();
-
- // add the full backup image as an ancestor until the last incremental backup
- if (backup.getType().equals(BackupType.FULL)) {
- // check the backup image coverage, if previous image could be covered by the newer ones,
- // then no need to add
- if (!BackupManifest.canCoverImage(ancestors, image)) {
- ancestors.add(image);
- }
- } else {
- // found last incremental backup, if previously added full backup ancestor images can cover
- // it, then this incremental ancestor is not the dependent of the current incremental
- // backup, that is to say, this is the backup scope boundary of current table set.
- // Otherwise, this incremental backup ancestor is the dependent ancestor of the ongoing
- // incremental backup
- if (BackupManifest.canCoverImage(ancestors, image)) {
- LOG.debug("Met the backup boundary of the current table set:");
- for (BackupImage image1 : ancestors) {
- LOG.debug(" BackupID=" + image1.getBackupId() + ", BackupDir=" + image1.getRootDir());
- }
- } else {
- Path logBackupPath =
- HBackupFileSystem.getBackupPath(backup.getBackupRootDir(), backup.getBackupId());
- LOG.debug("Current backup has an incremental backup ancestor, "
- + "touching its image manifest in " + logBackupPath.toString()
- + " to construct the dependency.");
- BackupManifest lastIncrImgManifest = new BackupManifest(conf, logBackupPath);
- BackupImage lastIncrImage = lastIncrImgManifest.getBackupImage();
- ancestors.add(lastIncrImage);
-
- LOG.debug("Last dependent incremental backup image: " + "{BackupID="
- + lastIncrImage.getBackupId() + "," + "BackupDir=" + lastIncrImage.getRootDir() + "}");
- }
- }
- }
- LOG.debug("Got " + ancestors.size() + " ancestors for the current backup.");
- return ancestors;
- }
-
- /**
- * Get the direct ancestors of this backup for one table involved.
- * @param backupInfo backup info
- * @param table table
- * @return backupImages on the dependency list
- * @throws BackupException exception
- * @throws IOException exception
- */
- public ArrayList<BackupImage> getAncestors(BackupInfo backupInfo, TableName table)
- throws BackupException, IOException {
- ArrayList<BackupImage> ancestors = getAncestors(backupInfo);
- ArrayList<BackupImage> tableAncestors = new ArrayList<BackupImage>();
- for (BackupImage image : ancestors) {
- if (image.hasTable(table)) {
- tableAncestors.add(image);
- if (image.getType() == BackupType.FULL) {
- break;
- }
- }
- }
- return tableAncestors;
- }
-
- /*
- * backup system table operations
- */
-
- /**
- * Updates status (state) of a backup session in a persistent store
- * @param context context
- * @throws IOException exception
- */
- public void updateBackupInfo(BackupInfo context) throws IOException {
- systemTable.updateBackupInfo(context);
- }
-
- /**
- * Starts new backup session
- * @throws IOException if active session already exists
- */
- public void startBackupSession() throws IOException {
- systemTable.startBackupExclusiveOperation();
- }
-
- /**
- * Finishes active backup session
- * @throws IOException if no active session
- */
- public void finishBackupSession() throws IOException {
- systemTable.finishBackupExclusiveOperation();
- }
-
- /**
- * Read the last backup start code (timestamp) of last successful backup. Will return null if
- * there is no startcode stored in backup system table or the value is of length 0. These two
- * cases indicate there is no successful backup completed so far.
- * @return the timestamp of a last successful backup
- * @throws IOException exception
- */
- public String readBackupStartCode() throws IOException {
- return systemTable.readBackupStartCode(backupInfo.getBackupRootDir());
- }
-
- /**
- * Write the start code (timestamp) to backup system table. If passed in null, then write 0 byte.
- * @param startCode start code
- * @throws IOException exception
- */
- public void writeBackupStartCode(Long startCode) throws IOException {
- systemTable.writeBackupStartCode(startCode, backupInfo.getBackupRootDir());
- }
-
- /**
- * Get the RS log information after the last log roll from backup system table.
- * @return RS log info
- * @throws IOException exception
- */
- public HashMap<String, Long> readRegionServerLastLogRollResult() throws IOException {
- return systemTable.readRegionServerLastLogRollResult(backupInfo.getBackupRootDir());
- }
-
- public Pair<Map<TableName, Map<String, Map<String, List<Pair<String, Boolean>>>>>, List<byte[]>>
- readBulkloadRows(List<TableName> tableList) throws IOException {
- return systemTable.readBulkloadRows(tableList);
- }
-
- public void removeBulkLoadedRows(List<TableName> lst, List<byte[]> rows) throws IOException {
- systemTable.removeBulkLoadedRows(lst, rows);
- }
-
- public void writeBulkLoadedFiles(List<TableName> sTableList, Map<byte[], List<Path>>[] maps)
- throws IOException {
- systemTable.writeBulkLoadedFiles(sTableList, maps, backupInfo.getBackupId());
- }
-
- /**
- * Get all completed backup information (in desc order by time)
- * @return history info of BackupCompleteData
- * @throws IOException exception
- */
- public List<BackupInfo> getBackupHistory() throws IOException {
- return systemTable.getBackupHistory();
- }
-
- public ArrayList<BackupInfo> getBackupHistory(boolean completed) throws IOException {
- return systemTable.getBackupHistory(completed);
- }
-
- /**
- * Write the current timestamps for each regionserver to backup system table after a successful
- * full or incremental backup. Each table may have a different set of log timestamps. The saved
- * timestamp is of the last log file that was backed up already.
- * @param tables tables
- * @throws IOException exception
- */
- public void writeRegionServerLogTimestamp(Set<TableName> tables,
- HashMap<String, Long> newTimestamps) throws IOException {
- systemTable.writeRegionServerLogTimestamp(tables, newTimestamps, backupInfo.getBackupRootDir());
- }
-
- /**
- * Read the timestamp for each region server log after the last successful backup. Each table has
- * its own set of the timestamps.
- * @return the timestamp for each region server. key: tableName value:
- * RegionServer,PreviousTimeStamp
- * @throws IOException exception
- */
- public HashMap<TableName, HashMap<String, Long>> readLogTimestampMap() throws IOException {
- return systemTable.readLogTimestampMap(backupInfo.getBackupRootDir());
- }
-
- /**
- * Return the current tables covered by incremental backup.
- * @return set of tableNames
- * @throws IOException exception
- */
- public Set<TableName> getIncrementalBackupTableSet() throws IOException {
- return systemTable.getIncrementalBackupTableSet(backupInfo.getBackupRootDir());
- }
-
- /**
- * Adds set of tables to overall incremental backup table set
- * @param tables tables
- * @throws IOException exception
- */
- public void addIncrementalBackupTableSet(Set<TableName> tables) throws IOException {
- systemTable.addIncrementalBackupTableSet(tables, backupInfo.getBackupRootDir());
- }
-
- /**
- * Saves list of WAL files after incremental backup operation. These files will be stored until
- * TTL expiration and are used by Backup Log Cleaner plug-in to determine which WAL files can be
- * safely purged.
- */
- public void recordWALFiles(List<String> files) throws IOException {
- systemTable.addWALFiles(files, backupInfo.getBackupId(), backupInfo.getBackupRootDir());
- }
-
- /**
- * Get WAL files iterator
- * @return WAL files iterator from backup system table
- * @throws IOException
- */
- public Iterator<BackupSystemTable.WALItem> getWALFilesFromBackupSystem() throws IOException {
- return systemTable.getWALFilesIterator(backupInfo.getBackupRootDir());
- }
-
- public Connection getConnection() {
- return conn;
- }
-}