You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2016/10/05 23:30:08 UTC
[02/10] hbase git commit: HBASE-16727 Backup refactoring: remove MR
dependencies from HMaster (Vladimir Rodionov)
http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java
deleted file mode 100644
index 2d41423..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/FullTableBackupProcedure.java
+++ /dev/null
@@ -1,777 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.master;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.NotServingRegionException;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupCopyService;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase;
-import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
-import org.apache.hadoop.hbase.backup.BackupRestoreServerFactory;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.HBackupFileSystem;
-import org.apache.hadoop.hbase.backup.impl.BackupException;
-import org.apache.hadoop.hbase.backup.impl.BackupManager;
-import org.apache.hadoop.hbase.backup.impl.BackupManifest;
-import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
-import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants;
-import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
-import org.apache.hadoop.hbase.backup.util.BackupServerUtil;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
-import org.apache.hadoop.hbase.master.procedure.MasterProcedureUtil;
-import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface;
-import org.apache.hadoop.hbase.procedure.MasterProcedureManager;
-import org.apache.hadoop.hbase.procedure.ProcedureUtil;
-import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
-import org.apache.hadoop.hbase.protobuf.generated.BackupProtos;
-import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.FullTableBackupState;
-import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.ServerTimestamp;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos.SnapshotDescription;
-import org.apache.hadoop.hbase.snapshot.SnapshotDescriptionUtils;
-import org.apache.hadoop.hbase.util.EnvironmentEdgeManager;
-import org.apache.hadoop.hbase.util.FSUtils;
-
-@InterfaceAudience.Private
-public class FullTableBackupProcedure
- extends StateMachineProcedure<MasterProcedureEnv, FullTableBackupState>
- implements TableProcedureInterface {
- private static final Log LOG = LogFactory.getLog(FullTableBackupProcedure.class);
-
- private static final String SNAPSHOT_BACKUP_MAX_ATTEMPTS_KEY = "hbase.backup.snapshot.attempts.max";
- private static final int DEFAULT_SNAPSHOT_BACKUP_MAX_ATTEMPTS = 10;
-
- private static final String SNAPSHOT_BACKUP_ATTEMPTS_DELAY_KEY = "hbase.backup.snapshot.attempts.delay";
- private static final int DEFAULT_SNAPSHOT_BACKUP_ATTEMPTS_DELAY = 10000;
-
- private final AtomicBoolean aborted = new AtomicBoolean(false);
- private Configuration conf;
- private String backupId;
- private List<TableName> tableList;
- private String targetRootDir;
- HashMap<String, Long> newTimestamps = null;
-
- private BackupManager backupManager;
- private BackupInfo backupContext;
-
- public FullTableBackupProcedure() {
- // Required by the Procedure framework to create the procedure on replay
- }
-
- public FullTableBackupProcedure(final MasterProcedureEnv env,
- final String backupId, List<TableName> tableList, String targetRootDir, final int workers,
- final long bandwidth) throws IOException {
- backupManager = new BackupManager(env.getMasterServices().getConnection(),
- env.getMasterConfiguration());
- this.backupId = backupId;
- this.tableList = tableList;
- this.targetRootDir = targetRootDir;
- backupContext =
- backupManager.createBackupContext(backupId, BackupType.FULL,
- tableList, targetRootDir, workers, bandwidth);
- if (tableList == null || tableList.isEmpty()) {
- this.tableList = new ArrayList<>(backupContext.getTables());
- }
- this.setOwner(env.getRequestUser().getUGI().getShortUserName());
- }
-
- @Override
- public byte[] getResult() {
- return backupId.getBytes();
- }
-
- /**
- * Begin the overall backup.
- * @param backupContext backup context
- * @throws IOException exception
- */
- static void beginBackup(BackupManager backupManager, BackupInfo backupContext)
- throws IOException {
- backupManager.setBackupContext(backupContext);
- // set the start timestamp of the overall backup
- long startTs = EnvironmentEdgeManager.currentTime();
- backupContext.setStartTs(startTs);
- // set overall backup status: ongoing
- backupContext.setState(BackupState.RUNNING);
- LOG.info("Backup " + backupContext.getBackupId() + " started at " + startTs + ".");
-
- backupManager.updateBackupInfo(backupContext);
- if (LOG.isDebugEnabled()) {
- LOG.debug("Backup session " + backupContext.getBackupId() + " has been started.");
- }
- }
-
- private static String getMessage(Exception e) {
- String msg = e.getMessage();
- if (msg == null || msg.equals("")) {
- msg = e.getClass().getName();
- }
- return msg;
- }
-
- /**
- * Delete HBase snapshot for backup.
- * @param backupCtx backup context
- * @throws Exception exception
- */
- private static void deleteSnapshot(final MasterProcedureEnv env,
- BackupInfo backupCtx, Configuration conf)
- throws IOException {
- LOG.debug("Trying to delete snapshot for full backup.");
- for (String snapshotName : backupCtx.getSnapshotNames()) {
- if (snapshotName == null) {
- continue;
- }
- LOG.debug("Trying to delete snapshot: " + snapshotName);
- HBaseProtos.SnapshotDescription.Builder builder =
- HBaseProtos.SnapshotDescription.newBuilder();
- builder.setName(snapshotName);
- try {
- env.getMasterServices().getSnapshotManager().deleteSnapshot(builder.build());
- } catch (IOException ioe) {
- LOG.debug("when deleting snapshot " + snapshotName, ioe);
- }
- LOG.debug("Deleting the snapshot " + snapshotName + " for backup "
- + backupCtx.getBackupId() + " succeeded.");
- }
- }
-
- /**
- * Clean up directories with prefix "exportSnapshot-", which are generated when exporting
- * snapshots.
- * @throws IOException exception
- */
- private static void cleanupExportSnapshotLog(Configuration conf) throws IOException {
- FileSystem fs = FSUtils.getCurrentFileSystem(conf);
- Path stagingDir =
- new Path(conf.get(BackupRestoreConstants.CONF_STAGING_ROOT, fs.getWorkingDirectory()
- .toString()));
- FileStatus[] files = FSUtils.listStatus(fs, stagingDir);
- if (files == null) {
- return;
- }
- for (FileStatus file : files) {
- if (file.getPath().getName().startsWith("exportSnapshot-")) {
- LOG.debug("Delete log files of exporting snapshot: " + file.getPath().getName());
- if (FSUtils.delete(fs, file.getPath(), true) == false) {
- LOG.warn("Can not delete " + file.getPath());
- }
- }
- }
- }
-
- /**
- * Clean up the uncompleted data at target directory if the ongoing backup has already entered the
- * copy phase.
- */
- static void cleanupTargetDir(BackupInfo backupContext, Configuration conf) {
- try {
- // clean up the uncompleted data at target directory if the ongoing backup has already entered
- // the copy phase
- LOG.debug("Trying to cleanup up target dir. Current backup phase: "
- + backupContext.getPhase());
- if (backupContext.getPhase().equals(BackupPhase.SNAPSHOTCOPY)
- || backupContext.getPhase().equals(BackupPhase.INCREMENTAL_COPY)
- || backupContext.getPhase().equals(BackupPhase.STORE_MANIFEST)) {
- FileSystem outputFs =
- FileSystem.get(new Path(backupContext.getTargetRootDir()).toUri(), conf);
-
- // now treat one backup as a transaction, clean up data that has been partially copied at
- // table level
- for (TableName table : backupContext.getTables()) {
- Path targetDirPath =
- new Path(HBackupFileSystem.getTableBackupDir(backupContext.getTargetRootDir(),
- backupContext.getBackupId(), table));
- if (outputFs.delete(targetDirPath, true)) {
- LOG.info("Cleaning up uncompleted backup data at " + targetDirPath.toString()
- + " done.");
- } else {
- LOG.info("No data has been copied to " + targetDirPath.toString() + ".");
- }
-
- Path tableDir = targetDirPath.getParent();
- FileStatus[] backups = FSUtils.listStatus(outputFs, tableDir);
- if (backups == null || backups.length == 0) {
- outputFs.delete(tableDir, true);
- LOG.debug(tableDir.toString() + " is empty, remove it.");
- }
- }
- }
-
- } catch (IOException e1) {
- LOG.error("Cleaning up uncompleted backup data of " + backupContext.getBackupId() + " at "
- + backupContext.getTargetRootDir() + " failed due to " + e1.getMessage() + ".");
- }
- }
-
- /**
- * Fail the overall backup.
- * @param backupContext backup context
- * @param e exception
- * @throws Exception exception
- */
- static void failBackup(final MasterProcedureEnv env, BackupInfo backupContext,
- BackupManager backupManager, Exception e,
- String msg, BackupType type, Configuration conf) throws IOException {
- LOG.error(msg + getMessage(e));
- // If this is a cancel exception, then we've already cleaned.
-
- // set the failure timestamp of the overall backup
- backupContext.setEndTs(EnvironmentEdgeManager.currentTime());
-
- // set failure message
- backupContext.setFailedMsg(e.getMessage());
-
- // set overall backup status: failed
- backupContext.setState(BackupState.FAILED);
-
- // compose the backup failed data
- String backupFailedData =
- "BackupId=" + backupContext.getBackupId() + ",startts=" + backupContext.getStartTs()
- + ",failedts=" + backupContext.getEndTs() + ",failedphase=" + backupContext.getPhase()
- + ",failedmessage=" + backupContext.getFailedMsg();
- LOG.error(backupFailedData);
-
- backupManager.updateBackupInfo(backupContext);
-
- // if full backup, then delete HBase snapshots if there already are snapshots taken
- // and also clean up export snapshot log files if exist
- if (type == BackupType.FULL) {
- deleteSnapshot(env, backupContext, conf);
- cleanupExportSnapshotLog(conf);
- }
-
- // clean up the uncompleted data at target directory if the ongoing backup has already entered
- // the copy phase
- // For incremental backup, DistCp logs will be cleaned with the targetDir.
- cleanupTargetDir(backupContext, conf);
-
- LOG.info("Backup " + backupContext.getBackupId() + " failed.");
- }
-
- /**
- * Do snapshot copy.
- * @param backupContext backup context
- * @throws Exception exception
- */
- private void snapshotCopy(BackupInfo backupContext) throws Exception {
- LOG.info("Snapshot copy is starting.");
-
- // set overall backup phase: snapshot_copy
- backupContext.setPhase(BackupPhase.SNAPSHOTCOPY);
-
- // call ExportSnapshot to copy files based on hbase snapshot for backup
- // ExportSnapshot only support single snapshot export, need loop for multiple tables case
- BackupCopyService copyService = BackupRestoreServerFactory.getBackupCopyService(conf);
-
- // number of snapshots matches number of tables
- float numOfSnapshots = backupContext.getSnapshotNames().size();
-
- LOG.debug("There are " + (int) numOfSnapshots + " snapshots to be copied.");
-
- for (TableName table : backupContext.getTables()) {
- // Currently we simply set the sub copy tasks by counting the table snapshot number, we can
- // calculate the real files' size for the percentage in the future.
- // backupCopier.setSubTaskPercntgInWholeTask(1f / numOfSnapshots);
- int res = 0;
- String[] args = new String[4];
- args[0] = "-snapshot";
- args[1] = backupContext.getSnapshotName(table);
- args[2] = "-copy-to";
- args[3] = backupContext.getBackupStatus(table).getTargetDir();
-
- LOG.debug("Copy snapshot " + args[1] + " to " + args[3]);
- res = copyService.copy(backupContext, backupManager, conf, BackupCopyService.Type.FULL, args);
- // if one snapshot export failed, do not continue for remained snapshots
- if (res != 0) {
- LOG.error("Exporting Snapshot " + args[1] + " failed with return code: " + res + ".");
-
- throw new IOException("Failed of exporting snapshot " + args[1] + " to " + args[3]
- + " with reason code " + res);
- }
- LOG.info("Snapshot copy " + args[1] + " finished.");
- }
- }
-
- /**
- * Add manifest for the current backup. The manifest is stored
- * within the table backup directory.
- * @param backupContext The current backup context
- * @throws IOException exception
- * @throws BackupException exception
- */
- private static void addManifest(BackupInfo backupContext, BackupManager backupManager,
- BackupType type, Configuration conf) throws IOException, BackupException {
- // set the overall backup phase : store manifest
- backupContext.setPhase(BackupPhase.STORE_MANIFEST);
-
- BackupManifest manifest;
-
- // Since we have each table's backup in its own directory structure,
- // we'll store its manifest with the table directory.
- for (TableName table : backupContext.getTables()) {
- manifest = new BackupManifest(backupContext, table);
- ArrayList<BackupImage> ancestors = backupManager.getAncestors(backupContext, table);
- for (BackupImage image : ancestors) {
- manifest.addDependentImage(image);
- }
-
- if (type == BackupType.INCREMENTAL) {
- // We'll store the log timestamps for this table only in its manifest.
- HashMap<TableName, HashMap<String, Long>> tableTimestampMap =
- new HashMap<TableName, HashMap<String, Long>>();
- tableTimestampMap.put(table, backupContext.getIncrTimestampMap().get(table));
- manifest.setIncrTimestampMap(tableTimestampMap);
- ArrayList<BackupImage> ancestorss = backupManager.getAncestors(backupContext);
- for (BackupImage image : ancestorss) {
- manifest.addDependentImage(image);
- }
- }
- manifest.store(conf);
- }
-
- // For incremental backup, we store a overall manifest in
- // <backup-root-dir>/WALs/<backup-id>
- // This is used when created the next incremental backup
- if (type == BackupType.INCREMENTAL) {
- manifest = new BackupManifest(backupContext);
- // set the table region server start and end timestamps for incremental backup
- manifest.setIncrTimestampMap(backupContext.getIncrTimestampMap());
- ArrayList<BackupImage> ancestors = backupManager.getAncestors(backupContext);
- for (BackupImage image : ancestors) {
- manifest.addDependentImage(image);
- }
- manifest.store(conf);
- }
- }
-
- /**
- * Get backup request meta data dir as string.
- * @param backupContext backup context
- * @return meta data dir
- */
- private static String obtainBackupMetaDataStr(BackupInfo backupContext) {
- StringBuffer sb = new StringBuffer();
- sb.append("type=" + backupContext.getType() + ",tablelist=");
- for (TableName table : backupContext.getTables()) {
- sb.append(table + ";");
- }
- if (sb.lastIndexOf(";") > 0) {
- sb.delete(sb.lastIndexOf(";"), sb.lastIndexOf(";") + 1);
- }
- sb.append(",targetRootDir=" + backupContext.getTargetRootDir());
-
- return sb.toString();
- }
-
- /**
- * Clean up directories with prefix "_distcp_logs-", which are generated when DistCp copying
- * hlogs.
- * @throws IOException exception
- */
- private static void cleanupDistCpLog(BackupInfo backupContext, Configuration conf)
- throws IOException {
- Path rootPath = new Path(backupContext.getHLogTargetDir()).getParent();
- FileSystem fs = FileSystem.get(rootPath.toUri(), conf);
- FileStatus[] files = FSUtils.listStatus(fs, rootPath);
- if (files == null) {
- return;
- }
- for (FileStatus file : files) {
- if (file.getPath().getName().startsWith("_distcp_logs")) {
- LOG.debug("Delete log files of DistCp: " + file.getPath().getName());
- FSUtils.delete(fs, file.getPath(), true);
- }
- }
- }
-
- /**
- * Complete the overall backup.
- * @param backupContext backup context
- * @throws Exception exception
- */
- static void completeBackup(final MasterProcedureEnv env, BackupInfo backupContext,
- BackupManager backupManager, BackupType type, Configuration conf) throws IOException {
- // set the complete timestamp of the overall backup
- backupContext.setEndTs(EnvironmentEdgeManager.currentTime());
- // set overall backup status: complete
- backupContext.setState(BackupState.COMPLETE);
- backupContext.setProgress(100);
- // add and store the manifest for the backup
- addManifest(backupContext, backupManager, type, conf);
-
- // after major steps done and manifest persisted, do convert if needed for incremental backup
- /* in-fly convert code here, provided by future jira */
- LOG.debug("in-fly convert code here, provided by future jira");
-
- // compose the backup complete data
- String backupCompleteData =
- obtainBackupMetaDataStr(backupContext) + ",startts=" + backupContext.getStartTs()
- + ",completets=" + backupContext.getEndTs() + ",bytescopied="
- + backupContext.getTotalBytesCopied();
- if (LOG.isDebugEnabled()) {
- LOG.debug("Backup " + backupContext.getBackupId() + " finished: " + backupCompleteData);
- }
- backupManager.updateBackupInfo(backupContext);
-
- // when full backup is done:
- // - delete HBase snapshot
- // - clean up directories with prefix "exportSnapshot-", which are generated when exporting
- // snapshots
- if (type == BackupType.FULL) {
- deleteSnapshot(env, backupContext, conf);
- cleanupExportSnapshotLog(conf);
- } else if (type == BackupType.INCREMENTAL) {
- cleanupDistCpLog(backupContext, conf);
- }
-
- LOG.info("Backup " + backupContext.getBackupId() + " completed.");
- }
-
- /**
- * Wrap a SnapshotDescription for a target table.
- * @param table table
- * @return a SnapshotDescription especially for backup.
- */
- static SnapshotDescription wrapSnapshotDescription(TableName tableName, String snapshotName) {
- // Mock a SnapshotDescription from backupContext to call SnapshotManager function,
- // Name it in the format "snapshot_<timestamp>_<table>"
- HBaseProtos.SnapshotDescription.Builder builder = HBaseProtos.SnapshotDescription.newBuilder();
- builder.setTable(tableName.getNameAsString());
- builder.setName(snapshotName);
- HBaseProtos.SnapshotDescription backupSnapshot = builder.build();
-
- LOG.debug("Wrapped a SnapshotDescription " + backupSnapshot.getName()
- + " from backupContext to request snapshot for backup.");
-
- return backupSnapshot;
- }
-
- @Override
- protected Flow executeFromState(final MasterProcedureEnv env, final FullTableBackupState state)
- throws InterruptedException {
- if (conf == null) {
- conf = env.getMasterConfiguration();
- }
- if (backupManager == null) {
- try {
- backupManager = new BackupManager(env.getMasterServices().getConnection(),
- env.getMasterConfiguration());
- } catch (IOException ioe) {
- setFailure("full backup", ioe);
- return Flow.NO_MORE_STATE;
- }
- }
- if (LOG.isTraceEnabled()) {
- LOG.trace(this + " execute state=" + state);
- }
- try {
- switch (state) {
- case PRE_SNAPSHOT_TABLE:
- beginBackup(backupManager, backupContext);
- String savedStartCode = null;
- boolean firstBackup = false;
- // do snapshot for full table backup
-
- try {
- savedStartCode = backupManager.readBackupStartCode();
- firstBackup = savedStartCode == null || Long.parseLong(savedStartCode) == 0L;
- if (firstBackup) {
- // This is our first backup. Let's put some marker on ZK so that we can hold the logs
- // while we do the backup.
- backupManager.writeBackupStartCode(0L);
- }
- // We roll log here before we do the snapshot. It is possible there is duplicate data
- // in the log that is already in the snapshot. But if we do it after the snapshot, we
- // could have data loss.
- // A better approach is to do the roll log on each RS in the same global procedure as
- // the snapshot.
- LOG.info("Execute roll log procedure for full backup ...");
- MasterProcedureManager mpm = env.getMasterServices().getMasterProcedureManagerHost()
- .getProcedureManager(LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE);
- Map<String, String> props= new HashMap<String, String>();
- props.put("backupRoot", backupContext.getTargetRootDir());
- long waitTime = ProcedureUtil.execProcedure(mpm,
- LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE,
- LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props);
- ProcedureUtil.waitForProcedure(mpm,
- LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_SIGNATURE,
- LogRollMasterProcedureManager.ROLLLOG_PROCEDURE_NAME, props, waitTime,
- conf.getInt(HConstants.HBASE_CLIENT_RETRIES_NUMBER,
- HConstants.DEFAULT_HBASE_CLIENT_RETRIES_NUMBER),
- conf.getLong(HConstants.HBASE_CLIENT_PAUSE,
- HConstants.DEFAULT_HBASE_CLIENT_PAUSE));
-
- newTimestamps = backupManager.readRegionServerLastLogRollResult();
- if (firstBackup) {
- // Updates registered log files
- // We record ALL old WAL files as registered, because
- // this is a first full backup in the system and these
- // files are not needed for next incremental backup
- List<String> logFiles = BackupServerUtil.getWALFilesOlderThan(conf, newTimestamps);
- backupManager.recordWALFiles(logFiles);
- }
- } catch (BackupException e) {
- setFailure("Failure in full-backup: pre-snapshot phase", e);
- // fail the overall backup and return
- failBackup(env, backupContext, backupManager, e, "Unexpected BackupException : ",
- BackupType.FULL, conf);
- return Flow.NO_MORE_STATE;
- }
- setNextState(FullTableBackupState.SNAPSHOT_TABLES);
- break;
- case SNAPSHOT_TABLES:
- for (TableName tableName : tableList) {
- String snapshotName = "snapshot_" + Long.toString(EnvironmentEdgeManager.currentTime())
- + "_" + tableName.getNamespaceAsString() + "_" + tableName.getQualifierAsString();
- HBaseProtos.SnapshotDescription backupSnapshot;
-
- // wrap a SnapshotDescription for offline/online snapshot
- backupSnapshot = wrapSnapshotDescription(tableName,snapshotName);
- try {
- env.getMasterServices().getSnapshotManager().deleteSnapshot(backupSnapshot);
- } catch (IOException e) {
- LOG.debug("Unable to delete " + snapshotName, e);
- }
- // Kick off snapshot for backup
- snapshotTable(env, backupSnapshot);
- backupContext.setSnapshotName(tableName, backupSnapshot.getName());
- }
- setNextState(FullTableBackupState.SNAPSHOT_COPY);
- break;
- case SNAPSHOT_COPY:
- // do snapshot copy
- LOG.debug("snapshot copy for " + backupId);
- try {
- this.snapshotCopy(backupContext);
- } catch (Exception e) {
- setFailure("Failure in full-backup: snapshot copy phase" + backupId, e);
- // fail the overall backup and return
- failBackup(env, backupContext, backupManager, e, "Unexpected BackupException : ",
- BackupType.FULL, conf);
- return Flow.NO_MORE_STATE;
- }
- // Updates incremental backup table set
- backupManager.addIncrementalBackupTableSet(backupContext.getTables());
- setNextState(FullTableBackupState.BACKUP_COMPLETE);
- break;
-
- case BACKUP_COMPLETE:
- // set overall backup status: complete. Here we make sure to complete the backup.
- // After this checkpoint, even if entering cancel process, will let the backup finished
- backupContext.setState(BackupState.COMPLETE);
- // The table list in backupContext is good for both full backup and incremental backup.
- // For incremental backup, it contains the incremental backup table set.
- backupManager.writeRegionServerLogTimestamp(backupContext.getTables(), newTimestamps);
-
- HashMap<TableName, HashMap<String, Long>> newTableSetTimestampMap =
- backupManager.readLogTimestampMap();
-
- Long newStartCode =
- BackupClientUtil.getMinValue(BackupServerUtil.getRSLogTimestampMins(newTableSetTimestampMap));
- backupManager.writeBackupStartCode(newStartCode);
-
- // backup complete
- completeBackup(env, backupContext, backupManager, BackupType.FULL, conf);
- return Flow.NO_MORE_STATE;
-
- default:
- throw new UnsupportedOperationException("unhandled state=" + state);
- }
- } catch (IOException e) {
- LOG.error("Backup failed in " + state);
- setFailure("snapshot-table", e);
- }
- return Flow.HAS_MORE_STATE;
- }
-
- private void snapshotTable(final MasterProcedureEnv env, SnapshotDescription backupSnapshot)
- throws IOException
- {
-
- int maxAttempts = env.getMasterConfiguration().getInt(SNAPSHOT_BACKUP_MAX_ATTEMPTS_KEY,
- DEFAULT_SNAPSHOT_BACKUP_MAX_ATTEMPTS);
- int delay = env.getMasterConfiguration().getInt(SNAPSHOT_BACKUP_ATTEMPTS_DELAY_KEY,
- DEFAULT_SNAPSHOT_BACKUP_ATTEMPTS_DELAY);
- int attempts = 0;
-
- while (attempts++ < maxAttempts) {
- try {
- env.getMasterServices().getSnapshotManager().takeSnapshot(backupSnapshot);
- long waitTime = SnapshotDescriptionUtils.getMaxMasterTimeout(
- env.getMasterConfiguration(),
- backupSnapshot.getType(), SnapshotDescriptionUtils.DEFAULT_MAX_WAIT_TIME);
- BackupServerUtil.waitForSnapshot(backupSnapshot, waitTime,
- env.getMasterServices().getSnapshotManager(), env.getMasterConfiguration());
- break;
- } catch( NotServingRegionException ee) {
- LOG.warn("Snapshot attempt "+attempts +" failed for table "+backupSnapshot.getTable() +
- ", sleeping for " + delay+"ms", ee);
- if(attempts < maxAttempts) {
- try {
- Thread.sleep(delay);
- } catch (InterruptedException e) {
- Thread.currentThread().interrupt();
- }
- }
- }
- }
- }
- @Override
- protected void rollbackState(final MasterProcedureEnv env, final FullTableBackupState state)
- throws IOException {
- if (state != FullTableBackupState.PRE_SNAPSHOT_TABLE) {
- deleteSnapshot(env, backupContext, conf);
- cleanupExportSnapshotLog(conf);
- }
-
- // clean up the uncompleted data at target directory if the ongoing backup has already entered
- // the copy phase
- // For incremental backup, DistCp logs will be cleaned with the targetDir.
- if (state == FullTableBackupState.SNAPSHOT_COPY) {
- cleanupTargetDir(backupContext, conf);
- }
- }
-
- @Override
- protected FullTableBackupState getState(final int stateId) {
- return FullTableBackupState.valueOf(stateId);
- }
-
- @Override
- protected int getStateId(final FullTableBackupState state) {
- return state.getNumber();
- }
-
- @Override
- protected FullTableBackupState getInitialState() {
- return FullTableBackupState.PRE_SNAPSHOT_TABLE;
- }
-
- @Override
- protected void setNextState(final FullTableBackupState state) {
- if (aborted.get()) {
- setAbortFailure("backup-table", "abort requested");
- } else {
- super.setNextState(state);
- }
- }
-
- @Override
- public boolean abort(final MasterProcedureEnv env) {
- aborted.set(true);
- return true;
- }
-
- @Override
- public void toStringClassDetails(StringBuilder sb) {
- sb.append(getClass().getSimpleName());
- sb.append(" (targetRootDir=");
- sb.append(targetRootDir);
- sb.append("; backupId=").append(backupId);
- sb.append("; tables=");
- int len = tableList.size();
- for (int i = 0; i < len-1; i++) {
- sb.append(tableList.get(i)).append(",");
- }
- sb.append(tableList.get(len-1));
- sb.append(")");
- }
-
- BackupProtos.BackupProcContext toBackupContext() {
- BackupProtos.BackupProcContext.Builder ctxBuilder = BackupProtos.BackupProcContext.newBuilder();
- ctxBuilder.setCtx(backupContext.toProtosBackupInfo());
- if (newTimestamps != null && !newTimestamps.isEmpty()) {
- BackupProtos.ServerTimestamp.Builder tsBuilder = ServerTimestamp.newBuilder();
- for (Entry<String, Long> entry : newTimestamps.entrySet()) {
- tsBuilder.clear().setServer(entry.getKey()).setTimestamp(entry.getValue());
- ctxBuilder.addServerTimestamp(tsBuilder.build());
- }
- }
- return ctxBuilder.build();
- }
-
- @Override
- public void serializeStateData(final OutputStream stream) throws IOException {
- super.serializeStateData(stream);
-
- BackupProtos.BackupProcContext backupProcCtx = toBackupContext();
- backupProcCtx.writeDelimitedTo(stream);
- }
-
- @Override
- public void deserializeStateData(final InputStream stream) throws IOException {
- super.deserializeStateData(stream);
-
- BackupProtos.BackupProcContext proto =BackupProtos.BackupProcContext.parseDelimitedFrom(stream);
- backupContext = BackupInfo.fromProto(proto.getCtx());
- backupId = backupContext.getBackupId();
- targetRootDir = backupContext.getTargetRootDir();
- tableList = backupContext.getTableNames();
- List<ServerTimestamp> svrTimestamps = proto.getServerTimestampList();
- if (svrTimestamps != null && !svrTimestamps.isEmpty()) {
- newTimestamps = new HashMap<>();
- for (ServerTimestamp ts : svrTimestamps) {
- newTimestamps.put(ts.getServer(), ts.getTimestamp());
- }
- }
- }
-
- @Override
- public TableName getTableName() {
- return TableName.BACKUP_TABLE_NAME;
- }
-
- @Override
- public TableOperationType getTableOperationType() {
- return TableOperationType.EDIT;
- }
-
- @Override
- protected boolean acquireLock(final MasterProcedureEnv env) {
- if (env.waitInitialized(this)) {
- return false;
- }
- return env.getProcedureQueue().tryAcquireTableExclusiveLock(this, TableName.BACKUP_TABLE_NAME);
- }
-
- @Override
- protected void releaseLock(final MasterProcedureEnv env) {
- env.getProcedureQueue().releaseTableExclusiveLock(this, TableName.BACKUP_TABLE_NAME);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java
deleted file mode 100644
index e877ebd..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/IncrementalTableBackupProcedure.java
+++ /dev/null
@@ -1,400 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.master;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map.Entry;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupCopyService;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.hadoop.hbase.backup.BackupRestoreServerFactory;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.BackupCopyService.Type;
-import org.apache.hadoop.hbase.backup.BackupInfo.BackupPhase;
-import org.apache.hadoop.hbase.backup.BackupInfo.BackupState;
-import org.apache.hadoop.hbase.backup.impl.BackupManager;
-import org.apache.hadoop.hbase.backup.impl.IncrementalBackupManager;
-import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
-import org.apache.hadoop.hbase.backup.util.BackupServerUtil;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
-import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface;
-import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
-import org.apache.hadoop.hbase.protobuf.generated.BackupProtos;
-import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.IncrementalTableBackupState;
-import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.ServerTimestamp;
-import org.apache.hadoop.security.UserGroupInformation;
-
-@InterfaceAudience.Private
-public class IncrementalTableBackupProcedure
- extends StateMachineProcedure<MasterProcedureEnv, IncrementalTableBackupState>
- implements TableProcedureInterface {
- private static final Log LOG = LogFactory.getLog(IncrementalTableBackupProcedure.class);
-
- private final AtomicBoolean aborted = new AtomicBoolean(false);
- private Configuration conf;
- private String backupId;
- private List<TableName> tableList;
- private String targetRootDir;
- HashMap<String, Long> newTimestamps = null;
-
- private BackupManager backupManager;
- private BackupInfo backupContext;
-
- public IncrementalTableBackupProcedure() {
- // Required by the Procedure framework to create the procedure on replay
- }
-
- public IncrementalTableBackupProcedure(final MasterProcedureEnv env,
- final String backupId,
- List<TableName> tableList, String targetRootDir, final int workers,
- final long bandwidth) throws IOException {
- backupManager = new BackupManager(env.getMasterServices().getConnection(),
- env.getMasterConfiguration());
- this.backupId = backupId;
- this.tableList = tableList;
- this.targetRootDir = targetRootDir;
- backupContext = backupManager.createBackupContext(backupId,
- BackupType.INCREMENTAL, tableList, targetRootDir, workers, (int)bandwidth);
- this.setOwner(env.getRequestUser().getUGI().getShortUserName());
- }
-
- @Override
- public byte[] getResult() {
- return backupId.getBytes();
- }
-
- private List<String> filterMissingFiles(List<String> incrBackupFileList) throws IOException {
- FileSystem fs = FileSystem.get(conf);
- List<String> list = new ArrayList<String>();
- for (String file : incrBackupFileList) {
- if (fs.exists(new Path(file))) {
- list.add(file);
- } else {
- LOG.warn("Can't find file: " + file);
- }
- }
- return list;
- }
-
- private List<String> getMissingFiles(List<String> incrBackupFileList) throws IOException {
- FileSystem fs = FileSystem.get(conf);
- List<String> list = new ArrayList<String>();
- for (String file : incrBackupFileList) {
- if (!fs.exists(new Path(file))) {
- list.add(file);
- }
- }
- return list;
-
- }
-
- /**
- * Do incremental copy.
- * @param backupContext backup context
- */
- private void incrementalCopy(BackupInfo backupContext) throws Exception {
-
- LOG.info("Incremental copy is starting.");
- // set overall backup phase: incremental_copy
- backupContext.setPhase(BackupPhase.INCREMENTAL_COPY);
- // get incremental backup file list and prepare parms for DistCp
- List<String> incrBackupFileList = backupContext.getIncrBackupFileList();
- // filter missing files out (they have been copied by previous backups)
- incrBackupFileList = filterMissingFiles(incrBackupFileList);
- String[] strArr = incrBackupFileList.toArray(new String[incrBackupFileList.size() + 1]);
- strArr[strArr.length - 1] = backupContext.getHLogTargetDir();
-
- BackupCopyService copyService = BackupRestoreServerFactory.getBackupCopyService(conf);
- int counter = 0;
- int MAX_ITERAIONS = 2;
- while (counter++ < MAX_ITERAIONS) {
- // We run DistCp maximum 2 times
- // If it fails on a second time, we throw Exception
- int res = copyService.copy(backupContext, backupManager, conf,
- BackupCopyService.Type.INCREMENTAL, strArr);
-
- if (res != 0) {
- LOG.error("Copy incremental log files failed with return code: " + res + ".");
- throw new IOException("Failed of Hadoop Distributed Copy from "+
- StringUtils.join(incrBackupFileList, ",") +" to "
- + backupContext.getHLogTargetDir());
- }
- List<String> missingFiles = getMissingFiles(incrBackupFileList);
-
- if(missingFiles.isEmpty()) {
- break;
- } else {
- // Repeat DistCp, some files have been moved from WALs to oldWALs during previous run
- // update backupContext and strAttr
- if(counter == MAX_ITERAIONS){
- String msg = "DistCp could not finish the following files: " +
- StringUtils.join(missingFiles, ",");
- LOG.error(msg);
- throw new IOException(msg);
- }
- List<String> converted = convertFilesFromWALtoOldWAL(missingFiles);
- incrBackupFileList.removeAll(missingFiles);
- incrBackupFileList.addAll(converted);
- backupContext.setIncrBackupFileList(incrBackupFileList);
-
- // Run DistCp only for missing files (which have been moved from WALs to oldWALs
- // during previous run)
- strArr = converted.toArray(new String[converted.size() + 1]);
- strArr[strArr.length - 1] = backupContext.getHLogTargetDir();
- }
- }
-
-
- LOG.info("Incremental copy from " + StringUtils.join(incrBackupFileList, ",") + " to "
- + backupContext.getHLogTargetDir() + " finished.");
- }
-
-
- private List<String> convertFilesFromWALtoOldWAL(List<String> missingFiles) throws IOException {
- List<String> list = new ArrayList<String>();
- for(String path: missingFiles){
- if(path.indexOf(Path.SEPARATOR + HConstants.HREGION_LOGDIR_NAME) < 0) {
- LOG.error("Copy incremental log files failed, file is missing : " + path);
- throw new IOException("Failed of Hadoop Distributed Copy to "
- + backupContext.getHLogTargetDir()+", file is missing "+ path);
- }
- list.add(path.replace(Path.SEPARATOR + HConstants.HREGION_LOGDIR_NAME,
- Path.SEPARATOR + HConstants.HREGION_OLDLOGDIR_NAME));
- }
- return list;
- }
-
- @Override
- protected Flow executeFromState(final MasterProcedureEnv env,
- final IncrementalTableBackupState state)
- throws InterruptedException {
- if (conf == null) {
- conf = env.getMasterConfiguration();
- }
- if (backupManager == null) {
- try {
- backupManager = new BackupManager(env.getMasterServices().getConnection(),
- env.getMasterConfiguration());
- } catch (IOException ioe) {
- setFailure("incremental backup", ioe);
- }
- }
- if (LOG.isTraceEnabled()) {
- LOG.trace(this + " execute state=" + state);
- }
- try {
- switch (state) {
- case PREPARE_INCREMENTAL:
- FullTableBackupProcedure.beginBackup(backupManager, backupContext);
- LOG.debug("For incremental backup, current table set is "
- + backupManager.getIncrementalBackupTableSet());
- try {
- IncrementalBackupManager incrBackupManager =new IncrementalBackupManager(backupManager);
-
- newTimestamps = incrBackupManager.getIncrBackupLogFileList(env.getMasterServices(),
- backupContext);
- } catch (Exception e) {
- setFailure("Failure in incremental-backup: preparation phase " + backupId, e);
- // fail the overall backup and return
- FullTableBackupProcedure.failBackup(env, backupContext, backupManager, e,
- "Unexpected Exception : ", BackupType.INCREMENTAL, conf);
- }
-
- setNextState(IncrementalTableBackupState.INCREMENTAL_COPY);
- break;
- case INCREMENTAL_COPY:
- try {
- // copy out the table and region info files for each table
- BackupServerUtil.copyTableRegionInfo(env.getMasterServices(), backupContext, conf);
- incrementalCopy(backupContext);
- // Save list of WAL files copied
- backupManager.recordWALFiles(backupContext.getIncrBackupFileList());
- } catch (Exception e) {
- String msg = "Unexpected exception in incremental-backup: incremental copy " + backupId;
- setFailure(msg, e);
- // fail the overall backup and return
- FullTableBackupProcedure.failBackup(env, backupContext, backupManager, e,
- msg, BackupType.INCREMENTAL, conf);
- }
- setNextState(IncrementalTableBackupState.INCR_BACKUP_COMPLETE);
- break;
- case INCR_BACKUP_COMPLETE:
- // set overall backup status: complete. Here we make sure to complete the backup.
- // After this checkpoint, even if entering cancel process, will let the backup finished
- backupContext.setState(BackupState.COMPLETE);
- // Set the previousTimestampMap which is before this current log roll to the manifest.
- HashMap<TableName, HashMap<String, Long>> previousTimestampMap =
- backupManager.readLogTimestampMap();
- backupContext.setIncrTimestampMap(previousTimestampMap);
-
- // The table list in backupContext is good for both full backup and incremental backup.
- // For incremental backup, it contains the incremental backup table set.
- backupManager.writeRegionServerLogTimestamp(backupContext.getTables(), newTimestamps);
-
- HashMap<TableName, HashMap<String, Long>> newTableSetTimestampMap =
- backupManager.readLogTimestampMap();
-
- Long newStartCode = BackupClientUtil
- .getMinValue(BackupServerUtil.getRSLogTimestampMins(newTableSetTimestampMap));
- backupManager.writeBackupStartCode(newStartCode);
- // backup complete
- FullTableBackupProcedure.completeBackup(env, backupContext, backupManager,
- BackupType.INCREMENTAL, conf);
- return Flow.NO_MORE_STATE;
-
- default:
- throw new UnsupportedOperationException("unhandled state=" + state);
- }
- } catch (IOException e) {
- setFailure("snapshot-table", e);
- }
- return Flow.HAS_MORE_STATE;
- }
-
- @Override
- protected void rollbackState(final MasterProcedureEnv env,
- final IncrementalTableBackupState state) throws IOException {
- // clean up the uncompleted data at target directory if the ongoing backup has already entered
- // the copy phase
- // For incremental backup, DistCp logs will be cleaned with the targetDir.
- FullTableBackupProcedure.cleanupTargetDir(backupContext, conf);
- }
-
- @Override
- protected IncrementalTableBackupState getState(final int stateId) {
- return IncrementalTableBackupState.valueOf(stateId);
- }
-
- @Override
- protected int getStateId(final IncrementalTableBackupState state) {
- return state.getNumber();
- }
-
- @Override
- protected IncrementalTableBackupState getInitialState() {
- return IncrementalTableBackupState.PREPARE_INCREMENTAL;
- }
-
- @Override
- protected void setNextState(final IncrementalTableBackupState state) {
- if (aborted.get()) {
- setAbortFailure("snapshot-table", "abort requested");
- } else {
- super.setNextState(state);
- }
- }
-
- @Override
- public boolean abort(final MasterProcedureEnv env) {
- aborted.set(true);
- return true;
- }
-
- @Override
- public void toStringClassDetails(StringBuilder sb) {
- sb.append(getClass().getSimpleName());
- sb.append(" (targetRootDir=");
- sb.append(targetRootDir);
- sb.append("; backupId=").append(backupId);
- sb.append("; tables=");
- int len = tableList.size();
- for (int i = 0; i < len-1; i++) {
- sb.append(tableList.get(i)).append(",");
- }
- sb.append(tableList.get(len-1));
- sb.append(")");
- }
-
- BackupProtos.BackupProcContext toBackupContext() {
- BackupProtos.BackupProcContext.Builder ctxBuilder = BackupProtos.BackupProcContext.newBuilder();
- ctxBuilder.setCtx(backupContext.toProtosBackupInfo());
- if (newTimestamps != null && !newTimestamps.isEmpty()) {
- BackupProtos.ServerTimestamp.Builder tsBuilder = ServerTimestamp.newBuilder();
- for (Entry<String, Long> entry : newTimestamps.entrySet()) {
- tsBuilder.clear().setServer(entry.getKey()).setTimestamp(entry.getValue());
- ctxBuilder.addServerTimestamp(tsBuilder.build());
- }
- }
- return ctxBuilder.build();
- }
-
- @Override
- public void serializeStateData(final OutputStream stream) throws IOException {
- super.serializeStateData(stream);
-
- BackupProtos.BackupProcContext backupProcCtx = toBackupContext();
- backupProcCtx.writeDelimitedTo(stream);
- }
-
- @Override
- public void deserializeStateData(final InputStream stream) throws IOException {
- super.deserializeStateData(stream);
-
- BackupProtos.BackupProcContext proto =BackupProtos.BackupProcContext.parseDelimitedFrom(stream);
- backupContext = BackupInfo.fromProto(proto.getCtx());
- backupId = backupContext.getBackupId();
- targetRootDir = backupContext.getTargetRootDir();
- tableList = backupContext.getTableNames();
- List<ServerTimestamp> svrTimestamps = proto.getServerTimestampList();
- if (svrTimestamps != null && !svrTimestamps.isEmpty()) {
- newTimestamps = new HashMap<>();
- for (ServerTimestamp ts : svrTimestamps) {
- newTimestamps.put(ts.getServer(), ts.getTimestamp());
- }
- }
- }
-
- @Override
- public TableName getTableName() {
- return TableName.BACKUP_TABLE_NAME;
- }
-
- @Override
- public TableOperationType getTableOperationType() {
- return TableOperationType.EDIT;
- }
-
- @Override
- protected boolean acquireLock(final MasterProcedureEnv env) {
- if (env.waitInitialized(this)) {
- return false;
- }
- return env.getProcedureQueue().tryAcquireTableExclusiveLock(this, TableName.BACKUP_TABLE_NAME);
- }
-
- @Override
- protected void releaseLock(final MasterProcedureEnv env) {
- env.getProcedureQueue().releaseTableExclusiveLock(this, TableName.BACKUP_TABLE_NAME);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/RestoreTablesProcedure.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/RestoreTablesProcedure.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/RestoreTablesProcedure.java
deleted file mode 100644
index 8fd7621..0000000
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/master/RestoreTablesProcedure.java
+++ /dev/null
@@ -1,387 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.master;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.io.OutputStream;
-import java.util.ArrayList;
-import java.util.HashMap;
-import java.util.List;
-import java.util.TreeSet;
-import java.util.concurrent.atomic.AtomicBoolean;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.MetaTableAccessor;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.HBackupFileSystem;
-import org.apache.hadoop.hbase.backup.impl.BackupManifest;
-import org.apache.hadoop.hbase.backup.impl.BackupManifest.BackupImage;
-import org.apache.hadoop.hbase.backup.util.RestoreServerUtil;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.TableState;
-import org.apache.hadoop.hbase.master.MasterServices;
-import org.apache.hadoop.hbase.master.TableStateManager;
-import org.apache.hadoop.hbase.master.procedure.MasterProcedureEnv;
-import org.apache.hadoop.hbase.master.procedure.TableProcedureInterface;
-import org.apache.hadoop.hbase.procedure2.StateMachineProcedure;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProtos;
-import org.apache.hadoop.hbase.protobuf.generated.MasterProtos.RestoreTablesState;
-
-@InterfaceAudience.Private
-public class RestoreTablesProcedure
- extends StateMachineProcedure<MasterProcedureEnv, RestoreTablesState>
- implements TableProcedureInterface {
- private static final Log LOG = LogFactory.getLog(RestoreTablesProcedure.class);
-
- private final AtomicBoolean aborted = new AtomicBoolean(false);
- private Configuration conf;
- private String backupId;
- private List<TableName> sTableList;
- private List<TableName> tTableList;
- private String targetRootDir;
- private boolean isOverwrite;
-
- public RestoreTablesProcedure() {
- // Required by the Procedure framework to create the procedure on replay
- }
-
- public RestoreTablesProcedure(final MasterProcedureEnv env,
- final String targetRootDir, String backupId, List<TableName> sTableList,
- List<TableName> tTableList, boolean isOverwrite) throws IOException {
- this.targetRootDir = targetRootDir;
- this.backupId = backupId;
- this.sTableList = sTableList;
- this.tTableList = tTableList;
- if (tTableList == null || tTableList.isEmpty()) {
- this.tTableList = sTableList;
- }
- this.isOverwrite = isOverwrite;
- this.setOwner(env.getRequestUser().getUGI().getShortUserName());
- }
-
- @Override
- public byte[] getResult() {
- return null;
- }
-
- /**
- * Validate target Tables
- * @param conn connection
- * @param mgr table state manager
- * @param tTableArray: target tables
- * @param isOverwrite overwrite existing table
- * @throws IOException exception
- */
- private void checkTargetTables(Connection conn, TableStateManager mgr, TableName[] tTableArray,
- boolean isOverwrite)
- throws IOException {
- ArrayList<TableName> existTableList = new ArrayList<>();
- ArrayList<TableName> disabledTableList = new ArrayList<>();
-
- // check if the tables already exist
- for (TableName tableName : tTableArray) {
- if (MetaTableAccessor.tableExists(conn, tableName)) {
- existTableList.add(tableName);
- if (mgr.isTableState(tableName, TableState.State.DISABLED, TableState.State.DISABLING)) {
- disabledTableList.add(tableName);
- }
- } else {
- LOG.info("HBase table " + tableName
- + " does not exist. It will be created during restore process");
- }
- }
-
- if (existTableList.size() > 0) {
- if (!isOverwrite) {
- LOG.error("Existing table (" + existTableList + ") found in the restore target, please add "
- + "\"-overwrite\" option in the command if you mean to restore to these existing tables");
- throw new IOException("Existing table found in target while no \"-overwrite\" "
- + "option found");
- } else {
- if (disabledTableList.size() > 0) {
- LOG.error("Found offline table in the restore target, "
- + "please enable them before restore with \"-overwrite\" option");
- LOG.info("Offline table list in restore target: " + disabledTableList);
- throw new IOException(
- "Found offline table in the target when restore with \"-overwrite\" option");
- }
- }
- }
- }
-
- /**
- * Restore operation handle each backupImage in array
- * @param svc: master services
- * @param images: array BackupImage
- * @param sTable: table to be restored
- * @param tTable: table to be restored to
- * @param truncateIfExists: truncate table
- * @throws IOException exception
- */
-
- private void restoreImages(MasterServices svc, BackupImage[] images, TableName sTable, TableName tTable,
- boolean truncateIfExists) throws IOException {
-
- // First image MUST be image of a FULL backup
- BackupImage image = images[0];
- String rootDir = image.getRootDir();
- String backupId = image.getBackupId();
- Path backupRoot = new Path(rootDir);
- RestoreServerUtil restoreTool = new RestoreServerUtil(conf, backupRoot, backupId);
- Path tableBackupPath = HBackupFileSystem.getTableBackupPath(sTable, backupRoot, backupId);
- String lastIncrBackupId = images.length == 1 ? null : images[images.length - 1].getBackupId();
- // We need hFS only for full restore (see the code)
- BackupManifest manifest = HBackupFileSystem.getManifest(sTable, conf, backupRoot, backupId);
- if (manifest.getType() == BackupType.FULL) {
- LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from full"
- + " backup image " + tableBackupPath.toString());
- restoreTool.fullRestoreTable(svc, tableBackupPath, sTable, tTable, truncateIfExists,
- lastIncrBackupId);
- } else { // incremental Backup
- throw new IOException("Unexpected backup type " + image.getType());
- }
-
- if (images.length == 1) {
- // full backup restore done
- return;
- }
-
- List<Path> dirList = new ArrayList<Path>();
- // add full backup path
- // full backup path comes first
- for (int i = 1; i < images.length; i++) {
- BackupImage im = images[i];
- String logBackupDir = HBackupFileSystem.getLogBackupDir(im.getRootDir(), im.getBackupId());
- dirList.add(new Path(logBackupDir));
- }
-
- String dirs = StringUtils.join(dirList, ",");
- LOG.info("Restoring '" + sTable + "' to '" + tTable + "' from log dirs: " + dirs);
- Path[] paths = new Path[dirList.size()];
- dirList.toArray(paths);
- restoreTool.incrementalRestoreTable(svc, tableBackupPath, paths, new TableName[] { sTable },
- new TableName[] { tTable }, lastIncrBackupId);
- LOG.info(sTable + " has been successfully restored to " + tTable);
-
- }
-
- /**
- * Restore operation. Stage 2: resolved Backup Image dependency
- * @param svc: master services
- * @param backupManifestMap : tableName, Manifest
- * @param sTableArray The array of tables to be restored
- * @param tTableArray The array of mapping tables to restore to
- * @return set of BackupImages restored
- * @throws IOException exception
- */
- private void restore(MasterServices svc, HashMap<TableName, BackupManifest> backupManifestMap,
- TableName[] sTableArray, TableName[] tTableArray, boolean isOverwrite) throws IOException {
- TreeSet<BackupImage> restoreImageSet = new TreeSet<BackupImage>();
- boolean truncateIfExists = isOverwrite;
- try {
- for (int i = 0; i < sTableArray.length; i++) {
- TableName table = sTableArray[i];
- BackupManifest manifest = backupManifestMap.get(table);
- // Get the image list of this backup for restore in time order from old
- // to new.
- List<BackupImage> list = new ArrayList<BackupImage>();
- list.add(manifest.getBackupImage());
- TreeSet<BackupImage> set = new TreeSet<BackupImage>(list);
- List<BackupImage> depList = manifest.getDependentListByTable(table);
- set.addAll(depList);
- BackupImage[] arr = new BackupImage[set.size()];
- set.toArray(arr);
- restoreImages(svc, arr, table, tTableArray[i], truncateIfExists);
- restoreImageSet.addAll(list);
- if (restoreImageSet != null && !restoreImageSet.isEmpty()) {
- LOG.info("Restore includes the following image(s):");
- for (BackupImage image : restoreImageSet) {
- LOG.info("Backup: "
- + image.getBackupId()
- + " "
- + HBackupFileSystem.getTableBackupDir(image.getRootDir(), image.getBackupId(),
- table));
- }
- }
- }
- } catch (Exception e) {
- LOG.error("Failed", e);
- throw new IOException(e);
- }
- LOG.debug("restoreStage finished");
- }
-
- @Override
- protected Flow executeFromState(final MasterProcedureEnv env, final RestoreTablesState state)
- throws InterruptedException {
- if (conf == null) {
- conf = env.getMasterConfiguration();
- }
- if (LOG.isTraceEnabled()) {
- LOG.trace(this + " execute state=" + state);
- }
- TableName[] tTableArray = tTableList.toArray(new TableName[tTableList.size()]);
- try {
- switch (state) {
- case VALIDATION:
-
- // check the target tables
- checkTargetTables(env.getMasterServices().getConnection(),
- env.getMasterServices().getTableStateManager(), tTableArray, isOverwrite);
-
- setNextState(RestoreTablesState.RESTORE_IMAGES);
- break;
- case RESTORE_IMAGES:
- TableName[] sTableArray = sTableList.toArray(new TableName[sTableList.size()]);
- HashMap<TableName, BackupManifest> backupManifestMap = new HashMap<>();
- // check and load backup image manifest for the tables
- Path rootPath = new Path(targetRootDir);
- HBackupFileSystem.checkImageManifestExist(backupManifestMap, sTableArray, conf, rootPath,
- backupId);
- restore(env.getMasterServices(), backupManifestMap, sTableArray, tTableArray, isOverwrite);
- return Flow.NO_MORE_STATE;
- default:
- throw new UnsupportedOperationException("unhandled state=" + state);
- }
- } catch (IOException e) {
- setFailure("restore-table", e);
- }
- return Flow.HAS_MORE_STATE;
- }
-
- @Override
- protected void rollbackState(final MasterProcedureEnv env, final RestoreTablesState state)
- throws IOException {
- }
-
- @Override
- protected RestoreTablesState getState(final int stateId) {
- return RestoreTablesState.valueOf(stateId);
- }
-
- @Override
- protected int getStateId(final RestoreTablesState state) {
- return state.getNumber();
- }
-
- @Override
- protected RestoreTablesState getInitialState() {
- return RestoreTablesState.VALIDATION;
- }
-
- @Override
- protected void setNextState(final RestoreTablesState state) {
- if (aborted.get()) {
- setAbortFailure("snapshot-table", "abort requested");
- } else {
- super.setNextState(state);
- }
- }
-
- @Override
- public boolean abort(final MasterProcedureEnv env) {
- aborted.set(true);
- return true;
- }
-
- @Override
- public void toStringClassDetails(StringBuilder sb) {
- sb.append(getClass().getSimpleName());
- sb.append(" (targetRootDir=");
- sb.append(targetRootDir);
- sb.append(" isOverwrite= ");
- sb.append(isOverwrite);
- sb.append(" backupId= ");
- sb.append(backupId);
- sb.append(")");
- }
-
- MasterProtos.RestoreTablesRequest toRestoreTables() {
- MasterProtos.RestoreTablesRequest.Builder bldr = MasterProtos.RestoreTablesRequest.newBuilder();
- bldr.setOverwrite(isOverwrite).setBackupId(backupId);
- bldr.setBackupRootDir(targetRootDir);
- for (TableName table : sTableList) {
- bldr.addTables(ProtobufUtil.toProtoTableName(table));
- }
- for (TableName table : tTableList) {
- bldr.addTargetTables(ProtobufUtil.toProtoTableName(table));
- }
- return bldr.build();
- }
-
- @Override
- public void serializeStateData(final OutputStream stream) throws IOException {
- super.serializeStateData(stream);
-
- MasterProtos.RestoreTablesRequest restoreTables = toRestoreTables();
- restoreTables.writeDelimitedTo(stream);
- }
-
- @Override
- public void deserializeStateData(final InputStream stream) throws IOException {
- super.deserializeStateData(stream);
-
- MasterProtos.RestoreTablesRequest proto =
- MasterProtos.RestoreTablesRequest.parseDelimitedFrom(stream);
- backupId = proto.getBackupId();
- targetRootDir = proto.getBackupRootDir();
- isOverwrite = proto.getOverwrite();
- sTableList = new ArrayList<>(proto.getTablesList().size());
- for (HBaseProtos.TableName table : proto.getTablesList()) {
- sTableList.add(ProtobufUtil.toTableName(table));
- }
- tTableList = new ArrayList<>(proto.getTargetTablesList().size());
- for (HBaseProtos.TableName table : proto.getTargetTablesList()) {
- tTableList.add(ProtobufUtil.toTableName(table));
- }
- }
-
- @Override
- public TableName getTableName() {
- return TableName.BACKUP_TABLE_NAME;
- }
-
- @Override
- public TableOperationType getTableOperationType() {
- return TableOperationType.EDIT;
- }
-
- @Override
- protected boolean acquireLock(final MasterProcedureEnv env) {
- if (env.waitInitialized(this)) {
- return false;
- }
- return env.getProcedureQueue().tryAcquireTableExclusiveLock(this, TableName.BACKUP_TABLE_NAME);
- }
-
- @Override
- protected void releaseLock(final MasterProcedureEnv env) {
- env.getProcedureQueue().releaseTableExclusiveLock(this, TableName.BACKUP_TABLE_NAME);
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java
new file mode 100644
index 0000000..c22f51b
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupClientUtil.java
@@ -0,0 +1,437 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.hbase.backup.util;
+
+import java.io.FileNotFoundException;
+import java.io.IOException;
+import java.net.URLDecoder;
+import java.util.ArrayList;
+import java.util.Collections;
+import java.util.Comparator;
+import java.util.HashMap;
+import java.util.Iterator;
+import java.util.List;
+import java.util.TreeMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.fs.FileStatus;
+import org.apache.hadoop.fs.FileSystem;
+import org.apache.hadoop.fs.LocatedFileStatus;
+import org.apache.hadoop.fs.Path;
+import org.apache.hadoop.fs.PathFilter;
+import org.apache.hadoop.fs.RemoteIterator;
+import org.apache.hadoop.hbase.HConstants;
+import org.apache.hadoop.hbase.ServerName;
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.backup.BackupInfo;
+import org.apache.hadoop.hbase.backup.impl.BackupManifest;
+import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+
+/**
+ * A collection of methods used by multiple classes to backup HBase tables.
+ */
+@InterfaceAudience.Private
+@InterfaceStability.Evolving
+public final class BackupClientUtil {
+ protected static final Log LOG = LogFactory.getLog(BackupClientUtil.class);
+ public static final String LOGNAME_SEPARATOR = ".";
+
+ private BackupClientUtil() {
+ throw new AssertionError("Instantiating utility class...");
+ }
+
+ /**
+ * Check whether the backup path exist
+ * @param backupStr backup
+ * @param conf configuration
+ * @return Yes if path exists
+ * @throws IOException exception
+ */
+ public static boolean checkPathExist(String backupStr, Configuration conf) throws IOException {
+ boolean isExist = false;
+ Path backupPath = new Path(backupStr);
+ FileSystem fileSys = backupPath.getFileSystem(conf);
+ String targetFsScheme = fileSys.getUri().getScheme();
+ if (LOG.isTraceEnabled()) {
+ LOG.trace("Schema of given url: " + backupStr + " is: " + targetFsScheme);
+ }
+ if (fileSys.exists(backupPath)) {
+ isExist = true;
+ }
+ return isExist;
+ }
+
+ // check target path first, confirm it doesn't exist before backup
+ public static void checkTargetDir(String backupRootPath, Configuration conf) throws IOException {
+ boolean targetExists = false;
+ try {
+ targetExists = checkPathExist(backupRootPath, conf);
+ } catch (IOException e) {
+ String expMsg = e.getMessage();
+ String newMsg = null;
+ if (expMsg.contains("No FileSystem for scheme")) {
+ newMsg =
+ "Unsupported filesystem scheme found in the backup target url. Error Message: "
+ + newMsg;
+ LOG.error(newMsg);
+ throw new IOException(newMsg);
+ } else {
+ throw e;
+ }
+ }
+
+ if (targetExists) {
+ LOG.info("Using existing backup root dir: " + backupRootPath);
+ } else {
+ LOG.info("Backup root dir " + backupRootPath + " does not exist. Will be created.");
+ }
+ }
+
+ /**
+ * Get the min value for all the Values a map.
+ * @param map map
+ * @return the min value
+ */
+ public static <T> Long getMinValue(HashMap<T, Long> map) {
+ Long minTimestamp = null;
+ if (map != null) {
+ ArrayList<Long> timestampList = new ArrayList<Long>(map.values());
+ Collections.sort(timestampList);
+ // The min among all the RS log timestamps will be kept in hbase:backup table.
+ minTimestamp = timestampList.get(0);
+ }
+ return minTimestamp;
+ }
+
+ /**
+ * Parses host name:port from archived WAL path
+ * @param p path
+ * @return host name
+ * @throws IOException exception
+ */
+ public static String parseHostFromOldLog(Path p) {
+ try {
+ String n = p.getName();
+ int idx = n.lastIndexOf(LOGNAME_SEPARATOR);
+ String s = URLDecoder.decode(n.substring(0, idx), "UTF8");
+ return ServerName.parseHostname(s) + ":" + ServerName.parsePort(s);
+ } catch (Exception e) {
+ LOG.warn("Skip log file (can't parse): " + p);
+ return null;
+ }
+ }
+
+ /**
+ * Given the log file, parse the timestamp from the file name. The timestamp is the last number.
+ * @param p a path to the log file
+ * @return the timestamp
+ * @throws IOException exception
+ */
+ public static Long getCreationTime(Path p) throws IOException {
+ int idx = p.getName().lastIndexOf(LOGNAME_SEPARATOR);
+ if (idx < 0) {
+ throw new IOException("Cannot parse timestamp from path " + p);
+ }
+ String ts = p.getName().substring(idx + 1);
+ return Long.parseLong(ts);
+ }
+
+ public static List<String> getFiles(FileSystem fs, Path rootDir, List<String> files,
+ PathFilter filter) throws FileNotFoundException, IOException {
+ RemoteIterator<LocatedFileStatus> it = fs.listFiles(rootDir, true);
+
+ while (it.hasNext()) {
+ LocatedFileStatus lfs = it.next();
+ if (lfs.isDirectory()) {
+ continue;
+ }
+ // apply filter
+ if (filter.accept(lfs.getPath())) {
+ files.add(lfs.getPath().toString());
+ }
+ }
+ return files;
+ }
+
+ public static void cleanupBackupData(BackupInfo context, Configuration conf) throws IOException {
+ cleanupHLogDir(context, conf);
+ cleanupTargetDir(context, conf);
+ }
+
+ /**
+ * Clean up directories which are generated when DistCp copying hlogs.
+ * @throws IOException
+ */
+ private static void cleanupHLogDir(BackupInfo backupContext, Configuration conf)
+ throws IOException {
+
+ String logDir = backupContext.getHLogTargetDir();
+ if (logDir == null) {
+ LOG.warn("No log directory specified for " + backupContext.getBackupId());
+ return;
+ }
+
+ Path rootPath = new Path(logDir).getParent();
+ FileSystem fs = FileSystem.get(rootPath.toUri(), conf);
+ FileStatus[] files = listStatus(fs, rootPath, null);
+ if (files == null) {
+ return;
+ }
+ for (FileStatus file : files) {
+ LOG.debug("Delete log files: " + file.getPath().getName());
+ fs.delete(file.getPath(), true);
+ }
+ }
+
+ /**
+ * Clean up the data at target directory
+ */
+ private static void cleanupTargetDir(BackupInfo backupInfo, Configuration conf) {
+ try {
+ // clean up the data at target directory
+ LOG.debug("Trying to cleanup up target dir : " + backupInfo.getBackupId());
+ String targetDir = backupInfo.getTargetRootDir();
+ if (targetDir == null) {
+ LOG.warn("No target directory specified for " + backupInfo.getBackupId());
+ return;
+ }
+
+ FileSystem outputFs = FileSystem.get(new Path(backupInfo.getTargetRootDir()).toUri(), conf);
+
+ for (TableName table : backupInfo.getTables()) {
+ Path targetDirPath =
+ new Path(getTableBackupDir(backupInfo.getTargetRootDir(), backupInfo.getBackupId(),
+ table));
+ if (outputFs.delete(targetDirPath, true)) {
+ LOG.info("Cleaning up backup data at " + targetDirPath.toString() + " done.");
+ } else {
+ LOG.info("No data has been found in " + targetDirPath.toString() + ".");
+ }
+
+ Path tableDir = targetDirPath.getParent();
+ FileStatus[] backups = listStatus(outputFs, tableDir, null);
+ if (backups == null || backups.length == 0) {
+ outputFs.delete(tableDir, true);
+ LOG.debug(tableDir.toString() + " is empty, remove it.");
+ }
+ }
+ outputFs.delete(new Path(targetDir, backupInfo.getBackupId()), true);
+ } catch (IOException e1) {
+ LOG.error("Cleaning up backup data of " + backupInfo.getBackupId() + " at "
+ + backupInfo.getTargetRootDir() + " failed due to " + e1.getMessage() + ".");
+ }
+ }
+
+ /**
+ * Given the backup root dir, backup id and the table name, return the backup image location,
+ * which is also where the backup manifest file is. return value look like:
+ * "hdfs://backup.hbase.org:9000/user/biadmin/backup1/backup_1396650096738/default/t1_dn/"
+ * @param backupRootDir backup root directory
+ * @param backupId backup id
+ * @param table table name
+ * @return backupPath String for the particular table
+ */
+ public static String
+ getTableBackupDir(String backupRootDir, String backupId, TableName tableName) {
+ return backupRootDir + Path.SEPARATOR + backupId + Path.SEPARATOR
+ + tableName.getNamespaceAsString() + Path.SEPARATOR + tableName.getQualifierAsString()
+ + Path.SEPARATOR;
+ }
+
+ public static TableName[] parseTableNames(String tables) {
+ if (tables == null) {
+ return null;
+ }
+ String[] tableArray = tables.split(BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
+
+ TableName[] ret = new TableName[tableArray.length];
+ for (int i = 0; i < tableArray.length; i++) {
+ ret[i] = TableName.valueOf(tableArray[i]);
+ }
+ return ret;
+ }
+
+ /**
+ * Sort history list by start time in descending order.
+ * @param historyList history list
+ * @return sorted list of BackupCompleteData
+ */
+ public static ArrayList<BackupInfo> sortHistoryListDesc(ArrayList<BackupInfo> historyList) {
+ ArrayList<BackupInfo> list = new ArrayList<BackupInfo>();
+ TreeMap<String, BackupInfo> map = new TreeMap<String, BackupInfo>();
+ for (BackupInfo h : historyList) {
+ map.put(Long.toString(h.getStartTs()), h);
+ }
+ Iterator<String> i = map.descendingKeySet().iterator();
+ while (i.hasNext()) {
+ list.add(map.get(i.next()));
+ }
+ return list;
+ }
+
+ /**
+ * Returns WAL file name
+ * @param walFileName WAL file name
+ * @return WAL file name
+ * @throws IOException exception
+ * @throws IllegalArgumentException exception
+ */
+ public static String getUniqueWALFileNamePart(String walFileName) throws IOException {
+ return getUniqueWALFileNamePart(new Path(walFileName));
+ }
+
+ /**
+ * Returns WAL file name
+ * @param p - WAL file path
+ * @return WAL file name
+ * @throws IOException exception
+ */
+ public static String getUniqueWALFileNamePart(Path p) throws IOException {
+ return p.getName();
+ }
+
+ /**
+ * Calls fs.listStatus() and treats FileNotFoundException as non-fatal This accommodates
+ * differences between hadoop versions, where hadoop 1 does not throw a FileNotFoundException, and
+ * return an empty FileStatus[] while Hadoop 2 will throw FileNotFoundException.
+ * @param fs file system
+ * @param dir directory
+ * @param filter path filter
+ * @return null if dir is empty or doesn't exist, otherwise FileStatus array
+ */
+ public static FileStatus[]
+ listStatus(final FileSystem fs, final Path dir, final PathFilter filter) throws IOException {
+ FileStatus[] status = null;
+ try {
+ status = filter == null ? fs.listStatus(dir) : fs.listStatus(dir, filter);
+ } catch (FileNotFoundException fnfe) {
+ // if directory doesn't exist, return null
+ if (LOG.isTraceEnabled()) {
+ LOG.trace(dir + " doesn't exist");
+ }
+ }
+ if (status == null || status.length < 1) return null;
+ return status;
+ }
+
+ /**
+ * Return the 'path' component of a Path. In Hadoop, Path is an URI. This method returns the
+ * 'path' component of a Path's URI: e.g. If a Path is
+ * <code>hdfs://example.org:9000/hbase_trunk/TestTable/compaction.dir</code>, this method returns
+ * <code>/hbase_trunk/TestTable/compaction.dir</code>. This method is useful if you want to print
+ * out a Path without qualifying Filesystem instance.
+ * @param p Filesystem Path whose 'path' component we are to return.
+ * @return Path portion of the Filesystem
+ */
+ public static String getPath(Path p) {
+ return p.toUri().getPath();
+ }
+
+ /**
+ * Given the backup root dir and the backup id, return the log file location for an incremental
+ * backup.
+ * @param backupRootDir backup root directory
+ * @param backupId backup id
+ * @return logBackupDir: ".../user/biadmin/backup1/WALs/backup_1396650096738"
+ */
+ public static String getLogBackupDir(String backupRootDir, String backupId) {
+ return backupRootDir + Path.SEPARATOR + backupId + Path.SEPARATOR
+ + HConstants.HREGION_LOGDIR_NAME;
+ }
+
+ private static List<BackupInfo> getHistory(Configuration conf, Path backupRootPath)
+ throws IOException {
+ // Get all (n) history from backup root destination
+ FileSystem fs = FileSystem.get(conf);
+ RemoteIterator<LocatedFileStatus> it = fs.listLocatedStatus(backupRootPath);
+
+ List<BackupInfo> infos = new ArrayList<BackupInfo>();
+ while (it.hasNext()) {
+ LocatedFileStatus lfs = it.next();
+ if (!lfs.isDirectory()) continue;
+ String backupId = lfs.getPath().getName();
+ try {
+ BackupInfo info = loadBackupInfo(backupRootPath, backupId, fs);
+ infos.add(info);
+ } catch(IOException e) {
+ LOG.error("Can not load backup info from: "+ lfs.getPath(), e);
+ }
+ }
+ // Sort
+ Collections.sort(infos, new Comparator<BackupInfo>() {
+
+ @Override
+ public int compare(BackupInfo o1, BackupInfo o2) {
+ long ts1 = getTimestamp(o1.getBackupId());
+ long ts2 = getTimestamp(o2.getBackupId());
+ if (ts1 == ts2) return 0;
+ return ts1 < ts2 ? 1 : -1;
+ }
+
+ private long getTimestamp(String backupId) {
+ String[] split = backupId.split("_");
+ return Long.parseLong(split[1]);
+ }
+ });
+ return infos;
+ }
+
+ public static List<BackupInfo> getHistory(Configuration conf, int n, Path backupRootPath,
+ BackupInfo.Filter... filters) throws IOException {
+ List<BackupInfo> infos = getHistory(conf, backupRootPath);
+ List<BackupInfo> ret = new ArrayList<BackupInfo>();
+ for (BackupInfo info : infos) {
+ if (ret.size() == n) {
+ break;
+ }
+ boolean passed = true;
+ for (int i = 0; i < filters.length; i++) {
+ if (!filters[i].apply(info)) {
+ passed = false;
+ break;
+ }
+ }
+ if (passed) {
+ ret.add(info);
+ }
+ }
+ return ret;
+ }
+
+ public static BackupInfo loadBackupInfo(Path backupRootPath, String backupId, FileSystem fs)
+ throws IOException {
+ Path backupPath = new Path(backupRootPath, backupId);
+
+ RemoteIterator<LocatedFileStatus> it = fs.listFiles(backupPath, true);
+ while (it.hasNext()) {
+ LocatedFileStatus lfs = it.next();
+ if (lfs.getPath().getName().equals(BackupManifest.MANIFEST_FILE_NAME)) {
+ // Load BackupManifest
+ BackupManifest manifest = new BackupManifest(fs, lfs.getPath().getParent());
+ BackupInfo info = manifest.toBackupInfo();
+ return info;
+ }
+ }
+ return null;
+ }
+}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java
index 486fd2b..d9bf749 100644
--- a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupServerUtil.java
@@ -158,14 +158,13 @@ public final class BackupServerUtil {
* @throws IOException exception
* @throws InterruptedException exception
*/
- public static void copyTableRegionInfo(MasterServices svc, BackupInfo backupContext,
+ public static void copyTableRegionInfo(Connection conn, BackupInfo backupContext,
Configuration conf) throws IOException, InterruptedException {
Path rootDir = FSUtils.getRootDir(conf);
FileSystem fs = rootDir.getFileSystem(conf);
// for each table in the table set, copy out the table info and region
// info files in the correct directory structure
- Connection conn = svc.getConnection();
for (TableName table : backupContext.getTables()) {
if(!MetaTableAccessor.tableExists(conn, table)) {
http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java
----------------------------------------------------------------------
diff --git a/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java
new file mode 100644
index 0000000..76402c7
--- /dev/null
+++ b/hbase-server/src/main/java/org/apache/hadoop/hbase/backup/util/BackupSet.java
@@ -0,0 +1,62 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.util;
+import java.util.List;
+
+import org.apache.hadoop.hbase.TableName;
+import org.apache.hadoop.hbase.classification.InterfaceAudience;
+import org.apache.hadoop.hbase.classification.InterfaceStability;
+/**
+ * Backup set is a named group of HBase tables,
+ * which are managed together by Backup/Restore
+ * framework. Instead of using list of tables in backup or restore
+ * operation, one can use set's name instead.
+ */
+@InterfaceAudience.Public
+@InterfaceStability.Evolving
+public class BackupSet {
+ private final String name;
+ private final List<TableName> tables;
+
+ public BackupSet(String name, List<TableName> tables) {
+ this.name = name;
+ this.tables = tables;
+ }
+
+ public String getName() {
+ return name;
+ }
+
+ public List<TableName> getTables() {
+ return tables;
+ }
+
+ public String toString() {
+ StringBuilder sb = new StringBuilder();
+ sb.append(name).append("={");
+ for (int i = 0; i < tables.size(); i++) {
+ sb.append(tables.get(i));
+ if (i < tables.size() - 1) {
+ sb.append(",");
+ }
+ }
+ sb.append("}");
+ return sb.toString();
+ }
+
+}