You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2016/10/05 23:30:16 UTC
[10/10] hbase git commit: HBASE-16727 Backup refactoring: remove MR
dependencies from HMaster (Vladimir Rodionov)
HBASE-16727 Backup refactoring: remove MR dependencies from HMaster (Vladimir Rodionov)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/b14e2ab1
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/b14e2ab1
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/b14e2ab1
Branch: refs/heads/HBASE-7912
Commit: b14e2ab1c24e65ff88dd4c579acf83cb4ed0605e
Parents: e35f7b9
Author: tedyu <yu...@gmail.com>
Authored: Wed Oct 5 16:29:40 2016 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Wed Oct 5 16:29:40 2016 -0700
----------------------------------------------------------------------
.../apache/hadoop/hbase/backup/BackupInfo.java | 504 -
.../hadoop/hbase/backup/BackupRequest.java | 91 -
.../hadoop/hbase/backup/BackupStatus.java | 104 -
.../hadoop/hbase/backup/RestoreRequest.java | 94 -
.../hbase/backup/impl/BackupCommands.java | 717 -
.../hbase/backup/impl/BackupException.java | 86 -
.../hbase/backup/impl/BackupManifest.java | 791 -
.../backup/impl/BackupRestoreConstants.java | 47 -
.../hbase/backup/impl/BackupSystemTable.java | 873 -
.../backup/impl/BackupSystemTableHelper.java | 433 -
.../hbase/backup/util/BackupClientUtil.java | 437 -
.../hadoop/hbase/backup/util/BackupSet.java | 62 -
.../org/apache/hadoop/hbase/client/Admin.java | 9 -
.../apache/hadoop/hbase/client/BackupAdmin.java | 174 -
.../hbase/client/ConnectionImplementation.java | 13 -
.../apache/hadoop/hbase/client/HBaseAdmin.java | 231 +-
.../hadoop/hbase/client/HBaseBackupAdmin.java | 439 -
.../hadoop/hbase/protobuf/RequestConverter.java | 44 -
.../ClientSnapshotDescriptionUtils.java | 3 +-
.../hbase/IntegrationTestBackupRestore.java | 10 +-
.../hbase/protobuf/generated/BackupProtos.java | 15632 ++++++++---------
.../hbase/protobuf/generated/MasterProtos.java | 5893 +------
hbase-protocol/src/main/protobuf/Backup.proto | 8 +-
hbase-protocol/src/main/protobuf/Master.proto | 44 -
.../apache/hadoop/hbase/backup/BackupAdmin.java | 171 +
.../apache/hadoop/hbase/backup/BackupInfo.java | 504 +
.../hadoop/hbase/backup/BackupRequest.java | 91 +
.../hadoop/hbase/backup/BackupStatus.java | 104 +
.../hadoop/hbase/backup/RestoreDriver.java | 4 +-
.../hadoop/hbase/backup/RestoreRequest.java | 94 +
.../hbase/backup/impl/BackupCommands.java | 720 +
.../hbase/backup/impl/BackupException.java | 86 +
.../hbase/backup/impl/BackupManifest.java | 791 +
.../backup/impl/BackupRestoreConstants.java | 47 +
.../hbase/backup/impl/BackupSystemTable.java | 926 +
.../backup/impl/BackupSystemTableHelper.java | 433 +
.../backup/impl/FullTableBackupClient.java | 540 +
.../hbase/backup/impl/HBaseBackupAdmin.java | 555 +
.../backup/impl/IncrementalBackupManager.java | 27 +-
.../impl/IncrementalTableBackupClient.java | 235 +
.../hbase/backup/impl/RestoreTablesClient.java | 236 +
.../backup/master/FullTableBackupProcedure.java | 777 -
.../master/IncrementalTableBackupProcedure.java | 400 -
.../backup/master/RestoreTablesProcedure.java | 387 -
.../hbase/backup/util/BackupClientUtil.java | 437 +
.../hbase/backup/util/BackupServerUtil.java | 3 +-
.../hadoop/hbase/backup/util/BackupSet.java | 62 +
.../hbase/backup/util/RestoreServerUtil.java | 85 +-
.../org/apache/hadoop/hbase/master/HMaster.java | 119 +-
.../hadoop/hbase/master/MasterRpcServices.java | 42 -
.../hadoop/hbase/master/MasterServices.java | 30 -
.../hadoop/hbase/backup/TestBackupBase.java | 24 +-
.../hbase/backup/TestBackupBoundaryTests.java | 8 +-
.../hbase/backup/TestBackupDeleteRestore.java | 1 -
.../hbase/backup/TestBackupMultipleDeletes.java | 20 +-
.../hadoop/hbase/backup/TestFullRestore.java | 1 -
.../hbase/backup/TestIncrementalBackup.java | 61 +-
.../TestIncrementalBackupDeleteTable.java | 8 +-
.../hadoop/hbase/backup/TestRemoteBackup.java | 1 -
.../hadoop/hbase/master/TestCatalogJanitor.java | 17 -
60 files changed, 13746 insertions(+), 21040 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java
deleted file mode 100644
index be5ffea..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupInfo.java
+++ /dev/null
@@ -1,504 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.IOException;
-import java.io.InputStream;
-import java.util.ArrayList;
-import java.util.Calendar;
-import java.util.Date;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.Set;
-
-import org.apache.commons.lang.StringUtils;
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.BackupProtos;
-import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.BackupInfo.Builder;
-import org.apache.hadoop.hbase.protobuf.generated.BackupProtos.TableBackupStatus;
-import org.apache.hadoop.hbase.util.Bytes;
-
-
-/**
- * An object to encapsulate the information for each backup request
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class BackupInfo implements Comparable<BackupInfo> {
- private static final Log LOG = LogFactory.getLog(BackupInfo.class);
-
- public static interface Filter {
-
- /**
- * Filter interface
- * @param info: backup info
- * @return true if info passes filter, false otherwise
- */
- public boolean apply(BackupInfo info);
- }
- // backup status flag
- public static enum BackupState {
- WAITING, RUNNING, COMPLETE, FAILED, ANY;
- }
-
- // backup phase
- public static enum BackupPhase {
- SNAPSHOTCOPY, INCREMENTAL_COPY, STORE_MANIFEST;
- }
-
- // backup id: a timestamp when we request the backup
- private String backupId;
-
- // backup type, full or incremental
- private BackupType type;
-
- // target root directory for storing the backup files
- private String targetRootDir;
-
- // overall backup state
- private BackupState state;
-
- // overall backup phase
- private BackupPhase phase;
-
- // overall backup failure message
- private String failedMsg;
-
- // backup status map for all tables
- private Map<TableName, BackupStatus> backupStatusMap;
-
- // actual start timestamp of the backup process
- private long startTs;
-
- // actual end timestamp of the backup process, could be fail or complete
- private long endTs;
-
- // the total bytes of incremental logs copied
- private long totalBytesCopied;
-
- // for incremental backup, the location of the backed-up hlogs
- private String hlogTargetDir = null;
-
- // incremental backup file list
- transient private List<String> incrBackupFileList;
-
- // new region server log timestamps for table set after distributed log roll
- // key - table name, value - map of RegionServer hostname -> last log rolled timestamp
- transient private HashMap<TableName, HashMap<String, Long>> tableSetTimestampMap;
-
- // backup progress in %% (0-100)
- private int progress;
-
- // distributed job id
- private String jobId;
-
- // Number of parallel workers. -1 - system defined
- private int workers = -1;
-
- // Bandwidth per worker in MB per sec. -1 - unlimited
- private long bandwidth = -1;
-
- public BackupInfo() {
- backupStatusMap = new HashMap<TableName, BackupStatus>();
- }
-
- public BackupInfo(String backupId, BackupType type, TableName[] tables, String targetRootDir) {
- this();
- this.backupId = backupId;
- this.type = type;
- this.targetRootDir = targetRootDir;
- if (LOG.isDebugEnabled()) {
- LOG.debug("CreateBackupContext: " + tables.length + " " + tables[0]);
- }
- this.addTables(tables);
-
- if (type == BackupType.INCREMENTAL) {
- setHlogTargetDir(BackupClientUtil.getLogBackupDir(targetRootDir, backupId));
- }
-
- this.startTs = 0;
- this.endTs = 0;
- }
-
- public String getJobId() {
- return jobId;
- }
-
- public void setJobId(String jobId) {
- this.jobId = jobId;
- }
-
- public int getWorkers() {
- return workers;
- }
-
- public void setWorkers(int workers) {
- this.workers = workers;
- }
-
- public long getBandwidth() {
- return bandwidth;
- }
-
- public void setBandwidth(long bandwidth) {
- this.bandwidth = bandwidth;
- }
-
- public void setBackupStatusMap(Map<TableName, BackupStatus> backupStatusMap) {
- this.backupStatusMap = backupStatusMap;
- }
-
- public HashMap<TableName, HashMap<String, Long>> getTableSetTimestampMap() {
- return tableSetTimestampMap;
- }
-
- public void
- setTableSetTimestampMap(HashMap<TableName, HashMap<String, Long>> tableSetTimestampMap) {
- this.tableSetTimestampMap = tableSetTimestampMap;
- }
-
- public String getHlogTargetDir() {
- return hlogTargetDir;
- }
-
- public void setType(BackupType type) {
- this.type = type;
- }
-
- public void setTargetRootDir(String targetRootDir) {
- this.targetRootDir = targetRootDir;
- }
-
- public void setTotalBytesCopied(long totalBytesCopied) {
- this.totalBytesCopied = totalBytesCopied;
- }
-
- /**
- * Set progress (0-100%)
- * @param msg progress value
- */
-
- public void setProgress(int p) {
- this.progress = p;
- }
-
- /**
- * Get current progress
- */
- public int getProgress() {
- return progress;
- }
-
- public String getBackupId() {
- return backupId;
- }
-
- public void setBackupId(String backupId) {
- this.backupId = backupId;
- }
-
- public BackupStatus getBackupStatus(TableName table) {
- return this.backupStatusMap.get(table);
- }
-
- public String getFailedMsg() {
- return failedMsg;
- }
-
- public void setFailedMsg(String failedMsg) {
- this.failedMsg = failedMsg;
- }
-
- public long getStartTs() {
- return startTs;
- }
-
- public void setStartTs(long startTs) {
- this.startTs = startTs;
- }
-
- public long getEndTs() {
- return endTs;
- }
-
- public void setEndTs(long endTs) {
- this.endTs = endTs;
- }
-
- public long getTotalBytesCopied() {
- return totalBytesCopied;
- }
-
- public BackupState getState() {
- return state;
- }
-
- public void setState(BackupState flag) {
- this.state = flag;
- }
-
- public BackupPhase getPhase() {
- return phase;
- }
-
- public void setPhase(BackupPhase phase) {
- this.phase = phase;
- }
-
- public BackupType getType() {
- return type;
- }
-
- public void setSnapshotName(TableName table, String snapshotName) {
- this.backupStatusMap.get(table).setSnapshotName(snapshotName);
- }
-
- public String getSnapshotName(TableName table) {
- return this.backupStatusMap.get(table).getSnapshotName();
- }
-
- public List<String> getSnapshotNames() {
- List<String> snapshotNames = new ArrayList<String>();
- for (BackupStatus backupStatus : this.backupStatusMap.values()) {
- snapshotNames.add(backupStatus.getSnapshotName());
- }
- return snapshotNames;
- }
-
- public Set<TableName> getTables() {
- return this.backupStatusMap.keySet();
- }
-
- public List<TableName> getTableNames() {
- return new ArrayList<TableName>(backupStatusMap.keySet());
- }
-
- public void addTables(TableName[] tables) {
- for (TableName table : tables) {
- BackupStatus backupStatus = new BackupStatus(table, this.targetRootDir, this.backupId);
- this.backupStatusMap.put(table, backupStatus);
- }
- }
-
- public void setTables(List<TableName> tables) {
- this.backupStatusMap.clear();
- for (TableName table : tables) {
- BackupStatus backupStatus = new BackupStatus(table, this.targetRootDir, this.backupId);
- this.backupStatusMap.put(table, backupStatus);
- }
- }
-
- public String getTargetRootDir() {
- return targetRootDir;
- }
-
- public void setHlogTargetDir(String hlogTagetDir) {
- this.hlogTargetDir = hlogTagetDir;
- }
-
- public String getHLogTargetDir() {
- return hlogTargetDir;
- }
-
- public List<String> getIncrBackupFileList() {
- return incrBackupFileList;
- }
-
- public void setIncrBackupFileList(List<String> incrBackupFileList) {
- this.incrBackupFileList = incrBackupFileList;
- }
-
- /**
- * Set the new region server log timestamps after distributed log roll
- * @param newTableSetTimestampMap table timestamp map
- */
- public void
- setIncrTimestampMap(HashMap<TableName, HashMap<String, Long>> newTableSetTimestampMap) {
- this.tableSetTimestampMap = newTableSetTimestampMap;
- }
-
- /**
- * Get new region server log timestamps after distributed log roll
- * @return new region server log timestamps
- */
- public HashMap<TableName, HashMap<String, Long>> getIncrTimestampMap() {
- return this.tableSetTimestampMap;
- }
-
- public TableName getTableBySnapshot(String snapshotName) {
- for (Entry<TableName, BackupStatus> entry : this.backupStatusMap.entrySet()) {
- if (snapshotName.equals(entry.getValue().getSnapshotName())) {
- return entry.getKey();
- }
- }
- return null;
- }
-
- public BackupProtos.BackupInfo toProtosBackupInfo() {
- BackupProtos.BackupInfo.Builder builder = BackupProtos.BackupInfo.newBuilder();
- builder.setBackupId(getBackupId());
- setBackupStatusMap(builder);
- builder.setEndTs(getEndTs());
- if (getFailedMsg() != null) {
- builder.setFailedMessage(getFailedMsg());
- }
- if (getState() != null) {
- builder.setState(BackupProtos.BackupInfo.BackupState.valueOf(getState().name()));
- }
- if (getPhase() != null) {
- builder.setPhase(BackupProtos.BackupInfo.BackupPhase.valueOf(getPhase().name()));
- }
-
- builder.setProgress(getProgress());
- builder.setStartTs(getStartTs());
- builder.setTargetRootDir(getTargetRootDir());
- builder.setType(BackupProtos.BackupType.valueOf(getType().name()));
- builder.setWorkersNumber(workers);
- builder.setBandwidth(bandwidth);
- if (jobId != null) {
- builder.setJobId(jobId);
- }
- return builder.build();
- }
-
- @Override
- public boolean equals(Object obj) {
- if (obj instanceof BackupInfo) {
- BackupInfo other = (BackupInfo) obj;
- try {
- return Bytes.equals(toByteArray(), other.toByteArray());
- } catch (IOException e) {
- LOG.error(e);
- return false;
- }
- } else {
- return false;
- }
- }
-
- public byte[] toByteArray() throws IOException {
- return toProtosBackupInfo().toByteArray();
- }
-
- private void setBackupStatusMap(Builder builder) {
- for (Entry<TableName, BackupStatus> entry : backupStatusMap.entrySet()) {
- builder.addTableBackupStatus(entry.getValue().toProto());
- }
- }
-
- public static BackupInfo fromByteArray(byte[] data) throws IOException {
- return fromProto(BackupProtos.BackupInfo.parseFrom(data));
- }
-
- public static BackupInfo fromStream(final InputStream stream) throws IOException {
- return fromProto(BackupProtos.BackupInfo.parseDelimitedFrom(stream));
- }
-
- public static BackupInfo fromProto(BackupProtos.BackupInfo proto) {
- BackupInfo context = new BackupInfo();
- context.setBackupId(proto.getBackupId());
- context.setBackupStatusMap(toMap(proto.getTableBackupStatusList()));
- context.setEndTs(proto.getEndTs());
- if (proto.hasFailedMessage()) {
- context.setFailedMsg(proto.getFailedMessage());
- }
- if (proto.hasState()) {
- context.setState(BackupInfo.BackupState.valueOf(proto.getState().name()));
- }
-
- context.setHlogTargetDir(BackupClientUtil.getLogBackupDir(proto.getTargetRootDir(),
- proto.getBackupId()));
-
- if (proto.hasPhase()) {
- context.setPhase(BackupPhase.valueOf(proto.getPhase().name()));
- }
- if (proto.hasProgress()) {
- context.setProgress(proto.getProgress());
- }
- context.setStartTs(proto.getStartTs());
- context.setTargetRootDir(proto.getTargetRootDir());
- context.setType(BackupType.valueOf(proto.getType().name()));
- context.setWorkers(proto.getWorkersNumber());
- context.setBandwidth(proto.getBandwidth());
- if (proto.hasJobId()) {
- context.setJobId(proto.getJobId());
- }
- return context;
- }
-
- private static Map<TableName, BackupStatus> toMap(List<TableBackupStatus> list) {
- HashMap<TableName, BackupStatus> map = new HashMap<>();
- for (TableBackupStatus tbs : list) {
- map.put(ProtobufUtil.toTableName(tbs.getTable()), BackupStatus.convert(tbs));
- }
- return map;
- }
-
- public String getShortDescription() {
- StringBuilder sb = new StringBuilder();
- sb.append("ID : " + backupId).append("\n");
- sb.append("Type : " + getType()).append("\n");
- sb.append("Tables : " + getTableListAsString()).append("\n");
- sb.append("State : " + getState()).append("\n");
- Date date = null;
- Calendar cal = Calendar.getInstance();
- cal.setTimeInMillis(getStartTs());
- date = cal.getTime();
- sb.append("Start time : " + date).append("\n");
- if (state == BackupState.FAILED) {
- sb.append("Failed message : " + getFailedMsg()).append("\n");
- } else if (state == BackupState.RUNNING) {
- sb.append("Phase : " + getPhase()).append("\n");
- } else if (state == BackupState.COMPLETE) {
- cal = Calendar.getInstance();
- cal.setTimeInMillis(getEndTs());
- date = cal.getTime();
- sb.append("End time : " + date).append("\n");
- }
- sb.append("Progress : " + getProgress()).append("\n");
- return sb.toString();
- }
-
- public String getStatusAndProgressAsString() {
- StringBuilder sb = new StringBuilder();
- sb.append("id: ").append(getBackupId()).append(" state: ").append(getState())
- .append(" progress: ").append(getProgress());
- return sb.toString();
- }
-
- public String getTableListAsString() {
- return StringUtils.join(backupStatusMap.keySet(), ",");
- }
-
- @Override
- public int compareTo(BackupInfo o) {
- Long thisTS = new Long(this.getBackupId().substring(this.getBackupId().lastIndexOf("_") + 1));
- Long otherTS = new Long(o.getBackupId().substring(o.getBackupId().lastIndexOf("_") + 1));
- return thisTS.compareTo(otherTS);
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java
deleted file mode 100644
index d141239..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java
+++ /dev/null
@@ -1,91 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.util.List;
-
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-
-/**
- * POJO class for backup request
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public final class BackupRequest {
- private BackupType type;
- private List<TableName> tableList;
- private String targetRootDir;
- private int workers = -1;
- private long bandwidth = -1L;
- private String backupSetName;
-
- public BackupRequest() {
- }
-
- public BackupRequest setBackupType(BackupType type) {
- this.type = type;
- return this;
- }
- public BackupType getBackupType() {
- return this.type;
- }
-
- public BackupRequest setTableList(List<TableName> tableList) {
- this.tableList = tableList;
- return this;
- }
- public List<TableName> getTableList() {
- return this.tableList;
- }
-
- public BackupRequest setTargetRootDir(String targetRootDir) {
- this.targetRootDir = targetRootDir;
- return this;
- }
- public String getTargetRootDir() {
- return this.targetRootDir;
- }
-
- public BackupRequest setWorkers(int workers) {
- this.workers = workers;
- return this;
- }
- public int getWorkers() {
- return this.workers;
- }
-
- public BackupRequest setBandwidth(long bandwidth) {
- this.bandwidth = bandwidth;
- return this;
- }
- public long getBandwidth() {
- return this.bandwidth;
- }
-
- public String getBackupSetName() {
- return backupSetName;
- }
-
- public void setBackupSetName(String backupSetName) {
- this.backupSetName = backupSetName;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java
deleted file mode 100644
index c82e05a..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/BackupStatus.java
+++ /dev/null
@@ -1,104 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup;
-
-import java.io.Serializable;
-
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.BackupProtos;
-
-/**
- * Backup status and related information encapsulated for a table.
- * At this moment only TargetDir and SnapshotName is encapsulated here.
- */
-
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class BackupStatus implements Serializable {
-
- private static final long serialVersionUID = -5968397963548535982L;
-
- // table name for backup
- private TableName table;
-
- // target directory of the backup image for this table
- private String targetDir;
-
- // snapshot name for offline/online snapshot
- private String snapshotName = null;
-
- public BackupStatus() {
-
- }
-
- public BackupStatus(TableName table, String targetRootDir, String backupId) {
- this.table = table;
- this.targetDir = BackupClientUtil.getTableBackupDir(targetRootDir, backupId, table);
- }
-
- public String getSnapshotName() {
- return snapshotName;
- }
-
- public void setSnapshotName(String snapshotName) {
- this.snapshotName = snapshotName;
- }
-
- public String getTargetDir() {
- return targetDir;
- }
-
- public TableName getTable() {
- return table;
- }
-
- public void setTable(TableName table) {
- this.table = table;
- }
-
- public void setTargetDir(String targetDir) {
- this.targetDir = targetDir;
- }
-
- public static BackupStatus convert(BackupProtos.TableBackupStatus proto)
- {
- BackupStatus bs = new BackupStatus();
- bs.setTable(ProtobufUtil.toTableName(proto.getTable()));
- bs.setTargetDir(proto.getTargetDir());
- if(proto.hasSnapshot()){
- bs.setSnapshotName(proto.getSnapshot());
- }
- return bs;
- }
-
- public BackupProtos.TableBackupStatus toProto() {
- BackupProtos.TableBackupStatus.Builder builder =
- BackupProtos.TableBackupStatus.newBuilder();
- if(snapshotName != null) {
- builder.setSnapshot(snapshotName);
- }
- builder.setTable(ProtobufUtil.toProtoTableName(table));
- builder.setTargetDir(targetDir);
- return builder.build();
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java
deleted file mode 100644
index 7490d20..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/RestoreRequest.java
+++ /dev/null
@@ -1,94 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-package org.apache.hadoop.hbase.backup;
-
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-
-/**
- * POJO class for restore request
- */
-@InterfaceAudience.Public
-@InterfaceStability.Evolving
-public class RestoreRequest {
-
- private String backupRootDir;
- private String backupId;
- private boolean check = false;
- private TableName[] fromTables;
- private TableName[] toTables;
- private boolean overwrite = false;
-
- public RestoreRequest() {
- }
-
- public String getBackupRootDir() {
- return backupRootDir;
- }
-
- public RestoreRequest setBackupRootDir(String backupRootDir) {
- this.backupRootDir = backupRootDir;
- return this;
- }
-
- public String getBackupId() {
- return backupId;
- }
-
- public RestoreRequest setBackupId(String backupId) {
- this.backupId = backupId;
- return this;
- }
-
- public boolean isCheck() {
- return check;
- }
-
- public RestoreRequest setCheck(boolean check) {
- this.check = check;
- return this;
- }
-
- public TableName[] getFromTables() {
- return fromTables;
- }
-
- public RestoreRequest setFromTables(TableName[] fromTables) {
- this.fromTables = fromTables;
- return this;
- }
-
- public TableName[] getToTables() {
- return toTables;
- }
-
- public RestoreRequest setToTables(TableName[] toTables) {
- this.toTables = toTables;
- return this;
- }
-
- public boolean isOverwrite() {
- return overwrite;
- }
-
- public RestoreRequest setOverwrite(boolean overwrite) {
- this.overwrite = overwrite;
- return this;
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
deleted file mode 100644
index 2ff5756..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
+++ /dev/null
@@ -1,717 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.impl;
-
-import java.io.IOException;
-import java.util.List;
-
-import org.apache.commons.cli.CommandLine;
-import org.apache.commons.lang.StringUtils;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.conf.Configured;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HBaseConfiguration;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.hadoop.hbase.backup.BackupRequest;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.impl.BackupRestoreConstants.BackupCommand;
-import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
-import org.apache.hadoop.hbase.backup.util.BackupSet;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.client.Admin;
-import org.apache.hadoop.hbase.client.BackupAdmin;
-import org.apache.hadoop.hbase.client.Connection;
-import org.apache.hadoop.hbase.client.ConnectionFactory;
-
-import com.google.common.collect.Lists;
-
-/**
- * General backup commands, options and usage messages
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public final class BackupCommands {
-
- public final static String INCORRECT_USAGE = "Incorrect usage";
-
- public static final String USAGE = "Usage: hbase backup COMMAND [command-specific arguments]\n"
- + "where COMMAND is one of:\n"
- + " create create a new backup image\n"
- + " delete delete an existing backup image\n"
- + " describe show the detailed information of a backup image\n"
- + " history show history of all successful backups\n"
- + " progress show the progress of the latest backup request\n"
- + " set backup set management\n"
- + "Run \'hbase backup COMMAND -h\' to see help message for each command\n";
-
- public static final String CREATE_CMD_USAGE =
- "Usage: hbase backup create <type> <BACKUP_ROOT> [tables] [-set name] "
- + "[-w workers][-b bandwith]\n"
- + " type \"full\" to create a full backup image\n"
- + " \"incremental\" to create an incremental backup image\n"
- + " BACKUP_ROOT The full root path to store the backup image,\n"
- + " the prefix can be hdfs, webhdfs or gpfs\n"
- + "Options:\n"
- + " tables If no tables (\"\") are specified, all tables are backed up.\n"
- + " Otherwise it is a comma separated list of tables.\n"
- + " -w number of parallel workers (MapReduce tasks).\n"
- + " -b bandwith per one worker (MapReduce task) in MBs per sec\n"
- + " -set name of backup set to use (mutually exclusive with [tables])" ;
-
- public static final String PROGRESS_CMD_USAGE = "Usage: hbase backup progress <backupId>\n"
- + " backupId backup image id\n";
- public static final String NO_INFO_FOUND = "No info was found for backup id: ";
-
- public static final String DESCRIBE_CMD_USAGE = "Usage: hbase backup decsribe <backupId>\n"
- + " backupId backup image id\n";
-
- public static final String HISTORY_CMD_USAGE =
- "Usage: hbase backup history [-path BACKUP_ROOT] [-n N] [-t table]\n"
- + " -n N show up to N last backup sessions, default - 10\n"
- + " -path backup root path\n"
- + " -t table table name. If specified, only backup images which contain this table\n"
- + " will be listed." ;
-
-
- public static final String DELETE_CMD_USAGE = "Usage: hbase backup delete <backupId>\n"
- + " backupId backup image id\n";
-
- public static final String CANCEL_CMD_USAGE = "Usage: hbase backup cancel <backupId>\n"
- + " backupId backup image id\n";
-
- public static final String SET_CMD_USAGE = "Usage: hbase backup set COMMAND [name] [tables]\n"
- + " name Backup set name\n"
- + " tables If no tables (\"\") are specified, all tables will belong to the set.\n"
- + " Otherwise it is a comma separated list of tables.\n"
- + "COMMAND is one of:\n"
- + " add add tables to a set, create a set if needed\n"
- + " remove remove tables from a set\n"
- + " list list all backup sets in the system\n"
- + " describe describe set\n"
- + " delete delete backup set\n";
-
- public static abstract class Command extends Configured {
- CommandLine cmdline;
-
- Command(Configuration conf) {
- super(conf);
- }
-
- public void execute() throws IOException
- {
- if (cmdline.hasOption("h") || cmdline.hasOption("help")) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- }
-
- protected abstract void printUsage();
- }
-
- private BackupCommands() {
- throw new AssertionError("Instantiating utility class...");
- }
-
- public static Command createCommand(Configuration conf, BackupCommand type, CommandLine cmdline) {
- Command cmd = null;
- switch (type) {
- case CREATE:
- cmd = new CreateCommand(conf, cmdline);
- break;
- case DESCRIBE:
- cmd = new DescribeCommand(conf, cmdline);
- break;
- case PROGRESS:
- cmd = new ProgressCommand(conf, cmdline);
- break;
- case DELETE:
- cmd = new DeleteCommand(conf, cmdline);
- break;
- case CANCEL:
- cmd = new CancelCommand(conf, cmdline);
- break;
- case HISTORY:
- cmd = new HistoryCommand(conf, cmdline);
- break;
- case SET:
- cmd = new BackupSetCommand(conf, cmdline);
- break;
- case HELP:
- default:
- cmd = new HelpCommand(conf, cmdline);
- break;
- }
- return cmd;
- }
-
- static int numOfArgs(String[] args) {
- if (args == null) return 0;
- return args.length;
- }
-
- public static class CreateCommand extends Command {
-
- CreateCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- super.execute();
- if (cmdline == null || cmdline.getArgs() == null) {
- System.err.println("ERROR: missing arguments");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- String[] args = cmdline.getArgs();
- if (args.length < 3 || args.length > 4) {
- System.err.println("ERROR: wrong number of arguments: "+ args.length);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- if (!BackupType.FULL.toString().equalsIgnoreCase(args[1])
- && !BackupType.INCREMENTAL.toString().equalsIgnoreCase(args[1])) {
- System.err.println("ERROR: invalid backup type: "+ args[1]);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String tables = null;
- Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create();
-
- // Check backup set
- String setName = null;
- if (cmdline.hasOption("set")) {
- setName = cmdline.getOptionValue("set");
- tables = getTablesForSet(setName, conf);
-
- if (tables == null) {
- System.err.println("ERROR: Backup set '" + setName+ "' is either empty or does not exist");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- } else {
- tables = (args.length == 4) ? args[3] : null;
- }
- int bandwidth = cmdline.hasOption('b') ? Integer.parseInt(cmdline.getOptionValue('b')) : -1;
- int workers = cmdline.hasOption('w') ? Integer.parseInt(cmdline.getOptionValue('w')) : -1;
-
- try (Connection conn = ConnectionFactory.createConnection(getConf());
- Admin admin = conn.getAdmin();
- BackupAdmin backupAdmin = admin.getBackupAdmin();) {
- BackupRequest request = new BackupRequest();
- request.setBackupType(BackupType.valueOf(args[1].toUpperCase()))
- .setTableList(tables != null?Lists.newArrayList(BackupClientUtil.parseTableNames(tables)): null)
- .setTargetRootDir(args[2]).setWorkers(workers).setBandwidth(bandwidth)
- .setBackupSetName(setName);
- String backupId = backupAdmin.backupTables(request);
- System.out.println("Backup session "+ backupId+" finished. Status: SUCCESS");
- } catch (IOException e) {
- System.err.println("Backup session finished. Status: FAILURE");
- throw e;
- }
- }
- private String getTablesForSet(String name, Configuration conf)
- throws IOException {
- try (final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupSystemTable table = new BackupSystemTable(conn)) {
- List<TableName> tables = table.describeBackupSet(name);
- if (tables == null) return null;
- return StringUtils.join(tables, BackupRestoreConstants.TABLENAME_DELIMITER_IN_COMMAND);
- }
- }
-
- @Override
- protected void printUsage() {
- System.err.println(CREATE_CMD_USAGE);
- }
- }
-
- private static class HelpCommand extends Command {
-
- HelpCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- super.execute();
- if (cmdline == null) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String[] args = cmdline.getArgs();
- if (args == null || args.length == 0) {
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- if (args.length != 2) {
- System.err.println("Only supports help message of a single command type");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String type = args[1];
-
- if (BackupCommand.CREATE.name().equalsIgnoreCase(type)) {
- System.out.println(CREATE_CMD_USAGE);
- } else if (BackupCommand.DESCRIBE.name().equalsIgnoreCase(type)) {
- System.out.println(DESCRIBE_CMD_USAGE);
- } else if (BackupCommand.HISTORY.name().equalsIgnoreCase(type)) {
- System.out.println(HISTORY_CMD_USAGE);
- } else if (BackupCommand.PROGRESS.name().equalsIgnoreCase(type)) {
- System.out.println(PROGRESS_CMD_USAGE);
- } else if (BackupCommand.DELETE.name().equalsIgnoreCase(type)) {
- System.out.println(DELETE_CMD_USAGE);
- } else if (BackupCommand.CANCEL.name().equalsIgnoreCase(type)) {
- System.out.println(CANCEL_CMD_USAGE);
- } else if (BackupCommand.SET.name().equalsIgnoreCase(type)) {
- System.out.println(SET_CMD_USAGE);
- } else {
- System.out.println("Unknown command : " + type);
- printUsage();
- }
- }
-
- @Override
- protected void printUsage() {
- System.err.println(USAGE);
- }
- }
-
- private static class DescribeCommand extends Command {
-
- DescribeCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- super.execute();
- if (cmdline == null || cmdline.getArgs() == null) {
- System.err.println("ERROR: missing arguments");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- String[] args = cmdline.getArgs();
- if (args.length != 2) {
- System.err.println("ERROR: wrong number of arguments");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String backupId = args[1];
- Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
- try (final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupAdmin admin = conn.getAdmin().getBackupAdmin();) {
- BackupInfo info = admin.getBackupInfo(backupId);
- if (info == null) {
- System.err.println("ERROR: " + backupId + " does not exist");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- System.out.println(info.getShortDescription());
- }
- }
-
- @Override
- protected void printUsage() {
- System.err.println(DESCRIBE_CMD_USAGE);
- }
- }
-
- private static class ProgressCommand extends Command {
-
- ProgressCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- super.execute();
-
- if (cmdline == null || cmdline.getArgs() == null ||
- cmdline.getArgs().length == 1) {
- System.err.println("No backup id was specified, "
- + "will retrieve the most recent (ongoing) sessions");
- }
- String[] args = cmdline.getArgs();
- if (args.length > 2) {
- System.err.println("ERROR: wrong number of arguments: " + args.length);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String backupId = (args == null || args.length <= 1) ? null : args[1];
- Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create();
- try(final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){
- int progress = admin.getProgress(backupId);
- if(progress < 0){
- System.err.println(NO_INFO_FOUND + backupId);
- } else{
- System.out.println(backupId+" progress=" + progress+"%");
- }
- }
- }
-
- @Override
- protected void printUsage() {
- System.err.println(PROGRESS_CMD_USAGE);
- }
- }
-
- private static class DeleteCommand extends Command {
-
- DeleteCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- super.execute();
- if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
- System.err.println("No backup id(s) was specified");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String[] args = cmdline.getArgs();
-
- String[] backupIds = new String[args.length - 1];
- System.arraycopy(args, 1, backupIds, 0, backupIds.length);
- Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
- try (final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupAdmin admin = conn.getAdmin().getBackupAdmin();) {
- int deleted = admin.deleteBackups(args);
- System.out.println("Deleted " + deleted + " backups. Total requested: " + args.length);
- }
-
- }
-
- @Override
- protected void printUsage() {
- System.err.println(DELETE_CMD_USAGE);
- }
- }
-
-// TODO Cancel command
-
- private static class CancelCommand extends Command {
-
- CancelCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- super.execute();
- if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
- System.out.println("No backup id(s) was specified, will use the most recent one");
- }
- String[] args = cmdline.getArgs();
- String backupId = args == null || args.length == 0 ? null : args[1];
- Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
- try (final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupAdmin admin = conn.getAdmin().getBackupAdmin();) {
- // TODO cancel backup
- }
- }
-
- @Override
- protected void printUsage() {
- }
- }
-
- private static class HistoryCommand extends Command {
-
- private final static int DEFAULT_HISTORY_LENGTH = 10;
-
- HistoryCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
-
- super.execute();
-
- int n = parseHistoryLength();
- final TableName tableName = getTableName();
- final String setName = getTableSetName();
- BackupInfo.Filter tableNameFilter = new BackupInfo.Filter() {
- @Override
- public boolean apply(BackupInfo info) {
- if (tableName == null) return true;
- List<TableName> names = info.getTableNames();
- return names.contains(tableName);
- }
- };
- BackupInfo.Filter tableSetFilter = new BackupInfo.Filter() {
- @Override
- public boolean apply(BackupInfo info) {
- if (setName == null) return true;
- String backupId = info.getBackupId();
- return backupId.startsWith(setName);
- }
- };
- Path backupRootPath = getBackupRootPath();
- List<BackupInfo> history = null;
- Configuration conf = getConf() != null ? getConf() : HBaseConfiguration.create();
- if (backupRootPath == null) {
- // Load from hbase:backup
- try (final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupAdmin admin = conn.getAdmin().getBackupAdmin();) {
-
- history = admin.getHistory(n, tableNameFilter, tableSetFilter);
- }
- } else {
- // load from backup FS
- history = BackupClientUtil.getHistory(conf, n, backupRootPath,
- tableNameFilter, tableSetFilter);
- }
- for (BackupInfo info : history) {
- System.out.println(info.getShortDescription());
- }
- }
-
- private Path getBackupRootPath() throws IOException {
- String value = null;
- try{
- value = cmdline.getOptionValue("path");
- if (value == null) return null;
- return new Path(value);
- } catch (IllegalArgumentException e) {
- System.err.println("ERROR: Illegal argument for backup root path: "+ value);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- }
-
- private TableName getTableName() throws IOException {
- String value = cmdline.getOptionValue("t");
- if (value == null) return null;
- try{
- return TableName.valueOf(value);
- } catch (IllegalArgumentException e){
- System.err.println("Illegal argument for table name: "+ value);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- }
-
- private String getTableSetName() throws IOException {
- String value = cmdline.getOptionValue("set");
- return value;
- }
-
- private int parseHistoryLength() throws IOException {
- String value = cmdline.getOptionValue("n");
- try{
- if (value == null) return DEFAULT_HISTORY_LENGTH;
- return Integer.parseInt(value);
- } catch(NumberFormatException e) {
- System.err.println("Illegal argument for history length: "+ value);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- }
-
- @Override
- protected void printUsage() {
- System.err.println(HISTORY_CMD_USAGE);
- }
- }
-
- private static class BackupSetCommand extends Command {
- private final static String SET_ADD_CMD = "add";
- private final static String SET_REMOVE_CMD = "remove";
- private final static String SET_DELETE_CMD = "delete";
- private final static String SET_DESCRIBE_CMD = "describe";
- private final static String SET_LIST_CMD = "list";
-
- BackupSetCommand(Configuration conf, CommandLine cmdline) {
- super(conf);
- this.cmdline = cmdline;
- }
-
- @Override
- public void execute() throws IOException {
- super.execute();
- // Command-line must have at least one element
- if (cmdline == null || cmdline.getArgs() == null || cmdline.getArgs().length < 2) {
- System.err.println("ERROR: Command line format");
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String[] args = cmdline.getArgs();
- String cmdStr = args[1];
- BackupCommand cmd = getCommand(cmdStr);
-
- switch (cmd) {
- case SET_ADD:
- processSetAdd(args);
- break;
- case SET_REMOVE:
- processSetRemove(args);
- break;
- case SET_DELETE:
- processSetDelete(args);
- break;
- case SET_DESCRIBE:
- processSetDescribe(args);
- break;
- case SET_LIST:
- processSetList(args);
- break;
- default:
- break;
-
- }
- }
-
- private void processSetList(String[] args) throws IOException {
- // List all backup set names
- // does not expect any args
- Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create();
- try(final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){
- List<BackupSet> list = admin.listBackupSets();
- for(BackupSet bs: list){
- System.out.println(bs);
- }
- }
- }
-
- private void processSetDescribe(String[] args) throws IOException {
- if (args == null || args.length != 3) {
- System.err.println("ERROR: Wrong number of args for 'set describe' command: "
- + numOfArgs(args));
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- String setName = args[2];
- Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create();
- try(final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){
- BackupSet set = admin.getBackupSet(setName);
- if(set == null) {
- System.out.println("Set '"+setName+"' does not exist.");
- } else{
- System.out.println(set);
- }
- }
- }
-
- private void processSetDelete(String[] args) throws IOException {
- if (args == null || args.length != 3) {
- System.err.println("ERROR: Wrong number of args for 'set delete' command: "
- + numOfArgs(args));
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- String setName = args[2];
- Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create();
- try(final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){
- boolean result = admin.deleteBackupSet(setName);
- if(result){
- System.out.println("Delete set "+setName+" OK.");
- } else{
- System.out.println("Set "+setName+" does not exist");
- }
- }
- }
-
- private void processSetRemove(String[] args) throws IOException {
- if (args == null || args.length != 4) {
- System.err.println("ERROR: Wrong number of args for 'set remove' command: "
- + numOfArgs(args));
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
-
- String setName = args[2];
- String[] tables = args[3].split(",");
- Configuration conf = getConf() != null? getConf(): HBaseConfiguration.create();
- try(final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){
- admin.removeFromBackupSet(setName, tables);
- }
- }
-
- private void processSetAdd(String[] args) throws IOException {
- if (args == null || args.length != 4) {
- System.err.println("ERROR: Wrong number of args for 'set add' command: "
- + numOfArgs(args));
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- String setName = args[2];
- String[] tables = args[3].split(",");
- TableName[] tableNames = new TableName[tables.length];
- for(int i=0; i < tables.length; i++){
- tableNames[i] = TableName.valueOf(tables[i]);
- }
- Configuration conf = getConf() != null? getConf():HBaseConfiguration.create();
- try(final Connection conn = ConnectionFactory.createConnection(conf);
- final BackupAdmin admin = conn.getAdmin().getBackupAdmin();){
- admin.addToBackupSet(setName, tableNames);
- }
-
- }
-
- private BackupCommand getCommand(String cmdStr) throws IOException {
- if (cmdStr.equals(SET_ADD_CMD)) {
- return BackupCommand.SET_ADD;
- } else if (cmdStr.equals(SET_REMOVE_CMD)) {
- return BackupCommand.SET_REMOVE;
- } else if (cmdStr.equals(SET_DELETE_CMD)) {
- return BackupCommand.SET_DELETE;
- } else if (cmdStr.equals(SET_DESCRIBE_CMD)) {
- return BackupCommand.SET_DESCRIBE;
- } else if (cmdStr.equals(SET_LIST_CMD)) {
- return BackupCommand.SET_LIST;
- } else {
- System.err.println("ERROR: Unknown command for 'set' :" + cmdStr);
- printUsage();
- throw new IOException(INCORRECT_USAGE);
- }
- }
-
- @Override
- protected void printUsage() {
- System.err.println(SET_CMD_USAGE);
- }
-
- }
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java
deleted file mode 100644
index ca204b4..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupException.java
+++ /dev/null
@@ -1,86 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.impl;
-
-import org.apache.hadoop.hbase.HBaseIOException;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-
-/**
- * Backup exception
- */
-@SuppressWarnings("serial")
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class BackupException extends HBaseIOException {
- private BackupInfo description;
-
- /**
- * Some exception happened for a backup and don't even know the backup that it was about
- * @param msg Full description of the failure
- */
- public BackupException(String msg) {
- super(msg);
- }
-
- /**
- * Some exception happened for a backup with a cause
- * @param cause the cause
- */
- public BackupException(Throwable cause) {
- super(cause);
- }
-
- /**
- * Exception for the given backup that has no previous root cause
- * @param msg reason why the backup failed
- * @param desc description of the backup that is being failed
- */
- public BackupException(String msg, BackupInfo desc) {
- super(msg);
- this.description = desc;
- }
-
- /**
- * Exception for the given backup due to another exception
- * @param msg reason why the backup failed
- * @param cause root cause of the failure
- * @param desc description of the backup that is being failed
- */
- public BackupException(String msg, Throwable cause, BackupInfo desc) {
- super(msg, cause);
- this.description = desc;
- }
-
- /**
- * Exception when the description of the backup cannot be determined, due to some other root
- * cause
- * @param message description of what caused the failure
- * @param e root cause
- */
- public BackupException(String message, Exception e) {
- super(message, e);
- }
-
- public BackupInfo getBackupContext() {
- return this.description;
- }
-
-}
http://git-wip-us.apache.org/repos/asf/hbase/blob/b14e2ab1/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java
----------------------------------------------------------------------
diff --git a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java b/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java
deleted file mode 100644
index d10713d..0000000
--- a/hbase-client/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupManifest.java
+++ /dev/null
@@ -1,791 +0,0 @@
-/**
- * Licensed to the Apache Software Foundation (ASF) under one
- * or more contributor license agreements. See the NOTICE file
- * distributed with this work for additional information
- * regarding copyright ownership. The ASF licenses this file
- * to you under the Apache License, Version 2.0 (the
- * "License"); you may not use this file except in compliance
- * with the License. You may obtain a copy of the License at
- *
- * http://www.apache.org/licenses/LICENSE-2.0
- *
- * Unless required by applicable law or agreed to in writing, software
- * distributed under the License is distributed on an "AS IS" BASIS,
- * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
- * See the License for the specific language governing permissions and
- * limitations under the License.
- */
-
-package org.apache.hadoop.hbase.backup.impl;
-
-import java.io.IOException;
-import java.util.ArrayList;
-import java.util.Collections;
-import java.util.HashMap;
-import java.util.List;
-import java.util.Map;
-import java.util.Map.Entry;
-import java.util.TreeMap;
-
-import org.apache.commons.logging.Log;
-import org.apache.commons.logging.LogFactory;
-import org.apache.hadoop.conf.Configuration;
-import org.apache.hadoop.fs.FSDataInputStream;
-import org.apache.hadoop.fs.FSDataOutputStream;
-import org.apache.hadoop.fs.FileStatus;
-import org.apache.hadoop.fs.FileSystem;
-import org.apache.hadoop.fs.Path;
-import org.apache.hadoop.hbase.HConstants;
-import org.apache.hadoop.hbase.TableName;
-import org.apache.hadoop.hbase.backup.BackupInfo;
-import org.apache.hadoop.hbase.backup.BackupType;
-import org.apache.hadoop.hbase.backup.util.BackupClientUtil;
-import org.apache.hadoop.hbase.classification.InterfaceAudience;
-import org.apache.hadoop.hbase.classification.InterfaceStability;
-import org.apache.hadoop.hbase.exceptions.DeserializationException;
-import org.apache.hadoop.hbase.protobuf.ProtobufUtil;
-import org.apache.hadoop.hbase.protobuf.generated.BackupProtos;
-import org.apache.hadoop.hbase.protobuf.generated.HBaseProtos;
-
-import com.google.protobuf.InvalidProtocolBufferException;
-
-
-/**
- * Backup manifest Contains all the meta data of a backup image. The manifest info will be bundled
- * as manifest file together with data. So that each backup image will contain all the info needed
- * for restore.
- */
-@InterfaceAudience.Private
-@InterfaceStability.Evolving
-public class BackupManifest {
-
- private static final Log LOG = LogFactory.getLog(BackupManifest.class);
-
- // manifest file name
- public static final String MANIFEST_FILE_NAME = ".backup.manifest";
-
- // manifest file version, current is 1.0
- public static final String MANIFEST_VERSION = "1.0";
-
- // backup image, the dependency graph is made up by series of backup images
-
- public static class BackupImage implements Comparable<BackupImage> {
-
- private String backupId;
- private BackupType type;
- private String rootDir;
- private List<TableName> tableList;
- private long startTs;
- private long completeTs;
- private ArrayList<BackupImage> ancestors;
-
- public BackupImage() {
- super();
- }
-
- public BackupImage(String backupId, BackupType type, String rootDir,
- List<TableName> tableList, long startTs, long completeTs) {
- this.backupId = backupId;
- this.type = type;
- this.rootDir = rootDir;
- this.tableList = tableList;
- this.startTs = startTs;
- this.completeTs = completeTs;
- }
-
- static BackupImage fromProto(BackupProtos.BackupImage im) {
- String backupId = im.getBackupId();
- String rootDir = im.getRootDir();
- long startTs = im.getStartTs();
- long completeTs = im.getCompleteTs();
- List<HBaseProtos.TableName> tableListList = im.getTableListList();
- List<TableName> tableList = new ArrayList<TableName>();
- for(HBaseProtos.TableName tn : tableListList) {
- tableList.add(ProtobufUtil.toTableName(tn));
- }
-
- List<BackupProtos.BackupImage> ancestorList = im.getAncestorsList();
-
- BackupType type =
- im.getBackupType() == BackupProtos.BackupType.FULL ? BackupType.FULL:
- BackupType.INCREMENTAL;
-
- BackupImage image = new BackupImage(backupId, type, rootDir, tableList, startTs, completeTs);
- for(BackupProtos.BackupImage img: ancestorList) {
- image.addAncestor(fromProto(img));
- }
- return image;
- }
-
- BackupProtos.BackupImage toProto() {
- BackupProtos.BackupImage.Builder builder = BackupProtos.BackupImage.newBuilder();
- builder.setBackupId(backupId);
- builder.setCompleteTs(completeTs);
- builder.setStartTs(startTs);
- builder.setRootDir(rootDir);
- if (type == BackupType.FULL) {
- builder.setBackupType(BackupProtos.BackupType.FULL);
- } else{
- builder.setBackupType(BackupProtos.BackupType.INCREMENTAL);
- }
-
- for (TableName name: tableList) {
- builder.addTableList(ProtobufUtil.toProtoTableName(name));
- }
-
- if (ancestors != null){
- for (BackupImage im: ancestors){
- builder.addAncestors(im.toProto());
- }
- }
-
- return builder.build();
- }
-
- public String getBackupId() {
- return backupId;
- }
-
- public void setBackupId(String backupId) {
- this.backupId = backupId;
- }
-
- public BackupType getType() {
- return type;
- }
-
- public void setType(BackupType type) {
- this.type = type;
- }
-
- public String getRootDir() {
- return rootDir;
- }
-
- public void setRootDir(String rootDir) {
- this.rootDir = rootDir;
- }
-
- public List<TableName> getTableNames() {
- return tableList;
- }
-
- public void setTableList(List<TableName> tableList) {
- this.tableList = tableList;
- }
-
- public long getStartTs() {
- return startTs;
- }
-
- public void setStartTs(long startTs) {
- this.startTs = startTs;
- }
-
- public long getCompleteTs() {
- return completeTs;
- }
-
- public void setCompleteTs(long completeTs) {
- this.completeTs = completeTs;
- }
-
- public ArrayList<BackupImage> getAncestors() {
- if (this.ancestors == null) {
- this.ancestors = new ArrayList<BackupImage>();
- }
- return this.ancestors;
- }
-
- public void addAncestor(BackupImage backupImage) {
- this.getAncestors().add(backupImage);
- }
-
- public boolean hasAncestor(String token) {
- for (BackupImage image : this.getAncestors()) {
- if (image.getBackupId().equals(token)) {
- return true;
- }
- }
- return false;
- }
-
- public boolean hasTable(TableName table) {
- for (TableName t : tableList) {
- if (t.equals(table)) {
- return true;
- }
- }
- return false;
- }
-
- @Override
- public int compareTo(BackupImage other) {
- String thisBackupId = this.getBackupId();
- String otherBackupId = other.getBackupId();
- int index1 = thisBackupId.lastIndexOf("_");
- int index2 = otherBackupId.lastIndexOf("_");
- String name1 = thisBackupId.substring(0, index1);
- String name2 = otherBackupId.substring(0, index2);
- if(name1.equals(name2)) {
- Long thisTS = new Long(thisBackupId.substring(index1 + 1));
- Long otherTS = new Long(otherBackupId.substring(index2 + 1));
- return thisTS.compareTo(otherTS);
- } else {
- return name1.compareTo(name2);
- }
- }
- }
-
- // manifest version
- private String version = MANIFEST_VERSION;
-
- // hadoop hbase configuration
- protected Configuration config = null;
-
- // backup root directory
- private String rootDir = null;
-
- // backup image directory
- private String tableBackupDir = null;
-
- // backup log directory if this is an incremental backup
- private String logBackupDir = null;
-
- // backup token
- private String backupId;
-
- // backup type, full or incremental
- private BackupType type;
-
- // the table list for the backup
- private ArrayList<TableName> tableList;
-
- // actual start timestamp of the backup process
- private long startTs;
-
- // actual complete timestamp of the backup process
- private long completeTs;
-
- // the region server timestamp for tables:
- // <table, <rs, timestamp>>
- private Map<TableName, HashMap<String, Long>> incrTimeRanges;
-
- // dependency of this backup, including all the dependent images to do PIT recovery
- private Map<String, BackupImage> dependency;
-
- /**
- * Construct manifest for a ongoing backup.
- * @param backupCtx The ongoing backup context
- */
- public BackupManifest(BackupInfo backupCtx) {
- this.backupId = backupCtx.getBackupId();
- this.type = backupCtx.getType();
- this.rootDir = backupCtx.getTargetRootDir();
- if (this.type == BackupType.INCREMENTAL) {
- this.logBackupDir = backupCtx.getHLogTargetDir();
- }
- this.startTs = backupCtx.getStartTs();
- this.completeTs = backupCtx.getEndTs();
- this.loadTableList(backupCtx.getTableNames());
- }
-
-
- /**
- * Construct a table level manifest for a backup of the named table.
- * @param backupCtx The ongoing backup context
- */
- public BackupManifest(BackupInfo backupCtx, TableName table) {
- this.backupId = backupCtx.getBackupId();
- this.type = backupCtx.getType();
- this.rootDir = backupCtx.getTargetRootDir();
- this.tableBackupDir = backupCtx.getBackupStatus(table).getTargetDir();
- if (this.type == BackupType.INCREMENTAL) {
- this.logBackupDir = backupCtx.getHLogTargetDir();
- }
- this.startTs = backupCtx.getStartTs();
- this.completeTs = backupCtx.getEndTs();
- List<TableName> tables = new ArrayList<TableName>();
- tables.add(table);
- this.loadTableList(tables);
- }
-
- /**
- * Construct manifest from a backup directory.
- * @param conf configuration
- * @param backupPath backup path
- * @throws IOException
- */
-
- public BackupManifest(Configuration conf, Path backupPath) throws IOException {
- this(backupPath.getFileSystem(conf), backupPath);
- }
-
- /**
- * Construct manifest from a backup directory.
- * @param conf configuration
- * @param backupPath backup path
- * @throws BackupException exception
- */
-
- public BackupManifest(FileSystem fs, Path backupPath) throws BackupException {
- if (LOG.isDebugEnabled()) {
- LOG.debug("Loading manifest from: " + backupPath.toString());
- }
- // The input backupDir may not exactly be the backup table dir.
- // It could be the backup log dir where there is also a manifest file stored.
- // This variable's purpose is to keep the correct and original location so
- // that we can store/persist it.
- this.tableBackupDir = backupPath.toString();
- this.config = fs.getConf();
- try {
-
- FileStatus[] subFiles = BackupClientUtil.listStatus(fs, backupPath, null);
- if (subFiles == null) {
- String errorMsg = backupPath.toString() + " does not exist";
- LOG.error(errorMsg);
- throw new IOException(errorMsg);
- }
- for (FileStatus subFile : subFiles) {
- if (subFile.getPath().getName().equals(MANIFEST_FILE_NAME)) {
-
- // load and set manifest field from file content
- FSDataInputStream in = fs.open(subFile.getPath());
- long len = subFile.getLen();
- byte[] pbBytes = new byte[(int) len];
- in.readFully(pbBytes);
- BackupProtos.BackupManifest proto = null;
- try{
- proto = parseFrom(pbBytes);
- } catch(Exception e){
- throw new BackupException(e);
- }
- this.version = proto.getVersion();
- this.backupId = proto.getBackupId();
- this.type = BackupType.valueOf(proto.getType().name());
- // Here the parameter backupDir is where the manifest file is.
- // There should always be a manifest file under:
- // backupRootDir/namespace/table/backupId/.backup.manifest
- this.rootDir = backupPath.getParent().getParent().getParent().toString();
-
- Path p = backupPath.getParent();
- if (p.getName().equals(HConstants.HREGION_LOGDIR_NAME)) {
- this.rootDir = p.getParent().toString();
- } else {
- this.rootDir = p.getParent().getParent().toString();
- }
-
- loadTableList(proto);
- this.startTs = proto.getStartTs();
- this.completeTs = proto.getCompleteTs();
- loadIncrementalTimestampMap(proto);
- loadDependency(proto);
- //TODO: merge will be implemented by future jira
- LOG.debug("Loaded manifest instance from manifest file: "
- + BackupClientUtil.getPath(subFile.getPath()));
- return;
- }
- }
- String errorMsg = "No manifest file found in: " + backupPath.toString();
- throw new IOException(errorMsg);
-
- } catch (IOException e) {
- throw new BackupException(e.getMessage());
- }
- }
-
- private void loadIncrementalTimestampMap(BackupProtos.BackupManifest proto) {
- List<BackupProtos.TableServerTimestamp> list = proto.getTstMapList();
- if(list == null || list.size() == 0) return;
- this.incrTimeRanges = new HashMap<TableName, HashMap<String, Long>>();
- for(BackupProtos.TableServerTimestamp tst: list){
- TableName tn = ProtobufUtil.toTableName(tst.getTable());
- HashMap<String, Long> map = this.incrTimeRanges.get(tn);
- if(map == null){
- map = new HashMap<String, Long>();
- this.incrTimeRanges.put(tn, map);
- }
- List<BackupProtos.ServerTimestamp> listSt = tst.getServerTimestampList();
- for(BackupProtos.ServerTimestamp stm: listSt) {
- map.put(stm.getServer(), stm.getTimestamp());
- }
- }
- }
-
- private void loadDependency(BackupProtos.BackupManifest proto) {
- if(LOG.isDebugEnabled()) {
- LOG.debug("load dependency for: "+proto.getBackupId());
- }
-
- dependency = new HashMap<String, BackupImage>();
- List<BackupProtos.BackupImage> list = proto.getDependentBackupImageList();
- for (BackupProtos.BackupImage im : list) {
- BackupImage bim = BackupImage.fromProto(im);
- if(im.getBackupId() != null){
- dependency.put(im.getBackupId(), bim);
- } else{
- LOG.warn("Load dependency for backup manifest: "+ backupId+
- ". Null backup id in dependent image");
- }
- }
- }
-
- private void loadTableList(BackupProtos.BackupManifest proto) {
- this.tableList = new ArrayList<TableName>();
- List<HBaseProtos.TableName> list = proto.getTableListList();
- for (HBaseProtos.TableName name: list) {
- this.tableList.add(ProtobufUtil.toTableName(name));
- }
- }
-
- public BackupType getType() {
- return type;
- }
-
- public void setType(BackupType type) {
- this.type = type;
- }
-
- /**
- * Loads table list.
- * @param tableList Table list
- */
- private void loadTableList(List<TableName> tableList) {
-
- this.tableList = this.getTableList();
- if (this.tableList.size() > 0) {
- this.tableList.clear();
- }
- for (int i = 0; i < tableList.size(); i++) {
- this.tableList.add(tableList.get(i));
- }
-
- LOG.debug(tableList.size() + " tables exist in table set.");
- }
-
- /**
- * Get the table set of this image.
- * @return The table set list
- */
- public ArrayList<TableName> getTableList() {
- if (this.tableList == null) {
- this.tableList = new ArrayList<TableName>();
- }
- return this.tableList;
- }
-
- /**
- * Persist the manifest file.
- * @throws IOException IOException when storing the manifest file.
- */
-
- public void store(Configuration conf) throws BackupException {
- byte[] data = toByteArray();
-
- // write the file, overwrite if already exist
- Path manifestFilePath =
- new Path(new Path((this.tableBackupDir != null ? this.tableBackupDir : this.logBackupDir))
- ,MANIFEST_FILE_NAME);
- try {
- FSDataOutputStream out =
- manifestFilePath.getFileSystem(conf).create(manifestFilePath, true);
- out.write(data);
- out.close();
- } catch (IOException e) {
- throw new BackupException(e.getMessage());
- }
-
- LOG.info("Manifest file stored to " + manifestFilePath);
- }
-
- /**
- * Protobuf serialization
- * @return The filter serialized using pb
- */
- public byte[] toByteArray() {
- BackupProtos.BackupManifest.Builder builder = BackupProtos.BackupManifest.newBuilder();
- builder.setVersion(this.version);
- builder.setBackupId(this.backupId);
- builder.setType(BackupProtos.BackupType.valueOf(this.type.name()));
- setTableList(builder);
- builder.setStartTs(this.startTs);
- builder.setCompleteTs(this.completeTs);
- setIncrementalTimestampMap(builder);
- setDependencyMap(builder);
- return builder.build().toByteArray();
- }
-
- private void setIncrementalTimestampMap(BackupProtos.BackupManifest.Builder builder) {
- if (this.incrTimeRanges == null) {
- return;
- }
- for (Entry<TableName, HashMap<String,Long>> entry: this.incrTimeRanges.entrySet()) {
- TableName key = entry.getKey();
- HashMap<String, Long> value = entry.getValue();
- BackupProtos.TableServerTimestamp.Builder tstBuilder =
- BackupProtos.TableServerTimestamp.newBuilder();
- tstBuilder.setTable(ProtobufUtil.toProtoTableName(key));
-
- for (String s : value.keySet()) {
- BackupProtos.ServerTimestamp.Builder stBuilder = BackupProtos.ServerTimestamp.newBuilder();
- stBuilder.setServer(s);
- stBuilder.setTimestamp(value.get(s));
- tstBuilder.addServerTimestamp(stBuilder.build());
- }
- builder.addTstMap(tstBuilder.build());
- }
- }
-
- private void setDependencyMap(BackupProtos.BackupManifest.Builder builder) {
- for (BackupImage image: getDependency().values()) {
- builder.addDependentBackupImage(image.toProto());
- }
- }
-
- private void setTableList(BackupProtos.BackupManifest.Builder builder) {
- for(TableName name: tableList){
- builder.addTableList(ProtobufUtil.toProtoTableName(name));
- }
- }
-
- /**
- * Parse protobuf from byte array
- * @param pbBytes A pb serialized BackupManifest instance
- * @return An instance of made from <code>bytes</code>
- * @throws DeserializationException
- */
- private static BackupProtos.BackupManifest parseFrom(final byte[] pbBytes)
- throws DeserializationException {
- BackupProtos.BackupManifest proto;
- try {
- proto = BackupProtos.BackupManifest.parseFrom(pbBytes);
- } catch (InvalidProtocolBufferException e) {
- throw new DeserializationException(e);
- }
- return proto;
- }
-
- /**
- * Get manifest file version
- * @return version
- */
- public String getVersion() {
- return version;
- }
-
- /**
- * Get this backup image.
- * @return the backup image.
- */
- public BackupImage getBackupImage() {
- return this.getDependency().get(this.backupId);
- }
-
- /**
- * Add dependent backup image for this backup.
- * @param image The direct dependent backup image
- */
- public void addDependentImage(BackupImage image) {
- this.getDependency().get(this.backupId).addAncestor(image);
- this.setDependencyMap(this.getDependency(), image);
- }
-
-
-
- /**
- * Get all dependent backup images. The image of this backup is also contained.
- * @return The dependent backup images map
- */
- public Map<String, BackupImage> getDependency() {
- if (this.dependency == null) {
- this.dependency = new HashMap<String, BackupImage>();
- LOG.debug(this.rootDir + " " + this.backupId + " " + this.type);
- this.dependency.put(this.backupId,
- new BackupImage(this.backupId, this.type, this.rootDir, tableList, this.startTs,
- this.completeTs));
- }
- return this.dependency;
- }
-
- /**
- * Set the incremental timestamp map directly.
- * @param incrTimestampMap timestamp map
- */
- public void setIncrTimestampMap(HashMap<TableName, HashMap<String, Long>> incrTimestampMap) {
- this.incrTimeRanges = incrTimestampMap;
- }
-
-
- public Map<TableName, HashMap<String, Long>> getIncrTimestampMap() {
- if (this.incrTimeRanges == null) {
- this.incrTimeRanges = new HashMap<TableName, HashMap<String, Long>>();
- }
- return this.incrTimeRanges;
- }
-
-
- /**
- * Get the image list of this backup for restore in time order.
- * @param reverse If true, then output in reverse order, otherwise in time order from old to new
- * @return the backup image list for restore in time order
- */
- public ArrayList<BackupImage> getRestoreDependentList(boolean reverse) {
- TreeMap<Long, BackupImage> restoreImages = new TreeMap<Long, BackupImage>();
- for (BackupImage image : this.getDependency().values()) {
- restoreImages.put(Long.valueOf(image.startTs), image);
- }
- return new ArrayList<BackupImage>(reverse ? (restoreImages.descendingMap().values())
- : (restoreImages.values()));
- }
-
- /**
- * Get the dependent image list for a specific table of this backup in time order from old to new
- * if want to restore to this backup image level.
- * @param table table
- * @return the backup image list for a table in time order
- */
- public ArrayList<BackupImage> getDependentListByTable(TableName table) {
- ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>();
- ArrayList<BackupImage> imageList = getRestoreDependentList(true);
- for (BackupImage image : imageList) {
- if (image.hasTable(table)) {
- tableImageList.add(image);
- if (image.getType() == BackupType.FULL) {
- break;
- }
- }
- }
- Collections.reverse(tableImageList);
- return tableImageList;
- }
-
- /**
- * Get the full dependent image list in the whole dependency scope for a specific table of this
- * backup in time order from old to new.
- * @param table table
- * @return the full backup image list for a table in time order in the whole scope of the
- * dependency of this image
- */
- public ArrayList<BackupImage> getAllDependentListByTable(TableName table) {
- ArrayList<BackupImage> tableImageList = new ArrayList<BackupImage>();
- ArrayList<BackupImage> imageList = getRestoreDependentList(false);
- for (BackupImage image : imageList) {
- if (image.hasTable(table)) {
- tableImageList.add(image);
- }
- }
- return tableImageList;
- }
-
-
- /**
- * Recursively set the dependency map of the backup images.
- * @param map The dependency map
- * @param image The backup image
- */
- private void setDependencyMap(Map<String, BackupImage> map, BackupImage image) {
- if (image == null) {
- return;
- } else {
- map.put(image.getBackupId(), image);
- for (BackupImage img : image.getAncestors()) {
- setDependencyMap(map, img);
- }
- }
- }
-
- /**
- * Check whether backup image1 could cover backup image2 or not.
- * @param image1 backup image 1
- * @param image2 backup image 2
- * @return true if image1 can cover image2, otherwise false
- */
- public static boolean canCoverImage(BackupImage image1, BackupImage image2) {
- // image1 can cover image2 only when the following conditions are satisfied:
- // - image1 must not be an incremental image;
- // - image1 must be taken after image2 has been taken;
- // - table set of image1 must cover the table set of image2.
- if (image1.getType() == BackupType.INCREMENTAL) {
- return false;
- }
- if (image1.getStartTs() < image2.getStartTs()) {
- return false;
- }
- List<TableName> image1TableList = image1.getTableNames();
- List<TableName> image2TableList = image2.getTableNames();
- boolean found = false;
- for (int i = 0; i < image2TableList.size(); i++) {
- found = false;
- for (int j = 0; j < image1TableList.size(); j++) {
- if (image2TableList.get(i).equals(image1TableList.get(j))) {
- found = true;
- break;
- }
- }
- if (!found) {
- return false;
- }
- }
-
- LOG.debug("Backup image " + image1.getBackupId() + " can cover " + image2.getBackupId());
- return true;
- }
-
- /**
- * Check whether backup image set could cover a backup image or not.
- * @param fullImages The backup image set
- * @param image The target backup image
- * @return true if fullImages can cover image, otherwise false
- */
- public static boolean canCoverImage(ArrayList<BackupImage> fullImages, BackupImage image) {
- // fullImages can cover image only when the following conditions are satisfied:
- // - each image of fullImages must not be an incremental image;
- // - each image of fullImages must be taken after image has been taken;
- // - sum table set of fullImages must cover the table set of image.
- for (BackupImage image1 : fullImages) {
- if (image1.getType() == BackupType.INCREMENTAL) {
- return false;
- }
- if (image1.getStartTs() < image.getStartTs()) {
- return false;
- }
- }
-
- ArrayList<String> image1TableList = new ArrayList<String>();
- for (BackupImage image1 : fullImages) {
- List<TableName> tableList = image1.getTableNames();
- for (TableName table : tableList) {
- image1TableList.add(table.getNameAsString());
- }
- }
- ArrayList<String> image2TableList = new ArrayList<String>();
- List<TableName> tableList = image.getTableNames();
- for (TableName table : tableList) {
- image2TableList.add(table.getNameAsString());
- }
-
- for (int i = 0; i < image2TableList.size(); i++) {
- if (image1TableList.contains(image2TableList.get(i)) == false) {
- return false;
- }
- }
-
- LOG.debug("Full image set can cover image " + image.getBackupId());
- return true;
- }
-
- public BackupInfo toBackupInfo()
- {
- BackupInfo info = new BackupInfo();
- info.setType(type);
- TableName[] tables = new TableName[tableList.size()];
- info.addTables(getTableList().toArray(tables));
- info.setBackupId(backupId);
- info.setStartTs(startTs);
- info.setTargetRootDir(rootDir);
- if(type == BackupType.INCREMENTAL) {
- info.setHlogTargetDir(logBackupDir);
- }
- return info;
- }
-}