You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by te...@apache.org on 2017/08/30 23:11:45 UTC
hbase git commit: HBASE-17826 Backup: submit M/R job to a particular
Yarn queue (Vladimir Rodionov)
Repository: hbase
Updated Branches:
refs/heads/master b5942241c -> 645e5a5e2
HBASE-17826 Backup: submit M/R job to a particular Yarn queue (Vladimir Rodionov)
Project: http://git-wip-us.apache.org/repos/asf/hbase/repo
Commit: http://git-wip-us.apache.org/repos/asf/hbase/commit/645e5a5e
Tree: http://git-wip-us.apache.org/repos/asf/hbase/tree/645e5a5e
Diff: http://git-wip-us.apache.org/repos/asf/hbase/diff/645e5a5e
Branch: refs/heads/master
Commit: 645e5a5e2b598310513db7ccfa559f8961c65f05
Parents: b594224
Author: tedyu <yu...@gmail.com>
Authored: Wed Aug 30 16:11:32 2017 -0700
Committer: tedyu <yu...@gmail.com>
Committed: Wed Aug 30 16:11:32 2017 -0700
----------------------------------------------------------------------
.../org/apache/hadoop/hbase/backup/BackupRequest.java | 14 ++++++++++++++
.../hadoop/hbase/backup/BackupRestoreConstants.java | 4 ++++
.../org/apache/hadoop/hbase/backup/RestoreDriver.java | 12 +++++++++++-
.../hadoop/hbase/backup/impl/BackupCommands.java | 10 ++++++++++
4 files changed, 39 insertions(+), 1 deletion(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hbase/blob/645e5a5e/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java
index dadd861..8659868 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRequest.java
@@ -67,6 +67,11 @@ public final class BackupRequest {
return this;
}
+ public Builder withYarnPoolName(String name) {
+ request.setYarnPoolName(name);
+ return this;
+ }
+
public BackupRequest build() {
return request;
}
@@ -79,6 +84,7 @@ public final class BackupRequest {
private int totalTasks = -1;
private long bandwidth = -1L;
private String backupSetName;
+ private String yarnPoolName;
private BackupRequest() {
}
@@ -136,4 +142,12 @@ public final class BackupRequest {
this.backupSetName = backupSetName;
return this;
}
+
+ public String getYarnPoolName() {
+ return yarnPoolName;
+ }
+
+ public void setYarnPoolName(String yarnPoolName) {
+ this.yarnPoolName = yarnPoolName;
+ }
}
http://git-wip-us.apache.org/repos/asf/hbase/blob/645e5a5e/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
index 48e70a1..f511cc4 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/BackupRestoreConstants.java
@@ -89,6 +89,10 @@ public interface BackupRestoreConstants {
public static final String OPTION_TABLE_MAPPING_DESC =
"A comma separated list of target tables. "
+ "If specified, each table in <tables> must have a mapping";
+ public static final String OPTION_YARN_QUEUE_NAME = "q";
+ public static final String OPTION_YARN_QUEUE_NAME_DESC = "Yarn queue name to run backup create command on";
+ public static final String OPTION_YARN_QUEUE_NAME_RESTORE_DESC = "Yarn queue name to run backup restore command on";
+
public static final String BACKUP_CONFIG_STRING = BackupRestoreConstants.BACKUP_ENABLE_KEY + "=true\n"
+ "hbase.master.logcleaner.plugins="
http://git-wip-us.apache.org/repos/asf/hbase/blob/645e5a5e/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
index 82a1b56..2a1458c 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/RestoreDriver.java
@@ -29,6 +29,8 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_MAPPING;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_MAPPING_DESC;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_RESTORE_DESC;
import java.io.IOException;
import java.net.URI;
@@ -125,6 +127,13 @@ public class RestoreDriver extends AbstractHBaseTool {
printToolUsage();
return -1;
}
+
+ if (cmd.hasOption(OPTION_YARN_QUEUE_NAME)) {
+ String queueName = cmd.getOptionValue(OPTION_YARN_QUEUE_NAME);
+ // Set system property value for MR job
+ System.setProperty("mapreduce.job.queuename", queueName);
+ }
+
// parse main restore command options
String[] remainArgs = cmd.getArgs();
if (remainArgs.length != 2) {
@@ -195,8 +204,9 @@ public class RestoreDriver extends AbstractHBaseTool {
addOptNoArg(OPTION_DEBUG, OPTION_DEBUG_DESC);
addOptWithArg(OPTION_SET, OPTION_SET_RESTORE_DESC);
addOptWithArg(OPTION_TABLE, OPTION_TABLE_LIST_DESC);
-
addOptWithArg(OPTION_TABLE_MAPPING, OPTION_TABLE_MAPPING_DESC);
+ addOptWithArg(OPTION_YARN_QUEUE_NAME, OPTION_YARN_QUEUE_NAME_RESTORE_DESC);
+
}
@Override
http://git-wip-us.apache.org/repos/asf/hbase/blob/645e5a5e/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
----------------------------------------------------------------------
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
index 650ba2e..eaba049 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/impl/BackupCommands.java
@@ -32,6 +32,8 @@ import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_TABLE_LIST_DESC;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS;
import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_WORKERS_DESC;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME;
+import static org.apache.hadoop.hbase.backup.BackupRestoreConstants.OPTION_YARN_QUEUE_NAME_DESC;
import java.io.IOException;
import java.net.URI;
@@ -321,6 +323,12 @@ public final class BackupCommands {
cmdline.hasOption(OPTION_WORKERS) ? Integer.parseInt(cmdline
.getOptionValue(OPTION_WORKERS)) : -1;
+ if (cmdline.hasOption(OPTION_YARN_QUEUE_NAME)) {
+ String poolName = cmdline.getOptionValue(OPTION_YARN_QUEUE_NAME);
+ // Set system property value for MR job
+ System.setProperty("mapreduce.job.queuename", poolName);
+ }
+
try (BackupAdminImpl admin = new BackupAdminImpl(conn);) {
BackupRequest.Builder builder = new BackupRequest.Builder();
@@ -368,6 +376,8 @@ public final class BackupCommands {
options.addOption(OPTION_BANDWIDTH, true, OPTION_BANDWIDTH_DESC);
options.addOption(OPTION_SET, true, OPTION_SET_BACKUP_DESC);
options.addOption(OPTION_TABLE, true, OPTION_TABLE_LIST_DESC);
+ options.addOption(OPTION_YARN_QUEUE_NAME, true, OPTION_YARN_QUEUE_NAME_DESC);
+
HelpFormatter helpFormatter = new HelpFormatter();
helpFormatter.setLeftPadding(2);