You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@hbase.apache.org by nd...@apache.org on 2024/07/18 12:48:24 UTC
(hbase) branch branch-3 updated: HBASE-28686 MapReduceBackupCopyJob should support custom DistCp options (#6017)
This is an automated email from the ASF dual-hosted git repository.
ndimiduk pushed a commit to branch branch-3
in repository https://gitbox.apache.org/repos/asf/hbase.git
The following commit(s) were added to refs/heads/branch-3 by this push:
new eab2134b957 HBASE-28686 MapReduceBackupCopyJob should support custom DistCp options (#6017)
eab2134b957 is described below
commit eab2134b9571e2a163c9d8b9d161361252d871c4
Author: Ray Mattingly <rm...@gmail.com>
AuthorDate: Thu Jul 18 07:45:42 2024 -0400
HBASE-28686 MapReduceBackupCopyJob should support custom DistCp options (#6017)
Co-authored-by: Ray Mattingly <rm...@hubspot.com>
Signed-off-by: Duo Zhang <zh...@apache.org>
Signed-off-by: Nick Dimiduk <nd...@apache.org>
---
.../backup/mapreduce/MapReduceBackupCopyJob.java | 37 ++++++++++++++-
.../mapreduce/TestMapReduceBackupCopyJob.java | 55 ++++++++++++++++++++++
2 files changed, 91 insertions(+), 1 deletion(-)
diff --git a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyJob.java b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyJob.java
index 51a276df4c5..747bd3e217d 100644
--- a/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyJob.java
+++ b/hbase-backup/src/main/java/org/apache/hadoop/hbase/backup/mapreduce/MapReduceBackupCopyJob.java
@@ -22,6 +22,7 @@ import java.lang.reflect.Field;
import java.lang.reflect.InvocationTargetException;
import java.lang.reflect.Method;
import java.math.BigDecimal;
+import java.util.ArrayList;
import java.util.Arrays;
import java.util.List;
import java.util.Objects;
@@ -45,6 +46,7 @@ import org.apache.hadoop.mapreduce.JobID;
import org.apache.hadoop.tools.CopyListingFileStatus;
import org.apache.hadoop.tools.DistCp;
import org.apache.hadoop.tools.DistCpConstants;
+import org.apache.hadoop.tools.DistCpOptionSwitch;
import org.apache.hadoop.tools.DistCpOptions;
import org.apache.yetus.audience.InterfaceAudience;
import org.slf4j.Logger;
@@ -58,6 +60,10 @@ import org.slf4j.LoggerFactory;
@InterfaceAudience.Private
public class MapReduceBackupCopyJob implements BackupCopyJob {
public static final String NUMBER_OF_LEVELS_TO_PRESERVE_KEY = "num.levels.preserve";
+
+ // This prefix specifies the DistCp options to be used during backup copy
+ public static final String BACKUP_COPY_OPTION_PREFIX = "hbase.backup.copy.";
+
private static final Logger LOG = LoggerFactory.getLogger(MapReduceBackupCopyJob.class);
private Configuration conf;
@@ -394,7 +400,15 @@ public class MapReduceBackupCopyJob implements BackupCopyJob {
if (!destfs.exists(dest)) {
destfs.mkdirs(dest);
}
- res = distcp.run(newOptions);
+
+ List<String> distCpOptionsFromConf = parseDistCpOptions(conf);
+ String[] finalOptions = new String[newOptions.length + distCpOptionsFromConf.size()];
+ for (int i = 0; i < distCpOptionsFromConf.size(); i++) {
+ finalOptions[i] = distCpOptionsFromConf.get(i);
+ }
+ System.arraycopy(newOptions, 0, finalOptions, distCpOptionsFromConf.size(),
+ newOptions.length);
+ res = distcp.run(finalOptions);
}
return res;
@@ -425,4 +439,25 @@ public class MapReduceBackupCopyJob implements BackupCopyJob {
}
}
+ protected static List<String> parseDistCpOptions(Configuration conf) {
+ List<String> extraArgsFromConf = new ArrayList<>();
+
+ for (DistCpOptionSwitch optionSwitch : DistCpOptionSwitch.values()) {
+ String configLabel = BACKUP_COPY_OPTION_PREFIX + optionSwitch.getConfigLabel();
+ if (conf.get(configLabel) != null) {
+ if (optionSwitch.getOption().hasArg()) {
+ extraArgsFromConf.add("-" + optionSwitch.getOption().getOpt());
+ extraArgsFromConf.add(conf.get(configLabel));
+ } else {
+ boolean value = conf.getBoolean(configLabel, false);
+ if (value) {
+ extraArgsFromConf.add("-" + optionSwitch.getOption().getOpt());
+ }
+ }
+ }
+ }
+
+ return extraArgsFromConf;
+ }
+
}
diff --git a/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/mapreduce/TestMapReduceBackupCopyJob.java b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/mapreduce/TestMapReduceBackupCopyJob.java
new file mode 100644
index 00000000000..6e35815c84f
--- /dev/null
+++ b/hbase-backup/src/test/java/org/apache/hadoop/hbase/backup/mapreduce/TestMapReduceBackupCopyJob.java
@@ -0,0 +1,55 @@
+/*
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+package org.apache.hadoop.hbase.backup.mapreduce;
+
+import static org.apache.hadoop.hbase.backup.mapreduce.MapReduceBackupCopyJob.BACKUP_COPY_OPTION_PREFIX;
+import static org.apache.hadoop.tools.DistCpConstants.CONF_LABEL_DIRECT_WRITE;
+import static org.apache.hadoop.tools.DistCpConstants.CONF_LABEL_MAX_MAPS;
+import static org.junit.Assert.assertEquals;
+
+import java.util.List;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.hbase.HBaseClassTestRule;
+import org.apache.hadoop.hbase.testclassification.SmallTests;
+import org.junit.ClassRule;
+import org.junit.Test;
+import org.junit.experimental.categories.Category;
+
+import org.apache.hbase.thirdparty.com.google.common.collect.ImmutableList;
+
+@Category(SmallTests.class)
+public class TestMapReduceBackupCopyJob {
+
+ @ClassRule
+ public static final HBaseClassTestRule CLASS_RULE =
+ HBaseClassTestRule.forClass(TestMapReduceBackupCopyJob.class);
+
+ @Test
+ public void testDistCpOptionParsing() {
+ Configuration conf = new Configuration();
+ conf.setInt(BACKUP_COPY_OPTION_PREFIX + CONF_LABEL_MAX_MAPS, 1000);
+ conf.setBoolean(BACKUP_COPY_OPTION_PREFIX + CONF_LABEL_DIRECT_WRITE, true);
+ List<String> args = MapReduceBackupCopyJob.parseDistCpOptions(conf);
+
+ List<String> expectedArgs =
+ ImmutableList.<String> builder().add("-m", "1000").add("-direct").build();
+
+ assertEquals(args, expectedArgs);
+ }
+
+}