You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by su...@apache.org on 2016/09/14 02:43:52 UTC
[14/50] [abbrv] hadoop git commit: HADOOP-13587.
distcp.map.bandwidth.mb is overwritten even when -bandwidth flag isn't set.
Contributed by Zoran Dimitrijevic
HADOOP-13587. distcp.map.bandwidth.mb is overwritten even when -bandwidth flag isn't set. Contributed by Zoran Dimitrijevic
Project: http://git-wip-us.apache.org/repos/asf/hadoop/repo
Commit: http://git-wip-us.apache.org/repos/asf/hadoop/commit/9faccd10
Tree: http://git-wip-us.apache.org/repos/asf/hadoop/tree/9faccd10
Diff: http://git-wip-us.apache.org/repos/asf/hadoop/diff/9faccd10
Branch: refs/heads/YARN-2915
Commit: 9faccd104672dfef123735ca8ada178fc3a6196f
Parents: cc01ed70
Author: Ravi Prakash <ra...@altiscale.com>
Authored: Mon Sep 12 08:26:08 2016 -0700
Committer: Ravi Prakash <ra...@altiscale.com>
Committed: Mon Sep 12 08:26:08 2016 -0700
----------------------------------------------------------------------
.../java/org/apache/hadoop/tools/DistCp.java | 4 ++
.../org/apache/hadoop/tools/DistCpOptions.java | 8 ++-
.../apache/hadoop/tools/TestOptionsParser.java | 64 +++++++++++++++++++-
3 files changed, 70 insertions(+), 6 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9faccd10/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
index 36e5ee4..be58f13 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCp.java
@@ -71,6 +71,7 @@ public class DistCp extends Configured implements Tool {
private static final String PREFIX = "_distcp";
private static final String WIP_PREFIX = "._WIP_";
private static final String DISTCP_DEFAULT_XML = "distcp-default.xml";
+ private static final String DISTCP_SITE_XML = "distcp-site.xml";
static final Random rand = new Random();
private boolean submitted;
@@ -86,6 +87,7 @@ public class DistCp extends Configured implements Tool {
public DistCp(Configuration configuration, DistCpOptions inputOptions) throws Exception {
Configuration config = new Configuration(configuration);
config.addResource(DISTCP_DEFAULT_XML);
+ config.addResource(DISTCP_SITE_XML);
setConf(config);
this.inputOptions = inputOptions;
this.metaFolder = createMetaFolderPath();
@@ -393,10 +395,12 @@ public class DistCp extends Configured implements Tool {
* Loads properties from distcp-default.xml into configuration
* object
* @return Configuration which includes properties from distcp-default.xml
+ * and distcp-site.xml
*/
private static Configuration getDefaultConf() {
Configuration config = new Configuration();
config.addResource(DISTCP_DEFAULT_XML);
+ config.addResource(DISTCP_SITE_XML);
return config;
}
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9faccd10/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
index e6f53f5..4c5518f 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptions.java
@@ -47,7 +47,7 @@ public class DistCpOptions {
public static final int maxNumListstatusThreads = 40;
private int numListstatusThreads = 0; // Indicates that flag is not set.
private int maxMaps = DistCpConstants.DEFAULT_MAPS;
- private float mapBandwidth = DistCpConstants.DEFAULT_BANDWIDTH_MB;
+ private float mapBandwidth = 0; // Indicates that we should use the default.
private String copyStrategy = DistCpConstants.UNIFORMSIZE;
@@ -609,8 +609,10 @@ public class DistCpOptions {
String.valueOf(useDiff));
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.SKIP_CRC,
String.valueOf(skipCRC));
- DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH,
- String.valueOf(mapBandwidth));
+ if (mapBandwidth > 0) {
+ DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.BANDWIDTH,
+ String.valueOf(mapBandwidth));
+ }
DistCpOptionSwitch.addToConf(conf, DistCpOptionSwitch.PRESERVE_STATUS,
DistCpUtils.packAttributes(preserveStatus));
if (filtersFile != null) {
http://git-wip-us.apache.org/repos/asf/hadoop/blob/9faccd10/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
----------------------------------------------------------------------
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
index c46bcd9..218de4e 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestOptionsParser.java
@@ -107,7 +107,7 @@ public class TestOptionsParser {
DistCpOptions options = OptionsParser.parse(new String[] {
"hdfs://localhost:9820/source/first",
"hdfs://localhost:9820/target/"});
- Assert.assertEquals(options.getMapBandwidth(), DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA);
+ Assert.assertEquals(options.getMapBandwidth(), 0, DELTA);
options = OptionsParser.parse(new String[] {
"-bandwidth",
@@ -389,7 +389,7 @@ public class TestOptionsParser {
+ "deleteMissing=false, ignoreFailures=false, overwrite=false, "
+ "append=false, useDiff=false, fromSnapshot=null, toSnapshot=null, "
+ "skipCRC=false, blocking=true, numListstatusThreads=0, maxMaps=20, "
- + "mapBandwidth=100.0, "
+ + "mapBandwidth=0.0, "
+ "copyStrategy='uniformsize', preserveStatus=[], "
+ "preserveRawXattrs=false, atomicWorkPath=null, logPath=null, "
+ "sourceFileListing=abc, sourcePaths=null, targetPath=xyz, "
@@ -572,6 +572,8 @@ public class TestOptionsParser {
Configuration conf = new Configuration();
Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false));
Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false));
+ Assert.assertEquals(
+ conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
DistCpOptions options = OptionsParser.parse(new String[] {
"-atomic",
"-i",
@@ -581,7 +583,7 @@ public class TestOptionsParser {
Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.IGNORE_FAILURES.getConfigLabel(), false));
Assert.assertTrue(conf.getBoolean(DistCpOptionSwitch.ATOMIC_COMMIT.getConfigLabel(), false));
Assert.assertEquals(conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1),
- DistCpConstants.DEFAULT_BANDWIDTH_MB, DELTA);
+ -1.0, DELTA);
conf = new Configuration();
Assert.assertFalse(conf.getBoolean(DistCpOptionSwitch.SYNC_FOLDERS.getConfigLabel(), false));
@@ -603,6 +605,62 @@ public class TestOptionsParser {
}
@Test
+ public void testOptionsAppendToConfDoesntOverwriteBandwidth() {
+ Configuration conf = new Configuration();
+ Assert.assertEquals(
+ conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
+ DistCpOptions options = OptionsParser.parse(new String[] {
+ "hdfs://localhost:8020/source/first",
+ "hdfs://localhost:8020/target/"});
+ options.appendToConf(conf);
+ Assert.assertEquals(
+ conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), -1.0,
+ DELTA);
+
+ conf = new Configuration();
+ Assert.assertEquals(
+ conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), null);
+ options = OptionsParser.parse(new String[] {
+ "-update",
+ "-delete",
+ "-pu",
+ "-bandwidth",
+ "77",
+ "hdfs://localhost:8020/source/first",
+ "hdfs://localhost:8020/target/"});
+ options.appendToConf(conf);
+ Assert.assertEquals(
+ conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 77.0,
+ DELTA);
+
+ conf = new Configuration();
+ conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88");
+ Assert.assertEquals(
+ conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88");
+ options = OptionsParser.parse(new String[] {
+ "hdfs://localhost:8020/source/first",
+ "hdfs://localhost:8020/target/"});
+ options.appendToConf(conf);
+ Assert.assertEquals(
+ conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 88.0,
+ DELTA);
+
+ conf = new Configuration();
+ conf.set(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), "88.0");
+ Assert.assertEquals(
+ conf.getRaw(DistCpOptionSwitch.BANDWIDTH.getConfigLabel()), "88.0");
+ options = OptionsParser.parse(new String[] {
+ "-bandwidth",
+ "99",
+ "hdfs://localhost:8020/source/first",
+ "hdfs://localhost:8020/target/"});
+ options.appendToConf(conf);
+ Assert.assertEquals(
+ conf.getFloat(DistCpOptionSwitch.BANDWIDTH.getConfigLabel(), -1), 99.0,
+ DELTA);
+ }
+
+ @Test
public void testAppendOption() {
Configuration conf = new Configuration();
Assert.assertFalse(conf.getBoolean(
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org