You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by st...@apache.org on 2019/02/27 21:51:10 UTC
[hadoop] branch branch-2.9 updated: HADOOP-16018. DistCp won't
reassemble chunks when blocks per chunk > 0.
This is an automated email from the ASF dual-hosted git repository.
stevel pushed a commit to branch branch-2.9
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/branch-2.9 by this push:
new 6619f43 HADOOP-16018. DistCp won't reassemble chunks when blocks per chunk > 0.
6619f43 is described below
commit 6619f4333dffbf2c4497219ecd220f81d997d2a6
Author: Kai Xie <gi...@gmail.com>
AuthorDate: Wed Feb 27 21:50:45 2019 +0000
HADOOP-16018. DistCp won't reassemble chunks when blocks per chunk > 0.
Contributed by Kai Xie.
(cherry picked from commit a49cb4465e6849a4346dcfa6f4a235d6fde917d3)
---
.../org/apache/hadoop/tools/DistCpConstants.java | 4 ++++
.../apache/hadoop/tools/DistCpOptionSwitch.java | 2 +-
.../org/apache/hadoop/tools/TestDistCpOptions.java | 28 ++++++++++++++++++++++
3 files changed, 33 insertions(+), 1 deletion(-)
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
index 48b3590..8a4023a 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
@@ -118,6 +118,10 @@ public class DistCpConstants {
public static final String CONF_LABEL_COPY_BUFFER_SIZE =
"distcp.copy.buffer.size";
+ /** DistCp Blocks Per Chunk: {@value}. */
+ public static final String CONF_LABEL_BLOCKS_PER_CHUNK =
+ "distcp.blocks.per.chunk";
+
/**
* Conf label for SSL Trust-store location.
*/
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
index 56362e2..2b1b301 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
@@ -186,7 +186,7 @@ public enum DistCpOptionSwitch {
new Option("sizelimit", true, "(Deprecated!) Limit number of files " +
"copied to <= n bytes")),
- BLOCKS_PER_CHUNK("",
+ BLOCKS_PER_CHUNK(DistCpConstants.CONF_LABEL_BLOCKS_PER_CHUNK,
new Option("blocksperchunk", true, "If set to a positive value, files"
+ "with more blocks than this value will be split into chunks of "
+ "<blocksperchunk> blocks to be transferred in parallel, and "
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
index 0211e06..7291965 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
@@ -21,6 +21,7 @@ package org.apache.hadoop.tools;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.tools.DistCpOptions.FileAttribute;
+import org.apache.hadoop.conf.Configuration;
import org.junit.Assert;
import org.junit.Test;
@@ -530,4 +531,31 @@ public class TestDistCpOptions {
options.setVerboseLog(true);
Assert.assertTrue(options.shouldVerboseLog());
}
+
+ @Test
+ public void testAppendToConf() {
+ final int expectedBlocksPerChunk = 999;
+ final String expectedValForEmptyConfigKey = "VALUE_OF_EMPTY_CONFIG_KEY";
+
+ DistCpOptions options = new DistCpOptions(
+ Collections.singletonList(
+ new Path("hdfs://localhost:8020/source")),
+ new Path("hdfs://localhost:8020/target/"));
+ options.setBlocksPerChunk(expectedBlocksPerChunk);
+
+ Configuration config = new Configuration();
+ config.set("", expectedValForEmptyConfigKey);
+
+ options.appendToConf(config);
+ Assert.assertEquals(expectedBlocksPerChunk,
+ config.getInt(
+ DistCpOptionSwitch
+ .BLOCKS_PER_CHUNK
+ .getConfigLabel(), 0));
+ Assert.assertEquals(
+ "Some DistCpOptionSwitch's config label is empty! " +
+ "Pls ensure the config label is provided when apply to config, " +
+ "otherwise it may not be fetched properly",
+ expectedValForEmptyConfigKey, config.get(""));
+ }
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org