[GitHub] kai33 closed pull request #451: HADOOP-16018: Fix DistCp not reassemble chunks when blocks per chunk > 0

[GitBox <gi...@apache.org>]

kai33 closed pull request #451: HADOOP-16018: Fix DistCp not reassemble chunks when blocks per chunk > 0
URL: https://github.com/apache/hadoop/pull/451
 
 
   

This is a PR merged from a forked repository.
As GitHub hides the original diff on merge, it is displayed below for
the sake of provenance:

As this is a foreign pull request (from a fork), the diff is supplied
below (as it won't show otherwise due to GitHub magic):

diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
index 212256ccfd70..8f0f79d84233 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpConstants.java
@@ -123,6 +123,10 @@ private DistCpConstants() {
   public static final String CONF_LABEL_COPY_BUFFER_SIZE =
       "distcp.copy.buffer.size";
 
+  /* DistCp Blocks Per Chunk */
+  public static final String CONF_LABEL_BLOCKS_PER_CHUNK =
+      "distcp.blocks.per.chunk";
+
   /**
    * Constants for DistCp return code to shell / consumer of ToolRunner's run
    */
diff --git a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
index 3ce12b264d32..e57e413de332 100644
--- a/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
+++ b/hadoop-tools/hadoop-distcp/src/main/java/org/apache/hadoop/tools/DistCpOptionSwitch.java
@@ -192,7 +192,7 @@
       new Option("sizelimit", true, "(Deprecated!) Limit number of files " +
               "copied to <= n bytes")),
 
-  BLOCKS_PER_CHUNK("",
+  BLOCKS_PER_CHUNK(DistCpConstants.CONF_LABEL_BLOCKS_PER_CHUNK,
       new Option("blocksperchunk", true, "If set to a positive value, files"
           + "with more blocks than this value will be split into chunks of "
           + "<blocksperchunk> blocks to be transferred in parallel, and "
diff --git a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
index dd8ec697b650..62a2e6d7514a 100644
--- a/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
+++ b/hadoop-tools/hadoop-distcp/src/test/java/org/apache/hadoop/tools/TestDistCpOptions.java
@@ -20,6 +20,7 @@
 
 import java.util.Collections;
 
+import org.apache.hadoop.conf.Configuration;
 import org.junit.Assert;
 import org.junit.Test;
 
@@ -533,4 +534,32 @@ public void testVerboseLog() {
     builder.withLogPath(logPath).withVerboseLog(true);
     Assert.assertTrue(builder.build().shouldVerboseLog());
   }
+
+  @Test
+  public void testAppendToConf() {
+    final int expectedBlocksPerChunk = 999;
+    final String expectedValForEmptyConfigKey = "VALUE_OF_EMPTY_CONFIG_KEY";
+
+    DistCpOptions options = new DistCpOptions.Builder(
+        Collections.singletonList(
+            new Path("hdfs://localhost:8020/source")),
+        new Path("hdfs://localhost:8020/target/"))
+        .withBlocksPerChunk(expectedBlocksPerChunk)
+        .build();
+
+    Configuration config = new Configuration();
+    config.set("", expectedValForEmptyConfigKey);
+
+    options.appendToConf(config);
+    Assert.assertEquals(expectedBlocksPerChunk,
+        config.getInt(
+            DistCpOptionSwitch
+                .BLOCKS_PER_CHUNK
+                .getConfigLabel(), 0));
+    Assert.assertEquals(
+        "Some DistCpOptionSwitch's config label is empty! " +
+            "Pls ensure the config label is provided when apply to config, " +
+            "otherwise it may not be fetched properly",
+            expectedValForEmptyConfigKey, config.get(""));
+  }
 }


 

----------------------------------------------------------------
This is an automated message from the Apache Git Service.
To respond to the message, please log on GitHub and use the
URL above to go to the specific comment.
 
For queries about this service, please contact Infrastructure at:
users@infra.apache.org


With regards,
Apache Git Services

---------------------------------------------------------------------
To unsubscribe, e-mail: common-issues-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-issues-help@hadoop.apache.org