You are viewing a plain text version of this content. The canonical link for it is here.
Posted to common-commits@hadoop.apache.org by qu...@apache.org on 2022/08/31 15:33:42 UTC
[hadoop] branch trunk updated: MAPREDUCE-7409. Make shuffle key length configurable. Contributed by Ashutosh Gupta.
This is an automated email from the ASF dual-hosted git repository.
quapaw pushed a commit to branch trunk
in repository https://gitbox.apache.org/repos/asf/hadoop.git
The following commit(s) were added to refs/heads/trunk by this push:
new 84081a8cae7 MAPREDUCE-7409. Make shuffle key length configurable. Contributed by Ashutosh Gupta.
84081a8cae7 is described below
commit 84081a8cae7d050cc3dc2b3bddd298d769032120
Author: 9uapaw <gy...@gmail.com>
AuthorDate: Wed Aug 31 17:30:24 2022 +0200
MAPREDUCE-7409. Make shuffle key length configurable. Contributed by Ashutosh Gupta.
---
.../src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java | 5 +++--
.../src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java | 2 ++
2 files changed, 5 insertions(+), 2 deletions(-)
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
index 4c983178a7f..232b8bf13bc 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/JobSubmitter.java
@@ -71,7 +71,6 @@ class JobSubmitter {
protected static final Logger LOG =
LoggerFactory.getLogger(JobSubmitter.class);
private static final String SHUFFLE_KEYGEN_ALGORITHM = "HmacSHA1";
- private static final int SHUFFLE_KEY_LENGTH = 64;
private FileSystem jtFs;
private ClientProtocol submitClient;
private String submitHostName;
@@ -177,7 +176,9 @@ class JobSubmitter {
KeyGenerator keyGen;
try {
keyGen = KeyGenerator.getInstance(SHUFFLE_KEYGEN_ALGORITHM);
- keyGen.init(SHUFFLE_KEY_LENGTH);
+ int shuffleKeyLength =
+ conf.getInt(MRJobConfig.SHUFFLE_KEY_LENGTH, MRJobConfig.DEFAULT_SHUFFLE_KEY_LENGTH);
+ keyGen.init(shuffleKeyLength);
} catch (NoSuchAlgorithmException e) {
throw new IOException("Error generating shuffle secret key", e);
}
diff --git a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
index 15d57a6746b..4523565f300 100644
--- a/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
+++ b/hadoop-mapreduce-project/hadoop-mapreduce-client/hadoop-mapreduce-client-core/src/main/java/org/apache/hadoop/mapreduce/MRJobConfig.java
@@ -1302,4 +1302,6 @@ public interface MRJobConfig {
*/
@Unstable
String INPUT_FILE_MANDATORY_PREFIX = "mapreduce.job.input.file.must.";
+ String SHUFFLE_KEY_LENGTH = "mapreduce.shuffle-key-length";
+ int DEFAULT_SHUFFLE_KEY_LENGTH = 64;
}
---------------------------------------------------------------------
To unsubscribe, e-mail: common-commits-unsubscribe@hadoop.apache.org
For additional commands, e-mail: common-commits-help@hadoop.apache.org