You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by je...@apache.org on 2016/07/27 19:50:29 UTC
tez git commit: TEZ-3378. Move Shuffle Handler configuration into the
Tez namespace (Eric Badger via jeagles)
Repository: tez
Updated Branches:
refs/heads/TEZ-3334 333d19322 -> 50541ffff
TEZ-3378. Move Shuffle Handler configuration into the Tez namespace (Eric Badger via jeagles)
Project: http://git-wip-us.apache.org/repos/asf/tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/tez/commit/50541fff
Tree: http://git-wip-us.apache.org/repos/asf/tez/tree/50541fff
Diff: http://git-wip-us.apache.org/repos/asf/tez/diff/50541fff
Branch: refs/heads/TEZ-3334
Commit: 50541ffffdd84d599571791b758ca8184e558632
Parents: 333d193
Author: Jonathan Eagles <je...@yahoo-inc.com>
Authored: Wed Jul 27 14:50:16 2016 -0500
Committer: Jonathan Eagles <je...@yahoo-inc.com>
Committed: Wed Jul 27 14:50:16 2016 -0500
----------------------------------------------------------------------
TEZ-3334-CHANGES.txt | 1 +
.../org/apache/tez/dag/api/TezConstants.java | 2 +-
.../apache/tez/auxservices/ShuffleHandler.java | 36 ++++++++++----------
.../library/common/shuffle/ShuffleUtils.java | 2 +-
4 files changed, 21 insertions(+), 20 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/tez/blob/50541fff/TEZ-3334-CHANGES.txt
----------------------------------------------------------------------
diff --git a/TEZ-3334-CHANGES.txt b/TEZ-3334-CHANGES.txt
index c5a16b3..7a41a1f 100644
--- a/TEZ-3334-CHANGES.txt
+++ b/TEZ-3334-CHANGES.txt
@@ -4,6 +4,7 @@ Apache Tez Change Log
INCOMPATIBLE CHANGES:
ALL CHANGES:
+ TEZ-3378. Move Shuffle Handler configuration into the Tez namespace
TEZ-3377. Remove ShuffleHandler dependency on mapred.FadvisedChunkedFile and mapred.FadvisedFileRegion
TEZ-3380. Shuffle Handler: Replace primitive wrapper's valueOf method with parse* method to avoid unnecessary boxing/unboxing
TEZ-3355. Tez Custom Shuffle Handler POC
http://git-wip-us.apache.org/repos/asf/tez/blob/50541fff/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
index 6e1cb2d..43f09a2 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConstants.java
@@ -74,7 +74,7 @@ public class TezConstants {
* The service id for the NodeManager plugin used to share intermediate data
* between vertices.
*/
- public static final String TEZ_SHUFFLE_HANDLER_SERVICE_ID = "mapreduce_shuffle";
+ public static final String TEZ_SHUFFLE_HANDLER_SERVICE_ID = "tez_shuffle";
public static final String TEZ_PREWARM_DAG_NAME_PREFIX = "TezPreWarmDAG";
http://git-wip-us.apache.org/repos/asf/tez/blob/50541fff/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java
----------------------------------------------------------------------
diff --git a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java
index cccd1a0..ca49ecc 100644
--- a/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java
+++ b/tez-plugins/tez-aux-services/src/main/java/org/apache/tez/auxservices/ShuffleHandler.java
@@ -146,10 +146,10 @@ public class ShuffleHandler extends AuxiliaryService {
private static final Log LOG = LogFactory.getLog(ShuffleHandler.class);
private static final Log AUDITLOG =
LogFactory.getLog(ShuffleHandler.class.getName()+".audit");
- public static final String SHUFFLE_MANAGE_OS_CACHE = "mapreduce.shuffle.manage.os.cache";
+ public static final String SHUFFLE_MANAGE_OS_CACHE = "tez.shuffle.manage.os.cache";
public static final boolean DEFAULT_SHUFFLE_MANAGE_OS_CACHE = true;
- public static final String SHUFFLE_READAHEAD_BYTES = "mapreduce.shuffle.readahead.bytes";
+ public static final String SHUFFLE_READAHEAD_BYTES = "tez.shuffle.readahead.bytes";
public static final int DEFAULT_SHUFFLE_READAHEAD_BYTES = 4 * 1024 * 1024;
public static final String USERCACHE = "usercache";
public static final String APPCACHE = "appcache";
@@ -160,7 +160,7 @@ public class ShuffleHandler extends AuxiliaryService {
"^.*(?:connection.*reset|connection.*closed|broken.*pipe).*$",
Pattern.CASE_INSENSITIVE);
- private static final String STATE_DB_NAME = "mapreduce_shuffle_state";
+ private static final String STATE_DB_NAME = "tez_shuffle_state";
private static final String STATE_DB_SCHEMA_VERSION_KEY = "shuffle-schema-version";
protected static final Version CURRENT_VERSION_INFO =
Version.newInstance(1, 0);
@@ -191,45 +191,45 @@ public class ShuffleHandler extends AuxiliaryService {
private DB stateDb = null;
- public static final String MAPREDUCE_SHUFFLE_SERVICEID =
- "mapreduce_shuffle";
+ public static final String TEZ_SHUFFLE_SERVICEID =
+ "tez_shuffle";
- public static final String SHUFFLE_PORT_CONFIG_KEY = "mapreduce.shuffle.port";
- public static final int DEFAULT_SHUFFLE_PORT = 13562;
+ public static final String SHUFFLE_PORT_CONFIG_KEY = "tez.shuffle.port";
+ public static final int DEFAULT_SHUFFLE_PORT = 13563;
public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED =
- "mapreduce.shuffle.connection-keep-alive.enable";
+ "tez.shuffle.connection-keep-alive.enable";
public static final boolean DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_ENABLED = false;
public static final String SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT =
- "mapreduce.shuffle.connection-keep-alive.timeout";
+ "tez.shuffle.connection-keep-alive.timeout";
public static final int DEFAULT_SHUFFLE_CONNECTION_KEEP_ALIVE_TIME_OUT = 5; //seconds
public static final String SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE =
- "mapreduce.shuffle.mapoutput-info.meta.cache.size";
+ "tez.shuffle.mapoutput-info.meta.cache.size";
public static final int DEFAULT_SHUFFLE_MAPOUTPUT_META_INFO_CACHE_SIZE =
1000;
public static final String CONNECTION_CLOSE = "close";
public static final String SUFFLE_SSL_FILE_BUFFER_SIZE_KEY =
- "mapreduce.shuffle.ssl.file.buffer.size";
+ "tez.shuffle.ssl.file.buffer.size";
public static final int DEFAULT_SUFFLE_SSL_FILE_BUFFER_SIZE = 60 * 1024;
- public static final String MAX_SHUFFLE_CONNECTIONS = "mapreduce.shuffle.max.connections";
+ public static final String MAX_SHUFFLE_CONNECTIONS = "tez.shuffle.max.connections";
public static final int DEFAULT_MAX_SHUFFLE_CONNECTIONS = 0; // 0 implies no limit
- public static final String MAX_SHUFFLE_THREADS = "mapreduce.shuffle.max.threads";
+ public static final String MAX_SHUFFLE_THREADS = "tez.shuffle.max.threads";
// 0 implies Netty default of 2 * number of available processors
public static final int DEFAULT_MAX_SHUFFLE_THREADS = 0;
public static final String SHUFFLE_BUFFER_SIZE =
- "mapreduce.shuffle.transfer.buffer.size";
+ "tez.shuffle.transfer.buffer.size";
public static final int DEFAULT_SHUFFLE_BUFFER_SIZE = 128 * 1024;
public static final String SHUFFLE_TRANSFERTO_ALLOWED =
- "mapreduce.shuffle.transferTo.allowed";
+ "tez.shuffle.transferTo.allowed";
public static final boolean DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED = true;
public static final boolean WINDOWS_DEFAULT_SHUFFLE_TRANSFERTO_ALLOWED =
false;
@@ -238,14 +238,14 @@ public class ShuffleHandler extends AuxiliaryService {
open simultaneously during shuffle
*/
public static final String SHUFFLE_MAX_SESSION_OPEN_FILES =
- "mapreduce.shuffle.max.session-open-files";
+ "tez.shuffle.max.session-open-files";
public static final int DEFAULT_SHUFFLE_MAX_SESSION_OPEN_FILES = 3;
boolean connectionKeepAliveEnabled = false;
int connectionKeepAliveTimeOut;
int mapOutputMetaInfoCacheSize;
- @Metrics(about="Shuffle output metrics", context="mapred")
+ @Metrics(about="Shuffle output metrics", context="mapred", name="tez")
static class ShuffleMetrics implements ChannelFutureListener {
@Metric("Shuffle output in bytes")
MutableCounterLong shuffleOutputBytes;
@@ -368,7 +368,7 @@ public class ShuffleHandler extends AuxiliaryService {
}
ShuffleHandler(MetricsSystem ms) {
- super(MAPREDUCE_SHUFFLE_SERVICEID);
+ super(TEZ_SHUFFLE_SERVICEID);
metrics = ms.register(new ShuffleMetrics());
}
http://git-wip-us.apache.org/repos/asf/tez/blob/50541fff/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java
----------------------------------------------------------------------
diff --git a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java
index d74e447..e194298 100644
--- a/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java
+++ b/tez-runtime-library/src/main/java/org/apache/tez/runtime/library/common/shuffle/ShuffleUtils.java
@@ -73,7 +73,7 @@ import org.apache.tez.runtime.library.shuffle.impl.ShuffleUserPayloads.DetailedP
public class ShuffleUtils {
private static final Logger LOG = LoggerFactory.getLogger(ShuffleUtils.class);
- public static final String SHUFFLE_HANDLER_SERVICE_ID = "mapreduce_shuffle";
+ public static final String SHUFFLE_HANDLER_SERVICE_ID = "tez_shuffle";
private static final long MB = 1024l * 1024l;
//Shared by multiple threads