You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2014/02/04 04:44:05 UTC
git commit: TEZ-761. Replace MRJobConfig parameter references in the
tez-dag module with Tez equivalents. (Mohammad Kamrul Islam via sseth)
Updated Branches:
refs/heads/master 847068a50 -> 6e44345cc
TEZ-761. Replace MRJobConfig parameter references in the tez-dag module
with Tez equivalents. (Mohammad Kamrul Islam via sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/6e44345c
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/6e44345c
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/6e44345c
Branch: refs/heads/master
Commit: 6e44345ccc8b281938f8c4e51700cb987b1c1243
Parents: 847068a
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Feb 3 19:43:17 2014 -0800
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Feb 3 19:43:17 2014 -0800
----------------------------------------------------------------------
.../org/apache/tez/client/TezClientUtils.java | 2 +-
.../org/apache/tez/common/TezJobConfig.java | 1 -
.../apache/tez/dag/api/TezConfiguration.java | 30 ++++++++++++++++++--
.../tez/dag/app/ContainerHeartbeatHandler.java | 6 ++--
.../tez/dag/app/TaskHeartbeatHandler.java | 6 ++--
.../java/org/apache/tez/dag/app/dag/DAG.java | 1 -
.../apache/tez/dag/app/dag/impl/DAGImpl.java | 9 +-----
.../dag/app/launcher/ContainerLauncherImpl.java | 6 ++--
.../security/authorize/TezAMPolicyProvider.java | 10 +++----
9 files changed, 44 insertions(+), 27 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6e44345c/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
index 26af839..c6a3c3b 100644
--- a/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
+++ b/tez-api/src/main/java/org/apache/tez/client/TezClientUtils.java
@@ -341,7 +341,7 @@ public class TezClientUtils {
addLog4jSystemProperties(amLogLevel, vargs);
vargs.add(amConfig.getAMConf().get(TezConfiguration.TEZ_AM_JAVA_OPTS,
- TezConfiguration.DEFAULT_TEZ_AM_JAVA_OPTS));
+ TezConfiguration.TEZ_AM_JAVA_OPTS_DEFAULT));
// FIX sun bug mentioned in TEZ-327
vargs.add("-Dsun.nio.ch.bugLevel=''");
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6e44345c/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java b/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
index f21a55f..711a13a 100644
--- a/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
+++ b/tez-api/src/main/java/org/apache/tez/common/TezJobConfig.java
@@ -310,5 +310,4 @@ public class TezJobConfig {
public static final String TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_MAX_SIZE = "tez.runtime.broadcast.data-via-events.max-size";
public static final int TEZ_RUNTIME_BROADCAST_DATA_VIA_EVENTS_MAX_SIZE_DEFAULT = 200 << 10;// 200KB
-
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6e44345c/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
----------------------------------------------------------------------
diff --git a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
index c1154a1..6f646b5 100644
--- a/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
+++ b/tez-api/src/main/java/org/apache/tez/dag/api/TezConfiguration.java
@@ -54,10 +54,10 @@ public class TezConfiguration extends Configuration {
public static final String TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE =
TEZ_AM_PREFIX + "abort-all-outputs-on-dag-failure";
public static final boolean TEZ_AM_ABORT_ALL_OUTPUTS_ON_DAG_FAILURE_DEFAULT = true;
-
+
public static final String TEZ_AM_JAVA_OPTS = TEZ_AM_PREFIX
+ "java.opts";
- public static final String DEFAULT_TEZ_AM_JAVA_OPTS = " -Xmx1024m ";
+ public static final String TEZ_AM_JAVA_OPTS_DEFAULT = " -Xmx1024m ";
public static final String TEZ_AM_CANCEL_DELEGATION_TOKEN = TEZ_AM_PREFIX +
"am.complete.cancel.delegation.tokens";
@@ -67,6 +67,29 @@ public class TezConfiguration extends Configuration {
TEZ_AM_PREFIX + "task.listener.thread-count";
public static final int TEZ_AM_TASK_LISTENER_THREAD_COUNT_DEFAULT = 30;
+ /*
+ * MR AM Service Authorization
+ * These are the same as MR which allows Tez to run in secure
+ * mode without configuring service ACLs
+ */
+ public static final String
+ TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL =
+ "security.job.task.protocol.acl";
+ public static final String
+ TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT =
+ "security.job.client.protocol.acl";
+
+ /**
+ * Upper limit on the number of threads user to launch containers in the app
+ * master. Expect level config, you shouldn't be needing it in most cases.
+ */
+ public static final String TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT =
+ TEZ_AM_PREFIX+"containerlauncher.thread-count-limit";
+
+ public static final int TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT_DEFAULT =
+ 500;
+
+
// TODO Some of the DAG properties are job specific and not AM specific. Rename accordingly.
// TODO Are any of these node blacklisting properties required. (other than for MR compat)
public static final String TEZ_AM_MAX_TASK_FAILURES_PER_NODE = TEZ_AM_PREFIX
@@ -139,6 +162,9 @@ public class TezConfiguration extends Configuration {
+ "max-events-per-heartbeat.max";
public static final int TEZ_TASK_MAX_EVENTS_PER_HEARTBEAT_DEFAULT = 100;
+ public static final String TASK_TIMEOUT = TEZ_TASK_PREFIX + "timeout";
+
+ public static final String TASK_HEARTBEAT_TIMEOUT_MS = TEZ_TASK_PREFIX + "heartbeat.timeout-ms";
/**
* Configuration to specify whether container should be reused.
*/
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6e44345c/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java
index 55ce7da..c89370f 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/ContainerHeartbeatHandler.java
@@ -20,9 +20,9 @@ package org.apache.tez.dag.app;
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.app.rm.container.AMContainerEvent;
import org.apache.tez.dag.app.rm.container.AMContainerEventType;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
public class ContainerHeartbeatHandler extends
HeartbeatHandlerBase<ContainerId> {
@@ -36,12 +36,12 @@ public class ContainerHeartbeatHandler extends
@Override
protected int getConfiguredTimeout(Configuration conf) {
// TODO Maybe define separate timeouts for Containers and tasks.
- return conf.getInt(MRJobConfig.TASK_TIMEOUT, 5 * 60 * 1000);
+ return conf.getInt(TezConfiguration.TASK_TIMEOUT, 5 * 60 * 1000);
}
@Override
protected int getConfiguredTimeoutCheckInterval(Configuration conf) {
- return conf.getInt(MRJobConfig.TASK_TIMEOUT_CHECK_INTERVAL_MS, 30 * 1000);
+ return conf.getInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 30 * 1000);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6e44345c/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
index c41244d..d5fb133 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/TaskHeartbeatHandler.java
@@ -20,10 +20,10 @@ package org.apache.tez.dag.app;
import org.apache.hadoop.conf.Configuration;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventAttemptFailed;
import org.apache.tez.dag.app.dag.event.TaskAttemptEventType;
import org.apache.tez.dag.records.TezTaskAttemptID;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
/**
@@ -41,12 +41,12 @@ public class TaskHeartbeatHandler extends HeartbeatHandlerBase<TezTaskAttemptID>
@Override
protected int getConfiguredTimeout(Configuration conf) {
- return conf.getInt(MRJobConfig.TASK_TIMEOUT, 5 * 60 * 1000);
+ return conf.getInt(TezConfiguration.TASK_TIMEOUT, 5 * 60 * 1000);
}
@Override
protected int getConfiguredTimeoutCheckInterval(Configuration conf) {
- return conf.getInt(MRJobConfig.TASK_TIMEOUT_CHECK_INTERVAL_MS, 30 * 1000);
+ return conf.getInt(TezConfiguration.TASK_HEARTBEAT_TIMEOUT_MS, 30 * 1000);
}
@Override
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6e44345c/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
index ec4aaef..8117619 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/DAG.java
@@ -64,7 +64,6 @@ public interface DAG {
float getProgress();
boolean isUber();
String getUserName();
- String getQueueName();
Configuration getConf();
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6e44345c/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
index 4027fc5..1c558d1 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/dag/impl/DAGImpl.java
@@ -50,6 +50,7 @@ import org.apache.hadoop.yarn.state.SingleArcTransition;
import org.apache.hadoop.yarn.state.StateMachine;
import org.apache.hadoop.yarn.state.StateMachineFactory;
import org.apache.hadoop.yarn.util.Clock;
+import org.apache.tez.common.TezJobConfig;
import org.apache.tez.common.counters.TezCounters;
import org.apache.tez.dag.api.DagTypeConverters;
import org.apache.tez.dag.api.EdgeManagerDescriptor;
@@ -98,7 +99,6 @@ import org.apache.tez.dag.history.events.DAGStartedEvent;
import org.apache.tez.dag.records.TezDAGID;
import org.apache.tez.dag.records.TezVertexID;
import org.apache.tez.dag.utils.TezBuilderUtils;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
import org.apache.tez.runtime.api.OutputCommitter;
import com.google.common.annotations.VisibleForTesting;
@@ -142,7 +142,6 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
// TODO Metrics
//private final MRAppMetrics metrics;
private final String userName;
- private final String queueName;
private final AppContext appContext;
private final UserGroupInformation dagUGI;
@@ -383,7 +382,6 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
this.userName = appUserName;
this.clock = clock;
this.appContext = appContext;
- this.queueName = conf.get(MRJobConfig.QUEUE_NAME, "default");
this.taskAttemptListener = taskAttemptListener;
this.taskHeartbeatHandler = thh;
@@ -986,11 +984,6 @@ public class DAGImpl implements org.apache.tez.dag.app.dag.DAG,
}
@Override
- public String getQueueName() {
- return queueName;
- }
-
- @Override
public String getName() {
return dagName;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6e44345c/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
index ecee425..75f67ad 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
@@ -45,6 +45,7 @@ import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy;
import org.apache.hadoop.yarn.client.api.impl.ContainerManagementProtocolProxy.ContainerManagementProtocolProxyData;
import org.apache.hadoop.yarn.util.Clock;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.TezUncheckedException;
import org.apache.tez.dag.app.AppContext;
import org.apache.tez.dag.app.rm.NMCommunicatorEvent;
@@ -56,7 +57,6 @@ import org.apache.tez.dag.app.rm.container.AMContainerEventStopFailed;
import org.apache.tez.dag.app.rm.container.AMContainerEventType;
import org.apache.tez.dag.history.DAGHistoryEvent;
import org.apache.tez.dag.history.events.ContainerLaunchedEvent;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
import com.google.common.util.concurrent.ThreadFactoryBuilder;
@@ -250,8 +250,8 @@ public class ContainerLauncherImpl extends AbstractService implements
CommonConfigurationKeysPublic.IPC_CLIENT_CONNECTION_MAXIDLETIME_KEY,
0);
this.limitOnPoolSize = conf.getInt(
- MRJobConfig.MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT,
- MRJobConfig.DEFAULT_MR_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT);
+ TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT,
+ TezConfiguration.TEZ_AM_CONTAINERLAUNCHER_THREAD_COUNT_LIMIT_DEFAULT);
LOG.info("Upper limit on the thread pool size is " + this.limitOnPoolSize);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/6e44345c/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java b/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java
index d72db04..3dca327 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/security/authorize/TezAMPolicyProvider.java
@@ -22,8 +22,8 @@ import org.apache.hadoop.classification.InterfaceStability;
import org.apache.hadoop.security.authorize.PolicyProvider;
import org.apache.hadoop.security.authorize.Service;
import org.apache.tez.common.TezTaskUmbilicalProtocol;
+import org.apache.tez.dag.api.TezConfiguration;
import org.apache.tez.dag.api.client.rpc.DAGClientAMProtocolBlockingPB;
-import org.apache.tez.mapreduce.hadoop.MRJobConfig;
/**
* {@link PolicyProvider} for YARN MapReduce protocols.
@@ -35,11 +35,11 @@ public class TezAMPolicyProvider extends PolicyProvider {
private static final Service[] tezApplicationMasterServices =
new Service[] {
new Service(
- MRJobConfig.MR_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL,
- TezTaskUmbilicalProtocol.class),
+ TezConfiguration.TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_TASK_UMBILICAL,
+ TezTaskUmbilicalProtocol.class),
new Service(
- MRJobConfig.MR_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT,
- DAGClientAMProtocolBlockingPB.class)
+ TezConfiguration.TEZ_AM_SECURITY_SERVICE_AUTHORIZATION_CLIENT,
+ DAGClientAMProtocolBlockingPB.class)
};
@Override