You are viewing a plain text version of this content. The canonical link for it is here.
Posted to commits@tez.apache.org by ss...@apache.org on 2013/06/03 18:26:09 UTC
git commit: Fix build after YARN-717, YARN-720. (sseth)
Updated Branches:
refs/heads/TEZ-1 eeaaf232f -> 865232927
Fix build after YARN-717, YARN-720. (sseth)
Project: http://git-wip-us.apache.org/repos/asf/incubator-tez/repo
Commit: http://git-wip-us.apache.org/repos/asf/incubator-tez/commit/86523292
Tree: http://git-wip-us.apache.org/repos/asf/incubator-tez/tree/86523292
Diff: http://git-wip-us.apache.org/repos/asf/incubator-tez/diff/86523292
Branch: refs/heads/TEZ-1
Commit: 8652329273978266490806028a4c274af33d213e
Parents: eeaaf23
Author: Siddharth Seth <ss...@apache.org>
Authored: Mon Jun 3 09:25:46 2013 -0700
Committer: Siddharth Seth <ss...@apache.org>
Committed: Mon Jun 3 09:25:46 2013 -0700
----------------------------------------------------------------------
.../dag/app/launcher/ContainerLauncherImpl.java | 13 ++++++-------
.../apache/tez/dag/app/rm/NMCommunicatorEvent.java | 8 ++++----
.../dag/app/rm/NMCommunicatorStopRequestEvent.java | 4 ++--
.../java/org/apache/tez/mapreduce/YARNRunner.java | 7 ++-----
4 files changed, 14 insertions(+), 18 deletions(-)
----------------------------------------------------------------------
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
index 5b23c75..b71af02 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/launcher/ContainerLauncherImpl.java
@@ -36,7 +36,6 @@ import org.apache.hadoop.fs.CommonConfigurationKeysPublic;
import org.apache.hadoop.mapred.ShuffleHandler;
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.security.UserGroupInformation;
-import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.util.StringUtils;
import org.apache.hadoop.yarn.Clock;
import org.apache.hadoop.yarn.YarnException;
@@ -46,7 +45,7 @@ import org.apache.hadoop.yarn.api.protocolrecords.StartContainerResponse;
import org.apache.hadoop.yarn.api.protocolrecords.StopContainerRequest;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.ipc.YarnRPC;
import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
import org.apache.hadoop.yarn.service.AbstractService;
@@ -123,10 +122,10 @@ public class ContainerLauncherImpl extends AbstractService implements
// store enough information to be able to cleanup the container
private ContainerId containerID;
final private String containerMgrAddress;
- private ContainerToken containerToken;
+ private Token containerToken;
public Container(ContainerId containerID,
- String containerMgrAddress, ContainerToken containerToken) {
+ String containerMgrAddress, Token containerToken) {
this.state = ContainerState.PREP;
this.containerMgrAddress = containerMgrAddress;
this.containerID = containerID;
@@ -355,7 +354,7 @@ public class ContainerLauncherImpl extends AbstractService implements
}
protected ContainerManager getCMProxy(ContainerId containerID,
- final String containerManagerBindAddr, ContainerToken containerToken)
+ final String containerManagerBindAddr, Token containerToken)
throws IOException {
final InetSocketAddress cmAddr =
@@ -364,8 +363,8 @@ public class ContainerLauncherImpl extends AbstractService implements
// the user in createRemoteUser in this context has to be ContainerID
UserGroupInformation user = UserGroupInformation
.createRemoteUser(containerID.toString());
- Token<ContainerTokenIdentifier> token = ProtoUtils.convertFromProtoFormat(
- containerToken, cmAddr);
+ org.apache.hadoop.security.token.Token<ContainerTokenIdentifier> token = ProtoUtils
+ .convertFromProtoFormat(containerToken, cmAddr);
user.addToken(token);
ContainerManager proxy = user
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
index 38f54c6..8bdeb28 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorEvent.java
@@ -19,18 +19,18 @@
package org.apache.tez.dag.app.rm;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.event.AbstractEvent;
public class NMCommunicatorEvent extends AbstractEvent<NMCommunicatorEventType> {
private final ContainerId containerId;
private final NodeId nodeId;
- private final ContainerToken containerToken;
+ private final Token containerToken;
public NMCommunicatorEvent(ContainerId containerId, NodeId nodeId,
- ContainerToken containerToken, NMCommunicatorEventType type) {
+ Token containerToken, NMCommunicatorEventType type) {
super(type);
this.containerId = containerId;
this.nodeId = nodeId;
@@ -45,7 +45,7 @@ public class NMCommunicatorEvent extends AbstractEvent<NMCommunicatorEventType>
return this.nodeId;
}
- public ContainerToken getContainerToken() {
+ public Token getContainerToken() {
return this.containerToken;
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
----------------------------------------------------------------------
diff --git a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
index 8f09a48..277d1e7 100644
--- a/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
+++ b/tez-dag/src/main/java/org/apache/tez/dag/app/rm/NMCommunicatorStopRequestEvent.java
@@ -19,13 +19,13 @@
package org.apache.tez.dag.app.rm;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.api.records.ContainerToken;
import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Token;
public class NMCommunicatorStopRequestEvent extends NMCommunicatorEvent {
public NMCommunicatorStopRequestEvent(ContainerId containerId, NodeId nodeId,
- ContainerToken containerToken) {
+ Token containerToken) {
super(containerId, nodeId, containerToken,
NMCommunicatorEventType.CONTAINER_STOP_REQUEST);
}
http://git-wip-us.apache.org/repos/asf/incubator-tez/blob/86523292/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
----------------------------------------------------------------------
diff --git a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
index 801ce8a..572437b 100644
--- a/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
+++ b/tez-yarn-client/src/main/java/org/apache/tez/mapreduce/YARNRunner.java
@@ -70,7 +70,6 @@ import org.apache.hadoop.mapreduce.v2.LogParams;
import org.apache.hadoop.mapreduce.v2.jobhistory.JobHistoryUtils;
import org.apache.hadoop.mapreduce.v2.util.MRApps;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.security.authorize.AccessControlList;
import org.apache.hadoop.security.token.Token;
import org.apache.hadoop.yarn.YarnException;
@@ -1141,13 +1140,11 @@ public class YARNRunner implements ClientProtocol {
private static void addLog4jSystemProperties(String logLevel,
List<String> vargs) {
vargs.add("-Dlog4j.configuration=container-log4j.properties");
- // TODO Fix the remaining properties after YARN-720 is fixed.
- // May need to introduce a log4j properties file for tez tasks.
- vargs.add("-D" + MRJobConfig.TASK_LOG_DIR + "="
+ vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_DIR + "="
+ ApplicationConstants.LOG_DIR_EXPANSION_VAR);
// Setting this to 0 to avoid log size restrictions.
// Should be enforced by YARN.
- vargs.add("-D" + MRJobConfig.TASK_LOG_SIZE + "=" + 0);
+ vargs.add("-D" + YarnConfiguration.YARN_APP_CONTAINER_LOG_SIZE + "=" + 0);
vargs.add("-Dhadoop.root.logger=" + logLevel + ",CLA");
}