You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by cm...@apache.org on 2014/08/20 01:51:01 UTC
svn commit: r1619012 [4/26] - in
/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project: ./
hadoop-yarn/bin/ hadoop-yarn/conf/ hadoop-yarn/dev-support/
hadoop-yarn/hadoop-yarn-api/
hadoop-yarn/hadoop-yarn-api/src/main/java/org/apache/hadoop/yarn/api/...
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestAMRMClient.java Tue Aug 19 23:49:39 2014
@@ -18,6 +18,7 @@
package org.apache.hadoop.yarn.client.api.impl;
+import com.google.common.base.Supplier;
import static org.junit.Assert.assertEquals;
import static org.junit.Assert.assertTrue;
import static org.junit.Assert.fail;
@@ -27,19 +28,23 @@ import static org.mockito.Mockito.when;
import java.io.IOException;
import java.nio.ByteBuffer;
+import java.security.PrivilegedAction;
import java.util.ArrayList;
import java.util.Arrays;
import java.util.Collection;
import java.util.Collections;
import java.util.HashMap;
+import java.util.Iterator;
import java.util.List;
import java.util.Set;
import java.util.TreeSet;
import org.junit.Assert;
-
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager.InvalidToken;
import org.apache.hadoop.service.Service.STATE;
import org.apache.hadoop.yarn.api.ApplicationMasterProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateRequest;
@@ -71,9 +76,12 @@ import org.apache.hadoop.yarn.client.api
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.ipc.YarnRPC;
+import org.apache.hadoop.yarn.security.AMRMTokenIdentifier;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttempt;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.attempt.RMAppAttemptState;
+import org.apache.hadoop.yarn.server.resourcemanager.security.AMRMTokenSecretManager;
import org.apache.hadoop.yarn.server.utils.BuilderUtils;
import org.apache.hadoop.yarn.util.Records;
import org.junit.After;
@@ -93,6 +101,9 @@ public class TestAMRMClient {
static ApplicationAttemptId attemptId = null;
static int nodeCount = 3;
+ static final int rolling_interval_sec = 13;
+ static final long am_expire_ms = 4000;
+
static Resource capability;
static Priority priority;
static Priority priority2;
@@ -106,6 +117,10 @@ public class TestAMRMClient {
public static void setup() throws Exception {
// start minicluster
conf = new YarnConfiguration();
+ conf.setLong(
+ YarnConfiguration.RM_AMRM_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS,
+ rolling_interval_sec);
+ conf.setLong(YarnConfiguration.RM_AM_EXPIRY_INTERVAL_MS, am_expire_ms);
conf.setInt(YarnConfiguration.RM_NM_HEARTBEAT_INTERVAL_MS, 100);
conf.setLong(YarnConfiguration.NM_LOG_RETAIN_SECONDS, 1);
yarnCluster = new MiniYARNCluster(TestAMRMClient.class.getName(), nodeCount, 1, 1);
@@ -800,6 +815,40 @@ public class TestAMRMClient {
assertEquals(0, amClient.ask.size());
assertEquals(0, amClient.release.size());
}
+
+ class CountDownSupplier implements Supplier<Boolean> {
+ int counter = 0;
+ @Override
+ public Boolean get() {
+ counter++;
+ if (counter >= 3) {
+ return true;
+ } else {
+ return false;
+ }
+ }
+ };
+
+ @Test
+ public void testWaitFor() throws InterruptedException {
+ AMRMClientImpl<ContainerRequest> amClient = null;
+ CountDownSupplier countDownChecker = new CountDownSupplier();
+
+ try {
+ // start am rm client
+ amClient =
+ (AMRMClientImpl<ContainerRequest>) AMRMClient
+ .<ContainerRequest> createAMRMClient();
+ amClient.init(new YarnConfiguration());
+ amClient.start();
+ amClient.waitFor(countDownChecker, 1000);
+ assertEquals(3, countDownChecker.counter);
+ } finally {
+ if (amClient != null) {
+ amClient.stop();
+ }
+ }
+ }
private void sleep(int sleepTime) {
try {
@@ -809,4 +858,123 @@ public class TestAMRMClient {
}
}
+ @Test(timeout = 60000)
+ public void testAMRMClientOnAMRMTokenRollOver() throws YarnException,
+ IOException {
+ AMRMClient<ContainerRequest> amClient = null;
+ try {
+ AMRMTokenSecretManager amrmTokenSecretManager =
+ yarnCluster.getResourceManager().getRMContext()
+ .getAMRMTokenSecretManager();
+
+ // start am rm client
+ amClient = AMRMClient.<ContainerRequest> createAMRMClient();
+
+ amClient.init(conf);
+ amClient.start();
+
+ Long startTime = System.currentTimeMillis();
+ amClient.registerApplicationMaster("Host", 10000, "");
+
+ org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_1 =
+ getAMRMToken();
+ Assert.assertNotNull(amrmToken_1);
+ Assert.assertEquals(amrmToken_1.decodeIdentifier().getKeyId(),
+ amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
+
+ // Wait for enough time and make sure the roll_over happens
+ // At mean time, the old AMRMToken should continue to work
+ while (System.currentTimeMillis() - startTime <
+ rolling_interval_sec * 1000) {
+ amClient.allocate(0.1f);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // TODO Auto-generated catch block
+ e.printStackTrace();
+ }
+ }
+ amClient.allocate(0.1f);
+
+ org.apache.hadoop.security.token.Token<AMRMTokenIdentifier> amrmToken_2 =
+ getAMRMToken();
+ Assert.assertNotNull(amrmToken_2);
+ Assert.assertEquals(amrmToken_2.decodeIdentifier().getKeyId(),
+ amrmTokenSecretManager.getMasterKey().getMasterKey().getKeyId());
+
+ Assert.assertNotEquals(amrmToken_1, amrmToken_2);
+
+ // can do the allocate call with latest AMRMToken
+ amClient.allocate(0.1f);
+
+ // Make sure previous token has been rolled-over
+ // and can not use this rolled-over token to make a allocate all.
+ while (true) {
+ if (amrmToken_2.decodeIdentifier().getKeyId() != amrmTokenSecretManager
+ .getCurrnetMasterKeyData().getMasterKey().getKeyId()) {
+ if (amrmTokenSecretManager.getNextMasterKeyData() == null) {
+ break;
+ } else if (amrmToken_2.decodeIdentifier().getKeyId() !=
+ amrmTokenSecretManager.getNextMasterKeyData().getMasterKey()
+ .getKeyId()) {
+ break;
+ }
+ }
+ amClient.allocate(0.1f);
+ try {
+ Thread.sleep(1000);
+ } catch (InterruptedException e) {
+ // DO NOTHING
+ }
+ }
+
+ try {
+ UserGroupInformation testUser =
+ UserGroupInformation.createRemoteUser("testUser");
+ SecurityUtil.setTokenService(amrmToken_2, yarnCluster
+ .getResourceManager().getApplicationMasterService().getBindAddress());
+ testUser.addToken(amrmToken_2);
+ testUser.doAs(new PrivilegedAction<ApplicationMasterProtocol>() {
+ @Override
+ public ApplicationMasterProtocol run() {
+ return (ApplicationMasterProtocol) YarnRPC.create(conf).getProxy(
+ ApplicationMasterProtocol.class,
+ yarnCluster.getResourceManager().getApplicationMasterService()
+ .getBindAddress(), conf);
+ }
+ }).allocate(Records.newRecord(AllocateRequest.class));
+ Assert.fail("The old Token should not work");
+ } catch (Exception ex) {
+ Assert.assertTrue(ex instanceof InvalidToken);
+ Assert.assertTrue(ex.getMessage().contains(
+ "Invalid AMRMToken from "
+ + amrmToken_2.decodeIdentifier().getApplicationAttemptId()));
+ }
+
+ amClient.unregisterApplicationMaster(FinalApplicationStatus.SUCCEEDED,
+ null, null);
+
+ } finally {
+ if (amClient != null && amClient.getServiceState() == STATE.STARTED) {
+ amClient.stop();
+ }
+ }
+ }
+
+ @SuppressWarnings("unchecked")
+ private org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>
+ getAMRMToken() throws IOException {
+ Credentials credentials =
+ UserGroupInformation.getCurrentUser().getCredentials();
+ Iterator<org.apache.hadoop.security.token.Token<?>> iter =
+ credentials.getAllTokens().iterator();
+ while (iter.hasNext()) {
+ org.apache.hadoop.security.token.Token<?> token = iter.next();
+ if (token.getKind().equals(AMRMTokenIdentifier.KIND_NAME)) {
+ return (org.apache.hadoop.security.token.Token<AMRMTokenIdentifier>)
+ token;
+ }
+ }
+ return null;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestNMClient.java Tue Aug 19 23:49:39 2014
@@ -43,6 +43,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.ApplicationReport;
import org.apache.hadoop.yarn.api.records.ApplicationSubmissionContext;
import org.apache.hadoop.yarn.api.records.Container;
+import org.apache.hadoop.yarn.api.records.ContainerExitStatus;
import org.apache.hadoop.yarn.api.records.ContainerLaunchContext;
import org.apache.hadoop.yarn.api.records.ContainerState;
import org.apache.hadoop.yarn.api.records.ContainerStatus;
@@ -346,7 +347,7 @@ public class TestNMClient {
// 137 is possible if the container is not terminated but killed
testGetContainerStatus(container, i, ContainerState.COMPLETE,
"Container killed by the ApplicationMaster.", Arrays.asList(
- new Integer[] {137, 143, 0}));
+ new Integer[] {ContainerExitStatus.KILLED_BY_APPMASTER}));
} catch (YarnException e) {
// The exception is possible because, after the container is stopped,
// it may be removed from NM's context.
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-client/src/test/java/org/apache/hadoop/yarn/client/api/impl/TestYarnClient.java Tue Aug 19 23:49:39 2014
@@ -25,19 +25,26 @@ import static org.mockito.Mockito.verify
import static org.mockito.Mockito.when;
import java.io.IOException;
+import java.nio.ByteBuffer;
import java.security.PrivilegedExceptionAction;
import java.util.ArrayList;
+import java.util.Collection;
import java.util.EnumSet;
import java.util.HashMap;
import java.util.HashSet;
import java.util.List;
import java.util.Set;
-import org.junit.Assert;
-
import org.apache.commons.io.IOUtils;
import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.DataInputByteBuffer;
+import org.apache.hadoop.io.DataOutputBuffer;
+import org.apache.hadoop.security.Credentials;
+import org.apache.hadoop.security.SecurityUtil;
import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.UserGroupInformation.AuthenticationMethod;
+import org.apache.hadoop.security.token.Token;
+import org.apache.hadoop.security.token.TokenIdentifier;
import org.apache.hadoop.yarn.api.ApplicationClientProtocol;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportRequest;
import org.apache.hadoop.yarn.api.protocolrecords.GetApplicationAttemptReportResponse;
@@ -69,19 +76,23 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.Resource;
import org.apache.hadoop.yarn.api.records.YarnApplicationAttemptState;
import org.apache.hadoop.yarn.api.records.YarnApplicationState;
+import org.apache.hadoop.yarn.client.api.TimelineClient;
import org.apache.hadoop.yarn.client.api.YarnClient;
import org.apache.hadoop.yarn.client.api.YarnClientApplication;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.exceptions.ApplicationIdNotProvidedException;
import org.apache.hadoop.yarn.exceptions.YarnException;
+import org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier;
import org.apache.hadoop.yarn.server.MiniYARNCluster;
import org.apache.hadoop.yarn.server.resourcemanager.MockRM;
import org.apache.hadoop.yarn.server.resourcemanager.ResourceManager;
import org.apache.hadoop.yarn.server.resourcemanager.rmapp.RMApp;
import org.apache.hadoop.yarn.util.Records;
+import org.apache.hadoop.yarn.util.timeline.TimelineUtils;
import org.apache.log4j.Level;
import org.apache.log4j.LogManager;
import org.apache.log4j.Logger;
+import org.junit.Assert;
import org.junit.Test;
public class TestYarnClient {
@@ -725,4 +736,80 @@ public class TestYarnClient {
IOUtils.closeQuietly(client);
}
}
+
+ @Test
+ public void testAutomaticTimelineDelegationTokenLoading()
+ throws Exception {
+ Configuration conf = new YarnConfiguration();
+ conf.setBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED, true);
+ SecurityUtil.setAuthenticationMethod(AuthenticationMethod.KERBEROS, conf);
+ final Token<TimelineDelegationTokenIdentifier> dToken =
+ new Token<TimelineDelegationTokenIdentifier>();
+ // crate a mock client
+ YarnClientImpl client = new YarnClientImpl() {
+ @Override
+ protected void serviceInit(Configuration conf) throws Exception {
+ if (getConfig().getBoolean(YarnConfiguration.TIMELINE_SERVICE_ENABLED,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_ENABLED)) {
+ timelineServiceEnabled = true;
+ timelineClient = mock(TimelineClient.class);
+ when(timelineClient.getDelegationToken(any(String.class)))
+ .thenReturn(dToken);
+ timelineClient.init(getConfig());
+ timelineService = TimelineUtils.buildTimelineTokenService(getConfig());
+ }
+ this.setConfig(conf);
+ }
+
+ @Override
+ protected void serviceStart() throws Exception {
+ rmClient = mock(ApplicationClientProtocol.class);
+ }
+
+ @Override
+ protected void serviceStop() throws Exception {
+ }
+
+ @Override
+ public ApplicationReport getApplicationReport(ApplicationId appId) {
+ ApplicationReport report = mock(ApplicationReport.class);
+ when(report.getYarnApplicationState())
+ .thenReturn(YarnApplicationState.SUBMITTED);
+ return report;
+ }
+
+ @Override
+ public boolean isSecurityEnabled() {
+ return true;
+ }
+ };
+ client.init(conf);
+ client.start();
+ ApplicationSubmissionContext context =
+ mock(ApplicationSubmissionContext.class);
+ ApplicationId applicationId = ApplicationId.newInstance(0, 1);
+ when(context.getApplicationId()).thenReturn(applicationId);
+ DataOutputBuffer dob = new DataOutputBuffer();
+ Credentials credentials = new Credentials();
+ credentials.writeTokenStorageToStream(dob);
+ ByteBuffer tokens = ByteBuffer.wrap(dob.getData(), 0, dob.getLength());
+ ContainerLaunchContext clc = ContainerLaunchContext.newInstance(
+ null, null, null, null, tokens, null);
+ when(context.getAMContainerSpec()).thenReturn(clc);
+ client.submitApplication(context);
+ // Check whether token is added or not
+ credentials = new Credentials();
+ DataInputByteBuffer dibb = new DataInputByteBuffer();
+ tokens = clc.getTokens();
+ if (tokens != null) {
+ dibb.reset(tokens);
+ credentials.readTokenStorageStream(dibb);
+ tokens.rewind();
+ }
+ Collection<Token<? extends TokenIdentifier>> dTokens =
+ credentials.getAllTokens();
+ Assert.assertEquals(1, dTokens.size());
+ Assert.assertEquals(dToken, dTokens.iterator().next());
+ client.stop();
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/pom.xml Tue Aug 19 23:49:39 2014
@@ -38,24 +38,6 @@
<groupId>org.apache.hadoop</groupId>
<artifactId>hadoop-common</artifactId>
<scope>provided</scope>
- <exclusions>
- <exclusion>
- <groupId>commons-el</groupId>
- <artifactId>commons-el</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-runtime</artifactId>
- </exclusion>
- <exclusion>
- <groupId>tomcat</groupId>
- <artifactId>jasper-compiler</artifactId>
- </exclusion>
- <exclusion>
- <groupId>org.mortbay.jetty</groupId>
- <artifactId>jsp-2.1-jetty</artifactId>
- </exclusion>
- </exclusions>
</dependency>
<!-- 'mvn dependency:analyze' fails to detect use of this dependency -->
@@ -85,12 +67,32 @@
<artifactId>commons-codec</artifactId>
</dependency>
<dependency>
+ <groupId>org.mortbay.jetty</groupId>
+ <artifactId>jetty-util</artifactId>
+ </dependency>
+ <dependency>
<groupId>com.sun.jersey</groupId>
<artifactId>jersey-core</artifactId>
</dependency>
<dependency>
+ <groupId>com.sun.jersey</groupId>
+ <artifactId>jersey-client</artifactId>
+ </dependency>
+ <dependency>
<groupId>org.codehaus.jackson</groupId>
- <artifactId>jackson-mapper-asl</artifactId>
+ <artifactId>jackson-core-asl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-mapper-asl</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-jaxrs</artifactId>
+ </dependency>
+ <dependency>
+ <groupId>org.codehaus.jackson</groupId>
+ <artifactId>jackson-xc</artifactId>
</dependency>
<dependency>
<groupId>com.google.guava</groupId>
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/AllocateResponsePBImpl.java Tue Aug 19 23:49:39 2014
@@ -25,6 +25,7 @@ import java.util.List;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Unstable;
+import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
import org.apache.hadoop.yarn.api.protocolrecords.AllocateResponse;
import org.apache.hadoop.yarn.api.records.AMCommand;
import org.apache.hadoop.yarn.api.records.Container;
@@ -35,6 +36,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeReport;
import org.apache.hadoop.yarn.api.records.PreemptionMessage;
import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.api.records.Token;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerPBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceDecreasePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ContainerResourceIncreasePBImpl;
@@ -44,6 +46,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.impl.pb.PreemptionMessagePBImpl;
import org.apache.hadoop.yarn.api.records.impl.pb.ProtoUtils;
import org.apache.hadoop.yarn.api.records.impl.pb.ResourcePBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.TokenPBImpl;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceDecreaseProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerResourceIncreaseProto;
@@ -74,7 +77,7 @@ public class AllocateResponsePBImpl exte
private List<NodeReport> updatedNodes = null;
private PreemptionMessage preempt;
-
+ private Token amrmToken = null;
public AllocateResponsePBImpl() {
builder = AllocateResponseProto.newBuilder();
@@ -154,6 +157,9 @@ public class AllocateResponsePBImpl exte
getChangeProtoIterable(this.decreasedContainers);
builder.addAllDecreasedContainers(iterable);
}
+ if (this.amrmToken != null) {
+ builder.setAmRmToken(convertToProtoFormat(this.amrmToken));
+ }
}
private synchronized void mergeLocalToProto() {
@@ -357,6 +363,28 @@ public class AllocateResponsePBImpl exte
this.decreasedContainers.addAll(decreasedContainers);
}
+ @Override
+ public synchronized Token getAMRMToken() {
+ AllocateResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (amrmToken != null) {
+ return amrmToken;
+ }
+ if (!p.hasAmRmToken()) {
+ return null;
+ }
+ this.amrmToken = convertFromProtoFormat(p.getAmRmToken());
+ return amrmToken;
+ }
+
+ @Override
+ public synchronized void setAMRMToken(Token amRMToken) {
+ maybeInitBuilder();
+ if (amRMToken == null) {
+ builder.clearAmRmToken();
+ }
+ this.amrmToken = amRMToken;
+ }
+
private synchronized void initLocalIncreasedContainerList() {
if (this.increasedContainers != null) {
return;
@@ -699,4 +727,12 @@ public class AllocateResponsePBImpl exte
private synchronized NMToken convertFromProtoFormat(NMTokenProto proto) {
return new NMTokenPBImpl(proto);
}
+
+ private TokenPBImpl convertFromProtoFormat(TokenProto p) {
+ return new TokenPBImpl(p);
+ }
+
+ private TokenProto convertToProtoFormat(Token t) {
+ return ((TokenPBImpl)t).getProto();
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/protocolrecords/impl/pb/GetApplicationsRequestPBImpl.java Tue Aug 19 23:49:39 2014
@@ -20,7 +20,6 @@ package org.apache.hadoop.yarn.api.proto
import java.util.EnumSet;
import java.util.HashSet;
-import java.util.Iterator;
import java.util.List;
import java.util.Set;
@@ -35,6 +34,8 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProto;
import org.apache.hadoop.yarn.proto.YarnServiceProtos.GetApplicationsRequestProtoOrBuilder;
+import com.google.common.base.Function;
+import com.google.common.collect.Iterables;
import com.google.protobuf.TextFormat;
@Private
@@ -49,7 +50,8 @@ public class GetApplicationsRequestPBImp
Set<String> users = null;
Set<String> queues = null;
long limit = Long.MAX_VALUE;
- LongRange start = null, finish = null;
+ LongRange start = null;
+ LongRange finish = null;
private Set<String> applicationTags;
private ApplicationsRequestScope scope;
@@ -78,57 +80,46 @@ public class GetApplicationsRequestPBImp
}
private void mergeLocalToBuilder() {
- if (this.applicationTypes != null) {
- addLocalApplicationTypesToProto();
+ if (applicationTypes != null && !applicationTypes.isEmpty()) {
+ builder.clearApplicationTypes();
+ builder.addAllApplicationTypes(applicationTypes);
}
- if (this.applicationStates != null) {
- maybeInitBuilder();
+ if (applicationStates != null && !applicationStates.isEmpty()) {
builder.clearApplicationStates();
- Iterable<YarnApplicationStateProto> iterable =
- new Iterable<YarnApplicationStateProto>() {
-
+ builder.addAllApplicationStates(Iterables.transform(applicationStates,
+ new Function<YarnApplicationState, YarnApplicationStateProto>() {
@Override
- public Iterator<YarnApplicationStateProto> iterator() {
- return new Iterator<YarnApplicationStateProto>() {
-
- Iterator<YarnApplicationState> iter = applicationStates
- .iterator();
-
- @Override
- public boolean hasNext() {
- return iter.hasNext();
- }
-
- @Override
- public YarnApplicationStateProto next() {
- return ProtoUtils.convertToProtoFormat(iter.next());
- }
-
- @Override
- public void remove() {
- throw new UnsupportedOperationException();
-
- }
- };
-
+ public YarnApplicationStateProto apply(YarnApplicationState input) {
+ return ProtoUtils.convertToProtoFormat(input);
}
- };
- builder.addAllApplicationStates(iterable);
+ }));
}
- if (this.applicationTags != null && !this.applicationTags.isEmpty()) {
+ if (applicationTags != null && !applicationTags.isEmpty()) {
+ builder.clearApplicationTags();
builder.addAllApplicationTags(this.applicationTags);
}
- if (this.scope != null) {
+ if (scope != null) {
builder.setScope(ProtoUtils.convertToProtoFormat(scope));
}
- }
-
- private void addLocalApplicationTypesToProto() {
- maybeInitBuilder();
- builder.clearApplicationTypes();
- if (this.applicationTypes == null)
- return;
- builder.addAllApplicationTypes(applicationTypes);
+ if (start != null) {
+ builder.setStartBegin(start.getMinimumLong());
+ builder.setStartEnd(start.getMaximumLong());
+ }
+ if (finish != null) {
+ builder.setFinishBegin(finish.getMinimumLong());
+ builder.setFinishEnd(finish.getMaximumLong());
+ }
+ if (limit != Long.MAX_VALUE) {
+ builder.setLimit(limit);
+ }
+ if (users != null && !users.isEmpty()) {
+ builder.clearUsers();
+ builder.addAllUsers(users);
+ }
+ if (queues != null && !queues.isEmpty()) {
+ builder.clearQueues();
+ builder.addAllQueues(queues);
+ }
}
private void maybeInitBuilder() {
@@ -326,7 +317,7 @@ public class GetApplicationsRequestPBImp
public LongRange getStartRange() {
if (this.start == null) {
GetApplicationsRequestProtoOrBuilder p = viaProto ? proto: builder;
- if (p.hasStartBegin() || p.hasFinishBegin()) {
+ if (p.hasStartBegin() || p.hasStartEnd()) {
long begin = p.hasStartBegin() ? p.getStartBegin() : 0L;
long end = p.hasStartEnd() ? p.getStartEnd() : Long.MAX_VALUE;
this.start = new LongRange(begin, end);
@@ -336,6 +327,11 @@ public class GetApplicationsRequestPBImp
}
@Override
+ public void setStartRange(LongRange range) {
+ this.start = range;
+ }
+
+ @Override
public void setStartRange(long begin, long end)
throws IllegalArgumentException {
if (begin > end) {
@@ -359,6 +355,11 @@ public class GetApplicationsRequestPBImp
}
@Override
+ public void setFinishRange(LongRange range) {
+ this.finish = range;
+ }
+
+ @Override
public void setFinishRange(long begin, long end) {
if (begin > end) {
throw new IllegalArgumentException("begin > end in range (begin, " +
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationReportPBImpl.java Tue Aug 19 23:49:39 2014
@@ -479,6 +479,7 @@ public class ApplicationReportPBImpl ext
builder.setAmRmToken(convertToProtoFormat(this.amRmToken));
}
if (this.applicationTags != null && !this.applicationTags.isEmpty()) {
+ builder.clearApplicationTags();
builder.addAllApplicationTags(this.applicationTags);
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ApplicationSubmissionContextPBImpl.java Tue Aug 19 23:49:39 2014
@@ -107,6 +107,7 @@ extends ApplicationSubmissionContext {
builder.setResource(convertToProtoFormat(this.resource));
}
if (this.applicationTags != null && !this.applicationTags.isEmpty()) {
+ builder.clearApplicationTags();
builder.addAllApplicationTags(this.applicationTags);
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerReportPBImpl.java Tue Aug 19 23:49:39 2014
@@ -18,24 +18,19 @@
package org.apache.hadoop.yarn.api.records.impl.pb;
-import org.apache.hadoop.security.proto.SecurityProtos.TokenProto;
-import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.api.records.ContainerReport;
import org.apache.hadoop.yarn.api.records.ContainerState;
-import org.apache.hadoop.yarn.api.records.ContainerStatus;
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
-
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerReportProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerReportProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
import org.apache.hadoop.yarn.proto.YarnProtos.NodeIdProto;
import org.apache.hadoop.yarn.proto.YarnProtos.PriorityProto;
import org.apache.hadoop.yarn.proto.YarnProtos.ResourceProto;
-import org.apache.hadoop.yarn.proto.YarnProtos.ContainerStateProto;
import com.google.protobuf.TextFormat;
@@ -150,9 +145,9 @@ public class ContainerReportPBImpl exten
}
@Override
- public long getStartTime() {
+ public long getCreationTime() {
ContainerReportProtoOrBuilder p = viaProto ? proto : builder;
- return p.getStartTime();
+ return p.getCreationTime();
}
@Override
@@ -237,9 +232,9 @@ public class ContainerReportPBImpl exten
}
@Override
- public void setStartTime(long startTime) {
+ public void setCreationTime(long creationTime) {
maybeInitBuilder();
- builder.setStartTime(startTime);
+ builder.setCreationTime(creationTime);
}
public ContainerReportProto getProto() {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceBlacklistRequestPBImpl.java Tue Aug 19 23:49:39 2014
@@ -90,7 +90,7 @@ public class ResourceBlacklistRequestPBI
private void addBlacklistRemovalsToProto() {
maybeInitBuilder();
- builder.clearBlacklistAdditions();
+ builder.clearBlacklistRemovals();
if (this.blacklistRemovals == null) {
return;
}
@@ -159,5 +159,14 @@ public class ResourceBlacklistRequestPBI
public int hashCode() {
return getProto().hashCode();
}
-
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ResourceOptionPBImpl.java Tue Aug 19 23:49:39 2014
@@ -86,4 +86,19 @@ public class ResourceOptionPBImpl extend
builder = null;
}
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/SerializedExceptionPBImpl.java Tue Aug 19 23:49:39 2014
@@ -32,9 +32,9 @@ import org.apache.hadoop.yarn.proto.Yarn
public class SerializedExceptionPBImpl extends SerializedException {
- SerializedExceptionProto proto = SerializedExceptionProto
- .getDefaultInstance();
- SerializedExceptionProto.Builder builder = null;
+ SerializedExceptionProto proto = null;
+ SerializedExceptionProto.Builder builder =
+ SerializedExceptionProto.newBuilder();
boolean viaProto = false;
public SerializedExceptionPBImpl() {
@@ -135,6 +135,22 @@ public class SerializedExceptionPBImpl e
return proto;
}
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null) {
+ return false;
+ }
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
+
private void maybeInitBuilder() {
if (viaProto || builder == null) {
builder = SerializedExceptionProto.newBuilder(proto);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/TokenPBImpl.java Tue Aug 19 23:49:39 2014
@@ -48,7 +48,7 @@ public class TokenPBImpl extends Token {
}
public synchronized TokenProto getProto() {
- mergeLocalToProto();
+ mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/logaggregation/LogCLIHelpers.java Tue Aug 19 23:49:39 2014
@@ -109,7 +109,10 @@ public class LogCLIHelpers implements Co
remoteRootLogDir, appId, user, logDirSuffix);
RemoteIterator<FileStatus> nodeFiles;
try {
- nodeFiles = FileContext.getFileContext().listStatus(remoteAppLogDir);
+ Path qualifiedLogDir =
+ FileContext.getFileContext(getConf()).makeQualified(remoteAppLogDir);
+ nodeFiles = FileContext.getFileContext(qualifiedLogDir.toUri(),
+ getConf()).listStatus(remoteAppLogDir);
} catch (FileNotFoundException fnf) {
System.out.println("Logs not available at " + remoteAppLogDir.toString());
System.out
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/AMRMTokenIdentifier.java Tue Aug 19 23:49:39 2014
@@ -44,6 +44,7 @@ public class AMRMTokenIdentifier extends
public static final Text KIND_NAME = new Text("YARN_AM_RM_TOKEN");
private ApplicationAttemptId applicationAttemptId;
+ private int keyId = Integer.MIN_VALUE;
public AMRMTokenIdentifier() {
}
@@ -53,6 +54,13 @@ public class AMRMTokenIdentifier extends
this.applicationAttemptId = appAttemptId;
}
+ public AMRMTokenIdentifier(ApplicationAttemptId appAttemptId,
+ int masterKeyId) {
+ this();
+ this.applicationAttemptId = appAttemptId;
+ this.keyId = masterKeyId;
+ }
+
@Private
public ApplicationAttemptId getApplicationAttemptId() {
return this.applicationAttemptId;
@@ -64,6 +72,7 @@ public class AMRMTokenIdentifier extends
out.writeLong(appId.getClusterTimestamp());
out.writeInt(appId.getId());
out.writeInt(this.applicationAttemptId.getAttemptId());
+ out.writeInt(this.keyId);
}
@Override
@@ -75,6 +84,7 @@ public class AMRMTokenIdentifier extends
ApplicationId.newInstance(clusterTimeStamp, appId);
this.applicationAttemptId =
ApplicationAttemptId.newInstance(applicationId, attemptId);
+ this.keyId = in.readInt();
}
@Override
@@ -92,6 +102,10 @@ public class AMRMTokenIdentifier extends
.toString());
}
+ public int getKeyId() {
+ return this.keyId;
+ }
+
// TODO: Needed?
@InterfaceAudience.Private
public static class Renewer extends Token.TrivialRenewer {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java Tue Aug 19 23:49:39 2014
@@ -34,6 +34,7 @@ import org.apache.hadoop.security.token.
import org.apache.hadoop.yarn.api.records.ApplicationAttemptId;
import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.Priority;
import org.apache.hadoop.yarn.api.records.Resource;
/**
@@ -56,10 +57,12 @@ public class ContainerTokenIdentifier ex
private long expiryTimeStamp;
private int masterKeyId;
private long rmIdentifier;
+ private Priority priority;
+ private long creationTime;
- public ContainerTokenIdentifier(ContainerId containerID, String hostName,
- String appSubmitter, Resource r, long expiryTimeStamp, int masterKeyId,
- long rmIdentifier) {
+ public ContainerTokenIdentifier(ContainerId containerID,
+ String hostName, String appSubmitter, Resource r, long expiryTimeStamp,
+ int masterKeyId, long rmIdentifier, Priority priority, long creationTime) {
this.containerId = containerID;
this.nmHostAddr = hostName;
this.appSubmitter = appSubmitter;
@@ -67,6 +70,8 @@ public class ContainerTokenIdentifier ex
this.expiryTimeStamp = expiryTimeStamp;
this.masterKeyId = masterKeyId;
this.rmIdentifier = rmIdentifier;
+ this.priority = priority;
+ this.creationTime = creationTime;
}
/**
@@ -99,6 +104,13 @@ public class ContainerTokenIdentifier ex
return this.masterKeyId;
}
+ public Priority getPriority() {
+ return this.priority;
+ }
+
+ public long getCreationTime() {
+ return this.creationTime;
+ }
/**
* Get the RMIdentifier of RM in which containers are allocated
* @return RMIdentifier
@@ -124,6 +136,8 @@ public class ContainerTokenIdentifier ex
out.writeLong(this.expiryTimeStamp);
out.writeInt(this.masterKeyId);
out.writeLong(this.rmIdentifier);
+ out.writeInt(this.priority.getPriority());
+ out.writeLong(this.creationTime);
}
@Override
@@ -142,6 +156,8 @@ public class ContainerTokenIdentifier ex
this.expiryTimeStamp = in.readLong();
this.masterKeyId = in.readInt();
this.rmIdentifier = in.readLong();
+ this.priority = Priority.newInstance(in.readInt());
+ this.creationTime = in.readLong();
}
@Override
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/server/api/protocolrecords/impl/pb/UpdateNodeResourceRequestPBImpl.java Tue Aug 19 23:49:39 2014
@@ -162,5 +162,19 @@ public class UpdateNodeResourceRequestPB
};
this.builder.addAllNodeResourceMap(values);
}
-
+
+ @Override
+ public int hashCode() {
+ return getProto().hashCode();
+ }
+
+ @Override
+ public boolean equals(Object other) {
+ if (other == null)
+ return false;
+ if (other.getClass().isAssignableFrom(this.getClass())) {
+ return this.getProto().equals(this.getClass().cast(other).getProto());
+ }
+ return false;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/ApplicationClassLoader.java Tue Aug 19 23:49:39 2014
@@ -173,7 +173,7 @@ public class ApplicationClassLoader exte
}
@VisibleForTesting
- static boolean isSystemClass(String name, List<String> systemClasses) {
+ public static boolean isSystemClass(String name, List<String> systemClasses) {
if (systemClasses != null) {
String canonicalName = name.replace('/', '.');
while (canonicalName.startsWith(".")) {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/FSDownload.java Tue Aug 19 23:49:39 2014
@@ -177,9 +177,10 @@ public class FSDownload implements Calla
/**
* Returns true if all ancestors of the specified path have the 'execute'
* permission set for all users (i.e. that other users can traverse
- * the directory heirarchy to the given path)
+ * the directory hierarchy to the given path)
*/
- private static boolean ancestorsHaveExecutePermissions(FileSystem fs,
+ @VisibleForTesting
+ static boolean ancestorsHaveExecutePermissions(FileSystem fs,
Path path, LoadingCache<Path,Future<FileStatus>> statCache)
throws IOException {
Path current = path;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/Times.java Tue Aug 19 23:49:39 2014
@@ -21,10 +21,14 @@ package org.apache.hadoop.yarn.util;
import java.text.SimpleDateFormat;
import java.util.Date;
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.classification.InterfaceAudience.Private;
@Private
public class Times {
+ private static final Log LOG = LogFactory.getLog(Times.class);
+
static final ThreadLocal<SimpleDateFormat> dateFormat =
new ThreadLocal<SimpleDateFormat>() {
@Override protected SimpleDateFormat initialValue() {
@@ -36,12 +40,30 @@ public class Times {
return Times.elapsed(started, finished, true);
}
+ // A valid elapsed is supposed to be non-negative. If finished/current time
+ // is ahead of the started time, return -1 to indicate invalid elapsed time,
+ // and record a warning log.
public static long elapsed(long started, long finished, boolean isRunning) {
if (finished > 0 && started > 0) {
- return finished - started;
+ long elapsed = finished - started;
+ if (elapsed >= 0) {
+ return elapsed;
+ } else {
+ LOG.warn("Finished time " + finished
+ + " is ahead of started time " + started);
+ return -1;
+ }
}
if (isRunning) {
- return started > 0 ? System.currentTimeMillis() - started : 0;
+ long current = System.currentTimeMillis();
+ long elapsed = started > 0 ? current - started : 0;
+ if (elapsed >= 0) {
+ return elapsed;
+ } else {
+ LOG.warn("Current time " + current
+ + " is ahead of started time " + started);
+ return -1;
+ }
} else {
return -1;
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DefaultResourceCalculator.java Tue Aug 19 23:49:39 2014
@@ -42,6 +42,13 @@ public class DefaultResourceCalculator e
Resource numerator, Resource denominator) {
return ratio(numerator, denominator);
}
+
+ public boolean isInvalidDivisor(Resource r) {
+ if (r.getMemory() == 0.0f) {
+ return true;
+ }
+ return false;
+ }
@Override
public float ratio(Resource a, Resource b) {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/DominantResourceCalculator.java Tue Aug 19 23:49:39 2014
@@ -109,6 +109,14 @@ public class DominantResourceCalculator
getResourceAsValue(clusterResource, numerator, true) /
getResourceAsValue(clusterResource, denominator, true);
}
+
+ @Override
+ public boolean isInvalidDivisor(Resource r) {
+ if (r.getMemory() == 0.0f || r.getVirtualCores() == 0.0f) {
+ return true;
+ }
+ return false;
+ }
@Override
public float ratio(Resource a, Resource b) {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/ResourceCalculator.java Tue Aug 19 23:49:39 2014
@@ -150,6 +150,15 @@ public abstract class ResourceCalculator
Resource clusterResource, Resource numerator, Resource denominator);
/**
+ * Determine if a resource is not suitable for use as a divisor
+ * (will result in divide by 0, etc)
+ *
+ * @param r resource
+ * @return true if divisor is invalid (should not be used), false else
+ */
+ public abstract boolean isInvalidDivisor(Resource r);
+
+ /**
* Ratio of resource <code>a</code> to resource <code>b</code>.
*
* @param a resource
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/resource/Resources.java Tue Aug 19 23:49:39 2014
@@ -184,6 +184,11 @@ public class Resources {
return calculator.roundDown(lhs, factor);
}
+ public static boolean isInvalidDivisor(
+ ResourceCalculator resourceCalculator, Resource divisor) {
+ return resourceCalculator.isInvalidDivisor(divisor);
+ }
+
public static float ratio(
ResourceCalculator resourceCalculator, Resource lhs, Resource rhs) {
return resourceCalculator.ratio(lhs, rhs);
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/util/timeline/TimelineUtils.java Tue Aug 19 23:49:39 2014
@@ -19,15 +19,18 @@
package org.apache.hadoop.yarn.util.timeline;
import java.io.IOException;
+import java.net.InetSocketAddress;
import org.apache.hadoop.classification.InterfaceAudience.Public;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.io.Text;
+import org.apache.hadoop.security.SecurityUtil;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.webapp.YarnJacksonJaxbJsonProvider;
import org.codehaus.jackson.JsonGenerationException;
-import org.codehaus.jackson.map.AnnotationIntrospector;
import org.codehaus.jackson.map.JsonMappingException;
import org.codehaus.jackson.map.ObjectMapper;
-import org.codehaus.jackson.map.annotate.JsonSerialize.Inclusion;
-import org.codehaus.jackson.xc.JaxbAnnotationIntrospector;
/**
* The helper class for the timeline module.
@@ -41,9 +44,7 @@ public class TimelineUtils {
static {
mapper = new ObjectMapper();
- AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
- mapper.setAnnotationIntrospector(introspector);
- mapper.setSerializationInclusion(Inclusion.NON_NULL);
+ YarnJacksonJaxbJsonProvider.configObjectMapper(mapper);
}
/**
@@ -82,4 +83,26 @@ public class TimelineUtils {
}
}
+ public static InetSocketAddress getTimelineTokenServiceAddress(
+ Configuration conf) {
+ InetSocketAddress timelineServiceAddr = null;
+ if (YarnConfiguration.useHttps(conf)) {
+ timelineServiceAddr = conf.getSocketAddr(
+ YarnConfiguration.TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_HTTPS_PORT);
+ } else {
+ timelineServiceAddr = conf.getSocketAddr(
+ YarnConfiguration.TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_ADDRESS,
+ YarnConfiguration.DEFAULT_TIMELINE_SERVICE_WEBAPP_PORT);
+ }
+ return timelineServiceAddr;
+ }
+
+ public static Text buildTimelineTokenService(Configuration conf) {
+ InetSocketAddress timelineServiceAddr =
+ getTimelineTokenServiceAddress(conf);
+ return SecurityUtil.buildTokenService(timelineServiceAddr);
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/GenericExceptionHandler.java Tue Aug 19 23:49:39 2014
@@ -21,10 +21,12 @@ import java.io.FileNotFoundException;
import java.io.IOException;
import javax.servlet.http.HttpServletResponse;
+import javax.ws.rs.WebApplicationException;
import javax.ws.rs.core.Context;
import javax.ws.rs.core.Response;
import javax.ws.rs.ext.ExceptionMapper;
import javax.ws.rs.ext.Provider;
+import javax.xml.bind.UnmarshalException;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -79,6 +81,8 @@ public class GenericExceptionHandler imp
s = Response.Status.NOT_FOUND;
} else if (e instanceof IOException) {
s = Response.Status.NOT_FOUND;
+ } else if (e instanceof ForbiddenException) {
+ s = Response.Status.FORBIDDEN;
} else if (e instanceof UnsupportedOperationException) {
s = Response.Status.BAD_REQUEST;
} else if (e instanceof IllegalArgumentException) {
@@ -87,6 +91,9 @@ public class GenericExceptionHandler imp
s = Response.Status.BAD_REQUEST;
} else if (e instanceof BadRequestException) {
s = Response.Status.BAD_REQUEST;
+ } else if (e instanceof WebApplicationException
+ && e.getCause() instanceof UnmarshalException) {
+ s = Response.Status.BAD_REQUEST;
} else {
LOG.warn("INTERNAL_SERVER_ERROR", e);
s = Response.Status.INTERNAL_SERVER_ERROR;
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/YarnJacksonJaxbJsonProvider.java Tue Aug 19 23:49:39 2014
@@ -49,9 +49,14 @@ public class YarnJacksonJaxbJsonProvider
@Override
public ObjectMapper locateMapper(Class<?> type, MediaType mediaType) {
ObjectMapper mapper = super.locateMapper(type, mediaType);
+ configObjectMapper(mapper);
+ return mapper;
+ }
+
+ public static void configObjectMapper(ObjectMapper mapper) {
AnnotationIntrospector introspector = new JaxbAnnotationIntrospector();
mapper.setAnnotationIntrospector(introspector);
mapper.setSerializationInclusion(Inclusion.NON_NULL);
- return mapper;
}
+
}
\ No newline at end of file
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/util/WebAppUtils.java Tue Aug 19 23:49:39 2014
@@ -19,12 +19,12 @@ package org.apache.hadoop.yarn.webapp.ut
import static org.apache.hadoop.yarn.util.StringHelper.PATH_JOINER;
+import java.io.IOException;
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
import java.util.ArrayList;
import java.util.List;
-import java.util.Map;
import org.apache.hadoop.classification.InterfaceAudience.Private;
import org.apache.hadoop.classification.InterfaceStability.Evolving;
@@ -34,11 +34,18 @@ import org.apache.hadoop.http.HttpServer
import org.apache.hadoop.net.NetUtils;
import org.apache.hadoop.yarn.conf.YarnConfiguration;
import org.apache.hadoop.yarn.conf.HAUtil;
+import org.apache.hadoop.yarn.exceptions.YarnRuntimeException;
import org.apache.hadoop.yarn.util.RMHAUtils;
@Private
@Evolving
public class WebAppUtils {
+ public static final String WEB_APP_TRUSTSTORE_PASSWORD_KEY =
+ "ssl.server.truststore.password";
+ public static final String WEB_APP_KEYSTORE_PASSWORD_KEY =
+ "ssl.server.keystore.password";
+ public static final String WEB_APP_KEY_PASSWORD_KEY =
+ "ssl.server.keystore.keypassword";
public static final String HTTPS_PREFIX = "https://";
public static final String HTTP_PREFIX = "http://";
@@ -170,6 +177,37 @@ public class WebAppUtils {
return sb.toString();
}
+ /**
+ * Get the URL to use for binding where bind hostname can be specified
+ * to override the hostname in the webAppURLWithoutScheme. Port specified in the
+ * webAppURLWithoutScheme will be used.
+ *
+ * @param conf the configuration
+ * @param hostProperty bind host property name
+ * @param webAppURLWithoutScheme web app URL without scheme String
+ * @return String representing bind URL
+ */
+ public static String getWebAppBindURL(
+ Configuration conf,
+ String hostProperty,
+ String webAppURLWithoutScheme) {
+
+ // If the bind-host setting exists then it overrides the hostname
+ // portion of the corresponding webAppURLWithoutScheme
+ String host = conf.getTrimmed(hostProperty);
+ if (host != null && !host.isEmpty()) {
+ if (webAppURLWithoutScheme.contains(":")) {
+ webAppURLWithoutScheme = host + ":" + webAppURLWithoutScheme.split(":")[1];
+ }
+ else {
+ throw new YarnRuntimeException("webAppURLWithoutScheme must include port specification but doesn't: " +
+ webAppURLWithoutScheme);
+ }
+ }
+
+ return webAppURLWithoutScheme;
+ }
+
public static String getNMWebAppURLWithoutScheme(Configuration conf) {
if (YarnConfiguration.useHttps(conf)) {
return conf.get(YarnConfiguration.NM_WEBAPP_HTTPS_ADDRESS,
@@ -242,21 +280,56 @@ public class WebAppUtils {
/**
* Load the SSL keystore / truststore into the HttpServer builder.
+ * @param builder the HttpServer2.Builder to populate with ssl config
*/
public static HttpServer2.Builder loadSslConfiguration(
HttpServer2.Builder builder) {
- Configuration sslConf = new Configuration(false);
+ return loadSslConfiguration(builder, null);
+ }
+
+ /**
+ * Load the SSL keystore / truststore into the HttpServer builder.
+ * @param builder the HttpServer2.Builder to populate with ssl config
+ * @param sslConf the Configuration instance to use during loading of SSL conf
+ */
+ public static HttpServer2.Builder loadSslConfiguration(
+ HttpServer2.Builder builder, Configuration sslConf) {
+ if (sslConf == null) {
+ sslConf = new Configuration(false);
+ }
boolean needsClientAuth = YarnConfiguration.YARN_SSL_CLIENT_HTTPS_NEED_AUTH_DEFAULT;
sslConf.addResource(YarnConfiguration.YARN_SSL_SERVER_RESOURCE_DEFAULT);
return builder
.needsClientAuth(needsClientAuth)
- .keyPassword(sslConf.get("ssl.server.keystore.keypassword"))
+ .keyPassword(getPassword(sslConf, WEB_APP_KEY_PASSWORD_KEY))
.keyStore(sslConf.get("ssl.server.keystore.location"),
- sslConf.get("ssl.server.keystore.password"),
+ getPassword(sslConf, WEB_APP_KEYSTORE_PASSWORD_KEY),
sslConf.get("ssl.server.keystore.type", "jks"))
.trustStore(sslConf.get("ssl.server.truststore.location"),
- sslConf.get("ssl.server.truststore.password"),
+ getPassword(sslConf, WEB_APP_TRUSTSTORE_PASSWORD_KEY),
sslConf.get("ssl.server.truststore.type", "jks"));
}
+
+ /**
+ * Leverages the Configuration.getPassword method to attempt to get
+ * passwords from the CredentialProvider API before falling back to
+ * clear text in config - if falling back is allowed.
+ * @param conf Configuration instance
+ * @param alias name of the credential to retreive
+ * @return String credential value or null
+ */
+ static String getPassword(Configuration conf, String alias) {
+ String password = null;
+ try {
+ char[] passchars = conf.getPassword(alias);
+ if (passchars != null) {
+ password = new String(passchars);
+ }
+ }
+ catch (IOException ioe) {
+ password = null;
+ }
+ return password;
+ }
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/FooterBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/FooterBlock.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/FooterBlock.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/FooterBlock.java Tue Aug 19 23:49:39 2014
@@ -25,7 +25,6 @@ public class FooterBlock extends HtmlBlo
@Override protected void render(Block html) {
html.
- div("#footer.ui-widget").
- a("http://hadoop.apache.org/", "About Apache Hadoop")._();
+ div("#footer.ui-widget")._();
}
}
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlPage.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlPage.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlPage.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/HtmlPage.java Tue Aug 19 23:49:39 2014
@@ -79,7 +79,8 @@ public abstract class HtmlPage extends T
@Override
public void render() {
puts(DOCTYPE);
- render(page().html().meta_http("Content-type", MimeType.HTML));
+ render(page().html().meta_http("X-UA-Compatible", "IE=8")
+ .meta_http("Content-type", MimeType.HTML));
if (page().nestLevel() != 0) {
throw new WebAppException("Error rendering page: nestLevel="+
page().nestLevel());
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/webapp/view/InfoBlock.java Tue Aug 19 23:49:39 2014
@@ -62,11 +62,11 @@ public class InfoBlock extends HtmlBlock
DIV<TD<TR<TABLE<DIV<Hamlet>>>>> singleLineDiv;
for ( String line :lines) {
singleLineDiv = td.div();
- singleLineDiv._r(line);
+ singleLineDiv._(line);
singleLineDiv._();
}
} else {
- td._r(value);
+ td._(value);
}
td._();
} else {
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.SecurityInfo Tue Aug 19 23:49:39 2014
@@ -12,6 +12,7 @@
# limitations under the License.
#
org.apache.hadoop.yarn.security.client.ClientRMSecurityInfo
+org.apache.hadoop.yarn.security.client.ClientTimelineSecurityInfo
org.apache.hadoop.yarn.security.ContainerManagerSecurityInfo
org.apache.hadoop.yarn.security.SchedulerSecurityInfo
org.apache.hadoop.yarn.security.admin.AdminSecurityInfo
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenIdentifier Tue Aug 19 23:49:39 2014
@@ -15,4 +15,5 @@ org.apache.hadoop.yarn.security.Containe
org.apache.hadoop.yarn.security.AMRMTokenIdentifier
org.apache.hadoop.yarn.security.client.ClientToAMTokenIdentifier
org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier
+org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier
org.apache.hadoop.yarn.security.NMTokenIdentifier
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/META-INF/services/org.apache.hadoop.security.token.TokenRenewer Tue Aug 19 23:49:39 2014
@@ -14,3 +14,4 @@
org.apache.hadoop.yarn.security.AMRMTokenIdentifier$Renewer
org.apache.hadoop.yarn.security.ContainerTokenIdentifier$Renewer
org.apache.hadoop.yarn.security.client.RMDelegationTokenIdentifier$Renewer
+org.apache.hadoop.yarn.security.client.TimelineDelegationTokenIdentifier$Renewer
Modified: hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js
URL: http://svn.apache.org/viewvc/hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js?rev=1619012&r1=1619011&r2=1619012&view=diff
==============================================================================
--- hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js (original)
+++ hadoop/common/branches/HADOOP-10388/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/webapps/static/yarn.dt.plugins.js Tue Aug 19 23:49:39 2014
@@ -119,13 +119,7 @@ function parseHadoopID(data, type, full)
return data;
}
//Return the visible string rather than the entire HTML tag
- if (type === 'filter') {
- return data.split('>')[1].split('<')[0];
- }
- //Parse the ID for 'sort', 'type' and undefined
- //The number after the last '_' and before the end tag '<'
- var splits = data.split('_');
- return splits[parseInt(splits.length-1)].split('<')[0];
+ return data.split('>')[1].split('<')[0];
}
//JSON array element is "20000 attempt_1360183373897_0001_m_000002_0"