You are viewing a plain text version of this content. The canonical link for it is here.
Posted to yarn-commits@hadoop.apache.org by ss...@apache.org on 2012/08/25 04:26:15 UTC
svn commit: r1377183 [1/3] - in
/hadoop/common/branches/branch-0.23/hadoop-yarn-project: ./
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/
hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/
hadoop-y...
Author: sseth
Date: Sat Aug 25 02:26:13 2012
New Revision: 1377183
URL: http://svn.apache.org/viewvc?rev=1377183&view=rev
Log:
YARN-39. RM-NM secret-keys should be randomly generated and rolled every so often. (Contributed by Vinod Kumar Vavilapalli and Siddharth Seth)
Added:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/MasterKey.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/MasterKeyPBImpl.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/security/RMContainerTokenSecretManager.java
Removed:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/ContainerTokenSecretManager.java
Modified:
hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/HeartbeatResponse.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/RegistrationResponse.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/HeartbeatResponsePBImpl.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/RegistrationResponsePBImpl.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestEventFlow.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNMAuditLogger.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestNodeStatusUpdater.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/BaseContainerManagerTest.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/TestApplication.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServices.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesApps.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServicesContainers.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/AdminService.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContext.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/RMContextImpl.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceManager.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/ResourceTrackerService.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNode.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeImpl.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/rmnode/RMNodeStatusEvent.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/ResourceScheduler.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacityScheduler.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/CapacitySchedulerContext.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/LeafQueue.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/main/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/FifoScheduler.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/MockNodes.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestAppManager.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestFifoScheduler.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMAuditLogger.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/TestRMNodeTransitions.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestAMRestart.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/applicationsmanager/TestSchedulerNegotiator.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestNMExpiry.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/resourcetracker/TestRMNMRPCResponseId.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/TestRMAppTransitions.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/rmapp/attempt/TestRMAppAttemptTransitions.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestCapacityScheduler.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestLeafQueue.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestQueueParsing.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/capacity/TestUtils.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/scheduler/fifo/TestFifoScheduler.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebApp.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-resourcemanager/src/test/java/org/apache/hadoop/yarn/server/resourcemanager/webapp/TestRMWebServicesNodes.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/TestContainerManagerSecurity.java
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/CHANGES.txt Sat Aug 25 02:26:13 2012
@@ -55,3 +55,5 @@ Release 0.23.3 - Unreleased
MAPREDUCE-2374. "Text File Busy" errors launching MR tasks. (Andy Isaacson
via atm)
+ YARN-39. RM-NM secret-keys should be randomly generated and rolled every
+ so often. (vinodkv and sseth via sseth)
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/conf/YarnConfiguration.java Sat Aug 25 02:26:13 2012
@@ -18,16 +18,16 @@
package org.apache.hadoop.yarn.conf;
-import com.google.common.base.Joiner;
-import com.google.common.base.Splitter;
-
import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.net.UnknownHostException;
-import java.util.Iterator;
+
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.net.NetUtils;
+import com.google.common.base.Joiner;
+import com.google.common.base.Splitter;
+
public class YarnConfiguration extends Configuration {
private static final Splitter ADDR_SPLITTER = Splitter.on(':').trimResults();
private static final Joiner JOINER = Joiner.on("");
@@ -262,6 +262,12 @@ public class YarnConfiguration extends C
public static final String DEFAULT_RM_METRICS_RUNTIME_BUCKETS =
"60,300,1440";
+ public static final String RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
+ RM_PREFIX + "container-tokens.master-key-rolling-interval-secs";
+
+ public static final long DEFAULT_RM_CONTAINER_TOKEN_MASTER_KEY_ROLLING_INTERVAL_SECS =
+ 24 * 60 * 60;
+
////////////////////////////////
// Node Manager Configs
////////////////////////////////
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/java/org/apache/hadoop/yarn/security/ContainerTokenIdentifier.java Sat Aug 25 02:26:13 2012
@@ -50,13 +50,15 @@ public class ContainerTokenIdentifier ex
private String nmHostAddr;
private Resource resource;
private long expiryTimeStamp;
+ private int masterKeyId;
public ContainerTokenIdentifier(ContainerId containerID, String hostName,
- Resource r, long expiryTimeStamp) {
+ Resource r, long expiryTimeStamp, int masterKeyId) {
this.containerId = containerID;
this.nmHostAddr = hostName;
this.resource = r;
this.expiryTimeStamp = expiryTimeStamp;
+ this.masterKeyId = masterKeyId;
}
/**
@@ -81,6 +83,10 @@ public class ContainerTokenIdentifier ex
return this.expiryTimeStamp;
}
+ public int getMasterKeyId() {
+ return this.masterKeyId;
+ }
+
@Override
public void write(DataOutput out) throws IOException {
LOG.debug("Writing ContainerTokenIdentifier to RPC layer: " + this);
@@ -94,6 +100,7 @@ public class ContainerTokenIdentifier ex
out.writeUTF(this.nmHostAddr);
out.writeInt(this.resource.getMemory());
out.writeLong(this.expiryTimeStamp);
+ out.writeInt(this.masterKeyId);
}
@Override
@@ -107,6 +114,7 @@ public class ContainerTokenIdentifier ex
this.nmHostAddr = in.readUTF();
this.resource = BuilderUtils.newResource(in.readInt());
this.expiryTimeStamp = in.readLong();
+ this.masterKeyId = in.readInt();
}
@Override
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-common/src/main/resources/yarn-default.xml Sat Aug 25 02:26:13 2012
@@ -239,6 +239,17 @@
<value>86400</value>
</property>
+ <property>
+ <description>Interval for the roll over for the master key used to generate
+ container tokens. It is expected to be much greater than
+ yarn.nm.liveness-monitor.expiry-interval-ms and
+ yarn.rm.container-allocation.expiry-interval-ms. Otherwise the
+ behavior is undefined.
+ </description>
+ <name>yarn.resourcemanager.container-tokens.master-key-rolling-interval-secs</name>
+ <value>86400</value>
+ </property>
+
<!-- Node Manager Configs -->
<property>
<description>address of node manager IPC.</description>
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/HeartbeatResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/HeartbeatResponse.java?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/HeartbeatResponse.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/HeartbeatResponse.java Sat Aug 25 02:26:13 2012
@@ -36,7 +36,10 @@ public interface HeartbeatResponse {
void setResponseId(int responseId);
void setNodeAction(NodeAction action);
-
+
+ MasterKey getMasterKey();
+ void setMasterKey(MasterKey secretKey);
+
void addAllContainersToCleanup(List<ContainerId> containers);
void addContainerToCleanup(ContainerId container);
void removeContainerToCleanup(int index);
Added: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/MasterKey.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/MasterKey.java?rev=1377183&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/MasterKey.java (added)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/MasterKey.java Sat Aug 25 02:26:13 2012
@@ -0,0 +1,33 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.records;
+
+import java.nio.ByteBuffer;
+
+public interface MasterKey {
+
+ int getKeyId();
+
+ void setKeyId(int keyId);
+
+ ByteBuffer getBytes();
+
+ void setBytes(ByteBuffer bytes);
+
+}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/RegistrationResponse.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/RegistrationResponse.java?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/RegistrationResponse.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/RegistrationResponse.java Sat Aug 25 02:26:13 2012
@@ -17,14 +17,13 @@
*/
package org.apache.hadoop.yarn.server.api.records;
-import java.nio.ByteBuffer;
-
public interface RegistrationResponse {
- public abstract ByteBuffer getSecretKey();
+
+ MasterKey getMasterKey();
- public abstract void setSecretKey(ByteBuffer secretKey);
+ void setMasterKey(MasterKey secretKey);
- public abstract NodeAction getNodeAction();
+ NodeAction getNodeAction();
- public abstract void setNodeAction(NodeAction nodeAction);
+ void setNodeAction(NodeAction nodeAction);
}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/HeartbeatResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/HeartbeatResponsePBImpl.java?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/HeartbeatResponsePBImpl.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/HeartbeatResponsePBImpl.java Sat Aug 25 02:26:13 2012
@@ -32,8 +32,10 @@ import org.apache.hadoop.yarn.proto.Yarn
import org.apache.hadoop.yarn.proto.YarnProtos.ContainerIdProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.HeartbeatResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.HeartbeatResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto;
import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
+import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
public class HeartbeatResponsePBImpl extends
@@ -43,9 +45,8 @@ public class HeartbeatResponsePBImpl ext
boolean viaProto = false;
private List<ContainerId> containersToCleanup = null;
-
private List<ApplicationId> applicationsToCleanup = null;
-
+ private MasterKey masterKey = null;
public HeartbeatResponsePBImpl() {
builder = HeartbeatResponseProto.newBuilder();
@@ -71,6 +72,9 @@ public class HeartbeatResponsePBImpl ext
if (this.applicationsToCleanup != null) {
addApplicationsToCleanupToProto();
}
+ if (this.masterKey != null) {
+ builder.setMasterKey(convertToProtoFormat(this.masterKey));
+ }
}
private void mergeLocalToProto() {
@@ -100,6 +104,28 @@ public class HeartbeatResponsePBImpl ext
maybeInitBuilder();
builder.setResponseId((responseId));
}
+
+ @Override
+ public MasterKey getMasterKey() {
+ HeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.masterKey != null) {
+ return this.masterKey;
+ }
+ if (!p.hasMasterKey()) {
+ return null;
+ }
+ this.masterKey = convertFromProtoFormat(p.getMasterKey());
+ return this.masterKey;
+ }
+
+ @Override
+ public void setMasterKey(MasterKey masterKey) {
+ maybeInitBuilder();
+ if (masterKey == null)
+ builder.clearMasterKey();
+ this.masterKey = masterKey;
+ }
+
@Override
public NodeAction getNodeAction() {
HeartbeatResponseProtoOrBuilder p = viaProto ? proto : builder;
@@ -313,4 +339,12 @@ public class HeartbeatResponsePBImpl ext
private NodeActionProto convertToProtoFormat(NodeAction t) {
return NodeActionProto.valueOf(t.name());
}
+
+ private MasterKeyPBImpl convertFromProtoFormat(MasterKeyProto p) {
+ return new MasterKeyPBImpl(p);
+ }
+
+ private MasterKeyProto convertToProtoFormat(MasterKey t) {
+ return ((MasterKeyPBImpl)t).getProto();
+ }
}
Added: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/MasterKeyPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/MasterKeyPBImpl.java?rev=1377183&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/MasterKeyPBImpl.java (added)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/MasterKeyPBImpl.java Sat Aug 25 02:26:13 2012
@@ -0,0 +1,102 @@
+/**
+ * Licensed to the Apache Software Foundation (ASF) under one
+ * or more contributor license agreements. See the NOTICE file
+ * distributed with this work for additional information
+ * regarding copyright ownership. The ASF licenses this file
+ * to you under the Apache License, Version 2.0 (the
+ * "License"); you may not use this file except in compliance
+ * with the License. You may obtain a copy of the License at
+ *
+ * http://www.apache.org/licenses/LICENSE-2.0
+ *
+ * Unless required by applicable law or agreed to in writing, software
+ * distributed under the License is distributed on an "AS IS" BASIS,
+ * WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+ * See the License for the specific language governing permissions and
+ * limitations under the License.
+ */
+
+package org.apache.hadoop.yarn.server.api.records.impl.pb;
+
+import java.nio.ByteBuffer;
+
+import org.apache.hadoop.yarn.api.records.ProtoBase;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProtoOrBuilder;
+import org.apache.hadoop.yarn.server.api.records.MasterKey;
+
+public class MasterKeyPBImpl extends ProtoBase<MasterKeyProto> implements
+ MasterKey {
+ MasterKeyProto proto = MasterKeyProto.getDefaultInstance();
+ MasterKeyProto.Builder builder = null;
+ boolean viaProto = false;
+
+ public MasterKeyPBImpl() {
+ builder = MasterKeyProto.newBuilder();
+ }
+
+ public MasterKeyPBImpl(MasterKeyProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public synchronized MasterKeyProto getProto() {
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private synchronized void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = MasterKeyProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public synchronized int getKeyId() {
+ MasterKeyProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getKeyId());
+ }
+
+ @Override
+ public synchronized void setKeyId(int id) {
+ maybeInitBuilder();
+ builder.setKeyId((id));
+ }
+
+ @Override
+ public synchronized ByteBuffer getBytes() {
+ MasterKeyProtoOrBuilder p = viaProto ? proto : builder;
+ return convertFromProtoFormat(p.getBytes());
+ }
+
+ @Override
+ public synchronized void setBytes(ByteBuffer bytes) {
+ maybeInitBuilder();
+ builder.setBytes(convertToProtoFormat(bytes));
+ }
+
+ @Override
+ public int hashCode() {
+ return getKeyId();
+ }
+
+ @Override
+ public boolean equals(Object obj) {
+ if (this == obj)
+ return true;
+ if (!(obj instanceof MasterKey)) {
+ return false;
+ }
+ MasterKey other = (MasterKey) obj;
+ if (this.getKeyId() != other.getKeyId()) {
+ return false;
+ }
+ if (!this.getBytes().equals(other.getBytes())) {
+ return false;
+ }
+ return true;
+ }
+
+}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/RegistrationResponsePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/RegistrationResponsePBImpl.java?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/RegistrationResponsePBImpl.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/api/records/impl/pb/RegistrationResponsePBImpl.java Sat Aug 25 02:26:13 2012
@@ -19,12 +19,12 @@
package org.apache.hadoop.yarn.server.api.records.impl.pb;
-import java.nio.ByteBuffer;
-
import org.apache.hadoop.yarn.api.records.ProtoBase;
+import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.MasterKeyProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.NodeActionProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.RegistrationResponseProto;
import org.apache.hadoop.yarn.proto.YarnServerCommonProtos.RegistrationResponseProtoOrBuilder;
+import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.RegistrationResponse;
@@ -34,7 +34,7 @@ public class RegistrationResponsePBImpl
RegistrationResponseProto.Builder builder = null;
boolean viaProto = false;
- private ByteBuffer secretKey = null;
+ private MasterKey masterKey = null;
public RegistrationResponsePBImpl() {
builder = RegistrationResponseProto.newBuilder();
@@ -54,8 +54,8 @@ public class RegistrationResponsePBImpl
}
private void mergeLocalToBuilder() {
- if (this.secretKey != null) {
- builder.setSecretKey(convertToProtoFormat(this.secretKey));
+ if (this.masterKey != null) {
+ builder.setMasterKey(convertToProtoFormat(this.masterKey));
}
}
@@ -76,26 +76,26 @@ public class RegistrationResponsePBImpl
}
@Override
- public ByteBuffer getSecretKey() {
+ public MasterKey getMasterKey() {
RegistrationResponseProtoOrBuilder p = viaProto ? proto : builder;
- if (this.secretKey != null) {
- return this.secretKey;
+ if (this.masterKey != null) {
+ return this.masterKey;
}
- if (!p.hasSecretKey()) {
+ if (!p.hasMasterKey()) {
return null;
}
- this.secretKey = convertFromProtoFormat(p.getSecretKey());
- return this.secretKey;
+ this.masterKey = convertFromProtoFormat(p.getMasterKey());
+ return this.masterKey;
}
@Override
- public void setSecretKey(ByteBuffer secretKey) {
+ public void setMasterKey(MasterKey masterKey) {
maybeInitBuilder();
- if (secretKey == null)
- builder.clearSecretKey();
- this.secretKey = secretKey;
+ if (masterKey == null)
+ builder.clearMasterKey();
+ this.masterKey = masterKey;
}
-
+
@Override
public NodeAction getNodeAction() {
RegistrationResponseProtoOrBuilder p = viaProto ? proto : builder;
@@ -123,4 +123,11 @@ public class RegistrationResponsePBImpl
return NodeActionProto.valueOf(t.name());
}
+ private MasterKeyPBImpl convertFromProtoFormat(MasterKeyProto p) {
+ return new MasterKeyPBImpl(p);
+ }
+
+ private MasterKeyProto convertToProtoFormat(MasterKey t) {
+ return ((MasterKeyPBImpl)t).getProto();
+ }
}
Added: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java?rev=1377183&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java (added)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/java/org/apache/hadoop/yarn/server/security/BaseContainerTokenSecretManager.java Sat Aug 25 02:26:13 2012
@@ -0,0 +1,202 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.security;
+
+import java.nio.ByteBuffer;
+import java.security.SecureRandom;
+import java.util.concurrent.locks.Lock;
+import java.util.concurrent.locks.ReadWriteLock;
+import java.util.concurrent.locks.ReentrantReadWriteLock;
+
+import javax.crypto.SecretKey;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.api.records.ContainerToken;
+import org.apache.hadoop.yarn.api.records.NodeId;
+import org.apache.hadoop.yarn.api.records.Resource;
+import org.apache.hadoop.yarn.conf.YarnConfiguration;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.api.records.MasterKey;
+import org.apache.hadoop.yarn.util.BuilderUtils;
+import org.apache.hadoop.yarn.util.Records;
+
+/**
+ * SecretManager for ContainerTokens. Extended by both RM and NM and hence is
+ * present in yarn-server-common package.
+ *
+ */
+public class BaseContainerTokenSecretManager extends
+ SecretManager<ContainerTokenIdentifier> {
+
+ private static Log LOG = LogFactory
+ .getLog(BaseContainerTokenSecretManager.class);
+
+ private int serialNo = new SecureRandom().nextInt();
+
+ protected final ReadWriteLock readWriteLock = new ReentrantReadWriteLock();
+ protected final Lock readLock = readWriteLock.readLock();
+ protected final Lock writeLock = readWriteLock.writeLock();
+
+ /**
+ * THE masterKey. ResourceManager should persist this and recover it on
+ * restart instead of generating a new key. The NodeManagers get it from the
+ * ResourceManager and use it for validating container-tokens.
+ */
+ protected MasterKeyData currentMasterKey;
+
+ protected final class MasterKeyData {
+
+ private final MasterKey masterKeyRecord;
+ // Underlying secret-key also stored to avoid repetitive encoding and
+ // decoding the masterKeyRecord bytes.
+ private final SecretKey generatedSecretKey;
+
+ private MasterKeyData() {
+ this.masterKeyRecord = Records.newRecord(MasterKey.class);
+ this.masterKeyRecord.setKeyId(serialNo++);
+ this.generatedSecretKey = generateSecret();
+ this.masterKeyRecord.setBytes(ByteBuffer.wrap(generatedSecretKey
+ .getEncoded()));
+ }
+
+ public MasterKeyData(MasterKey masterKeyRecord) {
+ this.masterKeyRecord = masterKeyRecord;
+ this.generatedSecretKey =
+ SecretManager.createSecretKey(this.masterKeyRecord.getBytes().array()
+ .clone());
+ }
+
+ public MasterKey getMasterKey() {
+ return this.masterKeyRecord;
+ }
+
+ private SecretKey getSecretKey() {
+ return this.generatedSecretKey;
+ }
+ }
+
+ protected final long containerTokenExpiryInterval;
+
+ public BaseContainerTokenSecretManager(Configuration conf) {
+ this.containerTokenExpiryInterval =
+ conf.getInt(YarnConfiguration.RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS,
+ YarnConfiguration.DEFAULT_RM_CONTAINER_ALLOC_EXPIRY_INTERVAL_MS);
+ }
+
+ // Need lock as we increment serialNo etc.
+ protected MasterKeyData createNewMasterKey() {
+ this.writeLock.lock();
+ try {
+ return new MasterKeyData();
+ } finally {
+ this.writeLock.unlock();
+ }
+ }
+
+ @Private
+ public MasterKey getCurrentKey() {
+ this.readLock.lock();
+ try {
+ return this.currentMasterKey.getMasterKey();
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
+ public byte[] createPassword(ContainerTokenIdentifier identifier) {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Creating password for " + identifier.getContainerID()
+ + " to be run on NM " + identifier.getNmHostAddress());
+ }
+ this.readLock.lock();
+ try {
+ return createPassword(identifier.getBytes(),
+ this.currentMasterKey.getSecretKey());
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ @Override
+ public byte[] retrievePassword(ContainerTokenIdentifier identifier)
+ throws SecretManager.InvalidToken {
+ this.readLock.lock();
+ try {
+ return retrievePasswordInternal(identifier, this.currentMasterKey);
+ } finally {
+ this.readLock.unlock();
+ }
+ }
+
+ protected byte[] retrievePasswordInternal(ContainerTokenIdentifier identifier,
+ MasterKeyData masterKey)
+ throws org.apache.hadoop.security.token.SecretManager.InvalidToken {
+ if (LOG.isDebugEnabled()) {
+ LOG.debug("Retrieving password for " + identifier.getContainerID()
+ + " to be run on NM " + identifier.getNmHostAddress());
+ }
+ return createPassword(identifier.getBytes(), masterKey.getSecretKey());
+ }
+
+ /**
+ * Used by the RPC layer.
+ */
+ @Override
+ public ContainerTokenIdentifier createIdentifier() {
+ return new ContainerTokenIdentifier();
+ }
+
+ /**
+ * Helper function for creating ContainerTokens
+ *
+ * @param containerId
+ * @param nodeId
+ * @param capability
+ * @return the container-token
+ */
+ public ContainerToken createContainerToken(ContainerId containerId,
+ NodeId nodeId, Resource capability) {
+ byte[] password;
+ ContainerTokenIdentifier tokenIdentifier;
+ long expiryTimeStamp =
+ System.currentTimeMillis() + containerTokenExpiryInterval;
+
+ // Lock so that we use the same MasterKey's keyId and its bytes
+ this.readLock.lock();
+ try {
+ tokenIdentifier =
+ new ContainerTokenIdentifier(containerId, nodeId.toString(),
+ capability, expiryTimeStamp, this.currentMasterKey.getMasterKey()
+ .getKeyId());
+ password = this.createPassword(tokenIdentifier);
+
+ } finally {
+ this.readLock.unlock();
+ }
+
+ return BuilderUtils.newContainerToken(nodeId, ByteBuffer.wrap(password),
+ tokenIdentifier);
+ }
+}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-common/src/main/proto/yarn_server_common_protos.proto Sat Aug 25 02:26:13 2012
@@ -37,15 +37,21 @@ message NodeStatusProto {
repeated ApplicationIdProto keep_alive_applications = 5;
}
+message MasterKeyProto {
+ optional int32 key_id = 1;
+ optional bytes bytes = 2;
+}
+
message RegistrationResponseProto {
- optional bytes secret_key = 1;
+ optional MasterKeyProto master_key = 1;
optional NodeActionProto nodeAction = 2;
}
message HeartbeatResponseProto {
optional int32 response_id = 1;
- optional NodeActionProto nodeAction = 2;
- repeated ContainerIdProto containers_to_cleanup = 3;
- repeated ApplicationIdProto applications_to_cleanup = 4;
+ optional MasterKeyProto master_key = 2;
+ optional NodeActionProto nodeAction = 3;
+ repeated ContainerIdProto containers_to_cleanup = 4;
+ repeated ApplicationIdProto applications_to_cleanup = 5;
}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/Context.java Sat Aug 25 02:26:13 2012
@@ -26,6 +26,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.api.records.NodeId;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
/**
* Context interface for sharing information across components in the
@@ -44,5 +45,7 @@ public interface Context {
ConcurrentMap<ContainerId, Container> getContainers();
+ NMContainerTokenSecretManager getContainerTokenSecretManager();
+
NodeHealthStatus getNodeHealthStatus();
}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeManager.java Sat Aug 25 02:26:13 2012
@@ -46,9 +46,9 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.nodemanager.security.NMContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.webapp.WebServer;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.service.ServiceStateChangeListener;
@@ -64,7 +64,6 @@ public class NodeManager extends Composi
private static final Log LOG = LogFactory.getLog(NodeManager.class);
protected final NodeManagerMetrics metrics = NodeManagerMetrics.create();
- protected ContainerTokenSecretManager containerTokenSecretManager;
private ApplicationACLsManager aclsManager;
private NodeHealthCheckerService nodeHealthChecker;
private LocalDirsHandlerService dirsHandler;
@@ -75,10 +74,9 @@ public class NodeManager extends Composi
}
protected NodeStatusUpdater createNodeStatusUpdater(Context context,
- Dispatcher dispatcher, NodeHealthCheckerService healthChecker,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ Dispatcher dispatcher, NodeHealthCheckerService healthChecker) {
return new NodeStatusUpdaterImpl(context, dispatcher, healthChecker,
- metrics, containerTokenSecretManager);
+ metrics);
}
protected NodeResourceMonitor createNodeResourceMonitor() {
@@ -87,11 +85,10 @@ public class NodeManager extends Composi
protected ContainerManagerImpl createContainerManager(Context context,
ContainerExecutor exec, DeletionService del,
- NodeStatusUpdater nodeStatusUpdater, ContainerTokenSecretManager
- containerTokenSecretManager, ApplicationACLsManager aclsManager,
+ NodeStatusUpdater nodeStatusUpdater, ApplicationACLsManager aclsManager,
LocalDirsHandlerService dirsHandler) {
return new ContainerManagerImpl(context, exec, del, nodeStatusUpdater,
- metrics, containerTokenSecretManager, aclsManager, dirsHandler);
+ metrics, aclsManager, dirsHandler);
}
protected WebServer createWebServer(Context nmContext,
@@ -110,15 +107,16 @@ public class NodeManager extends Composi
conf.setBoolean(Dispatcher.DISPATCHER_EXIT_ON_ERROR_KEY, true);
- Context context = new NMContext();
-
// Create the secretManager if need be.
+ NMContainerTokenSecretManager containerTokenSecretManager = null;
if (UserGroupInformation.isSecurityEnabled()) {
LOG.info("Security is enabled on NodeManager. "
+ "Creating ContainerTokenSecretManager");
- this.containerTokenSecretManager = new ContainerTokenSecretManager(conf);
+ containerTokenSecretManager = new NMContainerTokenSecretManager(conf);
}
+ Context context = new NMContext(containerTokenSecretManager);
+
this.aclsManager = new ApplicationACLsManager(conf);
ContainerExecutor exec = ReflectionUtils.newInstance(
@@ -139,8 +137,8 @@ public class NodeManager extends Composi
addService(nodeHealthChecker);
dirsHandler = nodeHealthChecker.getDiskHandler();
- NodeStatusUpdater nodeStatusUpdater = createNodeStatusUpdater(context,
- dispatcher, nodeHealthChecker, this.containerTokenSecretManager);
+ NodeStatusUpdater nodeStatusUpdater =
+ createNodeStatusUpdater(context, dispatcher, nodeHealthChecker);
nodeStatusUpdater.register(this);
NodeResourceMonitor nodeResourceMonitor = createNodeResourceMonitor();
@@ -148,7 +146,7 @@ public class NodeManager extends Composi
ContainerManagerImpl containerManager =
createContainerManager(context, exec, del, nodeStatusUpdater,
- this.containerTokenSecretManager, this.aclsManager, dirsHandler);
+ this.aclsManager, dirsHandler);
addService(containerManager);
Service webServer = createWebServer(context, containerManager
@@ -192,10 +190,13 @@ public class NodeManager extends Composi
private final ConcurrentMap<ContainerId, Container> containers =
new ConcurrentSkipListMap<ContainerId, Container>();
+ private final NMContainerTokenSecretManager containerTokenSecretManager;
+
private final NodeHealthStatus nodeHealthStatus = RecordFactoryProvider
.getRecordFactory(null).newRecordInstance(NodeHealthStatus.class);
- public NMContext() {
+ public NMContext(NMContainerTokenSecretManager containerTokenSecretManager) {
+ this.containerTokenSecretManager = containerTokenSecretManager;
this.nodeHealthStatus.setIsNodeHealthy(true);
this.nodeHealthStatus.setHealthReport("Healthy");
this.nodeHealthStatus.setLastHealthReportTime(System.currentTimeMillis());
@@ -220,6 +221,10 @@ public class NodeManager extends Composi
}
@Override
+ public NMContainerTokenSecretManager getContainerTokenSecretManager() {
+ return this.containerTokenSecretManager;
+ }
+ @Override
public NodeHealthStatus getNodeHealthStatus() {
return this.nodeHealthStatus;
}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdater.java Sat Aug 25 02:26:13 2012
@@ -22,7 +22,5 @@ import org.apache.hadoop.yarn.service.Se
public interface NodeStatusUpdater extends Service {
- byte[] getRMNMSharedSecret();
-
void sendOutofBandHeartBeat();
}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/NodeStatusUpdaterImpl.java Sat Aug 25 02:26:13 2012
@@ -25,8 +25,8 @@ import java.util.HashMap;
import java.util.Iterator;
import java.util.List;
import java.util.Map;
-import java.util.Random;
import java.util.Map.Entry;
+import java.util.Random;
import org.apache.avro.AvroRuntimeException;
import org.apache.commons.logging.Log;
@@ -51,15 +51,14 @@ import org.apache.hadoop.yarn.server.api
import org.apache.hadoop.yarn.server.api.protocolrecords.NodeHeartbeatRequest;
import org.apache.hadoop.yarn.server.api.protocolrecords.RegisterNodeManagerRequest;
import org.apache.hadoop.yarn.server.api.records.HeartbeatResponse;
+import org.apache.hadoop.yarn.server.api.records.MasterKey;
import org.apache.hadoop.yarn.server.api.records.NodeAction;
import org.apache.hadoop.yarn.server.api.records.NodeStatus;
import org.apache.hadoop.yarn.server.api.records.RegistrationResponse;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.AbstractService;
-
public class NodeStatusUpdaterImpl extends AbstractService implements
NodeStatusUpdater {
@@ -71,13 +70,11 @@ public class NodeStatusUpdaterImpl exten
private final Dispatcher dispatcher;
private NodeId nodeId;
- private ContainerTokenSecretManager containerTokenSecretManager;
private long heartBeatInterval;
private ResourceTracker resourceTracker;
private InetSocketAddress rmAddress;
private Resource totalResource;
private int httpPort;
- private byte[] secretKeyBytes = new byte[0];
private boolean isStopped;
private RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
private boolean tokenKeepAliveEnabled;
@@ -93,14 +90,12 @@ public class NodeStatusUpdaterImpl exten
private boolean hasToRebootNode;
public NodeStatusUpdaterImpl(Context context, Dispatcher dispatcher,
- NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics,
- ContainerTokenSecretManager containerTokenSecretManager) {
+ NodeHealthCheckerService healthChecker, NodeManagerMetrics metrics) {
super(NodeStatusUpdaterImpl.class.getName());
this.healthChecker = healthChecker;
this.context = context;
this.dispatcher = dispatcher;
this.metrics = metrics;
- this.containerTokenSecretManager = containerTokenSecretManager;
}
@Override
@@ -194,30 +189,24 @@ public class NodeStatusUpdaterImpl exten
throw new YarnException(
"Recieved SHUTDOWN signal from Resourcemanager ,Registration of NodeManager failed");
}
-
- if (UserGroupInformation.isSecurityEnabled()) {
- this.secretKeyBytes = regResponse.getSecretKey().array();
- }
- // do this now so that its set before we start heartbeating to RM
if (UserGroupInformation.isSecurityEnabled()) {
+ MasterKey masterKey = regResponse.getMasterKey();
+ // do this now so that its set before we start heartbeating to RM
LOG.info("Security enabled - updating secret keys now");
// It is expected that status updater is started by this point and
- // RM gives the shared secret in registration during StatusUpdater#start().
- this.containerTokenSecretManager.setSecretKey(
- this.nodeId.toString(),
- this.getRMNMSharedSecret());
+ // RM gives the shared secret in registration during
+ // StatusUpdater#start().
+ if (masterKey != null) {
+ this.context.getContainerTokenSecretManager().setMasterKey(masterKey);
+ }
}
+
LOG.info("Registered with ResourceManager as " + this.nodeId
+ " with total resource of " + this.totalResource);
}
- @Override
- public byte[] getRMNMSharedSecret() {
- return this.secretKeyBytes.clone();
- }
-
private List<ApplicationId> createKeepAliveApplicationList() {
if (!tokenKeepAliveEnabled) {
return Collections.emptyList();
@@ -335,6 +324,17 @@ public class NodeStatusUpdaterImpl exten
request.setNodeStatus(nodeStatus);
HeartbeatResponse response =
resourceTracker.nodeHeartbeat(request).getHeartbeatResponse();
+
+ // See if the master-key has rolled over
+ if (isSecurityEnabled()) {
+ MasterKey updatedMasterKey = response.getMasterKey();
+ if (updatedMasterKey != null) {
+ // Will be non-null only on roll-over on RM side
+ context.getContainerTokenSecretManager().setMasterKey(
+ updatedMasterKey);
+ }
+ }
+
if (response.getNodeAction() == NodeAction.SHUTDOWN) {
LOG
.info("Recieved SHUTDOWN signal from Resourcemanager as part of heartbeat," +
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java Sat Aug 25 02:26:13 2012
@@ -21,10 +21,10 @@ package org.apache.hadoop.yarn.server.no
import static org.apache.hadoop.yarn.service.Service.STATE.STARTED;
import java.io.IOException;
-import java.net.InetAddress;
import java.net.InetSocketAddress;
import java.nio.ByteBuffer;
import java.util.Map;
+import java.util.Set;
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
@@ -96,7 +96,6 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
import org.apache.hadoop.yarn.server.nodemanager.security.authorize.NMPolicyProvider;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.service.CompositeService;
import org.apache.hadoop.yarn.service.Service;
import org.apache.hadoop.yarn.service.ServiceStateChangeListener;
@@ -110,14 +109,12 @@ public class ContainerManagerImpl extend
final Context context;
private final ContainersMonitor containersMonitor;
private Server server;
- private InetAddress resolvedAddress = null;
private final ResourceLocalizationService rsrcLocalizationSrvc;
private final ContainersLauncher containersLauncher;
private final AuxServices auxiliaryServices;
private final NodeManagerMetrics metrics;
private final NodeStatusUpdater nodeStatusUpdater;
- private ContainerTokenSecretManager containerTokenSecretManager;
private final RecordFactory recordFactory = RecordFactoryProvider.getRecordFactory(null);
@@ -129,8 +126,7 @@ public class ContainerManagerImpl extend
public ContainerManagerImpl(Context context, ContainerExecutor exec,
DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
- NodeManagerMetrics metrics, ContainerTokenSecretManager
- containerTokenSecretManager, ApplicationACLsManager aclsManager,
+ NodeManagerMetrics metrics, ApplicationACLsManager aclsManager,
LocalDirsHandlerService dirsHandler) {
super(ContainerManagerImpl.class.getName());
this.context = context;
@@ -149,7 +145,6 @@ public class ContainerManagerImpl extend
addService(containersLauncher);
this.nodeStatusUpdater = nodeStatusUpdater;
- this.containerTokenSecretManager = containerTokenSecretManager;
this.aclsManager = aclsManager;
// Start configurable services
@@ -232,7 +227,7 @@ public class ContainerManagerImpl extend
server =
rpc.getServer(ContainerManager.class, this, initialAddress, conf,
- this.containerTokenSecretManager,
+ this.context.getContainerTokenSecretManager(),
conf.getInt(YarnConfiguration.NM_CONTAINER_MGR_THREAD_COUNT,
YarnConfiguration.DEFAULT_NM_CONTAINER_MGR_THREAD_COUNT));
@@ -267,56 +262,78 @@ public class ContainerManagerImpl extend
super.stop();
}
+ // Get the remoteUGI corresponding to the api call.
+ private UserGroupInformation getRemoteUgi(String containerIDStr)
+ throws YarnRemoteException {
+ UserGroupInformation remoteUgi;
+ try {
+ remoteUgi = UserGroupInformation.getCurrentUser();
+ } catch (IOException e) {
+ String msg = "Cannot obtain the user-name for containerId: "
+ + containerIDStr + ". Got exception: "
+ + StringUtils.stringifyException(e);
+ LOG.warn(msg);
+ throw RPCUtil.getRemoteException(msg);
+ }
+ return remoteUgi;
+ }
+
+ // Obtain the needed ContainerTokenIdentifier from the remote-UGI. RPC layer
+ // currently sets only the required id, but iterate through anyways just to
+ // be sure.
+ private ContainerTokenIdentifier selectContainerTokenIdentifier(
+ UserGroupInformation remoteUgi) {
+ Set<TokenIdentifier> tokenIdentifiers = remoteUgi.getTokenIdentifiers();
+ ContainerTokenIdentifier resultId = null;
+ for (TokenIdentifier id : tokenIdentifiers) {
+ if (id instanceof ContainerTokenIdentifier) {
+ resultId = (ContainerTokenIdentifier) id;
+ break;
+ }
+ }
+ return resultId;
+ }
+
/**
* Authorize the request.
*
- * @param containerID
+ * @param containerIDStr
* of the container
* @param launchContext
* passed if verifying the startContainer, null otherwise.
+ * @param remoteUgi
+ * ugi corresponding to the remote end making the api-call
* @throws YarnRemoteException
*/
- private void authorizeRequest(ContainerId containerID,
- ContainerLaunchContext launchContext) throws YarnRemoteException {
+ private void authorizeRequest(String containerIDStr,
+ ContainerLaunchContext launchContext, UserGroupInformation remoteUgi)
+ throws YarnRemoteException {
if (!UserGroupInformation.isSecurityEnabled()) {
return;
}
- String containerIDStr = containerID.toString();
-
- UserGroupInformation remoteUgi;
- try {
- remoteUgi = UserGroupInformation.getCurrentUser();
- } catch (IOException e) {
- String msg = "Cannot obtain the user-name for containerId: "
- + containerIDStr + ". Got exception: "
- + StringUtils.stringifyException(e);
- LOG.warn(msg);
- throw RPCUtil.getRemoteException(msg);
- }
-
boolean unauthorized = false;
- StringBuilder messageBuilder = new StringBuilder(
- "Unauthorized request to start container. ");
+ StringBuilder messageBuilder =
+ new StringBuilder("Unauthorized request to start container. ");
if (!remoteUgi.getUserName().equals(containerIDStr)) {
unauthorized = true;
messageBuilder.append("\nExpected containerId: "
+ remoteUgi.getUserName() + " Found: " + containerIDStr);
- }
-
- if (launchContext != null) {
-
- // Verify other things for startContainer() request.
+ } else if (launchContext != null) {
+ // Verify other things also for startContainer() request.
if (LOG.isDebugEnabled()) {
- LOG.debug("Number of TokenIdentifiers in the UGI from RPC: "
- + remoteUgi.getTokenIdentifiers().size());
+ LOG.debug("Number of TokenIdentifiers in the UGI from RPC: "
+ + remoteUgi.getTokenIdentifiers().size());
}
- // We must and should get only one TokenIdentifier from the RPC.
- ContainerTokenIdentifier tokenId = (ContainerTokenIdentifier) remoteUgi
- .getTokenIdentifiers().iterator().next();
+
+
+ // Get the tokenId from the remote user ugi
+ ContainerTokenIdentifier tokenId =
+ selectContainerTokenIdentifier(remoteUgi);
+
if (tokenId == null) {
unauthorized = true;
messageBuilder
@@ -324,6 +341,15 @@ public class ContainerManagerImpl extend
+ containerIDStr);
} else {
+ // Is the container being relaunched? Or RPC layer let startCall with
+ // tokens generated off old-secrets through
+ if (!this.context.getContainerTokenSecretManager()
+ .isValidStartContainerRequest(tokenId)) {
+ unauthorized = true;
+ messageBuilder.append("\n Attempt to relaunch the same " +
+ "container with id " + containerIDStr + ".");
+ }
+
// Ensure the token is not expired.
// Token expiry is not checked for stopContainer/getContainerStatus
if (tokenId.getExpiryTimeStamp() < System.currentTimeMillis()) {
@@ -348,7 +374,7 @@ public class ContainerManagerImpl extend
throw RPCUtil.getRemoteException(msg);
}
}
-
+
/**
* Start a container on this NodeManager.
*/
@@ -359,10 +385,13 @@ public class ContainerManagerImpl extend
ContainerLaunchContext launchContext = request.getContainerLaunchContext();
ContainerId containerID = launchContext.getContainerId();
- authorizeRequest(containerID, launchContext);
+ String containerIDStr = containerID.toString();
+
+ UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ authorizeRequest(containerIDStr, launchContext, remoteUgi);
- LOG.info("Start request for " + launchContext.getContainerId()
- + " by user " + launchContext.getUser());
+ LOG.info("Start request for " + containerIDStr + " by user "
+ + launchContext.getUser());
// //////////// Parse credentials
ByteBuffer tokens = launchContext.getContainerTokens();
@@ -394,14 +423,14 @@ public class ContainerManagerImpl extend
AuditConstants.START_CONTAINER, "ContainerManagerImpl",
"Container already running on this node!",
applicationID, containerID);
- throw RPCUtil.getRemoteException("Container " + containerID
+ throw RPCUtil.getRemoteException("Container " + containerIDStr
+ " already is running on this node!!");
}
// Create the application
Application application =
new ApplicationImpl(dispatcher, this.aclsManager,
- launchContext.getUser(), applicationID, credentials, context);
+ launchContext.getUser(), applicationID, credentials, context);
if (null ==
context.getApplications().putIfAbsent(applicationID, application)) {
LOG.info("Creating a new application reference for app "
@@ -414,6 +443,12 @@ public class ContainerManagerImpl extend
// TODO: Validate the request
dispatcher.getEventHandler().handle(
new ApplicationContainerInitEvent(container));
+ if (UserGroupInformation.isSecurityEnabled()) {
+ ContainerTokenIdentifier tokenId =
+ selectContainerTokenIdentifier(remoteUgi);
+ this.context.getContainerTokenSecretManager().startContainerSuccessful(
+ tokenId);
+ }
NMAuditLogger.logSuccess(launchContext.getUser(),
AuditConstants.START_CONTAINER, "ContainerManageImpl",
@@ -438,8 +473,12 @@ public class ContainerManagerImpl extend
throws YarnRemoteException {
ContainerId containerID = request.getContainerId();
+ String containerIDStr = containerID.toString();
+
// TODO: Only the container's owner can kill containers today.
- authorizeRequest(containerID, null);
+
+ UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ authorizeRequest(containerIDStr, null, remoteUgi);
StopContainerResponse response =
recordFactory.newRecordInstance(StopContainerResponse.class);
@@ -476,10 +515,14 @@ public class ContainerManagerImpl extend
GetContainerStatusRequest request) throws YarnRemoteException {
ContainerId containerID = request.getContainerId();
+ String containerIDStr = containerID.toString();
+
// TODO: Only the container's owner can get containers' status today.
- authorizeRequest(containerID, null);
- LOG.info("Getting container-status for " + containerID);
+ UserGroupInformation remoteUgi = getRemoteUgi(containerIDStr);
+ authorizeRequest(containerIDStr, null, remoteUgi);
+
+ LOG.info("Getting container-status for " + containerIDStr);
Container container = this.context.getContainers().get(containerID);
if (container != null) {
ContainerStatus containerStatus = container.cloneAndGetContainerStatus();
@@ -490,7 +533,7 @@ public class ContainerManagerImpl extend
return response;
}
- throw RPCUtil.getRemoteException("Container " + containerID
+ throw RPCUtil.getRemoteException("Container " + containerIDStr
+ " is not handled by this NodeManager");
}
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java Sat Aug 25 02:26:13 2012
@@ -28,8 +28,9 @@ import java.util.concurrent.locks.Reentr
import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.hadoop.security.Credentials;
-import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.security.UserGroupInformation;
import org.apache.hadoop.yarn.api.records.ApplicationAccessType;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
import org.apache.hadoop.yarn.api.records.ContainerId;
import org.apache.hadoop.yarn.event.Dispatcher;
import org.apache.hadoop.yarn.logaggregation.ContainerLogsRetentionPolicy;
@@ -42,6 +43,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.ApplicationLocalizationEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.event.LocalizationEventType;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.logaggregation.LogAggregationService;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppFinishedEvent;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerAppStartedEvent;
import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
@@ -365,6 +367,10 @@ public class ApplicationImpl implements
@Override
public void transition(ApplicationImpl app, ApplicationEvent event) {
+ // Inform the ContainerTokenSecretManager
+ if (UserGroupInformation.isSecurityEnabled()) {
+ app.context.getContainerTokenSecretManager().appFinished(app.appId);
+ }
// Inform the logService
app.dispatcher.getEventHandler().handle(
new LogHandlerAppFinishedEvent(app.appId));
Added: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java?rev=1377183&view=auto
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java (added)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/security/NMContainerTokenSecretManager.java Sat Aug 25 02:26:13 2012
@@ -0,0 +1,189 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+
+package org.apache.hadoop.yarn.server.nodemanager.security;
+
+import java.util.HashMap;
+import java.util.Map;
+import java.util.concurrent.ConcurrentHashMap;
+import java.util.concurrent.ConcurrentMap;
+
+import org.apache.commons.logging.Log;
+import org.apache.commons.logging.LogFactory;
+import org.apache.hadoop.classification.InterfaceAudience.Private;
+import org.apache.hadoop.conf.Configuration;
+import org.apache.hadoop.security.UserGroupInformation;
+import org.apache.hadoop.security.token.SecretManager;
+import org.apache.hadoop.yarn.api.records.ApplicationId;
+import org.apache.hadoop.yarn.api.records.ContainerId;
+import org.apache.hadoop.yarn.security.ContainerTokenIdentifier;
+import org.apache.hadoop.yarn.server.api.records.MasterKey;
+import org.apache.hadoop.yarn.server.security.BaseContainerTokenSecretManager;
+
+/**
+ * The NM maintains only two master-keys. The current key that RM knows and the
+ * key from the previous rolling-interval.
+ *
+ */
+public class NMContainerTokenSecretManager extends
+ BaseContainerTokenSecretManager {
+
+ private static final Log LOG = LogFactory
+ .getLog(NMContainerTokenSecretManager.class);
+
+ private MasterKeyData previousMasterKey;
+
+ private final Map<ApplicationId, ConcurrentMap<ContainerId, MasterKeyData>> oldMasterKeys;
+
+ public NMContainerTokenSecretManager(Configuration conf) {
+ super(conf);
+ this.oldMasterKeys =
+ new HashMap<ApplicationId, ConcurrentMap<ContainerId, MasterKeyData>>();
+ }
+
+ /**
+ * Used by NodeManagers to create a token-secret-manager with the key obtained
+ * from the RM. This can happen during registration or when the RM rolls the
+ * master-key and signals the NM.
+ *
+ * @param masterKeyRecord
+ */
+ @Private
+ public synchronized void setMasterKey(MasterKey masterKeyRecord) {
+ LOG.info("Rolling master-key for container-tokens, got key with id "
+ + masterKeyRecord.getKeyId());
+ if (super.currentMasterKey == null) {
+ super.currentMasterKey = new MasterKeyData(masterKeyRecord);
+ } else {
+ if (super.currentMasterKey.getMasterKey().getKeyId() != masterKeyRecord
+ .getKeyId()) {
+ // Update keys only if the key has changed.
+ this.previousMasterKey = super.currentMasterKey;
+ super.currentMasterKey = new MasterKeyData(masterKeyRecord);
+ }
+ }
+ }
+
+ /**
+ * Override of this is to validate ContainerTokens generated by using
+ * different {@link MasterKey}s.
+ */
+ @Override
+ public synchronized byte[] retrievePassword(
+ ContainerTokenIdentifier identifier) throws SecretManager.InvalidToken {
+ int keyId = identifier.getMasterKeyId();
+ ContainerId containerId = identifier.getContainerID();
+ ApplicationId appId =
+ containerId.getApplicationAttemptId().getApplicationId();
+
+ MasterKeyData masterKeyToUse = null;
+
+ if (this.previousMasterKey != null
+ && keyId == this.previousMasterKey.getMasterKey().getKeyId()) {
+ // A container-launch has come in with a token generated off the last
+ // master-key
+ masterKeyToUse = this.previousMasterKey;
+ } else if (keyId == super.currentMasterKey.getMasterKey().getKeyId()) {
+ // A container-launch has come in with a token generated off the current
+ // master-key
+ masterKeyToUse = super.currentMasterKey;
+ } else if (this.oldMasterKeys.containsKey(appId)
+ && this.oldMasterKeys.get(appId).containsKey(containerId)) {
+ // This means on the following happened:
+ // (1) a stopContainer() or a getStatus() happened for a container with
+ // token generated off a master-key that is neither current nor the
+ // previous one.
+ // (2) a container-relaunch has come in with a token generated off a
+ // master-key that is neither current nor the previous one.
+ // This basically lets stop and getStatus() calls with old-tokens to pass
+ // through without any issue, i.e. (1).
+ // Start-calls for repetitive launches (2) also pass through RPC here, but
+ // get thwarted at the app-layer as part of startContainer() call.
+ masterKeyToUse = this.oldMasterKeys.get(appId).get(containerId);
+ }
+
+ if (masterKeyToUse != null) {
+ return retrievePasswordInternal(identifier, masterKeyToUse);
+ }
+
+ // Invalid request. Like startContainer() with token generated off
+ // old-master-keys.
+ throw new SecretManager.InvalidToken("Given Container "
+ + identifier.getContainerID().toString()
+ + " seems to have an illegally generated token.");
+ }
+
+ /**
+ * Container start has gone through. Store the corresponding keys so that
+ * stopContainer() and getContainerStatus() can be authenticated long after
+ * the container-start went through.
+ */
+ public synchronized void startContainerSuccessful(
+ ContainerTokenIdentifier tokenId) {
+ if (!UserGroupInformation.isSecurityEnabled()) {
+ return;
+ }
+
+ int keyId = tokenId.getMasterKeyId();
+ if (currentMasterKey.getMasterKey().getKeyId() == keyId) {
+ addKeyForContainerId(tokenId.getContainerID(), currentMasterKey);
+ } else if (previousMasterKey != null
+ && previousMasterKey.getMasterKey().getKeyId() == keyId) {
+ addKeyForContainerId(tokenId.getContainerID(), previousMasterKey);
+ }
+ }
+
+ /**
+ * Ensure the startContainer call is not using an older cached key. Will
+ * return false once startContainerSuccessful is called. Does not check
+ * the actual key being current since that is verified by the security layer
+ * via retrievePassword.
+ */
+ public synchronized boolean isValidStartContainerRequest(
+ ContainerTokenIdentifier tokenId) {
+ ContainerId containerID = tokenId.getContainerID();
+ ApplicationId applicationId =
+ containerID.getApplicationAttemptId().getApplicationId();
+ return !this.oldMasterKeys.containsKey(applicationId)
+ || !this.oldMasterKeys.get(applicationId).containsKey(containerID);
+ }
+
+ private synchronized void addKeyForContainerId(ContainerId containerId,
+ MasterKeyData masterKeyData) {
+ if (containerId != null) {
+ ApplicationId appId =
+ containerId.getApplicationAttemptId().getApplicationId();
+ if (!this.oldMasterKeys.containsKey(appId)) {
+ this.oldMasterKeys.put(appId,
+ new ConcurrentHashMap<ContainerId, MasterKeyData>());
+ }
+ ConcurrentMap<ContainerId, MasterKeyData> containerIdToKeysMapForThisApp =
+ this.oldMasterKeys.get(appId);
+ containerIdToKeysMapForThisApp.put(containerId, masterKeyData);
+ } else {
+ LOG.warn("Not adding key for null containerId");
+ }
+ }
+
+ // Holding on to master-keys corresponding to containers until the app is
+ // finished due to the multiple ways a container can finish. Avoid
+ // stopContainer calls seeing unnecessary authorization exceptions.
+ public synchronized void appFinished(ApplicationId appId) {
+ this.oldMasterKeys.remove(appId);
+ }
+}
\ No newline at end of file
Modified: hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
URL: http://svn.apache.org/viewvc/hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java?rev=1377183&r1=1377182&r2=1377183&view=diff
==============================================================================
--- hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java (original)
+++ hadoop/common/branches/branch-0.23/hadoop-yarn-project/hadoop-yarn/hadoop-yarn-server/hadoop-yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java Sat Aug 25 02:26:13 2012
@@ -27,8 +27,6 @@ import org.apache.commons.logging.LogFac
import org.apache.hadoop.conf.Configuration;
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
-import org.apache.hadoop.yarn.server.security.ContainerTokenSecretManager;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.ContainerManagerImpl;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.Application;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.application.ApplicationEvent;
@@ -50,6 +48,7 @@ import org.apache.hadoop.yarn.server.nod
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.LogHandler;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.loghandler.event.LogHandlerEvent;
import org.apache.hadoop.yarn.server.nodemanager.metrics.NodeManagerMetrics;
+import org.apache.hadoop.yarn.server.security.ApplicationACLsManager;
public class DummyContainerManager extends ContainerManagerImpl {
@@ -59,11 +58,10 @@ public class DummyContainerManager exten
public DummyContainerManager(Context context, ContainerExecutor exec,
DeletionService deletionContext, NodeStatusUpdater nodeStatusUpdater,
NodeManagerMetrics metrics,
- ContainerTokenSecretManager containerTokenSecretManager,
ApplicationACLsManager applicationACLsManager,
LocalDirsHandlerService dirsHandler) {
super(context, exec, deletionContext, nodeStatusUpdater, metrics,
- containerTokenSecretManager, applicationACLsManager, dirsHandler);
+ applicationACLsManager, dirsHandler);
}
@Override