You are viewing a plain text version of this content. The canonical link for it is here.
Posted to mapreduce-commits@hadoop.apache.org by cd...@apache.org on 2011/04/29 10:35:56 UTC
svn commit: r1097727 [1/5] - in /hadoop/mapreduce/branches/MR-279: ./
mr-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/
yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/
yarn/yarn-common/src/main/ja...
Author: cdouglas
Date: Fri Apr 29 08:35:53 2011
New Revision: 1097727
URL: http://svn.apache.org/viewvc?rev=1097727&view=rev
Log:
Reorient container localization to be per-container rather than per-application
Added:
hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/test/java/org/apache/hadoop/yarn/event/
hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerAction.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerHeartbeatResponse.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerStatus.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/ResourceStatusType.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalizerHeartbeatResponsePBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalizerStatusPBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerLocalization.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerLocalizationImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerInitEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResourceEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResourceFailedEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerResourceLocalizedEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ContainerLocalizer.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourceRequest.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizedResource.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalizerContext.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceState.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ApplicationLocalizationEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizationRequestEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizationEventType.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerResourceRequestEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceEventType.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceLocalizedEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceReleaseEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ResourceRequestEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBLocalizerRPC.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/TestPBRecordImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/TestContainer.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalResourceStatus.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalizerHeartbeatResponse.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/MockLocalizerStatus.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestContainerLocalizer.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalizedResource.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestResourceLocalizationService.java
Removed:
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/FailedLocalizationRequest.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/FailedLocalizationResponse.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/SuccessfulLocalizationRequest.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/SuccessfulLocalizationResponse.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/FailedLocalizationRequestPBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/FailedLocalizationResponsePBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/SuccessfulLocalizationRequestPBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/SuccessfulLocalizationResponsePBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/AppLocalizationRunner.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/AppLocalizationRunnerImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ApplicationLocalizer.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResource.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ApplicationLocalizerEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/ContainerLocalizerEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/SyntheticContainerLaunch.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestApplicationLocalizer.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResources.java
Modified:
hadoop/mapreduce/branches/MR-279/CHANGES.txt
hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.h
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/main.c
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/LocalizationProtocol.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/service/LocalizationProtocolPBServiceImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/ContainerManagerImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/Application.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/application/ApplicationInitedEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/Container.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerEventType.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/container/ContainerImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainerLaunch.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/launcher/ContainersLauncher.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/FSDownload.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTracker.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/LocalResourcesTrackerImpl.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/ResourceLocalizationService.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerEvent.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/event/LocalizerEventType.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/webapp/ContainerLogsPage.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/proto/LocalizationProtocol.proto
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/proto/yarn_server_nodemanager_service_protos.proto
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/DummyContainerManager.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestDefaultContainerExecutor.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestRPCFactories.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/TestRecordFactory.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/TestContainerManager.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestFSDownload.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/containermanager/localizer/TestLocalResource.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/test/java/org/apache/hadoop/yarn/server/nodemanager/webapp/TestNMWebServer.java
hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-tests/src/test/java/org/apache/hadoop/yarn/server/MiniYARNCluster.java
Modified: hadoop/mapreduce/branches/MR-279/CHANGES.txt
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/CHANGES.txt?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/CHANGES.txt (original)
+++ hadoop/mapreduce/branches/MR-279/CHANGES.txt Fri Apr 29 08:35:53 2011
@@ -4,6 +4,9 @@ Trunk (unreleased changes)
MAPREDUCE-279
+ Reorient container localization to be per-container rather than
+ per-application. (cdouglas)
+
Completing the ZooKeeper Store for ResourceManager state. (mahadev)
Added support High-RAM applications in CapacityScheduler. (acmurthy)
Modified: hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java (original)
+++ hadoop/mapreduce/branches/MR-279/mr-client/hadoop-mapreduce-client-shuffle/src/main/java/org/apache/hadoop/mapred/ShuffleHandler.java Fri Apr 29 08:35:53 2011
@@ -92,7 +92,7 @@ import org.apache.hadoop.yarn.api.record
import org.apache.hadoop.yarn.factory.providers.RecordFactoryProvider;
import org.apache.hadoop.yarn.server.nodemanager.NMConfig;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.AuxServices;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ApplicationLocalizer;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
import org.apache.hadoop.yarn.service.AbstractService;
import org.apache.hadoop.yarn.util.ConverterUtils;
import org.apache.hadoop.mapreduce.security.token.JobTokenIdentifier;
@@ -350,8 +350,8 @@ public class ShuffleHandler extends Abst
appID.setClusterTimestamp(Long.parseLong(jobID.getJtIdentifier()));
appID.setId(jobID.getId());
final String base =
- ApplicationLocalizer.USERCACHE + "/" + user + "/"
- + ApplicationLocalizer.APPCACHE + "/"
+ ContainerLocalizer.USERCACHE + "/" + user + "/"
+ + ContainerLocalizer.APPCACHE + "/"
+ ConverterUtils.toString(appID) + "/output" + "/" + mapId;
LOG.debug("DEBUG0 " + base);
// Index file
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/ContainerLaunchContextPBImpl.java Fri Apr 29 08:35:53 2011
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.yarn.api.records.impl.pb;
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-api/src/main/java/org/apache/hadoop/yarn/api/records/impl/pb/LocalResourcePBImpl.java Fri Apr 29 08:35:53 2011
@@ -1,3 +1,20 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.yarn.api.records.impl.pb;
@@ -33,7 +50,7 @@ public class LocalResourcePBImpl extends
}
public LocalResourceProto getProto() {
- mergeLocalToProto();
+ mergeLocalToProto();
proto = viaProto ? proto : builder.build();
viaProto = true;
return proto;
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/main/java/org/apache/hadoop/yarn/event/AsyncDispatcher.java Fri Apr 29 08:35:53 2011
@@ -33,29 +33,33 @@ import org.apache.hadoop.yarn.service.Ab
* Dispatches events in a separate thread. Currently only single thread does
* that. Potentially there could be multiple channels for each event type
* class and a thread pool can be used to dispatch the events.
- *
*/
+@SuppressWarnings("rawtypes")
public class AsyncDispatcher extends AbstractService implements Dispatcher {
private static final Log LOG = LogFactory.getLog(AsyncDispatcher.class);
- private BlockingQueue<Event> eventQueue = new LinkedBlockingQueue<Event>();
+ private final BlockingQueue<Event> eventQueue;
private volatile boolean stopped = false;
private Thread eventHandlingThread;
- protected Map<Class<? extends Enum>, EventHandler> eventDispatchers =
- new HashMap<Class<? extends Enum>, EventHandler>();
+ protected final Map<Class<? extends Enum>, EventHandler> eventDispatchers;
public AsyncDispatcher() {
- super("Dispatcher");
+ this(new HashMap<Class<? extends Enum>, EventHandler>(),
+ new LinkedBlockingQueue<Event>());
}
- @Override
- public void start() {
- //start all the components
- super.start();
+ AsyncDispatcher(
+ Map<Class<? extends Enum>, EventHandler> eventDispatchers,
+ BlockingQueue<Event> eventQueue) {
+ super("Dispatcher");
+ this.eventQueue = eventQueue;
+ this.eventDispatchers = eventDispatchers;
+ }
- eventHandlingThread = new Thread(new Runnable() {
+ Runnable createThread() {
+ return new Runnable() {
@Override
public void run() {
while (!stopped && !Thread.currentThread().isInterrupted()) {
@@ -71,7 +75,14 @@ public class AsyncDispatcher extends Abs
}
}
}
- });
+ };
+ }
+
+ @Override
+ public void start() {
+ //start all the components
+ super.start();
+ eventHandlingThread = new Thread(createThread());
eventHandlingThread.start();
}
@@ -89,6 +100,7 @@ public class AsyncDispatcher extends Abs
super.stop();
}
+ @SuppressWarnings("unchecked")
protected void dispatch(Event event) {
//all events go thru this loop
LOG.info("Dispatching the event " + event.toString());
@@ -104,8 +116,8 @@ public class AsyncDispatcher extends Abs
}
}
- @SuppressWarnings("rawtypes")
@Override
+ @SuppressWarnings("rawtypes")
public void register(Class<? extends Enum> eventType,
EventHandler handler) {
/* check to see if we have a listener registered */
@@ -131,20 +143,16 @@ public class AsyncDispatcher extends Abs
@Override
public EventHandler getEventHandler() {
- // TODO Auto-generated method stub
return new GenericEventHandler();
-
}
- public class GenericEventHandler implements EventHandler<Event> {
- @Override
+ class GenericEventHandler implements EventHandler<Event> {
public void handle(Event event) {
/* all this method does is enqueue all the events onto the queue */
eventQueue.offer(event);
- }
+ };
}
-
/**
* Multiplexing an event. Sending it to different handlers that
* are interested in the event.
@@ -170,4 +178,4 @@ public class AsyncDispatcher extends Abs
}
}
-}
\ No newline at end of file
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java?rev=1097727&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-common/src/test/java/org/apache/hadoop/yarn/event/DrainDispatcher.java Fri Apr 29 08:35:53 2011
@@ -0,0 +1,86 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.event;
+
+import java.util.HashMap;
+import java.util.concurrent.BlockingQueue;
+import java.util.concurrent.LinkedBlockingQueue;
+import java.util.concurrent.TimeUnit;
+
+@SuppressWarnings("rawtypes")
+public class DrainDispatcher extends AsyncDispatcher {
+
+// flagrant initialize abuse throughout, but safe per
+// http://java.sun.com/docs/books/jls/third_edition/html/typesValues.html#96595
+// and similar grotesqueries
+ private volatile boolean drained = false;
+ private final BlockingQueue<Event> queue;
+
+ public DrainDispatcher() {
+ this(new LinkedBlockingQueue<Event>());
+ }
+
+ private DrainDispatcher(BlockingQueue<Event> eventQueue) {
+ super(new HashMap<Class<? extends Enum>, EventHandler>(), eventQueue);
+ this.queue = eventQueue;
+ }
+
+ /**
+ * Busy loop waiting for all queued events to drain.
+ */
+ public void await() {
+ while (!drained) {
+ Thread.yield();
+ }
+ }
+
+ @Override
+ Runnable createThread() {
+ return new Runnable() {
+ @Override
+ public void run() {
+ while (!Thread.currentThread().isInterrupted()) {
+ // !drained if dispatch queued new events on this dispatcher
+ drained = queue.isEmpty();
+ Event event;
+ try {
+ event = queue.take();
+ } catch(InterruptedException ie) {
+ return;
+ }
+ if (event != null) {
+ dispatch(event);
+ }
+ }
+ }
+ };
+ }
+
+ @Override
+ public EventHandler getEventHandler() {
+ final EventHandler actual = super.getEventHandler();
+ return new EventHandler() {
+ @Override
+ public void handle(Event event) {
+ drained = false;
+ actual.handle(event);
+ }
+ };
+ }
+
+}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.c Fri Apr 29 08:35:53 2011
@@ -648,8 +648,7 @@ int initialize_user(const char *user) {
* Function to prepare the job directories for the task JVM.
*/
int initialize_job(const char *user, const char *jobid,
- const char* credentials, const char* job_xml,
- char* const* args) {
+ const char* credentials, char* const* args) {
if (jobid == NULL || user == NULL) {
fprintf(LOGFILE, "Either jobid is null or the user passed is null.\n");
return INVALID_ARGUMENT_NUMBER;
@@ -678,12 +677,6 @@ int initialize_job(const char *user, con
return -1;
}
- // open up jobFiles
- int job_file = open_file_as_task_tracker(job_xml);
- if (job_file == -1) {
- return -1;
- }
-
// give up root privs
if (change_user(user_detail->pw_uid, user_detail->pw_gid) != 0) {
return -1;
@@ -725,15 +718,7 @@ int initialize_job(const char *user, con
if (copy_file(cred_file, credentials, cred_file_name, S_IRUSR|S_IWUSR) != 0){
return -1;
}
- char *job_file_name = concatenate("%s/%s", "job file", 2,
- primary_job_dir, FILECACHE_FILENAME);
- if (job_file_name == NULL) {
- return -1;
- }
- if (copy_file(job_file, job_xml, job_file_name,
- S_IRUSR|S_IWUSR|S_IRGRP) != 0) {
- return -1;
- }
+
fclose(stdin);
fflush(LOGFILE);
if (LOGFILE != stdout) {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.h
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.h?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.h (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/container-executor.h Fri Apr 29 08:35:53 2011
@@ -85,8 +85,7 @@ int delete_log_directory(const char *log
// initialize the job directory
int initialize_job(const char *user, const char *jobid,
- const char *credentials,
- const char *job_xml, char* const* args);
+ const char *credentials, char* const* args);
// run the task as the user
int run_task_as_user(const char * user, const char *jobid, const char *taskid,
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/main.c
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/main.c?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/main.c (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/c/container-executor/impl/main.c Fri Apr 29 08:35:53 2011
@@ -37,7 +37,7 @@ void display_usage(FILE *stream) {
fprintf(stream,
"Usage: container-executor user command command-args\n");
fprintf(stream, "Commands:\n");
- fprintf(stream, " initialize job: %2d jobid jobTokens jobFiles cmd args\n",
+ fprintf(stream, " initialize job: %2d jobid tokens cmd args\n",
INITIALIZE_JOB);
fprintf(stream, " launch task: %2d jobid taskid workdir task-script jobTokens\n",
LAUNCH_TASK_JVM);
@@ -63,7 +63,6 @@ int main(int argc, char **argv) {
const char * cred_file = NULL;
const char * script_file = NULL;
const char * current_dir = NULL;
- const char * job_xml = NULL;
int exit_code = 0;
@@ -129,16 +128,15 @@ int main(int argc, char **argv) {
switch (command) {
case INITIALIZE_JOB:
- if (argc < 7) {
- fprintf(LOGFILE, "Too few arguments (%d vs 7) for initialize job\n",
+ if (argc < 6) {
+ fprintf(LOGFILE, "Too few arguments (%d vs 6) for initialize job\n",
argc);
return INVALID_ARGUMENT_NUMBER;
}
job_id = argv[optind++];
cred_file = argv[optind++];
- job_xml = argv[optind++];
exit_code = initialize_job(user_detail->pw_name, job_id, cred_file,
- job_xml, argv + optind);
+ argv + optind);
break;
case LAUNCH_TASK_JVM:
if (argc < 8) {
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/ContainerExecutor.java Fri Apr 29 08:35:53 2011
@@ -20,6 +20,8 @@ package org.apache.hadoop.yarn.server.no
import java.io.IOException;
import java.lang.reflect.Field;
+
+import java.net.InetSocketAddress;
import java.util.List;
import java.util.concurrent.ConcurrentHashMap;
import java.util.concurrent.ConcurrentMap;
@@ -32,7 +34,6 @@ import org.apache.hadoop.fs.Path;
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
import org.apache.hadoop.yarn.api.records.ContainerId;
-import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
public abstract class ContainerExecutor implements Configurable {
@@ -60,7 +61,6 @@ public abstract class ContainerExecutor
* For $x in local.dirs
* create $x/$user/$appId
* Copy $nmLocal/appTokens -> $N/$user/$appId
- * Copy $nmLocal/publicEnv.sh -> $N/$user/$appId
* For $rsrc in private resources
* Copy $rsrc -> $N/$user/filecache/[idef]
* For $rsrc in job resources
@@ -72,10 +72,10 @@ public abstract class ContainerExecutor
* @throws IOException For most application init failures
* @throws InterruptedException If application init thread is halted by NM
*/
- public abstract void initApplication(Path nmLocal,
- LocalizationProtocol localization,
- String user, String appId, Path logDir, List<Path> localDirs)
- throws IOException, InterruptedException;
+ public abstract void startLocalizer(Path nmLocal,
+ InetSocketAddress nmAddr, String user, String appId, String locId,
+ Path logDir, List<Path> localDirs)
+ throws IOException, InterruptedException;
/**
* Launch the container on the node. This is a blocking call and returns only
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DefaultContainerExecutor.java Fri Apr 29 08:35:53 2011
@@ -31,13 +31,10 @@ import org.apache.hadoop.fs.UnsupportedF
import org.apache.hadoop.fs.permission.FsPermission;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-import org.apache.hadoop.yarn.YarnException;
-import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ApplicationLocalizer;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
-
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.util.ConverterUtils;
public class DefaultContainerExecutor extends ContainerExecutor {
@@ -59,21 +56,12 @@ public class DefaultContainerExecutor ex
}
@Override
- public void initApplication(Path nmLocal, LocalizationProtocol localization,
- String user, String appId, Path logDir, List<Path> localDirs)
- throws IOException, InterruptedException {
- // TODO need a type
- InetSocketAddress nmAddr =
- ((ResourceLocalizationService) localization).getAddress();
- FileContext appContext;
- try {
- appContext = FileContext.getLocalFSFileContext();
- } catch (UnsupportedFileSystemException e) {
- throw new YarnException("Failed to get local context", e);
- }
+ public void startLocalizer(Path nmLocal, InetSocketAddress nmAddr,
+ String user, String appId, String locId, Path logDir,
+ List<Path> localDirs) throws IOException, InterruptedException {
- ApplicationLocalizer localizer =
- new ApplicationLocalizer(appContext, user, appId, logDir, localDirs);
+ ContainerLocalizer localizer =
+ new ContainerLocalizer(user, appId, locId, logDir, localDirs);
createUserLocalDirs(localDirs, user);
createUserCacheDirs(localDirs, user);
@@ -81,12 +69,10 @@ public class DefaultContainerExecutor ex
createAppLogDir(logDir, appId);
Path appStorageDir = getApplicationDir(localDirs, user, appId);
- Path filedesc = new Path(nmLocal, ApplicationLocalizer.FILECACHE_FILE);
- lfs.util().copy(filedesc,
- new Path(appStorageDir, ApplicationLocalizer.FILECACHE_FILE));
- Path appTokens = new Path(nmLocal, ApplicationLocalizer.APPTOKEN_FILE);
- Path tokenDst = new Path(appStorageDir, ApplicationLocalizer.APPTOKEN_FILE);
+ String tokenFn = String.format(ContainerLocalizer.TOKEN_FILE_FMT, locId);
+ Path appTokens = new Path(nmLocal, tokenFn);
+ Path tokenDst = new Path(appStorageDir, tokenFn);
lfs.util().copy(appTokens, tokenDst);
lfs.setWorkingDirectory(appStorageDir);
@@ -100,7 +86,9 @@ public class DefaultContainerExecutor ex
String stderr) throws IOException {
// create container dirs
for (Path p : appDirs) {
- lfs.mkdir(new Path(p, container.toString()), null, false);
+ lfs.mkdir(new Path(p,
+ ConverterUtils.toString(container.getContainerID())),
+ null, true);
}
// copy launch script to work dir
Path appWorkDir = new Path(appDirs.get(0), container.toString());
@@ -108,8 +96,11 @@ public class DefaultContainerExecutor ex
Path launchDst = new Path(appWorkDir, ContainerLaunch.CONTAINER_SCRIPT);
lfs.util().copy(launchScript, launchDst);
// copy container tokens to work dir
- Path appTokens = new Path(nmLocal, ApplicationLocalizer.APPTOKEN_FILE);
- Path tokenDst = new Path(appWorkDir, ApplicationLocalizer.APPTOKEN_FILE);
+ String tokenFn = String.format(
+ ContainerLocalizer.TOKEN_FILE_FMT,
+ ConverterUtils.toString(container.getContainerID()));
+ Path appTokens = new Path(nmLocal, tokenFn);
+ Path tokenDst = new Path(appWorkDir, tokenFn);
lfs.util().copy(appTokens, tokenDst);
// create log dir under app
// fork script
@@ -216,17 +207,17 @@ public class DefaultContainerExecutor ex
}
private Path getUserCacheDir(Path base, String user) {
- return new Path(new Path(base, ApplicationLocalizer.USERCACHE), user);
+ return new Path(new Path(base, ContainerLocalizer.USERCACHE), user);
}
private Path getAppcacheDir(Path base, String user) {
return new Path(getUserCacheDir(base, user),
- ApplicationLocalizer.APPCACHE);
+ ContainerLocalizer.APPCACHE);
}
private Path getFileCacheDir(Path base, String user) {
return new Path(getUserCacheDir(base, user),
- ApplicationLocalizer.FILECACHE);
+ ContainerLocalizer.FILECACHE);
}
/**
@@ -339,7 +330,7 @@ public class DefaultContainerExecutor ex
// create $local.dir/usercache/$user/appcache/$appId/work
Path workDir =
new Path(getApplicationDir(localDirs, user, appId),
- ApplicationLocalizer.WORKDIR);
+ ContainerLocalizer.WORKDIR);
lfs.mkdir(workDir, null, true);
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/DeletionService.java Fri Apr 29 08:35:53 2011
@@ -40,7 +40,7 @@ public class DeletionService extends Abs
static final Log LOG = LogFactory.getLog(DeletionService.class);
private final ThreadPoolExecutor sched =
- new ThreadPoolExecutor(1, 4, 60L, SECONDS,
+ new ThreadPoolExecutor(1, DEFAULT_MAX_DELETE_THREADS, 60L, SECONDS,
new LinkedBlockingQueue<Runnable>());
private final ContainerExecutor exec;
private final FileContext lfs = getLfs();
@@ -69,8 +69,10 @@ public class DeletionService extends Abs
@Override
public void init(Configuration conf) {
- sched.setMaximumPoolSize(
- conf.getInt(NM_MAX_DELETE_THREADS, DEFAULT_MAX_DELETE_THREADS));
+ if (conf != null) {
+ sched.setMaximumPoolSize(
+ conf.getInt(NM_MAX_DELETE_THREADS, DEFAULT_MAX_DELETE_THREADS));
+ }
super.init(conf);
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/LinuxContainerExecutor.java Fri Apr 29 08:35:53 2011
@@ -31,11 +31,10 @@ import org.apache.hadoop.conf.Configurat
import org.apache.hadoop.fs.Path;
import org.apache.hadoop.util.Shell.ExitCodeException;
import org.apache.hadoop.util.Shell.ShellCommandExecutor;
-import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.container.Container;
import org.apache.hadoop.yarn.server.nodemanager.containermanager.launcher.ContainerLaunch;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ApplicationLocalizer;
-import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ResourceLocalizationService;
+import org.apache.hadoop.yarn.server.nodemanager.containermanager.localizer.ContainerLocalizer;
+import org.apache.hadoop.yarn.util.ConverterUtils;
public class LinuxContainerExecutor extends ContainerExecutor {
@@ -101,29 +100,26 @@ public class LinuxContainerExecutor exte
}
@Override
- public void initApplication(Path nmLocal, LocalizationProtocol localization,
- String user, String appId, Path logDir, List<Path> localDirs)
- throws IOException, InterruptedException {
- // TODO need a type
- InetSocketAddress nmAddr = ((ResourceLocalizationService)localization).getAddress();
- Path appFiles = new Path(nmLocal, ApplicationLocalizer.FILECACHE_FILE);
- Path appTokens = new Path(nmLocal, ApplicationLocalizer.APPTOKEN_FILE);
+ public void startLocalizer(Path nmLocal, InetSocketAddress nmAddr,
+ String user, String appId, String locId, Path logDir,
+ List<Path> localDirs) throws IOException, InterruptedException {
+ Path appTokens = new Path(nmLocal, String.format(
+ ContainerLocalizer.TOKEN_FILE_FMT, locId));
List<String> command = new ArrayList<String>(
Arrays.asList(containerExecutorExe,
user,
Integer.toString(Commands.INITIALIZE_JOB.getValue()),
appId,
- appTokens.toUri().getPath().toString(),
- appFiles.toUri().getPath().toString()));
+ appTokens.toUri().getPath().toString()));
File jvm = // use same jvm as parent
new File(new File(System.getProperty("java.home"), "bin"), "java");
command.add(jvm.toString());
command.add("-classpath");
command.add(System.getProperty("java.class.path"));
- command.add(ApplicationLocalizer.class.getName());
+ command.add(ContainerLocalizer.class.getName());
command.add(user);
command.add(appId);
- // add the task tracker's reporting address
+ command.add(locId);
command.add(nmAddr.getHostName());
command.add(Integer.toString(nmAddr.getPort()));
command.add(logDir.toUri().getPath().toString());
@@ -156,7 +152,9 @@ public class LinuxContainerExecutor exte
String stderr) throws IOException {
Path appWorkDir = new Path(appDirs.get(0), container.toString());
Path launchScript = new Path(nmLocal, ContainerLaunch.CONTAINER_SCRIPT);
- Path appToken = new Path(nmLocal, ApplicationLocalizer.APPTOKEN_FILE);
+ Path appToken = new Path(nmLocal, String.format(
+ ContainerLocalizer.TOKEN_FILE_FMT,
+ ConverterUtils.toString(container.getContainerID())));
List<String> command = new ArrayList<String>(
Arrays.asList(containerExecutorExe,
user,
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/LocalizationProtocol.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/LocalizationProtocol.java?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/LocalizationProtocol.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/LocalizationProtocol.java Fri Apr 29 08:35:53 2011
@@ -1,12 +1,27 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.yarn.server.nodemanager.api;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.FailedLocalizationRequest;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.FailedLocalizationResponse;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.SuccessfulLocalizationRequest;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.SuccessfulLocalizationResponse;
+import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHeartbeatResponse;
+import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus;
public interface LocalizationProtocol {
- public SuccessfulLocalizationResponse successfulLocalization(SuccessfulLocalizationRequest request) throws YarnRemoteException;
- public FailedLocalizationResponse failedLocalization(FailedLocalizationRequest request) throws YarnRemoteException;
+ public LocalizerHeartbeatResponse heartbeat(LocalizerStatus status)
+ throws YarnRemoteException;
}
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/client/LocalizationProtocolPBClientImpl.java Fri Apr 29 08:35:53 2011
@@ -1,6 +1,24 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.yarn.server.nodemanager.api.impl.pb.client;
import java.io.IOException;
+
import java.lang.reflect.UndeclaredThrowableException;
import java.net.InetSocketAddress;
@@ -9,17 +27,12 @@ import org.apache.hadoop.ipc.RPC;
import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
import org.apache.hadoop.yarn.ipc.ProtoOverHadoopRpcEngine;
import org.apache.hadoop.yarn.proto.LocalizationProtocol.LocalizationProtocolService;
-import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.FailedLocalizationRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.SuccessfulLocalizationRequestProto;
import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.FailedLocalizationRequest;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.FailedLocalizationResponse;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.SuccessfulLocalizationRequest;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.SuccessfulLocalizationResponse;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb.FailedLocalizationRequestPBImpl;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb.FailedLocalizationResponsePBImpl;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb.SuccessfulLocalizationRequestPBImpl;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb.SuccessfulLocalizationResponsePBImpl;
+import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHeartbeatResponse;
+import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerStatus;
+import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb.LocalizerHeartbeatResponsePBImpl;
+import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb.LocalizerStatusPBImpl;
+import static org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalizerStatusProto;
import com.google.protobuf.ServiceException;
@@ -34,28 +47,12 @@ public class LocalizationProtocolPBClien
}
@Override
- public SuccessfulLocalizationResponse successfulLocalization(
- SuccessfulLocalizationRequest request) throws YarnRemoteException {
- SuccessfulLocalizationRequestProto requestProto = ((SuccessfulLocalizationRequestPBImpl)request).getProto();
- try {
- return new SuccessfulLocalizationResponsePBImpl(proxy.successfulLocalization(null, requestProto));
- } catch (ServiceException e) {
- if (e.getCause() instanceof YarnRemoteException) {
- throw (YarnRemoteException)e.getCause();
- } else if (e.getCause() instanceof UndeclaredThrowableException) {
- throw (UndeclaredThrowableException)e.getCause();
- } else {
- throw new UndeclaredThrowableException(e);
- }
- }
- }
-
- @Override
- public FailedLocalizationResponse failedLocalization(
- FailedLocalizationRequest request) throws YarnRemoteException {
- FailedLocalizationRequestProto requestProto = ((FailedLocalizationRequestPBImpl)request).getProto();
+ public LocalizerHeartbeatResponse heartbeat(LocalizerStatus status)
+ throws YarnRemoteException {
+ LocalizerStatusProto statusProto = ((LocalizerStatusPBImpl)status).getProto();
try {
- return new FailedLocalizationResponsePBImpl(proxy.failedLocalization(null, requestProto));
+ return new LocalizerHeartbeatResponsePBImpl(
+ proxy.heartbeat(null, statusProto));
} catch (ServiceException e) {
if (e.getCause() instanceof YarnRemoteException) {
throw (YarnRemoteException)e.getCause();
Modified: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/service/LocalizationProtocolPBServiceImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/service/LocalizationProtocolPBServiceImpl.java?rev=1097727&r1=1097726&r2=1097727&view=diff
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/service/LocalizationProtocolPBServiceImpl.java (original)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/impl/pb/service/LocalizationProtocolPBServiceImpl.java Fri Apr 29 08:35:53 2011
@@ -1,22 +1,35 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
package org.apache.hadoop.yarn.server.nodemanager.api.impl.pb.service;
-import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
-import org.apache.hadoop.yarn.proto.LocalizationProtocol.LocalizationProtocolService.BlockingInterface;
-import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.FailedLocalizationRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.FailedLocalizationResponseProto;
-import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.SuccessfulLocalizationRequestProto;
-import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.SuccessfulLocalizationResponseProto;
-import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.FailedLocalizationResponse;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.SuccessfulLocalizationResponse;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb.FailedLocalizationRequestPBImpl;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb.FailedLocalizationResponsePBImpl;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb.SuccessfulLocalizationRequestPBImpl;
-import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb.SuccessfulLocalizationResponsePBImpl;
+import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb.LocalizerHeartbeatResponsePBImpl;
+import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb.LocalizerStatusPBImpl;
import com.google.protobuf.RpcController;
import com.google.protobuf.ServiceException;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.proto.LocalizationProtocol.LocalizationProtocolService.BlockingInterface;
+import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalizerHeartbeatResponseProto;
+import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalizerStatusProto;
+import org.apache.hadoop.yarn.server.nodemanager.api.LocalizationProtocol;
+import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalizerHeartbeatResponse;
+
public class LocalizationProtocolPBServiceImpl implements BlockingInterface {
private LocalizationProtocol real;
@@ -26,26 +39,12 @@ public class LocalizationProtocolPBServi
}
@Override
- public SuccessfulLocalizationResponseProto successfulLocalization(
- RpcController controller, SuccessfulLocalizationRequestProto proto)
- throws ServiceException {
- SuccessfulLocalizationRequestPBImpl request = new SuccessfulLocalizationRequestPBImpl(proto);
- try {
- SuccessfulLocalizationResponse response = real.successfulLocalization(request);
- return ((SuccessfulLocalizationResponsePBImpl)response).getProto();
- } catch (YarnRemoteException e) {
- throw new ServiceException(e);
- }
- }
-
- @Override
- public FailedLocalizationResponseProto failedLocalization(
- RpcController controller, FailedLocalizationRequestProto proto)
- throws ServiceException {
- FailedLocalizationRequestPBImpl request = new FailedLocalizationRequestPBImpl(proto);
+ public LocalizerHeartbeatResponseProto heartbeat(RpcController controller,
+ LocalizerStatusProto proto) throws ServiceException {
+ LocalizerStatusPBImpl request = new LocalizerStatusPBImpl(proto);
try {
- FailedLocalizationResponse response = real.failedLocalization(request);
- return ((FailedLocalizationResponsePBImpl)response).getProto();
+ LocalizerHeartbeatResponse response = real.heartbeat(request);
+ return ((LocalizerHeartbeatResponsePBImpl)response).getProto();
} catch (YarnRemoteException e) {
throw new ServiceException(e);
}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java?rev=1097727&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalResourceStatus.java Fri Apr 29 08:35:53 2011
@@ -0,0 +1,36 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords;
+
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+
+public interface LocalResourceStatus {
+ public LocalResource getResource();
+ public ResourceStatusType getStatus();
+ public URL getLocalPath();
+ public long getLocalSize();
+ public YarnRemoteException getException();
+
+ public void setResource(LocalResource resource);
+ public void setStatus(ResourceStatusType status);
+ public void setLocalPath(URL localPath);
+ public void setLocalSize(long size);
+ public void setException(YarnRemoteException exception);
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerAction.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerAction.java?rev=1097727&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerAction.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerAction.java Fri Apr 29 08:35:53 2011
@@ -0,0 +1,22 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords;
+
+public enum LocalizerAction {
+ LIVE, DIE
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerHeartbeatResponse.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerHeartbeatResponse.java?rev=1097727&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerHeartbeatResponse.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerHeartbeatResponse.java Fri Apr 29 08:35:53 2011
@@ -0,0 +1,35 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords;
+
+import java.util.List;
+
+import org.apache.hadoop.yarn.api.records.LocalResource;
+
+public interface LocalizerHeartbeatResponse {
+ public LocalizerAction getLocalizerAction();
+ public List<LocalResource> getAllResources();
+ public LocalResource getLocalResource(int i);
+
+ public void setLocalizerAction(LocalizerAction action);
+
+ public void addAllResources(List<LocalResource> resources);
+ public void addResource(LocalResource resource);
+ public void removeResource(int index);
+ public void clearResources();
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerStatus.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerStatus.java?rev=1097727&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerStatus.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/LocalizerStatus.java Fri Apr 29 08:35:53 2011
@@ -0,0 +1,32 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords;
+
+import java.util.List;
+
+public interface LocalizerStatus {
+ public String getLocalizerId();
+ public List<LocalResourceStatus> getResources();
+
+ public void setLocalizerId(String id);
+ public void addAllResources(List<LocalResourceStatus> resources);
+ public void addResourceStatus(LocalResourceStatus resource);
+ public LocalResourceStatus getResourceStatus(int index);
+ public void removeResource(int index);
+ public void clearResources();
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/ResourceStatusType.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/ResourceStatusType.java?rev=1097727&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/ResourceStatusType.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/ResourceStatusType.java Fri Apr 29 08:35:53 2011
@@ -0,0 +1,24 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords;
+
+public enum ResourceStatusType {
+ FETCH_PENDING,
+ FETCH_SUCCESS,
+ FETCH_FAILURE,
+}
Added: hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java
URL: http://svn.apache.org/viewvc/hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java?rev=1097727&view=auto
==============================================================================
--- hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java (added)
+++ hadoop/mapreduce/branches/MR-279/yarn/yarn-server/yarn-server-nodemanager/src/main/java/org/apache/hadoop/yarn/server/nodemanager/api/protocolrecords/impl/pb/LocalResourceStatusPBImpl.java Fri Apr 29 08:35:53 2011
@@ -0,0 +1,224 @@
+/**
+* Licensed to the Apache Software Foundation (ASF) under one
+* or more contributor license agreements. See the NOTICE file
+* distributed with this work for additional information
+* regarding copyright ownership. The ASF licenses this file
+* to you under the Apache License, Version 2.0 (the
+* "License"); you may not use this file except in compliance
+* with the License. You may obtain a copy of the License at
+*
+* http://www.apache.org/licenses/LICENSE-2.0
+*
+* Unless required by applicable law or agreed to in writing, software
+* distributed under the License is distributed on an "AS IS" BASIS,
+* WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied.
+* See the License for the specific language governing permissions and
+* limitations under the License.
+*/
+package org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.impl.pb;
+
+import org.apache.hadoop.yarn.api.records.LocalResource;
+import org.apache.hadoop.yarn.api.records.ProtoBase;
+import org.apache.hadoop.yarn.api.records.URL;
+import org.apache.hadoop.yarn.api.records.impl.pb.LocalResourcePBImpl;
+import org.apache.hadoop.yarn.api.records.impl.pb.URLPBImpl;
+import org.apache.hadoop.yarn.exceptions.YarnRemoteException;
+import org.apache.hadoop.yarn.exceptions.impl.pb.YarnRemoteExceptionPBImpl;
+import org.apache.hadoop.yarn.proto.YarnProtos.LocalResourceProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.URLProto;
+import org.apache.hadoop.yarn.proto.YarnProtos.YarnRemoteExceptionProto;
+import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProto;
+import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.LocalResourceStatusProtoOrBuilder;
+import org.apache.hadoop.yarn.proto.YarnServerNodemanagerServiceProtos.ResourceStatusTypeProto;
+import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.LocalResourceStatus;
+import org.apache.hadoop.yarn.server.nodemanager.api.protocolrecords.ResourceStatusType;
+
+public class LocalResourceStatusPBImpl
+ extends ProtoBase<LocalResourceStatusProto> implements LocalResourceStatus {
+
+ LocalResourceStatusProto proto =
+ LocalResourceStatusProto.getDefaultInstance();
+ LocalResourceStatusProto.Builder builder = null;
+ boolean viaProto = false;
+
+ private LocalResource resource;
+ private URL localPath;
+ private YarnRemoteException exception;
+
+ public LocalResourceStatusPBImpl() {
+ builder = LocalResourceStatusProto.newBuilder();
+ }
+
+ public LocalResourceStatusPBImpl(LocalResourceStatusProto proto) {
+ this.proto = proto;
+ viaProto = true;
+ }
+
+ public LocalResourceStatusProto getProto() {
+ mergeLocalToProto();
+ proto = viaProto ? proto : builder.build();
+ viaProto = true;
+ return proto;
+ }
+
+ private void mergeLocalToBuilder() {
+ if (this.resource != null &&
+ !((LocalResourcePBImpl)this.resource).getProto()
+ .equals(builder.getResource())) {
+ builder.setResource(convertToProtoFormat(this.resource));
+ }
+ if (this.localPath != null &&
+ !((URLPBImpl)this.localPath).getProto()
+ .equals(builder.getLocalPath())) {
+ builder.setLocalPath(convertToProtoFormat(this.localPath));
+ }
+ if (this.exception != null &&
+ !((YarnRemoteExceptionPBImpl)this.exception).getProto()
+ .equals(builder.getException())) {
+ builder.setException(convertToProtoFormat(this.exception));
+ }
+ }
+
+ private void mergeLocalToProto() {
+ if (viaProto)
+ maybeInitBuilder();
+ mergeLocalToBuilder();
+ proto = builder.build();
+ viaProto = true;
+ }
+
+ private void maybeInitBuilder() {
+ if (viaProto || builder == null) {
+ builder = LocalResourceStatusProto.newBuilder(proto);
+ }
+ viaProto = false;
+ }
+
+ @Override
+ public LocalResource getResource() {
+ LocalResourceStatusProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.resource != null) {
+ return this.resource;
+ }
+ if (!p.hasResource()) {
+ return null;
+ }
+ this.resource = convertFromProtoFormat(p.getResource());
+ return this.resource;
+ }
+
+ @Override
+ public ResourceStatusType getStatus() {
+ LocalResourceStatusProtoOrBuilder p = viaProto ? proto : builder;
+ if (!p.hasStatus()) {
+ return null;
+ }
+ return convertFromProtoFormat(p.getStatus());
+ }
+
+ @Override
+ public URL getLocalPath() {
+ LocalResourceStatusProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.localPath != null) {
+ return this.localPath;
+ }
+ if (!p.hasLocalPath()) {
+ return null;
+ }
+ this.localPath = convertFromProtoFormat(p.getLocalPath());
+ return this.localPath;
+ }
+
+ @Override
+ public long getLocalSize() {
+ LocalResourceStatusProtoOrBuilder p = viaProto ? proto : builder;
+ return (p.getLocalSize());
+ }
+
+ @Override
+ public YarnRemoteException getException() {
+ LocalResourceStatusProtoOrBuilder p = viaProto ? proto : builder;
+ if (this.exception != null) {
+ return this.exception;
+ }
+ if (!p.hasException()) {
+ return null;
+ }
+ this.exception = convertFromProtoFormat(p.getException());
+ return this.exception;
+ }
+
+
+ @Override
+ public void setResource(LocalResource resource) {
+ maybeInitBuilder();
+ if (resource == null)
+ builder.clearResource();
+ this.resource = resource;
+ }
+
+ @Override
+ public void setStatus(ResourceStatusType status) {
+ maybeInitBuilder();
+ if (status == null) {
+ builder.clearStatus();
+ return;
+ }
+ builder.setStatus(convertToProtoFormat(status));
+ }
+
+ @Override
+ public void setLocalPath(URL localPath) {
+ maybeInitBuilder();
+ if (localPath == null)
+ builder.clearLocalPath();
+ this.localPath = localPath;
+ }
+
+ @Override
+ public void setLocalSize(long size) {
+ maybeInitBuilder();
+ builder.setLocalSize(size);
+ }
+
+ @Override
+ public void setException(YarnRemoteException exception) {
+ maybeInitBuilder();
+ if (exception == null)
+ builder.clearException();
+ this.exception = exception;
+ }
+
+ private LocalResourceProto convertToProtoFormat(LocalResource rsrc) {
+ return ((LocalResourcePBImpl)rsrc).getProto();
+ }
+
+ private LocalResourcePBImpl convertFromProtoFormat(LocalResourceProto rsrc) {
+ return new LocalResourcePBImpl(rsrc);
+ }
+
+ private URLPBImpl convertFromProtoFormat(URLProto p) {
+ return new URLPBImpl(p);
+ }
+
+ private URLProto convertToProtoFormat(URL t) {
+ return ((URLPBImpl)t).getProto();
+ }
+
+ private ResourceStatusTypeProto convertToProtoFormat(ResourceStatusType e) {
+ return ResourceStatusTypeProto.valueOf(e.name());
+ }
+
+ private ResourceStatusType convertFromProtoFormat(ResourceStatusTypeProto e) {
+ return ResourceStatusType.valueOf(e.name());
+ }
+
+ private YarnRemoteExceptionPBImpl convertFromProtoFormat(YarnRemoteExceptionProto p) {
+ return new YarnRemoteExceptionPBImpl(p);
+ }
+
+ private YarnRemoteExceptionProto convertToProtoFormat(YarnRemoteException t) {
+ return ((YarnRemoteExceptionPBImpl)t).getProto();
+ }
+
+}